MDEV-6247: Merge 10.0-galera to 10.1.

Merged lp:maria/maria-10.0-galera up to revision 3879.

Added a new functions to handler API to forcefully abort_transaction,
producing fake_trx_id, get_checkpoint and set_checkpoint for XA. These
were added for future possiblity to add more storage engines that
could use galera replication.

327 files changed, 28128 insertions, 333 deletions

diff --git a/.bzrignore b/.bzrignore
index d5874ebf0f2..ba23211d3a3 100644
--- a/.bzrignore
+++ b/.bzrignore
@@ -12,6 +12,7 @@
 *.dll
 *.dsp
 *.dylib
+*.diff
 *.exe
 *.exp
 *.gcda
diff --git a/BUILD/SETUP.sh b/BUILD/SETUP.sh
index bde095b0aa8..860560767a7 100755
--- a/BUILD/SETUP.sh
+++ b/BUILD/SETUP.sh
@@ -215,7 +215,7 @@ all_configs="$SSL_LIBRARY --with-plugins=max --with-plugin-ndbcluster --with-emb
 alpha_cflags="$check_cpu_cflags -Wa,-m$cpu_flag"
 amd64_cflags="$check_cpu_cflags"
 amd64_cxxflags=""  # If dropping '--with-big-tables', add here  "-DBIG_TABLES"
-pentium_cflags="$check_cpu_cflags"
+pentium_cflags="$check_cpu_cflags -m32"
 pentium64_cflags="$check_cpu_cflags -m64"
 ppc_cflags="$check_cpu_cflags"
 sparc_cflags=""
diff --git a/BUILD/compile-amd64-debug-wsrep b/BUILD/compile-amd64-debug-wsrep
new file mode 100644
index 00000000000..995a8afcca9
--- /dev/null
+++ b/BUILD/compile-amd64-debug-wsrep
@@ -0,0 +1,11 @@
+#! /bin/sh
+
+path=`dirname $0`
+. "$path/SETUP.sh"
+
+extra_flags="$amd64_cflags $debug_cflags -g -O0 $wsrep_cflags"
+c_warnings="$c_warnings $debug_extra_warnings"
+cxx_warnings="$cxx_warnings $debug_extra_warnings"
+extra_configs="$amd64_configs $debug_configs $wsrep_configs --with-wsrep"
+
+. "$path/FINISH.sh"
diff --git a/BUILD/compile-amd64-wsrep b/BUILD/compile-amd64-wsrep
new file mode 100644
index 00000000000..57dfbdd6da2
--- /dev/null
+++ b/BUILD/compile-amd64-wsrep
@@ -0,0 +1,9 @@
+#! /bin/sh
+
+path=`dirname $0`
+. "$path/SETUP.sh"
+
+extra_flags="$amd64_cflags $fast_cflags -g $wsrep_cflags"
+extra_configs="$amd64_configs $wsrep_configs --with-wsrep"
+
+. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-debug-wsrep b/BUILD/compile-pentium-debug-wsrep
new file mode 100644
index 00000000000..ee68e3fd0c1
--- /dev/null
+++ b/BUILD/compile-pentium-debug-wsrep
@@ -0,0 +1,12 @@
+#! /bin/sh -x
+
+path=`dirname $0`
+set -- "$@" --with-debug=full
+. "$path/SETUP.sh"
+
+extra_flags="$pentium_cflags $debug_cflags -g -O0 $wsrep_cflags"
+c_warnings="$c_warnings $debug_extra_warnings"
+cxx_warnings="$cxx_warnings $debug_extra_warnings"
+extra_configs="$pentium_configs $debug_configs $wsrep_configs --with-wsrep"
+
+. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-wsrep b/BUILD/compile-pentium-wsrep
new file mode 100644
index 00000000000..eeb14310e4e
--- /dev/null
+++ b/BUILD/compile-pentium-wsrep
@@ -0,0 +1,11 @@
+#! /bin/sh
+
+path=`dirname $0`
+. "$path/SETUP.sh"
+
+extra_flags="$pentium_cflags $fast_cflags $wsrep_cflags"
+extra_configs="$pentium_configs $wsrep_configs --with-wsrep"
+
+#strip=yes
+
+. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium64-wsrep b/BUILD/compile-pentium64-wsrep
new file mode 100644
index 00000000000..0bccb34e753
--- /dev/null
+++ b/BUILD/compile-pentium64-wsrep
@@ -0,0 +1,28 @@
+#! /bin/sh
+
+# Copyright (C) 2006, 2007 MySQL AB
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Library General Public
+# License as published by the Free Software Foundation; version 2
+# of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Library General Public License for more details.
+#
+# You should have received a copy of the GNU Library General Public
+# License along with this library; if not, write to the Free
+# Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+# MA 02111-1307, USA
+
+path=`dirname $0`
+. "$path/SETUP.sh"
+
+extra_flags="$pentium64_cflags $fast_cflags -g $wsrep_cflags"
+extra_configs="$pentium_configs $static_link $wsrep_configs --with-wsrep"
+CC="$CC --pipe"
+strip=yes
+
+. "$path/FINISH.sh"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9b5c73b6e6c..68f3b01eb28 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -158,6 +158,7 @@ INCLUDE(ctest)
 INCLUDE(plugin)
 INCLUDE(install_macros)
 INCLUDE(mysql_add_executable)
+INCLUDE(wsrep)
 
 # Handle options
 OPTION(DISABLE_SHARED 
@@ -235,6 +236,12 @@ OPTION(ENABLED_LOCAL_INFILE
 OPTION(WITH_FAST_MUTEXES "Compile with fast mutexes" OFF)
 MARK_AS_ADVANCED(WITH_FAST_MUTEXES)
 
+OPTION(WITH_INNODB_DISALLOW_WRITES "InnoDB freeze writes patch from Google" ${WITH_WSREP})
+IF (WITH_INNODB_DISALLOW_WRITES)
+  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWITH_INNODB_DISALLOW_WRITES")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWITH_INNODB_DISALLOW_WRITES")
+ENDIF()
+
 # Set DBUG_OFF and other optional release-only flags for non-debug project types
 FOREACH(BUILD_TYPE RELEASE RELWITHDEBINFO MINSIZEREL)
   FOREACH(LANG C CXX)
@@ -369,6 +376,9 @@ ADD_SUBDIRECTORY(vio)
 ADD_SUBDIRECTORY(mysys)
 ADD_SUBDIRECTORY(mysys_ssl)
 ADD_SUBDIRECTORY(libmysql)
+IF(WITH_WSREP)
+  ADD_SUBDIRECTORY(wsrep)
+ENDIF()
 ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(extra)
 ADD_SUBDIRECTORY(libservices)
@@ -447,6 +457,7 @@ INSTALL_DOCUMENTATION(${CMAKE_BINARY_DIR}/Docs/INFO_SRC
 
 IF(UNIX)
   INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY COMPONENT Readme)
+  INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY Docs/README-wsrep COMPONENT Readme)
 ENDIF()
 
 INCLUDE(CPack)
diff --git a/Docs/README-wsrep b/Docs/README-wsrep
new file mode 100644
index 00000000000..422ec52f48a
--- /dev/null
+++ b/Docs/README-wsrep
@@ -0,0 +1,488 @@
+Codership Oy
+http://www.codership.com
+<info@codership.com>
+
+DISCLAIMER
+
+THIS SOFTWARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER
+EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+IN NO EVENT SHALL CODERSHIP OY BE HELD LIABLE TO ANY PARTY FOR ANY DAMAGES
+RESULTING DIRECTLY OR INDIRECTLY FROM THE USE OF THIS SOFTWARE.
+
+Trademark Information.
+
+MySQL is a trademark or registered trademark of Oracle and/or its affiliates.
+Other trademarks are the property of their respective owners.
+
+Licensing Information.
+
+Please see file COPYING that came with this distribution
+
+Source code can be found at
+wsrep API:    https://launchpad.net/wsrep
+MySQL patch:  https://launchpad.net/codership-mysql
+
+
+ABOUT THIS DOCUMENT
+
+This document covers installation and configuration issues specific to this
+wsrep-patched MySQL distribution by Codership. It does not cover the use or
+administration of MySQL server per se. The reader is assumed to know how to
+install, configure, administer and use standard MySQL server version 5.1.xx.
+
+
+                        MYSQL-5.5.x/wsrep-23.x
+
+CONTENTS:
+=========
+1. WHAT IS WSREP PATCH FOR MYSQL
+2. INSTALLATION
+3. FIRST TIME SETUP
+   3.1 CONFIGURATION FILES
+   3.2 DATABASE PRIVILEGES
+   3.3 CHECK AND CORRECT FIREWALL SETTINGS
+   3.4 SELINUX
+   3.5 APPARMOR
+   3.6 CONNECT TO CLUSTER
+4. UPGRADING FROM MySQL 5.1.x
+5. CONFIGURATION OPTIONS
+   5.1 MANDATORY MYSQL OPTIONS
+   5.2 WSREP OPTIONS
+6. ONLINE SCHEMA UPGRADE
+   6.1 TOTAL ORDER ISOLATION (TOI)
+   6.2 ROLLING SCHEMA UPGRADE (RSU)
+7. LIMITATIONS
+
+
+1. WHAT IS WSREP PATCH FOR MYSQL/INNODB
+
+Wsrep API developed by Codership Oy is a modern generic (database-agnostic)
+replication API for transactional databases with a goal to make database
+replication/logging subsystem completely modular and pluggable. It is developed
+with flexibility and completeness in mind to satisfy broad range of modern
+replication scenarios. It is equally suitable for synchronous and asynchronous,
+master-slave and multi-master replication.
+
+wsrep stands for Write Set REPlication.
+
+Wsrep patch for MySQL/InnoDB allows MySQL server to load and use various wsrep
+API implementations ("wsrep providers") with different qualities of service.
+Without wsrep provider MySQL-wsrep server will function like a regular
+standalone server.
+
+
+2. INSTALLATION
+
+In the examples below mysql authentication options are omitted for brevity.
+
+2.1 Download and install mysql-wsrep package.
+
+Download binary package for your Linux distribution from
+https://launchpad.net/codership-mysql/
+
+2.1.1 On Debian and Debian-derived distributions.
+
+Upgrade from mysql-server-5.0 to mysql-wsrep is not supported yet, please 
+upgrade to mysql-server-5.1 first.
+
+If you're installing over an existing mysql installation, mysql-server-wsrep
+will conflict with mysql-server-5.1 package, so remove it first:
+
+$ sudo apt-get remove mysql-server-5.1 mysql-server-core-5.1
+
+mysql-server-wsrep requires psmisc and mysql-client-5.1.47 (or later).
+MySQL 5.1 packages can be found from backports repositories.
+For further information about configuring and using Debian or Ubuntu
+backports, see:
+
+* http://backports.debian.org
+
+* https://help.ubuntu.com/community/UbuntuBackports
+
+For example, installation of required packages on Debian Lenny:
+
+$ sudo apt-get install psmisc
+$ sudo apt-get -t lenny-backports install mysql-client-5.1
+
+Now you should be able to install mysql-wsrep package:
+
+$ sudo dpkg -i <mysql-server-wsrep DEB>
+
+2.1.2 On CentOS and similar RPM-based distributions.
+
+If you're migrating from existing MySQL installation, there are two variants:
+
+  a) If you're already using official MySQL-server-community 5.1.x RPM from
+     Oracle:
+
+     # rpm -e mysql-server
+
+  b) If you're upgrading from the stock mysql-5.0.77 on CentOS:
+
+     1) Make sure that the following packages are not installed:
+     # rpm --nodeps --allmatches -e mysql-server mysql-test mysql-bench
+
+     2) Install *official* MySQL-shared-compat-5.1.x from
+        http://dev.mysql.com/downloads/mysql/5.1.html
+
+Actual installation:
+
+   # rpm -Uvh <MySQL-server-wsrep RPM>
+
+   If this fails due to unsatisfied dependencies, install missing packages
+   (e.g. yum install perl-DBI) and retry.
+
+Additional packages to consider (if not yet installed):
+   * galera (multi-master replication provider, https://launchpad.net/galera)
+   * MySQL-client-community (for connecting to server and mysqldump-based SST)
+   * rsync (for rsync-based SST)
+   * xtrabackup and nc (for xtrabackup-based SST)
+
+2.2 Upgrade system tables.
+
+If you're upgrading a previous MySQL installation, it might be advisable to
+upgrade system tables. To do that start mysqld and run mysql_upgrade command.
+Consult MySQL documentation in case of errors. Normally they are not critical
+and can be ignored unless specific functionality is needed.
+
+
+3. FIRST TIME SETUP
+
+Unless you're upgrading an already installed mysql-wsrep package, you will need
+to set up a few things to prepare server for operation.
+
+3.1 CONFIGURATION FILES
+
+* Make sure system-wide my.cnf does not bind mysqld to 127.0.0.1. That is, if
+  you have the following line in [mysqld] section, comment it out:
+
+  #bind-address = 127.0.0.1
+
+* Make sure system-wide my.cnf contains "!includedir /etc/mysql/conf.d/" line.
+
+* Edit /etc/mysql/conf.d/wsrep.cnf and set wsrep_provider option by specifying
+  a path to provider library. If you don't have a provider, leave it as it is.
+
+* When a new node joins the cluster it'll have to receive a state snapshot from
+  one of the peers. This requires a privileged MySQL account with access from
+  the rest of the cluster. Edit /etc/mysql/conf.d/wsrep.cnf and set mysql
+  login/password pair for SST, for example:
+
+  wsrep_sst_auth=wsrep_sst:wspass
+
+* See CONFIGURATION section below about other configuration parameters that you
+  might want to change at this point.
+
+3.2 DATABASE PRIVILEGES
+
+Restart MySQL server and connect to it as root to grant privileges to SST
+account (empty users confuse MySQL authentication matching rules, we need to
+delete them too):
+
+$ mysql -e "SET wsrep_on=OFF; DELETE FROM mysql.user WHERE user='';"
+$ mysql -e "SET wsrep_on=OFF; GRANT ALL ON *.* TO wsrep_sst@'%' IDENTIFIED BY 'wspass'";
+
+3.3 CHECK AND CORRECT FIREWALL SETTINGS.
+
+MySQL-wsrep server needs to be accessible from other cluster members through
+its client listening socket and through wsrep provider socket. See your
+distribution and wsrep provider documentation for details. For example on
+CentOS you might need to do something along these lines:
+
+# iptables --insert RH-Firewall-1-INPUT 1 --proto tcp --source <my IP>/24 --destination <my IP>/32 --dport 3306 -j ACCEPT
+# iptables --insert RH-Firewall-1-INPUT 1 --proto tcp --source <my IP>/24 --destination <my IP>/32 --dport 4567 -j ACCEPT
+
+If there is a NAT firewall between the nodes, it must be configured to allow
+direct connections between the nodes (e.g. via port forwarding).
+
+3.4 SELINUX
+
+If you have SELinux enabled, it may block mysqld from doing required operations.
+You'll need to either disable it or configure to allow mysqld to run external
+programs and open listen sockets at unprivileged ports (i.e. things that
+an unprivileged user can do). See SELinux documentation about it.
+
+To quickly disable SELinux:
+1) run 'setenforce 0' as root.
+2) set 'SELINUX=permissive' in  /etc/selinux/config
+
+3.5 APPARMOR
+
+AppArmor automatically comes with Ubuntu and may also prevent mysqld to from
+opening additional ports or run scripts. See AppArmor documentation about its
+configuration. To disable AppArmor for mysqld:
+
+$ cd /etc/apparmor.d/disable/
+$ sudo ln -s /etc/apparmor.d/usr.sbin.mysqld
+$ sudo service apparmor restart
+
+
+3.6 CONNECT TO CLUSTER
+
+Now you're ready to connect to cluster by setting wsrep_cluster_address variable
+and monitor status of wsrep provider:
+
+mysql> SET GLOBAL wsrep_cluster_address='<cluster address string>';
+mysql> SHOW STATUS LIKE 'wsrep%';
+
+
+4 UPGRADING FROM MySQL 5.1.x
+
+!!! THESE INSTRUCTIONS ARE PRELIMINARY AND INCOMPLETE !!!
+
+1) BEFORE UPGRADE (while running 5.1.x):
+   - comment out 'wsrep_provider' setting from configuration files
+     (my.cnf and/or wsrep.cnf)
+   - If performing a rolling upgrade on a running cluster, set
+     wsrep_sst_method=mysqldump.
+     You might also need to configure wsrep_sst_receive_address and
+     wsrep_sst_auth appropriately. mysqldump is the only way to transfer data
+     from 5.1.x to 5.5.x reliably.
+   - remove innodb_plugin settings from configuration files.
+
+2) Perform upgrade as usual:
+   http://dev.mysql.com/doc/refman/5.5/en/upgrading-from-previous-series.html
+   Don't forget to run 'mysql_upgrade' command.
+
+3) AFTER UPGRADING individual node:
+   - uncomment 'wsrep_provider' line in configuration file.
+   - restart the server and join the cluster.
+
+4) AFTER UPGRADING the whole cluster:
+   - revert to usual wsrep SST settings if not 'mysqldump'.
+
+
+5. CONFIGURATION OPTIONS
+
+5.1 MANDATORY MYSQL OPTIONS
+
+binlog_format=ROW
+   This option is required to use row-level replication as opposed to
+   statement-level. For performance and consistency considerations don't change
+   that. As a side effect, binlog, if turned on, can be ROW only. In future this
+   option won't have special meaning.
+
+innodb_autoinc_lock_mode=2
+   This is a required parameter. Without it INSERTs into tables with
+   AUTO_INCREMENT column may fail.
+   autoinc lock modes 0 and 1 can cause unresolved deadlock, and make
+   system unresponsive.
+
+innodb_locks_unsafe_for_binlog=1
+   This option is required for parallel applying.
+
+5.2 WSREP OPTIONS
+
+All options are optional except for wsrep_provider, wsrep_cluster_address, and
+wsrep_sst_auth.
+
+wsrep_provider=none
+   A full path to the library that implements WSREP interface. If none is
+   specified, the server behaves like a regular mysqld.
+
+wsrep_provider_options=
+   Provider-specific option string. Check wsrep provider documentation or
+   http://www.codership.com/wiki
+
+wsrep_cluster_address=
+   Provider-specific cluster address string. This is used to connect a node to
+   the desired cluster. This option can be given either on mysqld startup or set
+   during runtime. See wsrep provider documentation for possible values.
+
+wsrep_cluster_name="my_wsrep_cluster"
+   Logical cluster name, must be the same for all nodes of the cluster.
+
+wsrep_node_address=
+   An option to explicitly specify the network address of the node in the form
+   <address>[:port] if autoguessing for some reason does not produce desirable
+   results (multiple network interfaces, NAT, etc.)
+   If not explicitly overridden by wsrep_sst_receive_address, the <address> part
+   will be used to listen for SST (see below). And the whole <address>[:port]
+   will be passed to wsrep provider to be used as a base address in its
+   communications.
+
+wsrep_node_name=
+   Human readable node name (for easier log reading only). Defaults to hostname.
+
+wsrep_slave_threads=1
+   Number of threads dedicated to processing of writesets from other nodes.
+   For best performance should be few per CPU core.
+
+wsrep_dbug_option
+   Options for the built-in DBUG library (independent from what MySQL uses).
+   Empty by default. Not currently in use.
+
+wsrep_debug=0
+   Enable debug-level logging.
+
+wsrep_convert_LOCK_to_trx=0
+   Implicitly convert locking sessions into transactions inside mysqld. By
+   itself it does not mean support for locking sessions, but it prevents the
+   database from going into logically inconsistent state. Note however, that
+   loading large database dump with LOCK statements might result in abnormally
+   large transactions and cause an out-of-memory condition
+
+wsrep_retry_autocommit=1
+   Retry autocommit queries and single statement transactions should they fail
+   certification test. This is analogous to rescheduling an autocommit query
+   should it go into deadlock with other transactions in the database lock
+   manager.
+
+wsrep_auto_increment_control=1
+   Automatically adjust auto_increment_increment and auto_increment_offset
+   variables based on the number of nodes in the cluster. Significantly reduces
+   certification conflict rate for INSERTS.
+
+wsrep_drupal_282555_workaround=1
+   MySQL seems to have an obscure bug when INSERT into table with
+   AUTO_INCREMENT column with NULL value for that column can fail with a 
+   duplicate key error. When this option is on, it retries such INSERTs.
+   Required for stable Drupal operation. Documented at:
+      http://bugs.mysql.com/bug.php?id=41984
+      http://drupal.org/node/282555
+
+wsrep_causal_reads=0
+   Enforce strict READ COMMITTED semantics on reads and transactions. May
+   result in additional latencies. It is a session variable.
+
+wsrep_OSU_method=TOI
+   Online Schema Upgrade (OSU) can be performed with two alternative  methods:
+   Total Order Isolation (TOI) runs DDL statement in all cluster nodes in
+   same total order sequence locking the affected table for the duration of the
+   operation. This may result in the whole cluster being blocked for the
+   duration of the operation.
+   Rolling Schema Upgrade (RSU) executes the DDL statement only locally, thus
+   blocking only one cluster node. During the DDL processing, the node
+   is not replicating and may be unable to process replication events (due to
+   table lock). Once DDL operation is complete, the node will catch up and sync
+   with the cluster to become fully operational again. The DDL statement or
+   its effects are not replicated, so it is user's responsibility to manually
+   perform this operation on each of the nodes.
+
+wsrep_forced_binlog_format=none
+   Force every transaction to use given binlog format. When this variable is 
+   set to something else than NONE, all transactions will use the given forced
+   format, regardless of what the client session has specified in binlog_format.
+   Valid choices for wsrep_forced_binlog_format are: ROW, STATEMENT, MIXED and
+   special value NONE, meaning that there is no forced binlog format in effect.
+   This variable was intruduced to support STATEMENT format replication during
+   rolling schema upgrade processing. However, in most cases ROW replication 
+   is valid for asymmetrict schema replication.
+
+State snapshot transfer options.
+
+When a new node joins the cluster it has to synchronize its initial state with
+the other cluster members by transferring state snapshot from one of them.
+The options below govern how this happens and should be set up before attempting
+to join or start a cluster.
+
+wsrep_sst_method=rsync
+   What method to use to copy database state to a newly joined node. Supported
+   methods:
+   - mysqldump:  slow (except for small datasets) but allows for upgrade
+                 between major MySQL versions or InnoDB features.
+   - rsync:      much faster on large datasets (default).
+   - rsync_wan:  same as rsync but with deltaxfer to minimize network traffic.
+   - xtrabackup: very fast and practically non-blocking SST method based on
+                 Percona's xtrabackup tool.
+
+   (for xtrabackup to work the following settings must be present in my.cnf
+    on all nodes:
+      [mysqld]
+      wsrep_sst_auth=root:<root password>
+      datadir=<path to data dir>
+      [client]
+      socket=<path to socket>
+   )
+
+wsrep_sst_receive_address=
+   Address (hostname:port) at which this node wants to receive state snapshot.
+   Defaults to mysqld bind address, and if that is not specified (0.0.0.0) -
+   to the first IP of eth0 + mysqld bind port.
+   NOTE: check that your firewall allows connections to this address from other
+         cluster nodes.
+
+wsrep_sst_auth=
+   Authentication information needed for state transfer. Depends on the state
+   transfer method. For mysqldump-based SST it is
+   <mysql_root_user>:<mysql_root_password>
+   and should be the same on all nodes - it is used to authenticate with both
+   state snapshot receiver and state snapshot donor.
+
+wsrep_sst_donor=
+   A name of the node which should serve as state snapshot donor. This allows
+   to control which node will serve state snapshot request. By default the
+   most suitable node is chosen by wsrep provider. This is the same as given in
+   wsrep_node_name.
+
+
+6. ONLINE SCHEMA UPGRADE
+
+   Schema upgrades mean any data definition statements (DDL statemnents) run
+   for the database. They change the database structure and are non-
+   transactional.
+
+   Release 22.3 brings a new method for performing schema upgrades. User can
+   now choose whether to use the traditional total order isolation or new
+   rolling schema upgrade method. The OSU method choice is done by global
+   parameter: 'wsrep_OSU_method'.
+
+6.1 Total Order Isolation (TOI)
+
+   With earlier releases, DDL processing happened always by Total Order
+   Isolation (TOI) method. With TOI, the DDL was scheduled to be processed in
+   same transaction seqeuncing 'slot' in each cluster node.
+   The processing is secured by locking the affected table from any other use.
+   With TOI method, the whole cluster has part of the database locked for the
+   duration of the DDL processing.
+
+6.2 Rolling Schema Upgrade (RSU)
+
+   Rolling schema upgrade is new DDL processing method, where DDL will be
+   processed locally for the node. The node is disconnected of the replication
+   for the duration of the DDL processing, so that there is only DDL statement
+   processing in the node and it does not block the rest of the cluster. When
+   the DDL processing is complete, the node applies delayed replication events
+   and synchronizes back with the cluster.
+   The DDL can then be executed cluster-wide by running the same DDL statement
+   for each node in turn. When this rolling schema upgrade proceeds, part of
+   the cluster will have old schema structure and part of the cluster will have
+   new schema structure.
+
+
+7. LIMITATIONS
+
+1) Currently replication works only with InnoDB storage engine. Any writes to 
+   tables of other types, including system (mysql.*) tables are not replicated. 
+   However, DDL statements are replicated in statement level, and changes
+   to mysql.* tables will get replicated that way.
+   So, you can safely issue: CREATE USER...,
+   but issuing: INSERT INTO mysql.user..., will not be replicated.
+
+2) DELETE operation is unsupported on tables without primary key. Also rows in
+   tables without primary key may appear in different order on different nodes.
+   As a result SELECT...LIMIT... may return slightly different sets.
+
+3) Unsupported queries:
+    * LOCK/UNLOCK TABLES cannot be supported in multi-master setups.
+    * lock functions (GET_LOCK(), RELEASE_LOCK()... )
+
+4) Query log cannot be directed to table. If you enable query logging,
+   you must forward the log to a file:
+       log_output = FILE
+   Use general_log and general_log_file to choose query logging and the 
+   log file name
+
+5) Maximum allowed transaction size is defined by wsrep_max_ws_rows and
+   wsrep_max_ws_size. Anything bigger (e.g. huge LOAD DATA) will be rejected.
+
+6) Due to cluster level optimistic concurrency control, transaction issuing
+   COMMIT may still be aborted at that stage. There can be two transactions.
+   writing to same rows and committing in separate cluster nodes, and only one
+   of the them can successfully commit. The failing one will be aborted.
+   For cluster level aborts, MySQL/galera cluster gives back deadlock error.
+   code (Error: 1213 SQLSTATE: 40001  (ER_LOCK_DEADLOCK)).
+
+7) XA transactions can not be supported due to possible rollback on commit.
+
diff --git a/client/client_priv.h b/client/client_priv.h
index 656c8fcf32a..ef93818829e 100644
--- a/client/client_priv.h
+++ b/client/client_priv.h
@@ -92,6 +92,7 @@ enum options_client
   OPT_REPORT_PROGRESS,
   OPT_SKIP_ANNOTATE_ROWS_EVENTS,
   OPT_SSL_CRL, OPT_SSL_CRLPATH,
+  OPT_GALERA_SST_MODE,
   OPT_MAX_CLIENT_OPTION /* should be always the last */
 };
 
diff --git a/client/mysql_upgrade.c b/client/mysql_upgrade.c
index 80d57ce9faa..bf33397efdb 100644
--- a/client/mysql_upgrade.c
+++ b/client/mysql_upgrade.c
@@ -523,7 +523,12 @@ static int run_query(const char *query, DYNAMIC_STRING *ds_res,
   int ret;
   File fd;
   char query_file_path[FN_REFLEN];
+#ifdef WITH_WSREP
+  /* Note: wsrep_on=ON implicitly enables binary logging. */
+  const uchar sql_log_bin[]= "SET SQL_LOG_BIN=0, WSREP_ON=OFF;";
+#else
   const uchar sql_log_bin[]= "SET SQL_LOG_BIN=0;";
+#endif /* WITH_WSREP */
 
   DBUG_ENTER("run_query");
   DBUG_PRINT("enter", ("query: %s", query));
diff --git a/client/mysqlcheck.c b/client/mysqlcheck.c
index 57b7ce09ab5..f7ae0783e5e 100644
--- a/client/mysqlcheck.c
+++ b/client/mysqlcheck.c
@@ -731,9 +731,15 @@ static int use_db(char *database)
   DBUG_RETURN(0);
 } /* use_db */
 
+/* Do not send commands to replication slaves. */
 static int disable_binlog()
 {
+#ifdef WITH_WSREP
+  /* Additionally turn off @@wsrep_on to disable implicit binary logging. */
+  const char *stmt= "SET SQL_LOG_BIN=0, WSREP_ON=OFF";
+#else
   const char *stmt= "SET SQL_LOG_BIN=0";
+#endif /* WITH_WSREP */
   return run_query(stmt);
 }
 
diff --git a/client/mysqldump.c b/client/mysqldump.c
index cb4fa022d4f..eba5bd93671 100644
--- a/client/mysqldump.c
+++ b/client/mysqldump.c
@@ -111,6 +111,7 @@ static my_bool  verbose= 0, opt_no_create_info= 0, opt_no_data= 0,
                 opt_slave_apply= 0, 
                 opt_include_master_host_port= 0,
                 opt_events= 0, opt_comments_used= 0,
+                opt_galera_sst_mode= 0,
                 opt_alltspcs=0, opt_notspcs= 0;
 static my_bool insert_pat_inited= 0, debug_info_flag= 0, debug_check_flag= 0;
 static ulong opt_max_allowed_packet, opt_net_buffer_length;
@@ -346,6 +347,14 @@ static struct my_option my_long_options[] =
   {"force", 'f', "Continue even if we get an SQL error.",
    &ignore_errors, &ignore_errors, 0, GET_BOOL, NO_ARG,
    0, 0, 0, 0, 0, 0},
+  {"galera-sst-mode", OPT_GALERA_SST_MODE,
+   "This mode should normally be used in mysqldump snapshot state transfer "
+   "(SST) in a Galera cluster. If enabled, mysqldump additionally dumps "
+   "commands to turn off binary logging and SET global gtid_binlog_state "
+   "with the current value. Note: RESET MASTER needs to be executed on the "
+   "server receiving the resulting dump.",
+   &opt_galera_sst_mode, &opt_galera_sst_mode, 0, GET_BOOL, NO_ARG, 0, 0, 0,
+   0, 0, 0},
   {"help", '?', "Display this help message and exit.", 0, 0, 0, GET_NO_ARG,
    NO_ARG, 0, 0, 0, 0, 0, 0},
   {"hex-blob", OPT_HEXBLOB, "Dump binary strings (BINARY, "
@@ -4799,6 +4808,42 @@ static int dump_selected_tables(char *db, char **table_names, int tables)
 } /* dump_selected_tables */
 
 
+/**
+  Add the following statements to the generated dump:
+  a) SET @@session.sql_log_bin=OFF;
+  b) SET @@global.gtid_binlog_state='[N-N-N,...]'
+*/
+static int wsrep_set_sst_cmds(MYSQL *mysql) {
+  MYSQL_RES *res;
+  MYSQL_ROW row;
+
+  if (mysql_get_server_version(mysql) < 100005) {
+    /* @@gtid_binlog_state does not exist. */
+    return 0;
+  }
+
+  if (mysql_query_with_error_report(mysql, &res, "SELECT "
+                                    "@@global.gtid_binlog_state"))
+    return 1;
+
+  if (mysql_num_rows(res) != 1)
+    /* No entry for @@global.gtid_binlog_state, nothing needs to be done. */
+    return 0;
+
+  if (!(row= mysql_fetch_row(res)) || !(char *)row[0])
+    return 1;
+
+  /* first, add a command to turn off binary logging, */
+  fprintf(md_result_file, "SET @@session.sql_log_bin=OFF;\n");
+
+  /* followed by, a command to set global gtid_binlog_state. */
+  fprintf(md_result_file, "SET @@global.gtid_binlog_state='%s';\n",
+          (char*)row[0]);
+
+  mysql_free_result(res);
+  return 0;
+}
+
 static int do_show_master_status(MYSQL *mysql_con, int consistent_binlog_pos)
 {
   MYSQL_ROW row;
@@ -5743,6 +5788,10 @@ int main(int argc, char **argv)
   /* Add 'STOP SLAVE to beginning of dump */
   if (opt_slave_apply && add_stop_slave())
     goto err;
+
+  if (opt_galera_sst_mode && wsrep_set_sst_cmds(mysql))
+    goto err;
+
   if (opt_master_data && do_show_master_status(mysql, consistent_binlog_pos))
     goto err;
   if (opt_slave_data && do_show_slave_status(mysql))
diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake
index 30924120526..d8f659ec5ea 100644
--- a/cmake/cpack_rpm.cmake
+++ b/cmake/cpack_rpm.cmake
@@ -147,9 +147,10 @@ SETA(CPACK_RPM_test_PACKAGE_OBSOLETES
 SETA(CPACK_RPM_test_PACKAGE_PROVIDES
   "MySQL-test")
 
-SETA(CPACK_RPM_server_PACKAGE_REQUIRES 
-   ${CPACK_RPM_PACKAGE_REQUIRES}
-   "MariaDB-client")
+SETA(CPACK_RPM_server_PACKAGE_REQUIRES
+  "${CPACK_RPM_PACKAGE_REQUIRES}"
+  "MariaDB-client" "galera" "rsync" "lsof" "socat" "grep" "gawk" "iproute"
+  "coreutils" "findutils")
 
 SET(CPACK_RPM_server_PRE_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-prein.sh)
 SET(CPACK_RPM_server_PRE_UNINSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-preun.sh)
diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake
index b550695b796..2dc88782937 100644
--- a/cmake/plugin.cmake
+++ b/cmake/plugin.cmake
@@ -153,6 +153,14 @@ MACRO(MYSQL_ADD_PLUGIN)
         ADD_DEPENDENCIES(${target}_embedded GenError)
       ENDIF()
     ENDIF()
+
+    IF (WITH_WSREP)
+      # Set compile definitions for non-embedded plugins
+      LIST(APPEND wsrep_definitions "WITH_WSREP")
+      LIST(APPEND wsrep_definitions "WSREP_PROC_INFO")
+      SET_TARGET_PROPERTIES(${target} PROPERTIES
+        COMPILE_DEFINITIONS "${wsrep_definitions}")
+    ENDIF()
     
     IF(ARG_STATIC_OUTPUT_NAME)
       SET_TARGET_PROPERTIES(${target} PROPERTIES 
@@ -185,8 +193,17 @@ MACRO(MYSQL_ADD_PLUGIN)
     ADD_VERSION_INFO(${target} MODULE SOURCES)
     ADD_LIBRARY(${target} MODULE ${SOURCES}) 
     DTRACE_INSTRUMENT(${target})
+
+    LIST(APPEND dyn_compile_definitions "MYSQL_DYNAMIC_PLUGIN")
+
+    IF (WITH_WSREP)
+      LIST(APPEND dyn_compile_definitions "WITH_WSREP")
+      LIST(APPEND dyn_compile_definitions "WSREP_PROC_INFO")
+    ENDIF()
+
     SET_TARGET_PROPERTIES (${target} PROPERTIES PREFIX ""
-      COMPILE_DEFINITIONS "MYSQL_DYNAMIC_PLUGIN")
+      COMPILE_DEFINITIONS "${dyn_compile_definitions}")
+
     TARGET_LINK_LIBRARIES (${target} mysqlservices ${ARG_LINK_LIBRARIES})
 
     # Plugin uses symbols defined in mysqld executable.
@@ -207,7 +224,8 @@ MACRO(MYSQL_ADD_PLUGIN)
       OUTPUT_NAME "${ARG_MODULE_OUTPUT_NAME}")  
     # Install dynamic library
     IF(ARG_COMPONENT)
-      IF(CPACK_COMPONENTS_ALL AND NOT CPACK_COMPONENTS_ALL MATCHES ${ARG_COMPONENT})
+      IF(CPACK_COMPONENTS_ALL AND
+         NOT CPACK_COMPONENTS_ALL MATCHES ${ARG_COMPONENT})
         SET(CPACK_COMPONENTS_ALL ${CPACK_COMPONENTS_ALL} ${ARG_COMPONENT} PARENT_SCOPE)
         SET(CPACK_RPM_${ARG_COMPONENT}_PACKAGE_REQUIRES "MariaDB-server" PARENT_SCOPE)
 
diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake
new file mode 100644
index 00000000000..7dc797e9a05
--- /dev/null
+++ b/cmake/wsrep.cmake
@@ -0,0 +1,76 @@
+# Copyright (c) 2011, Codership Oy <info@codership.com>.
+# Copyright (c) 2013, Monty Program Ab.
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA 
+
+# We need to generate a proper spec file even without --with-wsrep flag,
+# so WSREP_VERSION is produced regardless
+
+# Set the patch version
+SET(WSREP_PATCH_VERSION "10")
+
+# MariaDB addition: Revision number of the last revision merged from
+# codership branch visible in @@visible_comment.
+# Branch : codership-mysql/5.6
+SET(WSREP_PATCH_REVNO "4123")  # Should be updated on every merge.
+
+# MariaDB addition: Revision number of the last revision merged from
+# Branch : lp:maria/maria-10.0-galera
+SET(WSREP_PATCH_REVNO2 "3867")  # Should be updated on every merge.
+
+# MariaDB: Obtain patch revision number:
+# Update WSREP_PATCH_REVNO if WSREP_REV environment variable is set.
+IF (DEFINED ENV{WSREP_REV})
+  SET(WSREP_PATCH_REVNO $ENV{WSREP_REV})
+ENDIF()
+
+# Obtain wsrep API version
+EXECUTE_PROCESS(
+  COMMAND sh -c "grep WSREP_INTERFACE_VERSION ${MySQL_SOURCE_DIR}/wsrep/wsrep_api.h | cut -d '\"' -f 2"
+  OUTPUT_VARIABLE WSREP_API_VERSION
+  RESULT_VARIABLE RESULT
+)
+#FILE(WRITE "wsrep_config" "Debug: WSREP_API_VERSION result: ${RESULT}\n")
+STRING(REGEX REPLACE "(\r?\n)+$" "" WSREP_API_VERSION "${WSREP_API_VERSION}")
+
+IF(NOT WSREP_PATCH_REVNO)
+  MESSAGE(WARNING "Could not determine bzr revision number, WSREP_VERSION will "
+          "not contain the revision number.")
+  SET(WSREP_VERSION
+      "${WSREP_API_VERSION}.${WSREP_PATCH_VERSION}"
+  )
+ELSE()
+  SET(WSREP_VERSION
+      "${WSREP_API_VERSION}.${WSREP_PATCH_VERSION}.r${WSREP_PATCH_REVNO}"
+  )
+ENDIF()
+
+#
+# Galera library does not compile with windows and solaris
+#
+IF(UNIX)
+OPTION(WITH_WSREP "WSREP replication API (to use, e.g. Galera Replication library)" ON)
+ELSE()
+OPTION(WITH_WSREP "WSREP replication API (to use, e.g. Galera Replication library)" OFF)
+ENDIF()
+
+MACRO (BUILD_WITH_WSREP)   
+  SET(WSREP_C_FLAGS   "-DWITH_WSREP -DWSREP_PROC_INFO -DMYSQL_MAX_VARIABLE_VALUE_LEN=2048")
+  SET(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} ${WSREP_C_FLAGS}")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WSREP_C_FLAGS}")
+  SET(COMPILATION_COMMENT "${COMPILATION_COMMENT}, wsrep_${WSREP_VERSION}")
+  #SET(WITH_EMBEDDED_SERVER OFF CACHE INTERNAL "" FORCE)
+ENDMACRO()
+
+#
diff --git a/debian/dist/Debian/control b/debian/dist/Debian/control
index bbee264ffc3..869e9be032c 100644
--- a/debian/dist/Debian/control
+++ b/debian/dist/Debian/control
@@ -192,7 +192,7 @@ Architecture: any
 Suggests: tinyca, mailx, mariadb-test
 Recommends: libhtml-template-perl
 Pre-Depends: mariadb-common, adduser (>= 3.40), debconf
-Depends: mariadb-client-10.1 (>= ${source:Version}), libdbi-perl, perl (>= 5.6), ${shlibs:Depends}, ${misc:Depends}, psmisc, passwd, lsb-base (>= 3.0-10), mariadb-server-core-10.1 (>= ${binary:Version})
+Depends: mariadb-client-10.1 (>= ${source:Version}), libdbi-perl, perl (>= 5.6), ${shlibs:Depends}, ${misc:Depends}, psmisc, passwd, lsb-base (>= 3.0-10), mariadb-server-core-10.1 (>= ${binary:Version}), galera (>=25.2) rsync, lsof, socat, grep, gawk, iproute, coreutils, findutils
 Provides: mariadb-server, mysql-server, virtual-mysql-server
 Conflicts: mariadb-server (<< ${source:Version}), mysql-server (<< ${source:Version}),
     mysql-server-4.1, mysql-server-5.0, mysql-server-5.1, mysql-server-5.5,
diff --git a/debian/dist/Ubuntu/control b/debian/dist/Ubuntu/control
index ffcd1e5dca6..941f756ea3c 100644
--- a/debian/dist/Ubuntu/control
+++ b/debian/dist/Ubuntu/control
@@ -186,7 +186,7 @@ Architecture: any
 Suggests: tinyca, mailx, mariadb-test
 Recommends: libhtml-template-perl
 Pre-Depends: mariadb-common, adduser (>= 3.40), debconf
-Depends: mariadb-client-10.1 (>= ${source:Version}), libdbi-perl, perl (>= 5.6), ${shlibs:Depends}, ${misc:Depends}, psmisc, passwd, lsb-base (>= 3.0-10), mariadb-server-core-10.1 (>= ${binary:Version})
+Depends: mariadb-client-10.1 (>= ${source:Version}), libdbi-perl, perl (>= 5.6), ${shlibs:Depends}, ${misc:Depends}, psmisc, passwd, lsb-base (>= 3.0-10), mariadb-server-core-10.1 (>= ${binary:Version}), galera (>=25.2) rsync, lsof, socat, grep, gawk, iproute, coreutils, findutils
 Provides: mariadb-server, mysql-server, virtual-mysql-server
 Conflicts: mariadb-server (<< ${source:Version}), mysql-server (<< ${source:Version}),
     mysql-server-4.1, mysql-server-5.0, mysql-server-5.1, mysql-server-5.5,
diff --git a/include/mysql/service_logger.h b/include/mysql/service_logger.h
index 962ab2fbc0b..d14b28da712 100644
--- a/include/mysql/service_logger.h
+++ b/include/mysql/service_logger.h
@@ -71,7 +71,7 @@ extern struct logger_service_st {
   int (*rotate)(LOGGER_HANDLE *log);
 } *logger_service;
 
-#if MYSQL_DYNAMIC_PLUGIN
+#ifdef MYSQL_DYNAMIC_PLUGIN
 
 #define logger_init_mutexes logger_service->logger_init_mutexes
 #define logger_open(path, size_limit, rotations) \
diff --git a/include/mysqld_default_groups.h b/include/mysqld_default_groups.h
index a2e94ddd854..30dfdae1338 100644
--- a/include/mysqld_default_groups.h
+++ b/include/mysqld_default_groups.h
@@ -5,4 +5,7 @@ const char *load_default_groups[]= {
 "mysqld", "server", MYSQL_BASE_VERSION,
 "mariadb", MARIADB_BASE_VERSION,
 "client-server",
+#ifdef WITH_WSREP
+"galera",
+#endif
 0, 0};
diff --git a/include/thr_lock.h b/include/thr_lock.h
index 3f7a5ca988f..2561709285f 100644
--- a/include/thr_lock.h
+++ b/include/thr_lock.h
@@ -20,7 +20,6 @@
 #ifdef	__cplusplus
 extern "C" {
 #endif
-
 #include <my_pthread.h>
 #include <my_list.h>
 
@@ -95,6 +94,7 @@ typedef struct st_thr_lock_info
 {
   pthread_t thread;
   my_thread_id thread_id;
+  void *mysql_thd;        // THD pointer
 } THR_LOCK_INFO;
 
 
@@ -164,6 +164,17 @@ my_bool thr_reschedule_write_lock(THR_LOCK_DATA *data,
                                   ulong lock_wait_timeout);
 void thr_set_lock_wait_callback(void (*before_wait)(void),
                                 void (*after_wait)(void));
+
+#ifdef WITH_WSREP
+#include <my_sys.h>
+  typedef my_bool (* wsrep_thd_is_brute_force_fun)(void *, my_bool);
+  typedef int (* wsrep_abort_thd_fun)(void *, void *, my_bool);
+  typedef int (* wsrep_on_fun)(void *);
+  void wsrep_thr_lock_init(
+    wsrep_thd_is_brute_force_fun bf_fun, wsrep_abort_thd_fun abort_fun,
+    my_bool debug, my_bool convert_LOCK_to_trx, wsrep_on_fun on_fun);
+#endif
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/include/wsrep.h b/include/wsrep.h
new file mode 100644
index 00000000000..c261d9c4b5f
--- /dev/null
+++ b/include/wsrep.h
@@ -0,0 +1,58 @@
+/* Copyright 2014 Codership Oy <http://www.codership.com> & SkySQL Ab
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef WSREP_INCLUDED
+#define WSERP_INCLUDED
+
+#ifdef WITH_WSREP
+#define IF_WSREP(A,B) A
+#define DBUG_ASSERT_IF_WSREP(A) DBUG_ASSERT(A)
+
+#if !defined(EMBEDDED_LIBRARY)
+#define WSREP_FORMAT(my_format)                           \
+  ((wsrep_forced_binlog_format != BINLOG_FORMAT_UNSPEC) ? \
+   wsrep_forced_binlog_format : my_format)
+#else
+#define WSREP_FORMAT(my_format) my_format
+#endif /* && !EMBEDDED_LIBRARY */
+
+#define WSREP_MYSQL_DB (char *)"mysql"
+#define WSREP_TO_ISOLATION_BEGIN(db_, table_, table_list_)                   \
+  if (WSREP_ON && WSREP(thd) && wsrep_to_isolation_begin(thd, db_, table_, table_list_)) \
+    goto error;
+
+#define WSREP_TO_ISOLATION_END                                              \
+  if (WSREP_ON && (WSREP(thd) || (thd && thd->wsrep_exec_mode==TOTAL_ORDER))) \
+    wsrep_to_isolation_end(thd);
+
+#define WSREP_DEBUG(...)                                                \
+    if (wsrep_debug)     WSREP_LOG(sql_print_information, ##__VA_ARGS__)
+#define WSREP_INFO(...)  WSREP_LOG(sql_print_information, ##__VA_ARGS__)
+#define WSREP_WARN(...)  WSREP_LOG(sql_print_warning,     ##__VA_ARGS__)
+#define WSREP_ERROR(...) WSREP_LOG(sql_print_error,       ##__VA_ARGS__)
+
+#else
+#define IF_WSREP(A,B) B
+#define DBUG_ASSERT_IF_WSREP(A)
+#define WSREP_DEBUG(...)
+#define WSREP_INFO(...)
+#define WSREP_WARN(...)
+#define WSREP_ERROR(...)
+#define WSREP_FORMAT(my_format) my_format
+#define WSREP_TO_ISOLATION_BEGIN(db_, table_, table_list_)
+#define WSREP_TO_ISOLATION_END
+#endif
+
+#endif /* WSERP_INCLUDED */
diff --git a/mysql-test/extra/binlog_tests/binlog.test b/mysql-test/extra/binlog_tests/binlog.test
index ce5dde97894..831c6c886d5 100644
--- a/mysql-test/extra/binlog_tests/binlog.test
+++ b/mysql-test/extra/binlog_tests/binlog.test
@@ -372,7 +372,8 @@ dfLtTBcBAAAAIgAAAPkAAAAAABcAAAAAAAcAAf/+AQAAAA==
 
 SELECT * FROM t1;
 --echo # Their values should be ON
-SHOW SESSION VARIABLES LIKE "%_checks";
+SHOW SESSION VARIABLES LIKE "foreign_key_checks";
+SHOW SESSION VARIABLES LIKE "unique_checks";
 
 --echo
 SET @@SESSION.foreign_key_checks= OFF;
@@ -387,7 +388,8 @@ dfLtTBcBAAAAIgAAAM0BAAAAABcAAAAAAAEAAf/+AgAAAA==
 
 SELECT * FROM t1;
 --echo # Their values should be OFF
-SHOW SESSION VARIABLES LIKE "%_checks";
+SHOW SESSION VARIABLES LIKE "foreign_key_checks";
+SHOW SESSION VARIABLES LIKE "unique_checks";
 
 --echo # INSERT INTO t1 VALUES(2)
 --echo # foreign_key_checks=1 and unique_checks=1
@@ -401,7 +403,8 @@ dfLtTBcBAAAAIgAAAM0BAAAAABcAAAAAAAEAAf/+AgAAAA==
 
 SELECT * FROM t1;
 --echo # Their values should be OFF
-SHOW SESSION VARIABLES LIKE "%_checks";
+SHOW SESSION VARIABLES LIKE "foreign_key_checks";
+SHOW SESSION VARIABLES LIKE "unique_checks";
 
 DROP TABLE t1;
 
diff --git a/mysql-test/include/galera_cluster.inc b/mysql-test/include/galera_cluster.inc
new file mode 100644
index 00000000000..bc652225722
--- /dev/null
+++ b/mysql-test/include/galera_cluster.inc
@@ -0,0 +1,10 @@
+# galera_cluster.inc
+# ==================
+#
+# Description
+# -----------
+# Configure galera cluster with 2 nodes.
+#
+
+--let $galera_cluster_size = 2
+--source include/galera_init.inc
diff --git a/mysql-test/include/galera_connect.inc b/mysql-test/include/galera_connect.inc
new file mode 100644
index 00000000000..bfd9b188667
--- /dev/null
+++ b/mysql-test/include/galera_connect.inc
@@ -0,0 +1,45 @@
+# galera_connect.inc
+# ==================
+#
+# Description
+# -----------
+# Open a connection to the specified server number ($galera_server_number).
+# The connection itself would be identified by $galera_connection_name.
+#
+# Parameters
+# ----------
+# $galera_connection_name
+#   Name of the resulting connection.
+#
+# $galera_server_number
+#   Sequence number of the node in the galera cluster.
+#
+# $galera_debug
+#   Print debug information.
+#
+
+if (!$galera_connection_name)
+{
+  --die ERROR IN TEST: $galera_connection_name must be set before sourcing include/galera_connect.inc
+}
+
+if (!$galera_server_number)
+{
+  --die ERROR IN TEST: $galera_server_number must be set before sourcing include/galera_connect.inc
+}
+
+--let $_galera_port= \$NODE_MYPORT_$galera_server_number
+if (!$_galera_port)
+{
+  --echo Bug in test case: '\$NODE_MYPORT_$galera_server_number' not initialized. Check the test's .cfg file.
+  --die Not all NODE_MYPORT_* environment variables are setup correctly.
+}
+
+if ($galera_debug)
+{
+  --echo connect($galera_connection_name,127.0.0.1,root,,test,$_galera_port,)
+}
+
+# Open a connection
+--connect($galera_connection_name,127.0.0.1,root,,test,$_galera_port,)
+
diff --git a/mysql-test/include/galera_diff.inc b/mysql-test/include/galera_diff.inc
new file mode 100644
index 00000000000..6043b582647
--- /dev/null
+++ b/mysql-test/include/galera_diff.inc
@@ -0,0 +1,100 @@
+# galera_diff.inc
+# ===============
+#
+# Description
+# -----------
+# Compare the output of the given statement on all the nodes of the cluster.
+#
+# Parameters
+# ----------
+# $galera_diff_statement
+#   Statement for which the output would be compared.
+# 
+# $galera_diff_database
+#   Database against which the above statement would be executed.
+#   (Default : test)
+#
+# $galera_diff_servers
+#   Comma separated list of servers to executed the diff statement on. If not
+#   set, a list of servers will be generated based on $galera_cluster_size.
+#
+# $galerra_debug
+#   Print debug information.
+#
+
+if (!$galera_diff_statement)
+{
+  --die ERROR IN TEST: $galera_diff_statement must be set before sourcing include/galera_diff.inc
+}
+
+--let $_galera_diff_database = $galera_diff_database
+if (!$_galera_diff_database)
+{
+  --let $_galera_diff_database = test 
+}
+
+--let $_galera_diff_servers= $galera_diff_servers
+if (!$_galera_diff_servers)
+{
+  --let $_i= $galera_cluster_size
+  --let $_galera_diff_servers= 
+  while ($_i)
+  {
+    --let $_galera_diff_servers= $_i,$_galera_diff_servers
+    --dec $_i
+  }
+}
+if ($galera_debug)
+{
+  --echo \$galera_diff_servers= '$_galera_diff_servers'
+}
+
+if (!$galera_debug)
+{
+  --disable_query_log
+}
+
+# Generate file containing $galera_diff_statement. We don't pass the
+# statement on the command line, because it would be subject to shell
+# substitutions.
+--let $write_to_file= GENERATE
+--let $write_var= $galera_diff_statement
+--source include/write_var_to_file.inc
+--let $_galera_diff_statement_file= $write_to_file
+
+if (!$galera_debug)
+{
+  --enable_query_log
+}
+
+# Compare all servers.
+--let $_galera_diff_first= 1
+while ($_galera_diff_servers)
+{
+  # Set $_galera_diff_server_i to the first number in the list
+  --let $_galera_diff_server_i= `SELECT SUBSTRING_INDEX('$_galera_diff_servers', ',', 1)`
+  # Remove $_galera_diff_server_i from the list
+  --let $_galera_diff_servers= `SELECT SUBSTRING('$_galera_diff_servers', LENGTH('$_galera_diff_server_i') + 2)`
+
+  # Execute statement
+  --let $_galera_diff_file= $MYSQLTEST_VARDIR/tmp/_galera_diff_server-$_galera_diff_server_i.tmp
+  --exec $MYSQL --defaults-group-suffix=.$_galera_diff_server_i $_galera_diff_database < $_galera_diff_statement_file > $_galera_diff_file
+
+  # Compare
+  if (!$_galera_diff_first)
+  {
+    if ($galera_debug)
+    {
+      --echo diffing $_galera_diff_file and $_galera_diff_prev_file
+    }
+    --diff_files $_galera_diff_file $_galera_diff_prev_file
+    --remove_file $_galera_diff_prev_file
+  }
+  --let $_galera_diff_prev_file= $_galera_diff_file
+  --let $_galera_diff_first= 0
+}
+
+# Cleanup
+--remove_file $_galera_diff_prev_file
+--remove_file $_galera_diff_statement_file
+
diff --git a/mysql-test/include/galera_end.inc b/mysql-test/include/galera_end.inc
new file mode 100644
index 00000000000..0fb5479844e
--- /dev/null
+++ b/mysql-test/include/galera_end.inc
@@ -0,0 +1,25 @@
+# galera_end.inc
+# ==============
+#
+# Description
+# -----------
+# Closes the connections opened via include/galera_init.inc
+#
+# Parameters
+# ----------
+# $galera_cluster_size
+#   Number of nodes in the cluster.
+#
+
+--let $_galera_node= $galera_cluster_size
+
+while ($_galera_node)
+{
+  if ($galera_debug)
+  {
+    --echo Disconnecting node_$_galera_node
+  }
+  --disconnect node_$_galera_node
+  --dec $_galera_node
+}
+
diff --git a/mysql-test/include/galera_init.inc b/mysql-test/include/galera_init.inc
new file mode 100644
index 00000000000..79591973862
--- /dev/null
+++ b/mysql-test/include/galera_init.inc
@@ -0,0 +1,26 @@
+# galera_init.inc
+# ===============
+#
+# Description
+# -----------
+# Set up a Galera cluster with $wsrep_cluster_size nodes.
+#
+# Parameters
+# ----------
+# $galera_cluster_size
+#   Number of nodes in the cluster.
+#
+
+--source include/have_wsrep_enabled.inc
+
+--let $_galera_node= $galera_cluster_size
+
+while ($_galera_node)
+{
+  --let $galera_connection_name= node_$_galera_node
+  --let $galera_server_number= $_galera_node
+  --source include/galera_connect.inc
+
+  --dec $_galera_node
+}
+
diff --git a/mysql-test/include/have_innodb_disallow_writes.inc b/mysql-test/include/have_innodb_disallow_writes.inc
new file mode 100644
index 00000000000..83b516b7a34
--- /dev/null
+++ b/mysql-test/include/have_innodb_disallow_writes.inc
@@ -0,0 +1,6 @@
+--source include/have_innodb.inc
+
+if (`SELECT COUNT(*) = 0 from INFORMATION_SCHEMA.GLOBAL_VARIABLES
+    WHERE VARIABLE_NAME = 'INNODB_DISALLOW_WRITES'`) {
+  --skip Test requires 'innodb_disallow_writes'
+}
diff --git a/mysql-test/include/have_wsrep.inc b/mysql-test/include/have_wsrep.inc
new file mode 100644
index 00000000000..52220edf481
--- /dev/null
+++ b/mysql-test/include/have_wsrep.inc
@@ -0,0 +1,8 @@
+# To be used in a test which requires server to be compiled with wsrep support
+# (-DWITH_WSREP=ON) and wsrep plugin is ACTIVE.
+
+if (`SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'wsrep' AND PLUGIN_STATUS='ACTIVE'`)
+{
+  --skip Test requires wsrep plugin.
+}
+
diff --git a/mysql-test/include/have_wsrep_enabled.inc b/mysql-test/include/have_wsrep_enabled.inc
new file mode 100644
index 00000000000..edb919fd852
--- /dev/null
+++ b/mysql-test/include/have_wsrep_enabled.inc
@@ -0,0 +1,9 @@
+# To be used in a test which requires wsrep plugin to be ACTIVE and enabled
+# (i.e. wsrep_on=ON). It includes have_wsrep.inc.
+
+--source include/have_wsrep.inc
+
+--require r/have_wsrep.require
+disable_query_log;
+SHOW VARIABLES LIKE 'wsrep_on';
+enable_query_log;
diff --git a/mysql-test/include/mtr_check.sql b/mysql-test/include/mtr_check.sql
index e34e32ad1a6..edc15850d97 100644
--- a/mysql-test/include/mtr_check.sql
+++ b/mysql-test/include/mtr_check.sql
@@ -34,6 +34,7 @@ BEGIN
      AND variable_name != 'INNODB_USE_NATIVE_AIO'
      AND variable_name not like 'GTID%POS'
      AND variable_name != 'GTID_BINLOG_STATE'
+     AND variable_name != 'WSREP_DATA_HOME_DIR'
    ORDER BY variable_name;
 
   -- Dump all databases, there should be none
diff --git a/mysql-test/include/mtr_warnings.sql b/mysql-test/include/mtr_warnings.sql
index 0ad1079cd92..06a7b49e979 100644
--- a/mysql-test/include/mtr_warnings.sql
+++ b/mysql-test/include/mtr_warnings.sql
@@ -226,6 +226,14 @@ INSERT INTO global_suppressions VALUES
  ("Slave I/O: Notifying master by SET @master_binlog_checksum= @@global.binlog_checksum failed with error.*"),
  ("Slave I/O: Setting master-side filtering of @@skip_replication failed with error:.*"),
  ("Slave I/O: Setting @mariadb_slave_capability failed with error:.*"),
+
+ /*
+   Galera-related warnings.
+ */
+ ("WSREP: Could not open saved state file for reading: .*"),
+ ("WSREP: last inactive check more than .* skipping check"),
+ ("WSREP: Gap in state sequence. Need state transfer."),
+ ("WSREP: Failed to prepare for incremental state transfer: .*"),
  ("THE_LAST_SUPPRESSION")||
 
 
diff --git a/mysql-test/include/not_wsrep.inc b/mysql-test/include/not_wsrep.inc
new file mode 100644
index 00000000000..3314b5c8717
--- /dev/null
+++ b/mysql-test/include/not_wsrep.inc
@@ -0,0 +1,7 @@
+# To be used in a test which should be skipped if server is compiled with wsrep
+# support (-DWITH_WSREP=ON) and wsrep plugin is ACTIVE.
+
+-- require r/not_wsrep.require
+disable_query_log;
+SELECT VERSION() LIKE '%wsrep%' AS 'HAVE_WSREP';
+enable_query_log;
diff --git a/mysql-test/include/write_result_to_file.inc b/mysql-test/include/write_result_to_file.inc
index 3e2ddf48957..db5d546750c 100644
--- a/mysql-test/include/write_result_to_file.inc
+++ b/mysql-test/include/write_result_to_file.inc
@@ -33,7 +33,8 @@
 --let _WRTF_SERVER_NUMBER= $server_number
 if (!$server_number)
 {
-  --let _WRTF_SERVER_NUMBER= `SELECT 1 + @@PORT - $MASTER_MYPORT`
+  # Note: 2 extra ports are reserved per server for galera use.
+  --let _WRTF_SERVER_NUMBER= `SELECT 1 + FLOOR((@@PORT - $MASTER_MYPORT) / 3)`
 }
 
 --let $_write_result_msg= [server=$_WRTF_SERVER_NUMBER]
diff --git a/mysql-test/lib/My/ConfigFactory.pm b/mysql-test/lib/My/ConfigFactory.pm
index 4e8507a5c4a..488b0823763 100644
--- a/mysql-test/lib/My/ConfigFactory.pm
+++ b/mysql-test/lib/My/ConfigFactory.pm
@@ -238,6 +238,9 @@ my @mysqld_rules=
  { 'pid-file' => \&fix_pidfile },
  { '#host' => \&fix_host },
  { 'port' => \&fix_port },
+ # galera base_port and port used during SST
+ { '#galera_port' => \&fix_port },
+ { '#sst_port' => \&fix_port },
  { 'socket' => \&fix_socket },
  { '#log-error' => \&fix_log_error },
  { 'general-log' => 1 },
diff --git a/mysql-test/lib/mtr_cases.pm b/mysql-test/lib/mtr_cases.pm
index 441fd6e6559..7415b229862 100644
--- a/mysql-test/lib/mtr_cases.pm
+++ b/mysql-test/lib/mtr_cases.pm
@@ -861,6 +861,8 @@ sub collect_one_test_case {
       # Suite has no config, autodetect which one to use
       if ($tinfo->{rpl_test}) {
         $config= "suite/rpl/my.cnf";
+      } elsif ($tinfo->{galera_test}) {
+        $config= "suite/galera/my.cnf";
       } else {
         $config= "include/default_my.cnf";
       }
@@ -981,6 +983,7 @@ my $tags_map= {'big_test' => ['big_test', 1],
                'master-slave' => ['rpl_test', 1],
                'ndb_master-slave' => ['rpl_test', 1, 'ndb_test', 1],
                'long_test' => ['long_test', 1],
+               'galera_init' => ['galera_test', 1],
 };
 my $tags_regex_string= join('|', keys %$tags_map);
 my $tags_regex= qr:include/($tags_regex_string)\.inc:o;
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index c2d44bf4db1..61f9538c1b4 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -135,6 +135,7 @@ my $opt_start;
 my $opt_start_dirty;
 my $opt_start_exit;
 my $start_only;
+my $file_wsrep_provider;
 
 END {
   if ( defined $opt_tmpdir_pid and $opt_tmpdir_pid == $$ )
@@ -412,6 +413,7 @@ sub main {
   check_ndbcluster_support();
   check_ssl_support();
   check_debug_support();
+  check_wsrep_support();
 
   if (!$opt_suites) {
     $opt_suites= join ',', collect_default_suites(@DEFAULT_SUITES);
@@ -2398,6 +2400,24 @@ sub environment_setup {
   }
 
   # ----------------------------------------------------
+  # Setup env for wsrep
+  # ----------------------------------------------------
+  if (have_wsrep()) {
+    if (defined $ENV{'WSREP_PROVIDER'} ) {
+      # Nothing needs to be done! WSREP_PROVIDER env is already set & checked;
+      # will be used.
+    } else {
+      $ENV{'WSREP_PROVIDER'}=  $file_wsrep_provider;
+    }
+
+    if ($ENV{'WSREP_PROVIDER'} ne "") {
+      mtr_verbose("WSREP_PROVIDER set to $ENV{'WSREP_PROVIDER'}");
+    } else {
+      mtr_verbose("WSREP_PROVIDER isn't available");
+    }
+  }
+
+  # ----------------------------------------------------
   # mysql clients
   # ----------------------------------------------------
   $ENV{'MYSQL_CHECK'}=              client_arguments("mysqlcheck");
@@ -3174,6 +3194,40 @@ sub ndbcluster_start ($) {
   return 0;
 }
 
+sub have_wsrep() {
+  my $wsrep_on= $mysqld_variables{'wsrep-on'};
+  return defined $wsrep_on
+}
+
+sub check_wsrep_provider_env {
+  if (defined $ENV{'WSREP_PROVIDER'}) {
+      if (mtr_file_exists($ENV{'WSREP_PROVIDER'}) eq "") {
+        mtr_error("WSREP_PROVIDER env set to an invalid path");
+        return 0; # error
+      }
+      # Ok, WSREP_PROVIDER set to a valid path.
+      return 1;
+  }
+  # Ok, WSREP_PROVIDER not defined.
+  return 2;
+}
+
+sub check_wsrep_support() {
+  if (have_wsrep())
+  {
+    mtr_report(" - binaries built with wsrep patch");
+
+    $file_wsrep_provider=
+      mtr_file_exists("/usr/lib/galera/libgalera_smm.so",
+                      "/usr/lib64/galera/libgalera_smm.so");
+
+    if ((check_wsrep_provider_env() == 1) || ($file_wsrep_provider ne "")) {
+      # Add galera test suites
+      mtr_report(" - adding wsrep, galera to default test suites");
+      push @DEFAULT_SUITES, qw(wsrep galera);
+    }
+  }
+}
 
 sub mysql_server_start($) {
   my ($mysqld, $tinfo) = @_;
@@ -4837,6 +4891,10 @@ sub extract_warning_lines ($$) {
      qr|Plugin 'FEEDBACK' registration as a INFORMATION SCHEMA failed|,
      qr|'log-bin-use-v1-row-events' is MySQL 5.6 compatible option|,
      qr|InnoDB: Setting thread \d+ nice to \d+ failed, current nice \d+, errno 13|, # setpriority() fails under valgrind
+     # Galera-related warnings.
+     qr|WSREP:.*down context.*|,
+     qr|WSREP: Failed to send state UUID:.*|,
+     qr|WSREP: wsrep_sst_receive_address.*|,
     );
 
   my $matched_lines= [];
diff --git a/mysql-test/r/have_wsrep.require b/mysql-test/r/have_wsrep.require
new file mode 100644
index 00000000000..af32ac7dca7
--- /dev/null
+++ b/mysql-test/r/have_wsrep.require
@@ -0,0 +1,2 @@
+Variable_name	Value
+wsrep_on	ON
diff --git a/mysql-test/r/mysql_tzinfo_to_sql_symlink.result b/mysql-test/r/mysql_tzinfo_to_sql_symlink.result
index dda732937ee..484f71a4c2e 100644
--- a/mysql-test/r/mysql_tzinfo_to_sql_symlink.result
+++ b/mysql-test/r/mysql_tzinfo_to_sql_symlink.result
@@ -1,6 +1,7 @@
 #
 # MDEV-5226 mysql_tzinfo_to_sql errors with tzdata 2013f and above
 #
+SET SESSION wsrep_replicate_myisam=ON;
 # Verbose run
 TRUNCATE TABLE time_zone;
 TRUNCATE TABLE time_zone_name;
diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result
index 5a2142c402c..6953a05fd5c 100644
--- a/mysql-test/r/mysqld--help.result
+++ b/mysql-test/r/mysqld--help.result
@@ -1065,6 +1065,93 @@ The following options may be given as the first argument:
  -V, --version       Output version information and exit.
  --wait-timeout=#    The number of seconds the server waits for activity on a
  connection before closing it
+ --wsrep-OSU-method[=name] 
+ Method for Online Schema Upgrade
+ --wsrep-auto-increment-control 
+ To automatically control the assignment of autoincrement
+ variables
+ (Defaults to on; use --skip-wsrep-auto-increment-control to disable.)
+ --wsrep-causal-reads 
+ (DEPRECATED) Setting this variable is equivalent to
+ setting wsrep_sync_wait READ flag
+ --wsrep-certify-nonPK 
+ Certify tables with no primary key
+ (Defaults to on; use --skip-wsrep-certify-nonPK to disable.)
+ --wsrep-cluster-address=name 
+ Address to initially connect to cluster
+ --wsrep-cluster-name=name 
+ Name for the cluster
+ --wsrep-convert-LOCK-to-trx 
+ To convert locking sessions into transactions
+ --wsrep-data-home-dir=name 
+ home directory for wsrep provider
+ --wsrep-dbug-option=name 
+ DBUG options to provider library
+ --wsrep-debug       To enable debug level logging
+ --wsrep-desync      To desynchronize the node from the cluster
+ --wsrep-drupal-282555-workaround 
+ To use a workaround forbad autoincrement value
+ --wsrep-forced-binlog-format=name 
+ binlog format to take effect over user's choice
+ --wsrep-load-data-splitting 
+ To commit LOAD DATA transaction after every 10K rows
+ inserted
+ (Defaults to on; use --skip-wsrep-load-data-splitting to disable.)
+ --wsrep-log-conflicts 
+ To log multi-master conflicts
+ --wsrep-max-ws-rows=# 
+ Max number of rows in write set
+ --wsrep-max-ws-size=# 
+ Max write set size (bytes)
+ --wsrep-mysql-replication-bundle=# 
+ mysql replication group commit 
+ --wsrep-node-address=name 
+ Node address
+ --wsrep-node-incoming-address=name 
+ Client connection address
+ --wsrep-node-name=name 
+ Node name
+ --wsrep-notify-cmd=name 
+ --wsrep-on          To enable wsrep replication 
+ (Defaults to on; use --skip-wsrep-on to disable.)
+ --wsrep-provider=name 
+ Path to replication provider library
+ --wsrep-provider-options=name 
+ provider specific options
+ --wsrep-recover     Recover database state after crash and exit
+ --wsrep-replicate-myisam 
+ To enable myisam replication
+ --wsrep-restart-slave 
+ Should MySQL slave be restarted automatically, when node
+ joins back to cluster
+ --wsrep-retry-autocommit=# 
+ Max number of times to retry a failed autocommit
+ statement
+ --wsrep-slave-FK-checks 
+ Should slave thread do foreign key constraint checks
+ (Defaults to on; use --skip-wsrep-slave-FK-checks to disable.)
+ --wsrep-slave-UK-checks 
+ Should slave thread do secondary index uniqueness checks
+ --wsrep-slave-threads=# 
+ Number of slave appliers to launch
+ --wsrep-sst-auth=name 
+ Authentication for SST connection
+ --wsrep-sst-donor=name 
+ preferred donor node for the SST
+ --wsrep-sst-donor-rejects-queries 
+ Reject client queries when donating state snapshot
+ transfer
+ --wsrep-sst-method=name 
+ State snapshot transfer method
+ --wsrep-sst-receive-address=name 
+ Address where node is waiting for SST contact
+ --wsrep-start-position=name 
+ global transaction position to start from 
+ --wsrep-sync-wait[=#] 
+ Ensure "synchronous" read view before executing an
+ operation of the type specified by bitmask: 1 -
+ READ(includes SELECT, SHOW and BEGIN/START TRANSACTION);
+ 2 - UPDATE and DELETE; 4 - INSERT and REPLACE
 
 Variables (--variable-name=value)
 allow-suspicious-udfs FALSE
@@ -1364,6 +1451,44 @@ use-stat-tables NEVER
 userstat FALSE
 verbose TRUE
 wait-timeout 28800
+wsrep-OSU-method TOI
+wsrep-auto-increment-control TRUE
+wsrep-causal-reads FALSE
+wsrep-certify-nonPK TRUE
+wsrep-cluster-address 
+wsrep-cluster-name my_wsrep_cluster
+wsrep-convert-LOCK-to-trx FALSE
+wsrep-data-home-dir 
+wsrep-dbug-option 
+wsrep-debug FALSE
+wsrep-desync FALSE
+wsrep-drupal-282555-workaround FALSE
+wsrep-forced-binlog-format NONE
+wsrep-load-data-splitting TRUE
+wsrep-log-conflicts FALSE
+wsrep-max-ws-rows 131072
+wsrep-max-ws-size 1073741824
+wsrep-mysql-replication-bundle 0
+wsrep-node-address 
+wsrep-node-incoming-address AUTO
+wsrep-notify-cmd 
+wsrep-on FALSE
+wsrep-provider none
+wsrep-provider-options 
+wsrep-recover FALSE
+wsrep-replicate-myisam FALSE
+wsrep-restart-slave FALSE
+wsrep-retry-autocommit 1
+wsrep-slave-FK-checks TRUE
+wsrep-slave-UK-checks FALSE
+wsrep-slave-threads 1
+wsrep-sst-auth (No default value)
+wsrep-sst-donor 
+wsrep-sst-donor-rejects-queries FALSE
+wsrep-sst-method rsync
+wsrep-sst-receive-address AUTO
+wsrep-start-position 00000000-0000-0000-0000-000000000000:-1
+wsrep-sync-wait 0
 
 To see what values a running MySQL server is using, type
 'mysqladmin variables' instead of 'mysqld --verbose --help'.
diff --git a/mysql-test/r/not_wsrep.require b/mysql-test/r/not_wsrep.require
new file mode 100644
index 00000000000..7c8e74af144
--- /dev/null
+++ b/mysql-test/r/not_wsrep.require
@@ -0,0 +1,2 @@
+HAVE_WSREP
+0
diff --git a/mysql-test/suite/binlog/r/binlog_row_binlog.result b/mysql-test/suite/binlog/r/binlog_row_binlog.result
index 20a4b52f1a7..05009c5a570 100644
--- a/mysql-test/suite/binlog/r/binlog_row_binlog.result
+++ b/mysql-test/suite/binlog/r/binlog_row_binlog.result
@@ -806,9 +806,11 @@ SELECT * FROM t1;
 c1
 1
 # Their values should be ON
-SHOW SESSION VARIABLES LIKE "%_checks";
+SHOW SESSION VARIABLES LIKE "foreign_key_checks";
 Variable_name	Value
 foreign_key_checks	ON
+SHOW SESSION VARIABLES LIKE "unique_checks";
+Variable_name	Value
 unique_checks	ON
 
 SET @@SESSION.foreign_key_checks= OFF;
@@ -824,9 +826,11 @@ c1
 1
 2
 # Their values should be OFF
-SHOW SESSION VARIABLES LIKE "%_checks";
+SHOW SESSION VARIABLES LIKE "foreign_key_checks";
 Variable_name	Value
 foreign_key_checks	OFF
+SHOW SESSION VARIABLES LIKE "unique_checks";
+Variable_name	Value
 unique_checks	OFF
 # INSERT INTO t1 VALUES(2)
 # foreign_key_checks=1 and unique_checks=1
@@ -842,8 +846,10 @@ c1
 1
 2
 # Their values should be OFF
-SHOW SESSION VARIABLES LIKE "%_checks";
+SHOW SESSION VARIABLES LIKE "foreign_key_checks";
 Variable_name	Value
 foreign_key_checks	OFF
+SHOW SESSION VARIABLES LIKE "unique_checks";
+Variable_name	Value
 unique_checks	OFF
 DROP TABLE t1;
diff --git a/mysql-test/suite/binlog/r/binlog_stm_binlog.result b/mysql-test/suite/binlog/r/binlog_stm_binlog.result
index 824bf3ed2a0..3a6af15e88a 100644
--- a/mysql-test/suite/binlog/r/binlog_stm_binlog.result
+++ b/mysql-test/suite/binlog/r/binlog_stm_binlog.result
@@ -618,9 +618,11 @@ SELECT * FROM t1;
 c1
 1
 # Their values should be ON
-SHOW SESSION VARIABLES LIKE "%_checks";
+SHOW SESSION VARIABLES LIKE "foreign_key_checks";
 Variable_name	Value
 foreign_key_checks	ON
+SHOW SESSION VARIABLES LIKE "unique_checks";
+Variable_name	Value
 unique_checks	ON
 
 SET @@SESSION.foreign_key_checks= OFF;
@@ -636,9 +638,11 @@ c1
 1
 2
 # Their values should be OFF
-SHOW SESSION VARIABLES LIKE "%_checks";
+SHOW SESSION VARIABLES LIKE "foreign_key_checks";
 Variable_name	Value
 foreign_key_checks	OFF
+SHOW SESSION VARIABLES LIKE "unique_checks";
+Variable_name	Value
 unique_checks	OFF
 # INSERT INTO t1 VALUES(2)
 # foreign_key_checks=1 and unique_checks=1
@@ -654,8 +658,10 @@ c1
 1
 2
 # Their values should be OFF
-SHOW SESSION VARIABLES LIKE "%_checks";
+SHOW SESSION VARIABLES LIKE "foreign_key_checks";
 Variable_name	Value
 foreign_key_checks	OFF
+SHOW SESSION VARIABLES LIKE "unique_checks";
+Variable_name	Value
 unique_checks	OFF
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/galera_2nodes.cnf b/mysql-test/suite/galera/galera_2nodes.cnf
new file mode 100644
index 00000000000..5a08125089a
--- /dev/null
+++ b/mysql-test/suite/galera/galera_2nodes.cnf
@@ -0,0 +1,24 @@
+# Use default setting for mysqld processes
+!include include/default_mysqld.cnf
+
+[mysqld.1]
+binlog-format=row
+wsrep_provider=@ENV.WSREP_PROVIDER
+wsrep_cluster_address='gcomm://'
+wsrep_provider_options='base_port=@mysqld.1.#galera_port'
+wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
+
+[mysqld.2]
+binlog-format=row
+wsrep_provider=@ENV.WSREP_PROVIDER
+wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
+wsrep_provider_options='base_port=@mysqld.2.#galera_port'
+wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
+
+[ENV]
+NODE_MYPORT_1= @mysqld.1.port
+NODE_MYSOCK_1= @mysqld.1.socket
+
+NODE_MYPORT_2= @mysqld.2.port
+NODE_MYSOCK_2= @mysqld.2.socket
+
diff --git a/mysql-test/suite/galera/my.cnf b/mysql-test/suite/galera/my.cnf
new file mode 100644
index 00000000000..ca163a540d9
--- /dev/null
+++ b/mysql-test/suite/galera/my.cnf
@@ -0,0 +1 @@
+!include galera_2nodes.cnf
diff --git a/mysql-test/suite/galera/r/basic.result b/mysql-test/suite/galera/r/basic.result
new file mode 100644
index 00000000000..d4efe348b61
--- /dev/null
+++ b/mysql-test/suite/galera/r/basic.result
@@ -0,0 +1,30 @@
+USE test;
+CREATE TABLE t1(c1 INT PRIMARY KEY) ENGINE=INNODB;
+INSERT INTO t1 VALUES (1), (2), (3), (4), (5);
+SELECT * FROM t1;
+c1
+1
+2
+3
+4
+5
+
+# On node_1
+SELECT * FROM test.t1;
+c1
+1
+2
+3
+4
+5
+
+# On node_2
+SELECT * FROM test.t1;
+c1
+1
+2
+3
+4
+5
+DROP TABLE t1;
+# End of test
diff --git a/mysql-test/suite/galera/r/galera_sst_mode.result b/mysql-test/suite/galera/r/galera_sst_mode.result
new file mode 100644
index 00000000000..ea25b322449
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_sst_mode.result
@@ -0,0 +1,24 @@
+#
+# Test for mysqldump's galera-sst-mode option
+#
+#
+# MDEV-6490: mysqldump unknown option --galera-sst-mode
+#
+CREATE DATABASE bug6490;
+USE bug6490;
+CREATE TABLE t1(c1 INT);
+INSERT INTO t1 values (1);
+INSERT INTO t1 values (2);
+# Save the current gtid_binlog_state.
+# Take a dump of bug6490 database
+DROP TABLE t1;
+# Load the dump
+RESET MASTER;
+SELECT * from t1;
+c1
+1
+2
+# Compare the two gtid_binlog_state's
+# Cleanup
+DROP DATABASE bug6490;
+# End of test
diff --git a/mysql-test/suite/galera/r/grant.result b/mysql-test/suite/galera/r/grant.result
new file mode 100644
index 00000000000..8d257e7e8e2
--- /dev/null
+++ b/mysql-test/suite/galera/r/grant.result
@@ -0,0 +1,17 @@
+#
+# MDEV#6266: Changing password fails on galera cluster
+#
+
+# On node_1
+GRANT SELECT ON *.* TO 'user_6266'@'localhost' IDENTIFIED BY 'pass';
+
+# Now, try changing password for 'user_6266'. This command should also
+# execute successfully on the other node.
+SET PASSWORD FOR 'user_6266'@'localhost' = PASSWORD('newpass');
+
+# On node_2
+SELECT user FROM mysql.user WHERE user='user_6266';
+user
+user_6266
+DROP USER 'user_6266'@'localhost';
+# End of test
diff --git a/mysql-test/suite/galera/r/partition.result b/mysql-test/suite/galera/r/partition.result
new file mode 100644
index 00000000000..60fb2ed298d
--- /dev/null
+++ b/mysql-test/suite/galera/r/partition.result
@@ -0,0 +1,23 @@
+#
+# MDEV#4953 Galera: DELETE from a partitioned table is not replicated
+#
+USE test;
+CREATE TABLE t1 (pk INT PRIMARY KEY, i INT) ENGINE=INNODB PARTITION BY HASH(pk) PARTITIONS 2;
+INSERT INTO t1 VALUES (1,100), (2,200);
+SELECT * FROM t1;
+pk	i
+2	200
+1	100
+DELETE FROM t1;
+SELECT * FROM t1;
+pk	i
+
+# On node_1
+SELECT * FROM t1;
+pk	i
+
+# On node_2
+SELECT * FROM t1;
+pk	i
+DROP TABLE t1;
+# End of test
diff --git a/mysql-test/suite/galera/r/unique_key.result b/mysql-test/suite/galera/r/unique_key.result
new file mode 100644
index 00000000000..ffb4f01c1f8
--- /dev/null
+++ b/mysql-test/suite/galera/r/unique_key.result
@@ -0,0 +1,47 @@
+#
+# MDEV#5552 Deadlock when inserting NULL column value in column with
+# UNIQUE index
+#
+USE test;
+
+# On node_1
+CREATE TABLE t1(c1 INT DEFAULT NULL, UNIQUE KEY c1(c1)) ENGINE=INNODB;
+INSERT INTO t1 VALUES (NULL);
+INSERT INTO t1 VALUES (NULL);
+SELECT * FROM test.t1;
+c1
+NULL
+NULL
+
+# On node_2
+SELECT * FROM test.t1;
+c1
+NULL
+NULL
+
+# On node_1
+INSERT INTO t1 VALUES (1);
+UPDATE t1 SET c1=NULL WHERE c1=1;
+SELECT * FROM test.t1;
+c1
+NULL
+NULL
+NULL
+
+# On node_2
+SELECT * FROM test.t1;
+c1
+NULL
+NULL
+NULL
+
+# On node_1
+DELETE FROM t1 WHERE c1<=>NULL;
+SELECT * FROM test.t1;
+c1
+
+# On node_2
+SELECT * FROM test.t1;
+c1
+DROP TABLE t1;
+# End of test
diff --git a/mysql-test/suite/galera/t/basic.test b/mysql-test/suite/galera/t/basic.test
new file mode 100644
index 00000000000..8fc6eee3b3b
--- /dev/null
+++ b/mysql-test/suite/galera/t/basic.test
@@ -0,0 +1,26 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+USE test;
+CREATE TABLE t1(c1 INT PRIMARY KEY) ENGINE=INNODB;
+INSERT INTO t1 VALUES (1), (2), (3), (4), (5);
+SELECT * FROM t1;
+
+--echo
+--echo # On node_1
+--connection node_1
+SELECT * FROM test.t1;
+
+--echo
+--echo # On node_2
+--connection node_2
+SELECT * FROM test.t1;
+
+--let $galera_diff_statement = SELECT * FROM t1
+--source include/galera_diff.inc
+
+# Cleanup
+DROP TABLE t1;
+
+--source include/galera_end.inc
+--echo # End of test
diff --git a/mysql-test/suite/galera/t/galera_sst_mode.test b/mysql-test/suite/galera/t/galera_sst_mode.test
new file mode 100644
index 00000000000..dd8f9531c9d
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_sst_mode.test
@@ -0,0 +1,43 @@
+# Embedded server doesn't support external clients
+--source include/not_embedded.inc
+# Binlog is required
+--source include/have_log_bin.inc
+
+--echo #
+--echo # Test for mysqldump's galera-sst-mode option
+--echo #
+
+--echo #
+--echo # MDEV-6490: mysqldump unknown option --galera-sst-mode
+--echo #
+CREATE DATABASE bug6490;
+USE bug6490;
+CREATE TABLE t1(c1 INT);
+INSERT INTO t1 values (1);
+INSERT INTO t1 values (2);
+
+--echo # Save the current gtid_binlog_state.
+--let $before= `SELECT @@global.gtid_binlog_state`
+
+--echo # Take a dump of bug6490 database
+--exec $MYSQL_DUMP --galera-sst-mode bug6490 > $MYSQLTEST_VARDIR/tmp/bug6490.sql
+DROP TABLE t1;
+
+--echo # Load the dump
+RESET MASTER;
+--exec $MYSQL -uroot bug6490 < $MYSQLTEST_VARDIR/tmp/bug6490.sql
+
+SELECT * from t1;
+
+--echo # Compare the two gtid_binlog_state's
+--let $after= `SELECT @@global.gtid_binlog_state`
+if (`SELECT STRCMP($before, $after)`)
+{
+  --die ERROR: The two gtid_binlog_state's did not match.
+}
+
+--echo # Cleanup
+--remove_file $MYSQLTEST_VARDIR/tmp/bug6490.sql
+DROP DATABASE bug6490;
+
+--echo # End of test
diff --git a/mysql-test/suite/galera/t/grant.test b/mysql-test/suite/galera/t/grant.test
new file mode 100644
index 00000000000..de1c202cfbb
--- /dev/null
+++ b/mysql-test/suite/galera/t/grant.test
@@ -0,0 +1,25 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+--echo #
+--echo # MDEV#6266: Changing password fails on galera cluster
+--echo #
+
+--echo
+--echo # On node_1
+--connection node_1
+GRANT SELECT ON *.* TO 'user_6266'@'localhost' IDENTIFIED BY 'pass';
+--echo
+--echo # Now, try changing password for 'user_6266'. This command should also
+--echo # execute successfully on the other node.
+SET PASSWORD FOR 'user_6266'@'localhost' = PASSWORD('newpass');
+
+--echo
+--echo # On node_2
+--connection node_2
+SELECT user FROM mysql.user WHERE user='user_6266';
+# cleanup
+DROP USER 'user_6266'@'localhost';
+
+--source include/galera_end.inc
+--echo # End of test
diff --git a/mysql-test/suite/galera/t/partition.test b/mysql-test/suite/galera/t/partition.test
new file mode 100644
index 00000000000..048f35a9282
--- /dev/null
+++ b/mysql-test/suite/galera/t/partition.test
@@ -0,0 +1,31 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_partition.inc
+
+--echo #
+--echo # MDEV#4953 Galera: DELETE from a partitioned table is not replicated
+--echo #
+
+USE test;
+CREATE TABLE t1 (pk INT PRIMARY KEY, i INT) ENGINE=INNODB PARTITION BY HASH(pk) PARTITIONS 2;
+INSERT INTO t1 VALUES (1,100), (2,200);
+SELECT * FROM t1;
+
+DELETE FROM t1;
+SELECT * FROM t1;
+
+--echo
+--echo # On node_1
+--connection node_1
+SELECT * FROM t1;
+
+--echo
+--echo # On node_2
+--connection node_2
+SELECT * FROM t1;
+
+# Cleanup
+DROP TABLE t1;
+
+--source include/galera_end.inc
+--echo # End of test
diff --git a/mysql-test/suite/galera/t/unique_key.test b/mysql-test/suite/galera/t/unique_key.test
new file mode 100644
index 00000000000..00b85d57165
--- /dev/null
+++ b/mysql-test/suite/galera/t/unique_key.test
@@ -0,0 +1,54 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+--echo #
+--echo # MDEV#5552 Deadlock when inserting NULL column value in column with
+--echo # UNIQUE index
+--echo #
+
+USE test;
+--echo
+--echo # On node_1
+--connection node_1
+CREATE TABLE t1(c1 INT DEFAULT NULL, UNIQUE KEY c1(c1)) ENGINE=INNODB;
+INSERT INTO t1 VALUES (NULL);
+INSERT INTO t1 VALUES (NULL);
+SELECT * FROM test.t1;
+
+--echo
+--echo # On node_2
+--connection node_2
+SELECT * FROM test.t1;
+
+
+--echo
+--echo # On node_1
+--connection node_1
+INSERT INTO t1 VALUES (1);
+UPDATE t1 SET c1=NULL WHERE c1=1;
+SELECT * FROM test.t1;
+
+--echo
+--echo # On node_2
+--connection node_2
+SELECT * FROM test.t1;
+
+--echo
+--echo # On node_1
+--connection node_1
+DELETE FROM t1 WHERE c1<=>NULL;
+SELECT * FROM test.t1;
+
+--echo
+--echo # On node_2
+--connection node_2
+SELECT * FROM test.t1;
+
+--let $galera_diff_statement = SELECT * FROM t1
+--source include/galera_diff.inc
+
+# Cleanup
+DROP TABLE t1;
+
+--source include/galera_end.inc
+--echo # End of test
diff --git a/mysql-test/suite/innodb/r/innodb-autoinc.result b/mysql-test/suite/innodb/r/innodb-autoinc.result
index d5ad06e861f..6ecb6055f26 100644
--- a/mysql-test/suite/innodb/r/innodb-autoinc.result
+++ b/mysql-test/suite/innodb/r/innodb-autoinc.result
@@ -197,7 +197,7 @@ c1	c2
 5	9
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	100
 auto_increment_offset	10
@@ -230,7 +230,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -269,7 +269,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -282,7 +282,7 @@ SELECT * FROM t1;
 c1
 -1
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	100
 auto_increment_offset	10
@@ -315,7 +315,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -330,7 +330,7 @@ SELECT * FROM t1;
 c1
 1
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	100
 auto_increment_offset	10
@@ -370,7 +370,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -385,7 +385,7 @@ SELECT * FROM t1;
 c1
 1
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	100
 auto_increment_offset	10
@@ -419,7 +419,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -434,7 +434,7 @@ c1
 1
 9223372036854775794
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	2
 auto_increment_offset	10
@@ -452,7 +452,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -467,7 +467,7 @@ c1
 1
 18446744073709551603
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	2
 auto_increment_offset	10
@@ -480,7 +480,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -495,7 +495,7 @@ c1
 1
 18446744073709551603
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=5, @@SESSION.AUTO_INCREMENT_OFFSET=7;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	5
 auto_increment_offset	7
@@ -508,7 +508,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -527,7 +527,7 @@ c1
 -9223372036854775806
 1
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=3, @@SESSION.AUTO_INCREMENT_OFFSET=3;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	3
 auto_increment_offset	3
@@ -544,7 +544,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -562,7 +562,7 @@ SET @@SESSION.AUTO_INCREMENT_INCREMENT=1152921504606846976, @@SESSION.AUTO_INCRE
 Warnings:
 Warning	1292	Truncated incorrect auto_increment_increment value: '1152921504606846976'
 Warning	1292	Truncated incorrect auto_increment_offset value: '1152921504606846976'
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	65535
 auto_increment_offset	65535
@@ -575,7 +575,7 @@ c1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -862,7 +862,7 @@ ERROR 22003: Out of range value for column 'c1' at row 1
 DROP TABLE t1;
 DROP TABLE t2;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
@@ -1252,7 +1252,7 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=InnoDB AUTO_INCREMENT=18446744073709551615 DEFAULT CHARSET=latin1
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=256;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	256
@@ -1270,7 +1270,7 @@ c1	c2
 1	NULL
 DROP TABLE t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 Variable_name	Value
 auto_increment_increment	1
 auto_increment_offset	1
diff --git a/mysql-test/suite/innodb/t/innodb-autoinc.test b/mysql-test/suite/innodb/t/innodb-autoinc.test
index fd40b50ebbc..415ada2939a 100644
--- a/mysql-test/suite/innodb/t/innodb-autoinc.test
+++ b/mysql-test/suite/innodb/t/innodb-autoinc.test
@@ -161,7 +161,7 @@ DROP TABLE t1;
 #
 # Test changes to AUTOINC next value calculation
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (NULL),(5),(NULL);
@@ -178,7 +178,7 @@ DROP TABLE t1;
 # Reset the AUTOINC session variables
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(0);
@@ -198,13 +198,13 @@ DROP TABLE t1;
 # Reset the AUTOINC session variables
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(-1);
 SELECT * FROM t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 INSERT INTO t1 VALUES (-2), (NULL),(2),(NULL);
 INSERT INTO t1 VALUES (250),(NULL);
 SELECT * FROM t1;
@@ -219,13 +219,13 @@ DROP TABLE t1;
 # Reset the AUTOINC session variables
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 INT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(-1);
 SELECT * FROM t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 INSERT INTO t1 VALUES (-2);
 INSERT INTO t1 VALUES (NULL);
 INSERT INTO t1 VALUES (2);
@@ -245,13 +245,13 @@ DROP TABLE t1;
 # Reset the AUTOINC session variables
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 INT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(-1);
 SELECT * FROM t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 INSERT INTO t1 VALUES (-2),(NULL),(2),(NULL);
 INSERT INTO t1 VALUES (250),(NULL);
 SELECT * FROM t1;
@@ -267,7 +267,7 @@ DROP TABLE t1;
 # Check for overflow handling when increment is > 1
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 BIGINT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 # TODO: Fix the autoinc init code
@@ -276,7 +276,7 @@ INSERT INTO t1 VALUES(NULL);
 INSERT INTO t1 VALUES (9223372036854775794); #-- 2^63 - 14
 SELECT * FROM t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 # This should just fit
 INSERT INTO t1 VALUES (NULL),(NULL),(NULL),(NULL),(NULL),(NULL);
 SELECT * FROM t1;
@@ -286,7 +286,7 @@ DROP TABLE t1;
 # Check for overflow handling when increment and offser are > 1
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 BIGINT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 # TODO: Fix the autoinc init code
@@ -295,7 +295,7 @@ INSERT INTO t1 VALUES(NULL);
 INSERT INTO t1 VALUES (18446744073709551603); #-- 2^64 - 13
 SELECT * FROM t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 --error ER_AUTOINC_READ_FAILED
 INSERT INTO t1 VALUES (NULL),(NULL),(NULL),(NULL),(NULL),(NULL),(NULL);
 SELECT * FROM t1;
@@ -305,7 +305,7 @@ DROP TABLE t1;
 # Check for overflow handling when increment and offset are odd numbers
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 BIGINT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 # TODO: Fix the autoinc init code
@@ -314,7 +314,7 @@ INSERT INTO t1 VALUES(NULL);
 INSERT INTO t1 VALUES (18446744073709551603); #-- 2^64 - 13
 SELECT * FROM t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=5, @@SESSION.AUTO_INCREMENT_OFFSET=7;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 --error ER_AUTOINC_READ_FAILED
 INSERT INTO t1 VALUES (NULL),(NULL), (NULL);
 SELECT * FROM t1;
@@ -324,7 +324,7 @@ DROP TABLE t1;
 # and check for large -ve numbers
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 BIGINT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 # TODO: Fix the autoinc init code
@@ -335,7 +335,7 @@ INSERT INTO t1 VALUES(-9223372036854775807); #-- -2^63 + 1
 INSERT INTO t1 VALUES(-9223372036854775808); #-- -2^63
 SELECT * FROM t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=3, @@SESSION.AUTO_INCREMENT_OFFSET=3;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 INSERT INTO t1 VALUES (NULL),(NULL), (NULL);
 SELECT * FROM t1;
 DROP TABLE t1;
@@ -344,7 +344,7 @@ DROP TABLE t1;
 # large numbers 2^60
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (c1 BIGINT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB;
 # TODO: Fix the autoinc init code
@@ -353,7 +353,7 @@ INSERT INTO t1 VALUES(NULL);
 INSERT INTO t1 VALUES (18446744073709551610); #-- 2^64 - 2
 SELECT * FROM t1;
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1152921504606846976, @@SESSION.AUTO_INCREMENT_OFFSET=1152921504606846976;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 --error 167
 INSERT INTO t1 VALUES (NULL),(NULL);
 SELECT * FROM t1;
@@ -364,7 +364,7 @@ DROP TABLE t1;
 #
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
 SET @@INSERT_ID=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 CREATE TABLE t1 (c1 DOUBLE NOT NULL AUTO_INCREMENT, c2 INT, PRIMARY KEY (c1)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(NULL, 1);
 INSERT INTO t1 VALUES(NULL, 2);
@@ -450,7 +450,7 @@ DROP TABLE t2;
 # If the user has specified negative values for an AUTOINC column then
 # InnoDB should ignore those values when setting the table's max value.
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 # TINYINT
 CREATE TABLE t1 (c1 TINYINT PRIMARY KEY AUTO_INCREMENT, c2 VARCHAR(10)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1, NULL);
@@ -646,7 +646,7 @@ DROP TABLE t1;
 
 # Check if we handle offset > column max value properly
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=256;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 # TINYINT
 CREATE TABLE t1 (c1 TINYINT PRIMARY KEY AUTO_INCREMENT, c2 VARCHAR(10)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1, NULL);
@@ -658,7 +658,7 @@ DROP TABLE t1;
 # of the column. IMO, this should not be allowed and the assertion that fails
 # is actually an invariant.
 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1;
-SHOW VARIABLES LIKE "%auto_inc%";
+SHOW VARIABLES LIKE "auto_inc%";
 # TINYINT
 CREATE TABLE t1 (c1 INT UNSIGNED PRIMARY KEY AUTO_INCREMENT, c2 VARCHAR(10)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (2147483648, 'a');
diff --git a/mysql-test/suite/parts/r/partition_exch_qa_10.result b/mysql-test/suite/parts/r/partition_exch_qa_10.result
index 77b91f19e8f..7193a6c99a1 100755..100644
--- a/mysql-test/suite/parts/r/partition_exch_qa_10.result
+++ b/mysql-test/suite/parts/r/partition_exch_qa_10.result
@@ -23,7 +23,7 @@ a	b
 DROP PROCEDURE test_p1;
 SET @save_autocommit= @@autocommit;
 SET @@autocommit= OFF;
-SHOW VARIABLES LIKE '%autocommit%';
+SHOW VARIABLES LIKE 'autocommit%';
 Variable_name	Value
 autocommit	OFF
 CREATE TRIGGER test_trg_1 BEFORE UPDATE ON tp FOR EACH ROW
diff --git a/mysql-test/suite/parts/t/partition_exch_qa_10.test b/mysql-test/suite/parts/t/partition_exch_qa_10.test
index 4f569605f5f..a87d658cfb6 100644
--- a/mysql-test/suite/parts/t/partition_exch_qa_10.test
+++ b/mysql-test/suite/parts/t/partition_exch_qa_10.test
@@ -37,7 +37,7 @@ DROP PROCEDURE test_p1;
 
 SET @save_autocommit= @@autocommit;
 SET @@autocommit= OFF;
-SHOW VARIABLES LIKE '%autocommit%';
+SHOW VARIABLES LIKE 'autocommit%';
 DELIMITER |;
 --error ER_COMMIT_NOT_ALLOWED_IN_SF_OR_TRG
 CREATE TRIGGER test_trg_1 BEFORE UPDATE ON tp FOR EACH ROW
diff --git a/mysql-test/suite/percona/innodb_sys_index.result b/mysql-test/suite/percona/innodb_sys_index.result
index 67604236366..1c4cc63c467 100644
--- a/mysql-test/suite/percona/innodb_sys_index.result
+++ b/mysql-test/suite/percona/innodb_sys_index.result
@@ -3,7 +3,7 @@ Warnings:
 Note	1051	Unknown table 'test.t1'
 select @@version_comment limit 1 ;
 @@version_comment
-Source distribution
+Source distribution, wsrep_25.10.r3991
 SELECT COUNT(*) FROM `information_schema`.`INNODB_SYS_INDEXES` ;
 CREATE TABLE test.t1 (  `a` SERIAL NOT NULL ,  `b` VARCHAR( 255 ) NOT NULL ,  INDEX ( `b` )  ) ENGINE = InnoDB ;
 SHOW TABLE STATUS FROM `information_schema` LIKE 'INNODB\_SYS\_INDEXES%' ;
diff --git a/mysql-test/suite/percona/innodb_sys_index.test b/mysql-test/suite/percona/innodb_sys_index.test
index 212baeda663..a5a79611815 100644
--- a/mysql-test/suite/percona/innodb_sys_index.test
+++ b/mysql-test/suite/percona/innodb_sys_index.test
@@ -5,7 +5,8 @@ drop table if exists t1;
 #
 # test for bug LP#875797 "Using 'innodb_sys_indexes' causes core dump"
 #
-select @@version_comment limit 1 ;
+# disable version_commit as it could contain wsrep
+#select @@version_comment limit 1 ;
 --disable_result_log
 SELECT COUNT(*) FROM `information_schema`.`INNODB_SYS_INDEXES` ;
 CREATE TABLE test.t1 (  `a` SERIAL NOT NULL ,  `b` VARCHAR( 255 ) NOT NULL ,  INDEX ( `b` )  ) ENGINE = InnoDB ;
diff --git a/mysql-test/suite/perfschema/r/rpl_statements.result b/mysql-test/suite/perfschema/r/rpl_statements.result
index e271cd2a7fa..211a7d3398d 100644
--- a/mysql-test/suite/perfschema/r/rpl_statements.result
+++ b/mysql-test/suite/perfschema/r/rpl_statements.result
@@ -11,9 +11,10 @@ include/master-slave.inc
 
 *** Create test tables
 
-show variables like '%binlog_format%';
+show variables like 'binlog_format%';
 Variable_name	Value
 binlog_format	MIXED
+wsrep_forced_binlog_format	NONE
 drop table if exists test.marker;
 select thread_id into @my_thread_id
 from performance_schema.threads
@@ -55,9 +56,10 @@ Expect 1
 *** MASTER ***
 **************
 
-show variables like '%binlog_format%';
+show variables like 'binlog_format%';
 Variable_name	Value
 binlog_format	MIXED
+wsrep_forced_binlog_format	NONE
 *** Clear statement events
 
 *** Create/drop table, create/drop database
diff --git a/mysql-test/suite/perfschema/t/rpl_statements.test b/mysql-test/suite/perfschema/t/rpl_statements.test
index fa429cd2aa3..479805edccc 100644
--- a/mysql-test/suite/perfschema/t/rpl_statements.test
+++ b/mysql-test/suite/perfschema/t/rpl_statements.test
@@ -64,7 +64,7 @@ connection master;
 --echo *** Create test tables
 --echo
 
-show variables like '%binlog_format%';
+show variables like 'binlog_format%';
 
 --disable_warnings
 drop table if exists test.marker;
@@ -129,7 +129,7 @@ connection master;
 --echo *** MASTER ***
 --echo **************
 --echo
-show variables like '%binlog_format%';
+show variables like 'binlog_format%';
 
 --echo *** Clear statement events
 --source ../include/rpl_statements_truncate.inc
diff --git a/mysql-test/suite/sys_vars/r/innodb_disallow_writes_basic.result b/mysql-test/suite/sys_vars/r/innodb_disallow_writes_basic.result
new file mode 100644
index 00000000000..bfb6b67b5d8
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/innodb_disallow_writes_basic.result
@@ -0,0 +1,45 @@
+#
+# innodb_disallow_writes
+#
+# save the initial value
+SET @innodb_disallow_writes_global_saved = @@global.innodb_disallow_writes;
+# default
+SELECT @@global.innodb_disallow_writes;
+@@global.innodb_disallow_writes
+0
+
+# scope
+SELECT @@session.innodb_disallow_writes;
+ERROR HY000: Variable 'innodb_disallow_writes' is a GLOBAL variable
+SET @@global.innodb_disallow_writes=OFF;
+SELECT @@global.innodb_disallow_writes;
+@@global.innodb_disallow_writes
+0
+SET @@global.innodb_disallow_writes=ON;
+SELECT @@global.innodb_disallow_writes;
+@@global.innodb_disallow_writes
+1
+
+# valid values
+SET @@global.innodb_disallow_writes='OFF';
+SELECT @@global.innodb_disallow_writes;
+@@global.innodb_disallow_writes
+0
+SET @@global.innodb_disallow_writes=ON;
+SELECT @@global.innodb_disallow_writes;
+@@global.innodb_disallow_writes
+1
+SET @@global.innodb_disallow_writes=default;
+SELECT @@global.innodb_disallow_writes;
+@@global.innodb_disallow_writes
+0
+
+# invalid values
+SET @@global.innodb_disallow_writes=NULL;
+ERROR 42000: Variable 'innodb_disallow_writes' can't be set to the value of 'NULL'
+SET @@global.innodb_disallow_writes='junk';
+ERROR 42000: Variable 'innodb_disallow_writes' can't be set to the value of 'junk'
+
+# restore the initial value
+SET @@global.innodb_disallow_writes = @innodb_disallow_writes_global_saved;
+# End of test
diff --git a/mysql-test/suite/sys_vars/r/wsrep_auto_increment_control_basic.result b/mysql-test/suite/sys_vars/r/wsrep_auto_increment_control_basic.result
new file mode 100644
index 00000000000..d5affeaf5e4
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_auto_increment_control_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_auto_increment_control;
+set @@global.wsrep_auto_increment_control=ON;
+set @@global.wsrep_auto_increment_control=OFF;
+set @@global.wsrep_auto_increment_control=1;
+set @@global.wsrep_auto_increment_control=0;
+SET @@global.wsrep_auto_increment_control = -1;
+ERROR 42000: Variable 'wsrep_auto_increment_control' can't be set to the value of '-1'
+set @@global.wsrep_auto_increment_control = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_causal_reads_basic.result b/mysql-test/suite/sys_vars/r/wsrep_causal_reads_basic.result
new file mode 100644
index 00000000000..3b96654f8c7
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_causal_reads_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_causal_reads;
+set @@global.wsrep_causal_reads=ON;
+set @@global.wsrep_causal_reads=OFF;
+set @@global.wsrep_causal_reads=1;
+set @@global.wsrep_causal_reads=0;
+SET @@global.wsrep_causal_reads = -1;
+ERROR 42000: Variable 'wsrep_causal_reads' can't be set to the value of '-1'
+set @@global.wsrep_causal_reads = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_certify_nonpk_basic.result b/mysql-test/suite/sys_vars/r/wsrep_certify_nonpk_basic.result
new file mode 100644
index 00000000000..4b02f9fb61e
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_certify_nonpk_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_certify_nonpk;
+set @@global.wsrep_certify_nonpk=ON;
+set @@global.wsrep_certify_nonpk=OFF;
+set @@global.wsrep_certify_nonpk=1;
+set @@global.wsrep_certify_nonpk=0;
+SET @@global.wsrep_certify_nonpk = -1;
+ERROR 42000: Variable 'wsrep_certify_nonPK' can't be set to the value of '-1'
+set @@global.wsrep_certify_nonpk = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_cluster_address_basic.result b/mysql-test/suite/sys_vars/r/wsrep_cluster_address_basic.result
new file mode 100644
index 00000000000..734908d42e5
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_cluster_address_basic.result
@@ -0,0 +1,45 @@
+SELECT COUNT(@@GLOBAL.wsrep_cluster_address);
+COUNT(@@GLOBAL.wsrep_cluster_address)
+1
+1 Expected
+SELECT COUNT(@@GLOBAL.wsrep_cluster_address);
+COUNT(@@GLOBAL.wsrep_cluster_address)
+1
+1 Expected
+SELECT @@GLOBAL.wsrep_cluster_address = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_cluster_address';
+@@GLOBAL.wsrep_cluster_address = VARIABLE_VALUE
+1
+1 Expected
+SELECT COUNT(@@GLOBAL.wsrep_cluster_address);
+COUNT(@@GLOBAL.wsrep_cluster_address)
+1
+1 Expected
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_cluster_address';
+COUNT(VARIABLE_VALUE)
+1
+1 Expected
+SELECT @@wsrep_cluster_address = @@GLOBAL.wsrep_cluster_address;
+@@wsrep_cluster_address = @@GLOBAL.wsrep_cluster_address
+1
+1 Expected
+SELECT COUNT(@@wsrep_cluster_address);
+COUNT(@@wsrep_cluster_address)
+1
+1 Expected
+SELECT COUNT(@@local.wsrep_cluster_address);
+ERROR HY000: Variable 'wsrep_cluster_address' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@SESSION.wsrep_cluster_address);
+ERROR HY000: Variable 'wsrep_cluster_address' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@GLOBAL.wsrep_cluster_address);
+COUNT(@@GLOBAL.wsrep_cluster_address)
+1
+1 Expected
+SELECT wsrep_cluster_address = @@SESSION.wsrep_cluster_address;
+ERROR 42S22: Unknown column 'wsrep_cluster_address' in 'field list'
+Expected error 'Readonly variable'
diff --git a/mysql-test/suite/sys_vars/r/wsrep_cluster_name_basic.result b/mysql-test/suite/sys_vars/r/wsrep_cluster_name_basic.result
new file mode 100644
index 00000000000..59c3b9381d1
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_cluster_name_basic.result
@@ -0,0 +1,7 @@
+set @start_value = @@wsrep_cluster_name;
+set @@global.wsrep_cluster_name='test';
+set @@global.wsrep_cluster_name=NULL;
+ERROR 42000: Variable 'wsrep_cluster_name' can't be set to the value of 'NULL'
+SET @@global.wsrep_cluster_name = 1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_cluster_name'
+set @@global.wsrep_cluster_name = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_convert_lock_to_trx_basic.result b/mysql-test/suite/sys_vars/r/wsrep_convert_lock_to_trx_basic.result
new file mode 100644
index 00000000000..10043812289
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_convert_lock_to_trx_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_convert_lock_to_trx;
+set @@global.wsrep_convert_lock_to_trx=ON;
+set @@global.wsrep_convert_lock_to_trx=OFF;
+set @@global.wsrep_convert_lock_to_trx=1;
+set @@global.wsrep_convert_lock_to_trx=0;
+SET @@global.wsrep_convert_lock_to_trx = -1;
+ERROR 42000: Variable 'wsrep_convert_LOCK_to_trx' can't be set to the value of '-1'
+set @@global.wsrep_convert_lock_to_trx = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_data_home_dir_basic.result b/mysql-test/suite/sys_vars/r/wsrep_data_home_dir_basic.result
new file mode 100644
index 00000000000..668aebe30f1
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_data_home_dir_basic.result
@@ -0,0 +1,48 @@
+SELECT COUNT(@@GLOBAL.wsrep_data_home_dir);
+COUNT(@@GLOBAL.wsrep_data_home_dir)
+1
+1 Expected
+SET @@GLOBAL.wsrep_data_home_dir=1;
+ERROR HY000: Variable 'wsrep_data_home_dir' is a read only variable
+Expected error 'Read only variable'
+SELECT COUNT(@@GLOBAL.wsrep_data_home_dir);
+COUNT(@@GLOBAL.wsrep_data_home_dir)
+1
+1 Expected
+SELECT @@GLOBAL.wsrep_data_home_dir = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_data_home_dir';
+@@GLOBAL.wsrep_data_home_dir = VARIABLE_VALUE
+1
+1 Expected
+SELECT COUNT(@@GLOBAL.wsrep_data_home_dir);
+COUNT(@@GLOBAL.wsrep_data_home_dir)
+1
+1 Expected
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_data_home_dir';
+COUNT(VARIABLE_VALUE)
+1
+1 Expected
+SELECT @@wsrep_data_home_dir = @@GLOBAL.wsrep_data_home_dir;
+@@wsrep_data_home_dir = @@GLOBAL.wsrep_data_home_dir
+1
+1 Expected
+SELECT COUNT(@@wsrep_data_home_dir);
+COUNT(@@wsrep_data_home_dir)
+1
+1 Expected
+SELECT COUNT(@@local.wsrep_data_home_dir);
+ERROR HY000: Variable 'wsrep_data_home_dir' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@SESSION.wsrep_data_home_dir);
+ERROR HY000: Variable 'wsrep_data_home_dir' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@GLOBAL.wsrep_data_home_dir);
+COUNT(@@GLOBAL.wsrep_data_home_dir)
+1
+1 Expected
+SELECT wsrep_data_home_dir = @@SESSION.wsrep_data_home_dir;
+ERROR 42S22: Unknown column 'wsrep_data_home_dir' in 'field list'
+Expected error 'Readonly variable'
diff --git a/mysql-test/suite/sys_vars/r/wsrep_dbug_option_basic.result b/mysql-test/suite/sys_vars/r/wsrep_dbug_option_basic.result
new file mode 100644
index 00000000000..36ebcb17002
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_dbug_option_basic.result
@@ -0,0 +1,6 @@
+set @start_value = @@wsrep_dbug_option;
+set @@global.wsrep_dbug_option='foo:bar';
+set @@global.wsrep_dbug_option=NULL;
+SET @@global.wsrep_dbug_option = -1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_dbug_option'
+set @@global.wsrep_dbug_option = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_debug_basic.result b/mysql-test/suite/sys_vars/r/wsrep_debug_basic.result
new file mode 100644
index 00000000000..6bbe780316b
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_debug_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_debug;
+set @@global.wsrep_debug=ON;
+set @@global.wsrep_debug=OFF;
+set @@global.wsrep_debug=1;
+set @@global.wsrep_debug=0;
+SET @@global.wsrep_debug = -1;
+ERROR 42000: Variable 'wsrep_debug' can't be set to the value of '-1'
+set @@global.wsrep_debug = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_desync_basic.result b/mysql-test/suite/sys_vars/r/wsrep_desync_basic.result
new file mode 100644
index 00000000000..a61367ca200
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_desync_basic.result
@@ -0,0 +1,3 @@
+select @@global.wsrep_desync;
+@@global.wsrep_desync
+0
diff --git a/mysql-test/suite/sys_vars/r/wsrep_drupal_282555_workaround_basic.result b/mysql-test/suite/sys_vars/r/wsrep_drupal_282555_workaround_basic.result
new file mode 100644
index 00000000000..5a8d5a8abee
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_drupal_282555_workaround_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_drupal_282555_workaround;
+set @@global.wsrep_drupal_282555_workaround=ON;
+set @@global.wsrep_drupal_282555_workaround=OFF;
+set @@global.wsrep_drupal_282555_workaround=1;
+set @@global.wsrep_drupal_282555_workaround=0;
+SET @@global.wsrep_drupal_282555_workaround = -1;
+ERROR 42000: Variable 'wsrep_drupal_282555_workaround' can't be set to the value of '-1'
+set @@global.wsrep_drupal_282555_workaround = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_forced_binlog_format_basic.result b/mysql-test/suite/sys_vars/r/wsrep_forced_binlog_format_basic.result
new file mode 100644
index 00000000000..58bdb45c8de
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_forced_binlog_format_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_forced_binlog_format;
+set @@global.wsrep_forced_binlog_format = ROW;
+set @@global.wsrep_forced_binlog_format = MIXED;
+set @@global.wsrep_forced_binlog_format = STATEMENT;
+set @@global.wsrep_forced_binlog_format = NONE;
+set @@global.wsrep_forced_binlog_format = FOO;
+ERROR 42000: Variable 'wsrep_forced_binlog_format' can't be set to the value of 'FOO'
+set @@global.wsrep_forced_binlog_format = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_load_data_splitting_basic.result b/mysql-test/suite/sys_vars/r/wsrep_load_data_splitting_basic.result
new file mode 100644
index 00000000000..be73397f35e
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_load_data_splitting_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_load_data_splitting;
+set @@global.wsrep_load_data_splitting=ON;
+set @@global.wsrep_load_data_splitting=OFF;
+set @@global.wsrep_load_data_splitting=1;
+set @@global.wsrep_load_data_splitting=0;
+SET @@global.wsrep_load_data_splitting = -1;
+ERROR 42000: Variable 'wsrep_load_data_splitting' can't be set to the value of '-1'
+set @@global.wsrep_load_data_splitting = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_log_conflicts_basic.result b/mysql-test/suite/sys_vars/r/wsrep_log_conflicts_basic.result
new file mode 100644
index 00000000000..22d8cbb568a
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_log_conflicts_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_log_conflicts;
+set @@global.wsrep_log_conflicts=ON;
+set @@global.wsrep_log_conflicts=OFF;
+set @@global.wsrep_log_conflicts=1;
+set @@global.wsrep_log_conflicts=0;
+SET @@global.wsrep_log_conflicts = -1;
+ERROR 42000: Variable 'wsrep_log_conflicts' can't be set to the value of '-1'
+set @@global.wsrep_log_conflicts = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_max_ws_rows_basic.result b/mysql-test/suite/sys_vars/r/wsrep_max_ws_rows_basic.result
new file mode 100644
index 00000000000..fc4dd38ade3
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_max_ws_rows_basic.result
@@ -0,0 +1,17 @@
+set @start_value = @@wsrep_max_ws_rows;
+set @@global.wsrep_max_ws_rows=256000;
+set @@global.wsrep_max_ws_rows=0;
+Warnings:
+Warning	1292	Truncated incorrect wsrep_max_ws_rows value: '0'
+show warnings;
+Level	Code	Message
+Warning	1292	Truncated incorrect wsrep_max_ws_rows value: '0'
+set @@global.wsrep_max_ws_rows=-1;
+Warnings:
+Warning	1292	Truncated incorrect wsrep_max_ws_rows value: '-1'
+show warnings;
+Level	Code	Message
+Warning	1292	Truncated incorrect wsrep_max_ws_rows value: '-1'
+SET @@global.wsrep_max_ws_rows = r;
+ERROR 42000: Incorrect argument type to variable 'wsrep_max_ws_rows'
+set @@global.wsrep_max_ws_rows = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_max_ws_size_basic.result b/mysql-test/suite/sys_vars/r/wsrep_max_ws_size_basic.result
new file mode 100644
index 00000000000..292fd4e02d8
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_max_ws_size_basic.result
@@ -0,0 +1,17 @@
+set @start_value = @@wsrep_max_ws_size;
+set @@global.wsrep_max_ws_size=256000;
+set @@global.wsrep_max_ws_size=0;
+Warnings:
+Warning	1292	Truncated incorrect wsrep_max_ws_size value: '0'
+show warnings;
+Level	Code	Message
+Warning	1292	Truncated incorrect wsrep_max_ws_size value: '0'
+set @@global.wsrep_max_ws_size=-1;
+Warnings:
+Warning	1292	Truncated incorrect wsrep_max_ws_size value: '-1'
+show warnings;
+Level	Code	Message
+Warning	1292	Truncated incorrect wsrep_max_ws_size value: '-1'
+SET @@global.wsrep_max_ws_size = r;
+ERROR 42000: Incorrect argument type to variable 'wsrep_max_ws_size'
+set @@global.wsrep_max_ws_size = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_mysql_replication_bundle_basic.result b/mysql-test/suite/sys_vars/r/wsrep_mysql_replication_bundle_basic.result
new file mode 100644
index 00000000000..ad435a2c05f
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_mysql_replication_bundle_basic.result
@@ -0,0 +1,18 @@
+set @start_value = @@wsrep_mysql_replication_bundle;
+set @@global.wsrep_mysql_replication_bundle=0;
+set @@global.wsrep_mysql_replication_bundle=1000;
+set @@global.wsrep_mysql_replication_bundle=-1;
+Warnings:
+Warning	1292	Truncated incorrect wsrep_mysql_replication_bundle value: '-1'
+show warnings;
+Level	Code	Message
+Warning	1292	Truncated incorrect wsrep_mysql_replication_bundle value: '-1'
+set @@global.wsrep_mysql_replication_bundle=1001;
+Warnings:
+Warning	1292	Truncated incorrect wsrep_mysql_replication_bundle value: '1001'
+show warnings;
+Level	Code	Message
+Warning	1292	Truncated incorrect wsrep_mysql_replication_bundle value: '1001'
+SET @@global.wsrep_mysql_replication_bundle = r;
+ERROR 42000: Incorrect argument type to variable 'wsrep_mysql_replication_bundle'
+set @@global.wsrep_mysql_replication_bundle = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_node_address_basic.result b/mysql-test/suite/sys_vars/r/wsrep_node_address_basic.result
new file mode 100644
index 00000000000..96ae51cc70f
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_node_address_basic.result
@@ -0,0 +1,45 @@
+SELECT COUNT(@@GLOBAL.wsrep_node_address);
+COUNT(@@GLOBAL.wsrep_node_address)
+1
+1 Expected
+SET @@GLOBAL.wsrep_node_address=1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_node_address'
+Expected error 'Read only variable'
+SELECT COUNT(@@GLOBAL.wsrep_node_address);
+COUNT(@@GLOBAL.wsrep_node_address)
+1
+1 Expected
+SELECT @@GLOBAL.wsrep_node_address = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_node_address';
+@@GLOBAL.wsrep_node_address = VARIABLE_VALUE
+1
+1 Expected
+SELECT COUNT(@@GLOBAL.wsrep_node_address);
+COUNT(@@GLOBAL.wsrep_node_address)
+1
+1 Expected
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_node_address';
+COUNT(VARIABLE_VALUE)
+1
+1 Expected
+SELECT @@wsrep_node_address = @@GLOBAL.wsrep_node_address;
+@@wsrep_node_address = @@GLOBAL.wsrep_node_address
+1
+1 Expected
+SELECT COUNT(@@wsrep_node_address);
+COUNT(@@wsrep_node_address)
+1
+1 Expected
+SELECT COUNT(@@local.wsrep_node_address);
+ERROR HY000: Variable 'wsrep_node_address' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@SESSION.wsrep_node_address);
+ERROR HY000: Variable 'wsrep_node_address' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@GLOBAL.wsrep_node_address);
+COUNT(@@GLOBAL.wsrep_node_address)
+1
+1 Expected
diff --git a/mysql-test/suite/sys_vars/r/wsrep_node_incoming_address_basic.result b/mysql-test/suite/sys_vars/r/wsrep_node_incoming_address_basic.result
new file mode 100644
index 00000000000..9ccf9706484
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_node_incoming_address_basic.result
@@ -0,0 +1,45 @@
+SELECT COUNT(@@GLOBAL.wsrep_node_incoming_address);
+COUNT(@@GLOBAL.wsrep_node_incoming_address)
+1
+1 Expected
+SET @@GLOBAL.wsrep_node_incoming_address=1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_node_incoming_address'
+Expected error 'Read only variable'
+SELECT COUNT(@@GLOBAL.wsrep_node_incoming_address);
+COUNT(@@GLOBAL.wsrep_node_incoming_address)
+1
+1 Expected
+SELECT @@GLOBAL.wsrep_node_incoming_address = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_node_incoming_address';
+@@GLOBAL.wsrep_node_incoming_address = VARIABLE_VALUE
+1
+1 Expected
+SELECT COUNT(@@GLOBAL.wsrep_node_incoming_address);
+COUNT(@@GLOBAL.wsrep_node_incoming_address)
+1
+1 Expected
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_node_incoming_address';
+COUNT(VARIABLE_VALUE)
+1
+1 Expected
+SELECT @@wsrep_node_incoming_address = @@GLOBAL.wsrep_node_incoming_address;
+@@wsrep_node_incoming_address = @@GLOBAL.wsrep_node_incoming_address
+1
+1 Expected
+SELECT COUNT(@@wsrep_node_incoming_address);
+COUNT(@@wsrep_node_incoming_address)
+1
+1 Expected
+SELECT COUNT(@@local.wsrep_node_incoming_address);
+ERROR HY000: Variable 'wsrep_node_incoming_address' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@SESSION.wsrep_node_incoming_address);
+ERROR HY000: Variable 'wsrep_node_incoming_address' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@GLOBAL.wsrep_node_incoming_address);
+COUNT(@@GLOBAL.wsrep_node_incoming_address)
+1
+1 Expected
diff --git a/mysql-test/suite/sys_vars/r/wsrep_node_name_basic.result b/mysql-test/suite/sys_vars/r/wsrep_node_name_basic.result
new file mode 100644
index 00000000000..f3c03570b7b
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_node_name_basic.result
@@ -0,0 +1,6 @@
+set @start_value = @@wsrep_node_name;
+set @@global.wsrep_node_name='test';
+set @@global.wsrep_node_name=NULL;
+SET @@global.wsrep_node_name = 1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_node_name'
+set @@global.wsrep_node_name = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_notify_cmd_basic.result b/mysql-test/suite/sys_vars/r/wsrep_notify_cmd_basic.result
new file mode 100644
index 00000000000..d1d68ea036b
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_notify_cmd_basic.result
@@ -0,0 +1,6 @@
+set @start_value = @@wsrep_notify_cmd;
+set @@global.wsrep_notify_cmd='test';
+set @@global.wsrep_notify_cmd=NULL;
+SET @@global.wsrep_notify_cmd = 1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_notify_cmd'
+set @@global.wsrep_notify_cmd = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_on_basic.result b/mysql-test/suite/sys_vars/r/wsrep_on_basic.result
new file mode 100644
index 00000000000..629c0e866cb
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_on_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_on;
+set @@global.wsrep_on=ON;
+set @@global.wsrep_on=OFF;
+set @@global.wsrep_on=1;
+set @@global.wsrep_on=0;
+SET @@global.wsrep_on = -1;
+ERROR 42000: Variable 'wsrep_on' can't be set to the value of '-1'
+set @@global.wsrep_on = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_osu_method_basic.result b/mysql-test/suite/sys_vars/r/wsrep_osu_method_basic.result
new file mode 100644
index 00000000000..84c828e5965
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_osu_method_basic.result
@@ -0,0 +1,12 @@
+set @start_value = @@wsrep_osu_method;
+set @@global.wsrep_osu_method='TOI';
+set @@global.wsrep_osu_method='RSU';
+set @@global.wsrep_osu_method=TOI;
+set @@global.wsrep_osu_method=RSU;
+set @@global.wsrep_osu_method=TSU;
+ERROR 42000: Variable 'wsrep_OSU_method' can't be set to the value of 'TSU'
+set @@global.wsrep_osu_method='TSU';
+ERROR 42000: Variable 'wsrep_OSU_method' can't be set to the value of 'TSU'
+SET @@global.wsrep_on = -1;
+ERROR 42000: Variable 'wsrep_on' can't be set to the value of '-1'
+set @@global.wsrep_osu_method = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_provider_basic.result b/mysql-test/suite/sys_vars/r/wsrep_provider_basic.result
new file mode 100644
index 00000000000..2de1e84e6c0
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_provider_basic.result
@@ -0,0 +1,4 @@
+SELECT COUNT(@@GLOBAL.wsrep_provider);
+COUNT(@@GLOBAL.wsrep_provider)
+1
+1 Expected
diff --git a/mysql-test/suite/sys_vars/r/wsrep_provider_options_basic.result b/mysql-test/suite/sys_vars/r/wsrep_provider_options_basic.result
new file mode 100644
index 00000000000..28b55e782bc
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_provider_options_basic.result
@@ -0,0 +1,4 @@
+SELECT COUNT(@@GLOBAL.wsrep_provider_options);
+COUNT(@@GLOBAL.wsrep_provider_options)
+1
+1 Expected
diff --git a/mysql-test/suite/sys_vars/r/wsrep_recover_basic.result b/mysql-test/suite/sys_vars/r/wsrep_recover_basic.result
new file mode 100644
index 00000000000..b9f7e41047e
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_recover_basic.result
@@ -0,0 +1,49 @@
+SELECT COUNT(@@GLOBAL.wsrep_recover);
+COUNT(@@GLOBAL.wsrep_recover)
+1
+1 Expected
+set @@global.wsrep_recover=ON;
+ERROR HY000: Variable 'wsrep_recover' is a read only variable
+Expected error 'Readonly variable'
+set @@global.wsrep_recover=OFF;
+ERROR HY000: Variable 'wsrep_recover' is a read only variable
+Expected error 'Readonly variable'
+SELECT @@GLOBAL.wsrep_recover = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_recover';
+@@GLOBAL.wsrep_recover = VARIABLE_VALUE
+1
+Warnings:
+Warning	1292	Truncated incorrect DOUBLE value: 'OFF'
+1 Expected
+SELECT COUNT(@@GLOBAL.wsrep_recover);
+COUNT(@@GLOBAL.wsrep_recover)
+1
+1 Expected
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_recover';
+COUNT(VARIABLE_VALUE)
+1
+1 Expected
+SELECT @@wsrep_recover = @@GLOBAL.wsrep_recover;
+@@wsrep_recover = @@GLOBAL.wsrep_recover
+1
+1 Expected
+SELECT COUNT(@@wsrep_recover);
+COUNT(@@wsrep_recover)
+1
+1 Expected
+SELECT COUNT(@@local.wsrep_recover);
+ERROR HY000: Variable 'wsrep_recover' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@SESSION.wsrep_recover);
+ERROR HY000: Variable 'wsrep_recover' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+SELECT COUNT(@@GLOBAL.wsrep_recover);
+COUNT(@@GLOBAL.wsrep_recover)
+1
+1 Expected
+SELECT wsrep_recover = @@SESSION.wsrep_recover;
+ERROR 42S22: Unknown column 'wsrep_recover' in 'field list'
+Expected error 'Readonly variable'
diff --git a/mysql-test/suite/sys_vars/r/wsrep_replicate_myisam_basic.result b/mysql-test/suite/sys_vars/r/wsrep_replicate_myisam_basic.result
new file mode 100644
index 00000000000..644258206a5
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_replicate_myisam_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_replicate_myisam;
+set @@global.wsrep_replicate_myisam=ON;
+set @@global.wsrep_replicate_myisam=OFF;
+set @@global.wsrep_replicate_myisam=1;
+set @@global.wsrep_replicate_myisam=0;
+SET @@global.wsrep_replicate_myisam = -1;
+ERROR 42000: Variable 'wsrep_replicate_myisam' can't be set to the value of '-1'
+set @@global.wsrep_replicate_myisam = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_restart_slave_basic.result b/mysql-test/suite/sys_vars/r/wsrep_restart_slave_basic.result
new file mode 100644
index 00000000000..811981b6a37
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_restart_slave_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_restart_slave;
+set @@global.wsrep_restart_slave=ON;
+set @@global.wsrep_restart_slave=OFF;
+set @@global.wsrep_restart_slave=1;
+set @@global.wsrep_restart_slave=0;
+SET @@global.wsrep_restart_slave = -1;
+ERROR 42000: Variable 'wsrep_restart_slave' can't be set to the value of '-1'
+set @@global.wsrep_restart_slave = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_retry_autocommit_basic.result b/mysql-test/suite/sys_vars/r/wsrep_retry_autocommit_basic.result
new file mode 100644
index 00000000000..811981b6a37
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_retry_autocommit_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_restart_slave;
+set @@global.wsrep_restart_slave=ON;
+set @@global.wsrep_restart_slave=OFF;
+set @@global.wsrep_restart_slave=1;
+set @@global.wsrep_restart_slave=0;
+SET @@global.wsrep_restart_slave = -1;
+ERROR 42000: Variable 'wsrep_restart_slave' can't be set to the value of '-1'
+set @@global.wsrep_restart_slave = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_slave_fk_checks_basic.result b/mysql-test/suite/sys_vars/r/wsrep_slave_fk_checks_basic.result
new file mode 100644
index 00000000000..40b3270e221
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_slave_fk_checks_basic.result
@@ -0,0 +1,45 @@
+#
+# wsrep_slave_fk_checks
+#
+# save the initial value
+SET @wsrep_slave_fk_checks_global_saved = @@global.wsrep_slave_fk_checks;
+# default
+SELECT @@global.wsrep_slave_fk_checks;
+@@global.wsrep_slave_fk_checks
+1
+
+# scope
+SELECT @@session.wsrep_slave_fk_checks;
+ERROR HY000: Variable 'wsrep_slave_FK_checks' is a GLOBAL variable
+SET @@global.wsrep_slave_fk_checks=OFF;
+SELECT @@global.wsrep_slave_fk_checks;
+@@global.wsrep_slave_fk_checks
+0
+SET @@global.wsrep_slave_fk_checks=ON;
+SELECT @@global.wsrep_slave_fk_checks;
+@@global.wsrep_slave_fk_checks
+1
+
+# valid values
+SET @@global.wsrep_slave_fk_checks='OFF';
+SELECT @@global.wsrep_slave_fk_checks;
+@@global.wsrep_slave_fk_checks
+0
+SET @@global.wsrep_slave_fk_checks=ON;
+SELECT @@global.wsrep_slave_fk_checks;
+@@global.wsrep_slave_fk_checks
+1
+SET @@global.wsrep_slave_fk_checks=default;
+SELECT @@global.wsrep_slave_fk_checks;
+@@global.wsrep_slave_fk_checks
+1
+
+# invalid values
+SET @@global.wsrep_slave_fk_checks=NULL;
+ERROR 42000: Variable 'wsrep_slave_FK_checks' can't be set to the value of 'NULL'
+SET @@global.wsrep_slave_fk_checks='junk';
+ERROR 42000: Variable 'wsrep_slave_FK_checks' can't be set to the value of 'junk'
+
+# restore the initial value
+SET @@global.wsrep_slave_fk_checks = @wsrep_slave_fk_checks_global_saved;
+# End of test
diff --git a/mysql-test/suite/sys_vars/r/wsrep_slave_threads_basic.result b/mysql-test/suite/sys_vars/r/wsrep_slave_threads_basic.result
new file mode 100644
index 00000000000..f8105660c6e
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_slave_threads_basic.result
@@ -0,0 +1,20 @@
+set @start_value = @@wsrep_slave_threads;
+set @@global.wsrep_slave_threads=1;
+set @@global.wsrep_slave_threads=4;
+show warnings;
+Level	Code	Message
+set @@global.wsrep_slave_threads=0;
+Warnings:
+Warning	1292	Truncated incorrect wsrep_slave_threads value: '0'
+show warnings;
+Level	Code	Message
+Warning	1292	Truncated incorrect wsrep_slave_threads value: '0'
+set @@global.wsrep_slave_threads=-1;
+Warnings:
+Warning	1292	Truncated incorrect wsrep_slave_threads value: '-1'
+show warnings;
+Level	Code	Message
+Warning	1292	Truncated incorrect wsrep_slave_threads value: '-1'
+SET @@global.wsrep_slave_threads = r;
+ERROR 42000: Incorrect argument type to variable 'wsrep_slave_threads'
+set @@global.wsrep_slave_threads = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_slave_uk_checks_basic.result b/mysql-test/suite/sys_vars/r/wsrep_slave_uk_checks_basic.result
new file mode 100644
index 00000000000..b78a83b547d
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_slave_uk_checks_basic.result
@@ -0,0 +1,45 @@
+#
+# wsrep_slave_uk_checks
+#
+# save the initial value
+SET @wsrep_slave_uk_checks_global_saved = @@global.wsrep_slave_uk_checks;
+# default
+SELECT @@global.wsrep_slave_uk_checks;
+@@global.wsrep_slave_uk_checks
+0
+
+# scope
+SELECT @@session.wsrep_slave_uk_checks;
+ERROR HY000: Variable 'wsrep_slave_UK_checks' is a GLOBAL variable
+SET @@global.wsrep_slave_uk_checks=OFF;
+SELECT @@global.wsrep_slave_uk_checks;
+@@global.wsrep_slave_uk_checks
+0
+SET @@global.wsrep_slave_uk_checks=ON;
+SELECT @@global.wsrep_slave_uk_checks;
+@@global.wsrep_slave_uk_checks
+1
+
+# valid values
+SET @@global.wsrep_slave_uk_checks='OFF';
+SELECT @@global.wsrep_slave_uk_checks;
+@@global.wsrep_slave_uk_checks
+0
+SET @@global.wsrep_slave_uk_checks=ON;
+SELECT @@global.wsrep_slave_uk_checks;
+@@global.wsrep_slave_uk_checks
+1
+SET @@global.wsrep_slave_uk_checks=default;
+SELECT @@global.wsrep_slave_uk_checks;
+@@global.wsrep_slave_uk_checks
+0
+
+# invalid values
+SET @@global.wsrep_slave_uk_checks=NULL;
+ERROR 42000: Variable 'wsrep_slave_UK_checks' can't be set to the value of 'NULL'
+SET @@global.wsrep_slave_uk_checks='junk';
+ERROR 42000: Variable 'wsrep_slave_UK_checks' can't be set to the value of 'junk'
+
+# restore the initial value
+SET @@global.wsrep_slave_uk_checks = @wsrep_slave_uk_checks_global_saved;
+# End of test
diff --git a/mysql-test/suite/sys_vars/r/wsrep_sst_auth_basic.result b/mysql-test/suite/sys_vars/r/wsrep_sst_auth_basic.result
new file mode 100644
index 00000000000..a8a31dbea61
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_sst_auth_basic.result
@@ -0,0 +1,3 @@
+SELECT COUNT(@@wsrep_sst_auth);
+COUNT(@@wsrep_sst_auth)
+0
diff --git a/mysql-test/suite/sys_vars/r/wsrep_sst_donor_basic.result b/mysql-test/suite/sys_vars/r/wsrep_sst_donor_basic.result
new file mode 100644
index 00000000000..fc359690275
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_sst_donor_basic.result
@@ -0,0 +1,10 @@
+SELECT COUNT(@@wsrep_sst_donor);
+COUNT(@@wsrep_sst_donor)
+1
+set @start_value = @@wsrep_sst_donor;
+set @@global.wsrep_sst_donor='foo';
+set @@global.wsrep_sst_donor=NULL;
+set @@global.wsrep_sst_donor=r;
+set @@global.wsrep_sst_donor=1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_sst_donor'
+set @@global.wsrep_sst_donor = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_sst_donor_rejects_queries_basic.result b/mysql-test/suite/sys_vars/r/wsrep_sst_donor_rejects_queries_basic.result
new file mode 100644
index 00000000000..55aa56f27ed
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_sst_donor_rejects_queries_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_sst_donor_rejects_queries;
+set @@global.wsrep_sst_donor_rejects_queries=ON;
+set @@global.wsrep_sst_donor_rejects_queries=OFF;
+set @@global.wsrep_sst_donor_rejects_queries=1;
+set @@global.wsrep_sst_donor_rejects_queries=0;
+SET @@global.wsrep_sst_donor_rejects_queries = -1;
+ERROR 42000: Variable 'wsrep_sst_donor_rejects_queries' can't be set to the value of '-1'
+set @@global.wsrep_sst_donor_rejects_queries = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_sst_method_basic.result b/mysql-test/suite/sys_vars/r/wsrep_sst_method_basic.result
new file mode 100644
index 00000000000..682a5683026
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_sst_method_basic.result
@@ -0,0 +1,12 @@
+set @start_value = @@wsrep_sst_method;
+set @@global.wsrep_sst_method='xtrabackup';
+set @@global.wsrep_sst_method='xtrabackup-v2';
+set @@global.wsrep_sst_method='rsync';
+set @@global.wsrep_sst_method='mysqldump';
+set @@global.wsrep_sst_method='myscript';
+set @@global.wsrep_sst_method='skip';
+set @@global.wsrep_sst_method=NULL;
+ERROR 42000: Variable 'wsrep_sst_method' can't be set to the value of 'NULL'
+SET @@global.wsrep_sst_method = -1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_sst_method'
+set @@global.wsrep_sst_method = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_sst_receive_address_basic.result b/mysql-test/suite/sys_vars/r/wsrep_sst_receive_address_basic.result
new file mode 100644
index 00000000000..a23b7efafa8
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_sst_receive_address_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_sst_receive_address;
+set @@global.wsrep_sst_receive_address='128.0.2.1';
+set @@global.wsrep_sst_receive_address=AUTO;
+set @@global.wsrep_sst_receive_address='AUTO';
+set @@global.wsrep_sst_receive_address=NULL;
+SET @@global.wsrep_sst_receive_address = -1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_sst_receive_address'
+set @@global.wsrep_sst_receive_address = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_start_position_basic.result b/mysql-test/suite/sys_vars/r/wsrep_start_position_basic.result
new file mode 100644
index 00000000000..ad3606d6d55
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_start_position_basic.result
@@ -0,0 +1,8 @@
+set @start_value = @@wsrep_start_position;
+set @@global.wsrep_start_position='foo:bar';
+ERROR 42000: Variable 'wsrep_start_position' can't be set to the value of 'foo:bar'
+set @@global.wsrep_start_position=NULL;
+ERROR 42000: Variable 'wsrep_start_position' can't be set to the value of 'NULL'
+SET @@global.wsrep_start_position = -1;
+ERROR 42000: Incorrect argument type to variable 'wsrep_start_position'
+set @@global.wsrep_start_position = @start_value;
diff --git a/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result b/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result
new file mode 100644
index 00000000000..1e7b9364570
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result
@@ -0,0 +1,56 @@
+#
+# wsrep_sync_wait
+#
+# save the initial values
+SET @wsrep_sync_wait_global_saved = @@global.wsrep_sync_wait;
+SET @wsrep_sync_wait_session_saved = @@session.wsrep_sync_wait;
+# default
+SELECT @@global.wsrep_sync_wait;
+@@global.wsrep_sync_wait
+0
+SELECT @@session.wsrep_sync_wait;
+@@session.wsrep_sync_wait
+0
+
+# scope and valid values
+SET @@global.wsrep_sync_wait=0;
+SELECT @@global.wsrep_sync_wait;
+@@global.wsrep_sync_wait
+0
+SET @@global.wsrep_sync_wait=7;
+SELECT @@global.wsrep_sync_wait;
+@@global.wsrep_sync_wait
+7
+SET @@session.wsrep_sync_wait=0;
+SELECT @@session.wsrep_sync_wait;
+@@session.wsrep_sync_wait
+0
+SET @@session.wsrep_sync_wait=7;
+SELECT @@session.wsrep_sync_wait;
+@@session.wsrep_sync_wait
+7
+SET @@session.wsrep_sync_wait=default;
+SELECT @@session.wsrep_sync_wait;
+@@session.wsrep_sync_wait
+7
+SET @@session.wsrep_sync_wait=8;
+Warnings:
+Warning	1292	Truncated incorrect wsrep_sync_wait value: '8'
+SELECT @@session.wsrep_sync_wait;
+@@session.wsrep_sync_wait
+7
+
+# invalid values
+SET @@global.wsrep_sync_wait=NULL;
+ERROR 42000: Incorrect argument type to variable 'wsrep_sync_wait'
+SET @@global.wsrep_sync_wait='junk';
+ERROR 42000: Incorrect argument type to variable 'wsrep_sync_wait'
+SET @@session.wsrep_sync_wait=NULL;
+ERROR 42000: Incorrect argument type to variable 'wsrep_sync_wait'
+SET @@session.wsrep_sync_wait='junk';
+ERROR 42000: Incorrect argument type to variable 'wsrep_sync_wait'
+
+# restore the initial values
+SET @@global.wsrep_sync_wait = @wsrep_sync_wait_global_saved;
+SET @@session.wsrep_sync_wait = @wsrep_sync_wait_session_saved;
+# End of test
diff --git a/mysql-test/suite/sys_vars/t/innodb_disallow_writes_basic.test b/mysql-test/suite/sys_vars/t/innodb_disallow_writes_basic.test
new file mode 100644
index 00000000000..b8e5c127377
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/innodb_disallow_writes_basic.test
@@ -0,0 +1,42 @@
+--source include/have_innodb_disallow_writes.inc
+
+--echo #
+--echo # innodb_disallow_writes
+--echo #
+
+--echo # save the initial value
+SET @innodb_disallow_writes_global_saved = @@global.innodb_disallow_writes;
+
+--echo # default
+SELECT @@global.innodb_disallow_writes;
+
+--echo
+--echo # scope
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@session.innodb_disallow_writes;
+SET @@global.innodb_disallow_writes=OFF;
+SELECT @@global.innodb_disallow_writes;
+SET @@global.innodb_disallow_writes=ON;
+SELECT @@global.innodb_disallow_writes;
+
+--echo
+--echo # valid values
+SET @@global.innodb_disallow_writes='OFF';
+SELECT @@global.innodb_disallow_writes;
+SET @@global.innodb_disallow_writes=ON;
+SELECT @@global.innodb_disallow_writes;
+SET @@global.innodb_disallow_writes=default;
+SELECT @@global.innodb_disallow_writes;
+
+--echo
+--echo # invalid values
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@global.innodb_disallow_writes=NULL;
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@global.innodb_disallow_writes='junk';
+
+--echo
+--echo # restore the initial value
+SET @@global.innodb_disallow_writes = @innodb_disallow_writes_global_saved;
+
+--echo # End of test
diff --git a/mysql-test/suite/sys_vars/t/wsrep_auto_increment_control_basic.test b/mysql-test/suite/sys_vars/t/wsrep_auto_increment_control_basic.test
new file mode 100644
index 00000000000..91d3c578a2c
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_auto_increment_control_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_auto_increment_control;
+
+set @@global.wsrep_auto_increment_control=ON;
+set @@global.wsrep_auto_increment_control=OFF;
+set @@global.wsrep_auto_increment_control=1;
+set @@global.wsrep_auto_increment_control=0;
+--Error 1231
+SET @@global.wsrep_auto_increment_control = -1;
+
+set @@global.wsrep_auto_increment_control = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_causal_reads_basic.test b/mysql-test/suite/sys_vars/t/wsrep_causal_reads_basic.test
new file mode 100644
index 00000000000..2fb597e842e
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_causal_reads_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_causal_reads;
+
+set @@global.wsrep_causal_reads=ON;
+set @@global.wsrep_causal_reads=OFF;
+set @@global.wsrep_causal_reads=1;
+set @@global.wsrep_causal_reads=0;
+--Error 1231
+SET @@global.wsrep_causal_reads = -1;
+
+set @@global.wsrep_causal_reads = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_certify_nonpk b/mysql-test/suite/sys_vars/t/wsrep_certify_nonpk
new file mode 100644
index 00000000000..f4512aeb5da
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_certify_nonpk
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_certify_nonpk;
+
+set @@global.wsrep_certify_nonpk=ON;
+set @@global.wsrep_certify_nonpk=OFF;
+set @@global.wsrep_certify_nonpk=1;
+set @@global.wsrep_certify_nonpk=0;
+--Error ER_WRONG_TYPE_FOR_VAR
+SET @@global.wsrep_certify_nonpk = -1;
+
+set @@global.wsrep_certify_nonpk = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_certify_nonpk_basic.test b/mysql-test/suite/sys_vars/t/wsrep_certify_nonpk_basic.test
new file mode 100644
index 00000000000..b623d8eb073
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_certify_nonpk_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_certify_nonpk;
+
+set @@global.wsrep_certify_nonpk=ON;
+set @@global.wsrep_certify_nonpk=OFF;
+set @@global.wsrep_certify_nonpk=1;
+set @@global.wsrep_certify_nonpk=0;
+--Error 1231
+SET @@global.wsrep_certify_nonpk = -1;
+
+set @@global.wsrep_certify_nonpk = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_cluster_address_basic.test b/mysql-test/suite/sys_vars/t/wsrep_cluster_address_basic.test
new file mode 100644
index 00000000000..b61d5ea60c3
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_cluster_address_basic.test
@@ -0,0 +1,44 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(@@GLOBAL.wsrep_cluster_address);
+--echo 1 Expected
+
+SELECT COUNT(@@GLOBAL.wsrep_cluster_address);
+--echo 1 Expected
+
+SELECT @@GLOBAL.wsrep_cluster_address = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_cluster_address';
+--echo 1 Expected
+
+SELECT COUNT(@@GLOBAL.wsrep_cluster_address);
+--echo 1 Expected
+
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_cluster_address';
+--echo 1 Expected
+
+SELECT @@wsrep_cluster_address = @@GLOBAL.wsrep_cluster_address;
+--echo 1 Expected
+
+SELECT COUNT(@@wsrep_cluster_address);
+--echo 1 Expected
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@local.wsrep_cluster_address);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@SESSION.wsrep_cluster_address);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+SELECT COUNT(@@GLOBAL.wsrep_cluster_address);
+--echo 1 Expected
+
+--Error ER_BAD_FIELD_ERROR
+SELECT wsrep_cluster_address = @@SESSION.wsrep_cluster_address;
+--echo Expected error 'Readonly variable'
+
+
diff --git a/mysql-test/suite/sys_vars/t/wsrep_cluster_name_basic.test b/mysql-test/suite/sys_vars/t/wsrep_cluster_name_basic.test
new file mode 100644
index 00000000000..104c342bfe4
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_cluster_name_basic.test
@@ -0,0 +1,12 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_cluster_name;
+
+set @@global.wsrep_cluster_name='test';
+--Error 1231
+set @@global.wsrep_cluster_name=NULL;
+--Error 1232
+SET @@global.wsrep_cluster_name = 1;
+
+set @@global.wsrep_cluster_name = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_convert_lock_to_trx_basic.test b/mysql-test/suite/sys_vars/t/wsrep_convert_lock_to_trx_basic.test
new file mode 100644
index 00000000000..84b92085238
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_convert_lock_to_trx_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_convert_lock_to_trx;
+
+set @@global.wsrep_convert_lock_to_trx=ON;
+set @@global.wsrep_convert_lock_to_trx=OFF;
+set @@global.wsrep_convert_lock_to_trx=1;
+set @@global.wsrep_convert_lock_to_trx=0;
+--Error 1231
+SET @@global.wsrep_convert_lock_to_trx = -1;
+
+set @@global.wsrep_convert_lock_to_trx = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_data_home_dir_basic.test b/mysql-test/suite/sys_vars/t/wsrep_data_home_dir_basic.test
new file mode 100644
index 00000000000..fccf685193e
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_data_home_dir_basic.test
@@ -0,0 +1,48 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(@@GLOBAL.wsrep_data_home_dir);
+--echo 1 Expected
+
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SET @@GLOBAL.wsrep_data_home_dir=1;
+--echo Expected error 'Read only variable'
+
+SELECT COUNT(@@GLOBAL.wsrep_data_home_dir);
+--echo 1 Expected
+
+SELECT @@GLOBAL.wsrep_data_home_dir = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_data_home_dir';
+--echo 1 Expected
+
+SELECT COUNT(@@GLOBAL.wsrep_data_home_dir);
+--echo 1 Expected
+
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_data_home_dir';
+--echo 1 Expected
+
+SELECT @@wsrep_data_home_dir = @@GLOBAL.wsrep_data_home_dir;
+--echo 1 Expected
+
+SELECT COUNT(@@wsrep_data_home_dir);
+--echo 1 Expected
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@local.wsrep_data_home_dir);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@SESSION.wsrep_data_home_dir);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+SELECT COUNT(@@GLOBAL.wsrep_data_home_dir);
+--echo 1 Expected
+
+--Error ER_BAD_FIELD_ERROR
+SELECT wsrep_data_home_dir = @@SESSION.wsrep_data_home_dir;
+--echo Expected error 'Readonly variable'
+
+
diff --git a/mysql-test/suite/sys_vars/t/wsrep_dbug_option_basic.test b/mysql-test/suite/sys_vars/t/wsrep_dbug_option_basic.test
new file mode 100644
index 00000000000..0a559fe8d27
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_dbug_option_basic.test
@@ -0,0 +1,11 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_dbug_option;
+
+set @@global.wsrep_dbug_option='foo:bar';
+set @@global.wsrep_dbug_option=NULL;
+--Error 1232
+SET @@global.wsrep_dbug_option = -1;
+
+set @@global.wsrep_dbug_option = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_debug_basic.test b/mysql-test/suite/sys_vars/t/wsrep_debug_basic.test
new file mode 100644
index 00000000000..194c163baa6
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_debug_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_debug;
+
+set @@global.wsrep_debug=ON;
+set @@global.wsrep_debug=OFF;
+set @@global.wsrep_debug=1;
+set @@global.wsrep_debug=0;
+--Error 1231
+SET @@global.wsrep_debug = -1;
+
+set @@global.wsrep_debug = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_debug_option_basic.test b/mysql-test/suite/sys_vars/t/wsrep_debug_option_basic.test
new file mode 100644
index 00000000000..060e721b676
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_debug_option_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_debug_option;
+
+--error 1231
+set @@global.wsrep_debug_option='foo:bar';
+--error 1231
+set @@global.wsrep_debug_option=NULL;
+--Error 1232
+SET @@global.wsrep_debug_option = -1;
+
+set @@global.wsrep_debug_option = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_desync_basic.test b/mysql-test/suite/sys_vars/t/wsrep_desync_basic.test
new file mode 100644
index 00000000000..3c64d8cb30f
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_desync_basic.test
@@ -0,0 +1,4 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+select @@global.wsrep_desync;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_drupal_282555_workaround_basic.test b/mysql-test/suite/sys_vars/t/wsrep_drupal_282555_workaround_basic.test
new file mode 100644
index 00000000000..ada08bb125d
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_drupal_282555_workaround_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_drupal_282555_workaround;
+
+set @@global.wsrep_drupal_282555_workaround=ON;
+set @@global.wsrep_drupal_282555_workaround=OFF;
+set @@global.wsrep_drupal_282555_workaround=1;
+set @@global.wsrep_drupal_282555_workaround=0;
+--Error 1231
+SET @@global.wsrep_drupal_282555_workaround = -1;
+
+set @@global.wsrep_drupal_282555_workaround = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_forced_binlog_format_basic.test b/mysql-test/suite/sys_vars/t/wsrep_forced_binlog_format_basic.test
new file mode 100644
index 00000000000..5e5530a8f64
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_forced_binlog_format_basic.test
@@ -0,0 +1,14 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_forced_binlog_format;
+
+set @@global.wsrep_forced_binlog_format = ROW;
+set @@global.wsrep_forced_binlog_format = MIXED;
+set @@global.wsrep_forced_binlog_format = STATEMENT;
+set @@global.wsrep_forced_binlog_format = NONE;
+
+--error 1231
+set @@global.wsrep_forced_binlog_format = FOO;
+
+set @@global.wsrep_forced_binlog_format = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_load_data_splitting_basic.test b/mysql-test/suite/sys_vars/t/wsrep_load_data_splitting_basic.test
new file mode 100644
index 00000000000..61ff27d6894
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_load_data_splitting_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_load_data_splitting;
+
+set @@global.wsrep_load_data_splitting=ON;
+set @@global.wsrep_load_data_splitting=OFF;
+set @@global.wsrep_load_data_splitting=1;
+set @@global.wsrep_load_data_splitting=0;
+--Error 1231
+SET @@global.wsrep_load_data_splitting = -1;
+
+set @@global.wsrep_load_data_splitting = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_log_conflicts_basic.test b/mysql-test/suite/sys_vars/t/wsrep_log_conflicts_basic.test
new file mode 100644
index 00000000000..dbd696c45a2
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_log_conflicts_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_log_conflicts;
+
+set @@global.wsrep_log_conflicts=ON;
+set @@global.wsrep_log_conflicts=OFF;
+set @@global.wsrep_log_conflicts=1;
+set @@global.wsrep_log_conflicts=0;
+--Error 1231
+SET @@global.wsrep_log_conflicts = -1;
+
+set @@global.wsrep_log_conflicts = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_max_ws_rows_basic.test b/mysql-test/suite/sys_vars/t/wsrep_max_ws_rows_basic.test
new file mode 100644
index 00000000000..f607a567d80
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_max_ws_rows_basic.test
@@ -0,0 +1,14 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_max_ws_rows;
+
+set @@global.wsrep_max_ws_rows=256000;
+set @@global.wsrep_max_ws_rows=0;
+show warnings;
+set @@global.wsrep_max_ws_rows=-1;
+show warnings;
+--Error 1232
+SET @@global.wsrep_max_ws_rows = r;
+
+set @@global.wsrep_max_ws_rows = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_max_ws_size_basic.test b/mysql-test/suite/sys_vars/t/wsrep_max_ws_size_basic.test
new file mode 100644
index 00000000000..6b1d1f71090
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_max_ws_size_basic.test
@@ -0,0 +1,14 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_max_ws_size;
+
+set @@global.wsrep_max_ws_size=256000;
+set @@global.wsrep_max_ws_size=0;
+show warnings;
+set @@global.wsrep_max_ws_size=-1;
+show warnings;
+--Error 1232
+SET @@global.wsrep_max_ws_size = r;
+
+set @@global.wsrep_max_ws_size = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_mysql_replication_bundle_basic.test b/mysql-test/suite/sys_vars/t/wsrep_mysql_replication_bundle_basic.test
new file mode 100644
index 00000000000..20a0dc9bf9f
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_mysql_replication_bundle_basic.test
@@ -0,0 +1,16 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_mysql_replication_bundle;
+
+set @@global.wsrep_mysql_replication_bundle=0;
+set @@global.wsrep_mysql_replication_bundle=1000;
+
+set @@global.wsrep_mysql_replication_bundle=-1;
+show warnings;
+set @@global.wsrep_mysql_replication_bundle=1001;
+show warnings;
+--Error 1232
+SET @@global.wsrep_mysql_replication_bundle = r;
+
+set @@global.wsrep_mysql_replication_bundle = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_node_address_basic.test b/mysql-test/suite/sys_vars/t/wsrep_node_address_basic.test
new file mode 100644
index 00000000000..feace325044
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_node_address_basic.test
@@ -0,0 +1,42 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(@@GLOBAL.wsrep_node_address);
+--echo 1 Expected
+
+--error 1232
+SET @@GLOBAL.wsrep_node_address=1;
+--echo Expected error 'Read only variable'
+
+SELECT COUNT(@@GLOBAL.wsrep_node_address);
+--echo 1 Expected
+
+SELECT @@GLOBAL.wsrep_node_address = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_node_address';
+--echo 1 Expected
+
+SELECT COUNT(@@GLOBAL.wsrep_node_address);
+--echo 1 Expected
+
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_node_address';
+--echo 1 Expected
+
+SELECT @@wsrep_node_address = @@GLOBAL.wsrep_node_address;
+--echo 1 Expected
+
+SELECT COUNT(@@wsrep_node_address);
+--echo 1 Expected
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@local.wsrep_node_address);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@SESSION.wsrep_node_address);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+SELECT COUNT(@@GLOBAL.wsrep_node_address);
+--echo 1 Expected
diff --git a/mysql-test/suite/sys_vars/t/wsrep_node_incoming_address_basic.test b/mysql-test/suite/sys_vars/t/wsrep_node_incoming_address_basic.test
new file mode 100644
index 00000000000..188a5960eb9
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_node_incoming_address_basic.test
@@ -0,0 +1,42 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(@@GLOBAL.wsrep_node_incoming_address);
+--echo 1 Expected
+
+--error 1232
+SET @@GLOBAL.wsrep_node_incoming_address=1;
+--echo Expected error 'Read only variable'
+
+SELECT COUNT(@@GLOBAL.wsrep_node_incoming_address);
+--echo 1 Expected
+
+SELECT @@GLOBAL.wsrep_node_incoming_address = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_node_incoming_address';
+--echo 1 Expected
+
+SELECT COUNT(@@GLOBAL.wsrep_node_incoming_address);
+--echo 1 Expected
+
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_node_incoming_address';
+--echo 1 Expected
+
+SELECT @@wsrep_node_incoming_address = @@GLOBAL.wsrep_node_incoming_address;
+--echo 1 Expected
+
+SELECT COUNT(@@wsrep_node_incoming_address);
+--echo 1 Expected
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@local.wsrep_node_incoming_address);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@SESSION.wsrep_node_incoming_address);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+SELECT COUNT(@@GLOBAL.wsrep_node_incoming_address);
+--echo 1 Expected
diff --git a/mysql-test/suite/sys_vars/t/wsrep_node_name_basic.test b/mysql-test/suite/sys_vars/t/wsrep_node_name_basic.test
new file mode 100644
index 00000000000..3220ae373e2
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_node_name_basic.test
@@ -0,0 +1,11 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_node_name;
+
+set @@global.wsrep_node_name='test';
+set @@global.wsrep_node_name=NULL;
+--Error 1232
+SET @@global.wsrep_node_name = 1;
+
+set @@global.wsrep_node_name = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_notify_cmd_basic.test b/mysql-test/suite/sys_vars/t/wsrep_notify_cmd_basic.test
new file mode 100644
index 00000000000..d816453f8a3
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_notify_cmd_basic.test
@@ -0,0 +1,11 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_notify_cmd;
+
+set @@global.wsrep_notify_cmd='test';
+set @@global.wsrep_notify_cmd=NULL;
+--Error 1232
+SET @@global.wsrep_notify_cmd = 1;
+
+set @@global.wsrep_notify_cmd = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_on_basic.test b/mysql-test/suite/sys_vars/t/wsrep_on_basic.test
new file mode 100644
index 00000000000..5afe5c4451f
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_on_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_on;
+
+set @@global.wsrep_on=ON;
+set @@global.wsrep_on=OFF;
+set @@global.wsrep_on=1;
+set @@global.wsrep_on=0;
+--Error 1231
+SET @@global.wsrep_on = -1;
+
+set @@global.wsrep_on = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_osu_method_basic.test b/mysql-test/suite/sys_vars/t/wsrep_osu_method_basic.test
new file mode 100644
index 00000000000..9e1adde76a3
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_osu_method_basic.test
@@ -0,0 +1,18 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_osu_method;
+
+set @@global.wsrep_osu_method='TOI';
+set @@global.wsrep_osu_method='RSU';
+set @@global.wsrep_osu_method=TOI;
+set @@global.wsrep_osu_method=RSU;
+
+--Error 1231
+set @@global.wsrep_osu_method=TSU;
+--Error 1231
+set @@global.wsrep_osu_method='TSU';
+--Error 1231
+SET @@global.wsrep_on = -1;
+
+set @@global.wsrep_osu_method = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_provider_basic.test b/mysql-test/suite/sys_vars/t/wsrep_provider_basic.test
new file mode 100644
index 00000000000..aae122c42fe
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_provider_basic.test
@@ -0,0 +1,5 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(@@GLOBAL.wsrep_provider);
+--echo 1 Expected
diff --git a/mysql-test/suite/sys_vars/t/wsrep_provider_options_basic.test b/mysql-test/suite/sys_vars/t/wsrep_provider_options_basic.test
new file mode 100644
index 00000000000..e2d8b63b2fd
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_provider_options_basic.test
@@ -0,0 +1,5 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(@@GLOBAL.wsrep_provider_options);
+--echo 1 Expected
diff --git a/mysql-test/suite/sys_vars/t/wsrep_recover_basic.test b/mysql-test/suite/sys_vars/t/wsrep_recover_basic.test
new file mode 100644
index 00000000000..f4e1707a434
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_recover_basic.test
@@ -0,0 +1,47 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(@@GLOBAL.wsrep_recover);
+--echo 1 Expected
+
+--Error 1238
+set @@global.wsrep_recover=ON;
+--echo Expected error 'Readonly variable'
+--Error 1238
+set @@global.wsrep_recover=OFF;
+--echo Expected error 'Readonly variable'
+
+SELECT @@GLOBAL.wsrep_recover = VARIABLE_VALUE
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
+WHERE VARIABLE_NAME='wsrep_recover';
+--echo 1 Expected
+
+SELECT COUNT(@@GLOBAL.wsrep_recover);
+--echo 1 Expected
+
+SELECT COUNT(VARIABLE_VALUE)
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='wsrep_recover';
+--echo 1 Expected
+
+SELECT @@wsrep_recover = @@GLOBAL.wsrep_recover;
+--echo 1 Expected
+
+SELECT COUNT(@@wsrep_recover);
+--echo 1 Expected
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@local.wsrep_recover);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT COUNT(@@SESSION.wsrep_recover);
+--echo Expected error 'Variable is a GLOBAL variable'
+
+SELECT COUNT(@@GLOBAL.wsrep_recover);
+--echo 1 Expected
+
+--Error ER_BAD_FIELD_ERROR
+SELECT wsrep_recover = @@SESSION.wsrep_recover;
+--echo Expected error 'Readonly variable'
+
diff --git a/mysql-test/suite/sys_vars/t/wsrep_replicate_myisam_basic.test b/mysql-test/suite/sys_vars/t/wsrep_replicate_myisam_basic.test
new file mode 100644
index 00000000000..c03d76b5123
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_replicate_myisam_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_replicate_myisam;
+
+set @@global.wsrep_replicate_myisam=ON;
+set @@global.wsrep_replicate_myisam=OFF;
+set @@global.wsrep_replicate_myisam=1;
+set @@global.wsrep_replicate_myisam=0;
+--Error 1231
+SET @@global.wsrep_replicate_myisam = -1;
+
+set @@global.wsrep_replicate_myisam = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_restart_slave_basic.test b/mysql-test/suite/sys_vars/t/wsrep_restart_slave_basic.test
new file mode 100644
index 00000000000..82f5a97327d
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_restart_slave_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_restart_slave;
+
+set @@global.wsrep_restart_slave=ON;
+set @@global.wsrep_restart_slave=OFF;
+set @@global.wsrep_restart_slave=1;
+set @@global.wsrep_restart_slave=0;
+--Error 1231
+SET @@global.wsrep_restart_slave = -1;
+
+set @@global.wsrep_restart_slave = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_retry_autocommit_basic.test b/mysql-test/suite/sys_vars/t/wsrep_retry_autocommit_basic.test
new file mode 100644
index 00000000000..82f5a97327d
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_retry_autocommit_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_restart_slave;
+
+set @@global.wsrep_restart_slave=ON;
+set @@global.wsrep_restart_slave=OFF;
+set @@global.wsrep_restart_slave=1;
+set @@global.wsrep_restart_slave=0;
+--Error 1231
+SET @@global.wsrep_restart_slave = -1;
+
+set @@global.wsrep_restart_slave = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_slave_fk_checks_basic.test b/mysql-test/suite/sys_vars/t/wsrep_slave_fk_checks_basic.test
new file mode 100644
index 00000000000..dd60eb8694b
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_slave_fk_checks_basic.test
@@ -0,0 +1,42 @@
+--source include/have_wsrep.inc
+
+--echo #
+--echo # wsrep_slave_fk_checks
+--echo #
+
+--echo # save the initial value
+SET @wsrep_slave_fk_checks_global_saved = @@global.wsrep_slave_fk_checks;
+
+--echo # default
+SELECT @@global.wsrep_slave_fk_checks;
+
+--echo
+--echo # scope
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@session.wsrep_slave_fk_checks;
+SET @@global.wsrep_slave_fk_checks=OFF;
+SELECT @@global.wsrep_slave_fk_checks;
+SET @@global.wsrep_slave_fk_checks=ON;
+SELECT @@global.wsrep_slave_fk_checks;
+
+--echo
+--echo # valid values
+SET @@global.wsrep_slave_fk_checks='OFF';
+SELECT @@global.wsrep_slave_fk_checks;
+SET @@global.wsrep_slave_fk_checks=ON;
+SELECT @@global.wsrep_slave_fk_checks;
+SET @@global.wsrep_slave_fk_checks=default;
+SELECT @@global.wsrep_slave_fk_checks;
+
+--echo
+--echo # invalid values
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@global.wsrep_slave_fk_checks=NULL;
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@global.wsrep_slave_fk_checks='junk';
+
+--echo
+--echo # restore the initial value
+SET @@global.wsrep_slave_fk_checks = @wsrep_slave_fk_checks_global_saved;
+
+--echo # End of test
diff --git a/mysql-test/suite/sys_vars/t/wsrep_slave_threads_basic.test b/mysql-test/suite/sys_vars/t/wsrep_slave_threads_basic.test
new file mode 100644
index 00000000000..cff4f433846
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_slave_threads_basic.test
@@ -0,0 +1,16 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_slave_threads;
+
+set @@global.wsrep_slave_threads=1;
+set @@global.wsrep_slave_threads=4;
+show warnings;
+set @@global.wsrep_slave_threads=0;
+show warnings;
+set @@global.wsrep_slave_threads=-1;
+show warnings;
+--Error 1232
+SET @@global.wsrep_slave_threads = r;
+
+set @@global.wsrep_slave_threads = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_slave_uk_checks_basic.test b/mysql-test/suite/sys_vars/t/wsrep_slave_uk_checks_basic.test
new file mode 100644
index 00000000000..c9012954371
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_slave_uk_checks_basic.test
@@ -0,0 +1,42 @@
+--source include/have_wsrep.inc
+
+--echo #
+--echo # wsrep_slave_uk_checks
+--echo #
+
+--echo # save the initial value
+SET @wsrep_slave_uk_checks_global_saved = @@global.wsrep_slave_uk_checks;
+
+--echo # default
+SELECT @@global.wsrep_slave_uk_checks;
+
+--echo
+--echo # scope
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@session.wsrep_slave_uk_checks;
+SET @@global.wsrep_slave_uk_checks=OFF;
+SELECT @@global.wsrep_slave_uk_checks;
+SET @@global.wsrep_slave_uk_checks=ON;
+SELECT @@global.wsrep_slave_uk_checks;
+
+--echo
+--echo # valid values
+SET @@global.wsrep_slave_uk_checks='OFF';
+SELECT @@global.wsrep_slave_uk_checks;
+SET @@global.wsrep_slave_uk_checks=ON;
+SELECT @@global.wsrep_slave_uk_checks;
+SET @@global.wsrep_slave_uk_checks=default;
+SELECT @@global.wsrep_slave_uk_checks;
+
+--echo
+--echo # invalid values
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@global.wsrep_slave_uk_checks=NULL;
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@global.wsrep_slave_uk_checks='junk';
+
+--echo
+--echo # restore the initial value
+SET @@global.wsrep_slave_uk_checks = @wsrep_slave_uk_checks_global_saved;
+
+--echo # End of test
diff --git a/mysql-test/suite/sys_vars/t/wsrep_sst_auth_basic.test b/mysql-test/suite/sys_vars/t/wsrep_sst_auth_basic.test
new file mode 100644
index 00000000000..6db2a4cd844
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_sst_auth_basic.test
@@ -0,0 +1,12 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(@@wsrep_sst_auth);
+
+# Cause crash, fix later
+#set @start_value = @@wsrep_sst_auth;
+#set @@global.wsrep_sst_auth='root:pass';
+#set @@global.wsrep_sst_auth=NULL;
+#set @@global.wsrep_sst_auth=r;
+#set @@global.wsrep_sst_auth=1;
+#set @@global.wsrep_sst_auth = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_sst_donor_basic.test b/mysql-test/suite/sys_vars/t/wsrep_sst_donor_basic.test
new file mode 100644
index 00000000000..58d005282a0
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_sst_donor_basic.test
@@ -0,0 +1,12 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(@@wsrep_sst_donor);
+
+set @start_value = @@wsrep_sst_donor;
+set @@global.wsrep_sst_donor='foo';
+set @@global.wsrep_sst_donor=NULL;
+set @@global.wsrep_sst_donor=r;
+--error 1232
+set @@global.wsrep_sst_donor=1;
+set @@global.wsrep_sst_donor = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_sst_donor_rejects_queries_basic.test b/mysql-test/suite/sys_vars/t/wsrep_sst_donor_rejects_queries_basic.test
new file mode 100644
index 00000000000..fc8633ce00f
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_sst_donor_rejects_queries_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_sst_donor_rejects_queries;
+
+set @@global.wsrep_sst_donor_rejects_queries=ON;
+set @@global.wsrep_sst_donor_rejects_queries=OFF;
+set @@global.wsrep_sst_donor_rejects_queries=1;
+set @@global.wsrep_sst_donor_rejects_queries=0;
+--Error 1231
+SET @@global.wsrep_sst_donor_rejects_queries = -1;
+
+set @@global.wsrep_sst_donor_rejects_queries = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_sst_method_basic.test b/mysql-test/suite/sys_vars/t/wsrep_sst_method_basic.test
new file mode 100644
index 00000000000..dab5831ff01
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_sst_method_basic.test
@@ -0,0 +1,17 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_sst_method;
+
+set @@global.wsrep_sst_method='xtrabackup';
+set @@global.wsrep_sst_method='xtrabackup-v2';
+set @@global.wsrep_sst_method='rsync';
+set @@global.wsrep_sst_method='mysqldump';
+set @@global.wsrep_sst_method='myscript';
+set @@global.wsrep_sst_method='skip';
+--error 1231
+set @@global.wsrep_sst_method=NULL;
+--Error 1232
+SET @@global.wsrep_sst_method = -1;
+
+set @@global.wsrep_sst_method = @start_value;
\ No newline at end of file
diff --git a/mysql-test/suite/sys_vars/t/wsrep_sst_receive_address_basic.test b/mysql-test/suite/sys_vars/t/wsrep_sst_receive_address_basic.test
new file mode 100644
index 00000000000..0a18098d77f
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_sst_receive_address_basic.test
@@ -0,0 +1,13 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_sst_receive_address;
+
+set @@global.wsrep_sst_receive_address='128.0.2.1';
+set @@global.wsrep_sst_receive_address=AUTO;
+set @@global.wsrep_sst_receive_address='AUTO';
+set @@global.wsrep_sst_receive_address=NULL;
+--Error 1232
+SET @@global.wsrep_sst_receive_address = -1;
+
+set @@global.wsrep_sst_receive_address = @start_value;
diff --git a/mysql-test/suite/sys_vars/t/wsrep_start_position_basic.test b/mysql-test/suite/sys_vars/t/wsrep_start_position_basic.test
new file mode 100644
index 00000000000..3e30d10c016
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_start_position_basic.test
@@ -0,0 +1,14 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_start_position;
+
+--error 1231
+set @@global.wsrep_start_position='foo:bar';
+--error 1231
+set @@global.wsrep_start_position=NULL;
+--Error 1232
+SET @@global.wsrep_start_position = -1;
+
+set @@global.wsrep_start_position = @start_value;
+
diff --git a/mysql-test/suite/sys_vars/t/wsrep_sync_wait_basic.test b/mysql-test/suite/sys_vars/t/wsrep_sync_wait_basic.test
new file mode 100644
index 00000000000..cd0d545f80e
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_sync_wait_basic.test
@@ -0,0 +1,47 @@
+--source include/have_wsrep.inc
+
+--echo #
+--echo # wsrep_sync_wait
+--echo #
+
+--echo # save the initial values
+SET @wsrep_sync_wait_global_saved = @@global.wsrep_sync_wait;
+SET @wsrep_sync_wait_session_saved = @@session.wsrep_sync_wait;
+
+--echo # default
+SELECT @@global.wsrep_sync_wait;
+SELECT @@session.wsrep_sync_wait;
+
+--echo
+--echo # scope and valid values
+SET @@global.wsrep_sync_wait=0;
+SELECT @@global.wsrep_sync_wait;
+SET @@global.wsrep_sync_wait=7;
+SELECT @@global.wsrep_sync_wait;
+
+SET @@session.wsrep_sync_wait=0;
+SELECT @@session.wsrep_sync_wait;
+SET @@session.wsrep_sync_wait=7;
+SELECT @@session.wsrep_sync_wait;
+SET @@session.wsrep_sync_wait=default;
+SELECT @@session.wsrep_sync_wait;
+SET @@session.wsrep_sync_wait=8;
+SELECT @@session.wsrep_sync_wait;
+
+--echo
+--echo # invalid values
+--error ER_WRONG_TYPE_FOR_VAR
+SET @@global.wsrep_sync_wait=NULL;
+--error ER_WRONG_TYPE_FOR_VAR
+SET @@global.wsrep_sync_wait='junk';
+--error ER_WRONG_TYPE_FOR_VAR
+SET @@session.wsrep_sync_wait=NULL;
+--error ER_WRONG_TYPE_FOR_VAR
+SET @@session.wsrep_sync_wait='junk';
+
+--echo
+--echo # restore the initial values
+SET @@global.wsrep_sync_wait = @wsrep_sync_wait_global_saved;
+SET @@session.wsrep_sync_wait = @wsrep_sync_wait_session_saved;
+
+--echo # End of test
diff --git a/mysql-test/suite/sys_vars/t/wsrep_wsrep_provider_basic.test b/mysql-test/suite/sys_vars/t/wsrep_wsrep_provider_basic.test
new file mode 100644
index 00000000000..f4faaccc6a7
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/wsrep_wsrep_provider_basic.test
@@ -0,0 +1,11 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+set @start_value = @@wsrep_provider;
+
+set @@global.wsrep_provider=none;
+
+--Error 1231
+SET @@global.wsrep_provider = -1;
+
+set @@global.wsrep_provider = @start_value;
diff --git a/mysql-test/suite/wsrep/README b/mysql-test/suite/wsrep/README
new file mode 100644
index 00000000000..988096071a4
--- /dev/null
+++ b/mysql-test/suite/wsrep/README
@@ -0,0 +1,7 @@
+* 'wsrep' suite is designated for tests which do not require a multi-node
+  galera cluster.
+
+* As these tests are specific to wsrep-related functionalities, they must skip
+  on server built without wsrep patch (vanilla). (-DWITH_WSREP=OFF)
+  See : include/have_wsrep.inc, include/have_wsrep_enabled.inc, not_wsrep.inc
+
diff --git a/mysql-test/suite/wsrep/r/binlog_format.result b/mysql-test/suite/wsrep/r/binlog_format.result
new file mode 100644
index 00000000000..5b8da51f829
--- /dev/null
+++ b/mysql-test/suite/wsrep/r/binlog_format.result
@@ -0,0 +1,35 @@
+call mtr.add_suppression("WSREP: cannot get fake InnoDB transaction ID");
+call mtr.add_suppression("WSREP: Could not open saved state file for reading:.*");
+SHOW VARIABLES LIKE 'binlog_format';
+Variable_name	Value
+binlog_format	ROW
+SET binlog_format=STATEMENT;
+ERROR 42000: Variable 'binlog_format' can't be set to the value of 'STATEMENT'
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1105	MariaDB Galera does not support binlog format: STATEMENT
+Error	1231	Variable 'binlog_format' can't be set to the value of 'STATEMENT'
+SHOW VARIABLES LIKE 'binlog_format';
+Variable_name	Value
+binlog_format	ROW
+CREATE TABLE IF NOT EXISTS test.t1 AS SELECT * FROM information_schema.routines WHERE 1 = 0;
+SET binlog_format=MIXED;
+ERROR 42000: Variable 'binlog_format' can't be set to the value of 'MIXED'
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1105	MariaDB Galera does not support binlog format: MIXED
+Error	1231	Variable 'binlog_format' can't be set to the value of 'MIXED'
+SHOW VARIABLES LIKE 'binlog_format';
+Variable_name	Value
+binlog_format	ROW
+CREATE TABLE IF NOT EXISTS test.t2 AS SELECT * FROM information_schema.routines WHERE 1 = 0;
+SET binlog_format=ROW;
+SHOW WARNINGS;
+Level	Code	Message
+SHOW VARIABLES LIKE 'binlog_format';
+Variable_name	Value
+binlog_format	ROW
+CREATE TABLE IF NOT EXISTS test.t3 AS SELECT * FROM information_schema.routines WHERE 1 = 0;
+DROP TABLE IF EXISTS test.t1;
+DROP TABLE IF EXISTS test.t2;
+DROP TABLE IF EXISTS test.t3;
diff --git a/mysql-test/suite/wsrep/r/innodb_load_xa_with_wsrep.result b/mysql-test/suite/wsrep/r/innodb_load_xa_with_wsrep.result
new file mode 100644
index 00000000000..90faf766145
--- /dev/null
+++ b/mysql-test/suite/wsrep/r/innodb_load_xa_with_wsrep.result
@@ -0,0 +1,19 @@
+install plugin innodb soname 'ha_innodb';
+select engine,support,transactions,xa from information_schema.engines where engine='innodb';
+engine	support	transactions	xa
+InnoDB	YES	YES	YES
+create table t1 (a int) engine=innodb;
+start transaction;
+insert t1 values (1);
+insert t1 values (2);
+commit;
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+mysqld-bin.000001	#	Gtid	#	#	GTID #-#-#
+mysqld-bin.000001	#	Query	#	#	use `test`; create table t1 (a int) engine=innodb
+mysqld-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+mysqld-bin.000001	#	Query	#	#	use `test`; insert t1 values (1)
+mysqld-bin.000001	#	Query	#	#	use `test`; insert t1 values (2)
+mysqld-bin.000001	#	Xid	#	#	COMMIT /* XID */
+drop table t1;
+uninstall plugin innodb;
diff --git a/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink.result b/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink.result
new file mode 100644
index 00000000000..f77a655773a
--- /dev/null
+++ b/mysql-test/suite/wsrep/r/mysql_tzinfo_to_sql_symlink.result
@@ -0,0 +1,70 @@
+#
+# MDEV-5226 mysql_tzinfo_to_sql errors with tzdata 2013f and above
+#
+# Verbose run
+SET GLOBAL wsrep_replicate_myisam= ON;
+TRUNCATE TABLE time_zone;
+TRUNCATE TABLE time_zone_name;
+TRUNCATE TABLE time_zone_transition;
+TRUNCATE TABLE time_zone_transition_type;
+INSERT INTO time_zone (Use_leap_seconds) VALUES ('N');
+SET @time_zone_id= LAST_INSERT_ID();
+INSERT INTO time_zone_name (Name, Time_zone_id) VALUES ('GMT', @time_zone_id);
+INSERT INTO time_zone_transition_type (Time_zone_id, Transition_type_id, Offset, Is_DST, Abbreviation) VALUES
+ (@time_zone_id, 0, 0, 0, 'GMT')
+;
+Warning: Unable to load 'MYSQLTEST_VARDIR/zoneinfo/garbage' as time zone. Skipping it.
+Warning: Unable to load 'MYSQLTEST_VARDIR/zoneinfo/ignored.tab' as time zone. Skipping it.
+INSERT INTO time_zone (Use_leap_seconds) VALUES ('N');
+SET @time_zone_id= LAST_INSERT_ID();
+INSERT INTO time_zone_name (Name, Time_zone_id) VALUES ('posix/GMT', @time_zone_id);
+INSERT INTO time_zone_transition_type (Time_zone_id, Transition_type_id, Offset, Is_DST, Abbreviation) VALUES
+ (@time_zone_id, 0, 0, 0, 'GMT')
+;
+Warning: Unable to load 'MYSQLTEST_VARDIR/zoneinfo/posix/garbage' as time zone. Skipping it.
+Warning: Unable to load 'MYSQLTEST_VARDIR/zoneinfo/posix/ignored.tab' as time zone. Skipping it.
+Warning: Skipping directory 'MYSQLTEST_VARDIR/zoneinfo/posix/posix': to avoid infinite symlink recursion.
+ALTER TABLE time_zone_transition ORDER BY Time_zone_id, Transition_time;
+ALTER TABLE time_zone_transition_type ORDER BY Time_zone_id, Transition_type_id;
+SET GLOBAL wsrep_replicate_myisam= OFF;
+# Silent run
+SET GLOBAL wsrep_replicate_myisam= ON;
+TRUNCATE TABLE time_zone;
+TRUNCATE TABLE time_zone_name;
+TRUNCATE TABLE time_zone_transition;
+TRUNCATE TABLE time_zone_transition_type;
+INSERT INTO time_zone (Use_leap_seconds) VALUES ('N');
+SET @time_zone_id= LAST_INSERT_ID();
+INSERT INTO time_zone_name (Name, Time_zone_id) VALUES ('GMT', @time_zone_id);
+INSERT INTO time_zone_transition_type (Time_zone_id, Transition_type_id, Offset, Is_DST, Abbreviation) VALUES
+ (@time_zone_id, 0, 0, 0, 'GMT')
+;
+Warning: Unable to load 'MYSQLTEST_VARDIR/zoneinfo/garbage' as time zone. Skipping it.
+INSERT INTO time_zone (Use_leap_seconds) VALUES ('N');
+SET @time_zone_id= LAST_INSERT_ID();
+INSERT INTO time_zone_name (Name, Time_zone_id) VALUES ('posix/GMT', @time_zone_id);
+INSERT INTO time_zone_transition_type (Time_zone_id, Transition_type_id, Offset, Is_DST, Abbreviation) VALUES
+ (@time_zone_id, 0, 0, 0, 'GMT')
+;
+Warning: Unable to load 'MYSQLTEST_VARDIR/zoneinfo/posix/garbage' as time zone. Skipping it.
+ALTER TABLE time_zone_transition ORDER BY Time_zone_id, Transition_time;
+ALTER TABLE time_zone_transition_type ORDER BY Time_zone_id, Transition_type_id;
+SET GLOBAL wsrep_replicate_myisam= OFF;
+#
+# Testing with explicit timezonefile
+#
+SET GLOBAL wsrep_replicate_myisam= ON;
+INSERT INTO time_zone (Use_leap_seconds) VALUES ('N');
+SET @time_zone_id= LAST_INSERT_ID();
+INSERT INTO time_zone_name (Name, Time_zone_id) VALUES ('XXX', @time_zone_id);
+INSERT INTO time_zone_transition_type (Time_zone_id, Transition_type_id, Offset, Is_DST, Abbreviation) VALUES
+ (@time_zone_id, 0, 0, 0, 'GMT')
+;
+SET GLOBAL wsrep_replicate_myisam= OFF;
+#
+# Testing --leap
+#
+SET GLOBAL wsrep_replicate_myisam= ON;
+TRUNCATE TABLE time_zone_leap_second;
+ALTER TABLE time_zone_leap_second ORDER BY Transition_time;
+SET GLOBAL wsrep_replicate_myisam= OFF;
diff --git a/mysql-test/suite/wsrep/r/pool_of_threads.result b/mysql-test/suite/wsrep/r/pool_of_threads.result
new file mode 100644
index 00000000000..ffe309f2580
--- /dev/null
+++ b/mysql-test/suite/wsrep/r/pool_of_threads.result
@@ -0,0 +1,8 @@
+
+#
+# MDEV#5687: Maria doesn't shutdown following upgrade to 5.5.35-galera
+#
+SELECT @@GLOBAL.thread_handling;
+@@GLOBAL.thread_handling
+pool-of-threads
+# End of test.
diff --git a/mysql-test/suite/wsrep/r/trans.result b/mysql-test/suite/wsrep/r/trans.result
new file mode 100644
index 00000000000..bc225897103
--- /dev/null
+++ b/mysql-test/suite/wsrep/r/trans.result
@@ -0,0 +1,9 @@
+#
+# MDEV-4222 : Assertion `( ((global_system_variables.wsrep_on) &&
+#            (thd && thd->variables.wsrep_on)) && srep_emulate_bin_log)
+#            || mysql_bin_log .is_open()' fails on SAVEPOINT with
+#            disabled wsrep_provider
+#
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+SAVEPOINT A;
+End of test.
diff --git a/mysql-test/suite/wsrep/r/variables.result b/mysql-test/suite/wsrep/r/variables.result
new file mode 100644
index 00000000000..bd762f916bc
--- /dev/null
+++ b/mysql-test/suite/wsrep/r/variables.result
@@ -0,0 +1,222 @@
+
+# MDEV#5534: mysql_tzinfo_to_sql generates wrong query
+#
+# Testing wsrep_replicate_myisam variable.
+SELECT @@session.wsrep_replicate_myisam;
+ERROR HY000: Variable 'wsrep_replicate_myisam' is a GLOBAL variable
+SELECT @@global.wsrep_replicate_myisam;
+@@global.wsrep_replicate_myisam
+0
+SET SESSION wsrep_replicate_myisam= ON;
+ERROR HY000: Variable 'wsrep_replicate_myisam' is a GLOBAL variable and should be set with SET GLOBAL
+SET GLOBAL wsrep_replicate_myisam= ON;
+SET GLOBAL wsrep_replicate_myisam= OFF;
+SET GLOBAL wsrep_provider=none;
+#
+# MDEV#5790: SHOW GLOBAL STATUS LIKE does not show the correct list of
+# variables when using "_"
+#
+CALL mtr.add_suppression("WSREP: Could not open saved state file for reading.*");
+SHOW GLOBAL STATUS LIKE 'wsrep%';
+Variable_name	Value
+wsrep_local_state_uuid	#
+wsrep_protocol_version	#
+wsrep_last_committed	#
+wsrep_replicated	#
+wsrep_replicated_bytes	#
+wsrep_repl_keys	#
+wsrep_repl_keys_bytes	#
+wsrep_repl_data_bytes	#
+wsrep_repl_other_bytes	#
+wsrep_received	#
+wsrep_received_bytes	#
+wsrep_local_commits	#
+wsrep_local_cert_failures	#
+wsrep_local_replays	#
+wsrep_local_send_queue	#
+wsrep_local_send_queue_avg	#
+wsrep_local_recv_queue	#
+wsrep_local_recv_queue_avg	#
+wsrep_local_cached_downto	#
+wsrep_flow_control_paused_ns	#
+wsrep_flow_control_paused	#
+wsrep_flow_control_sent	#
+wsrep_flow_control_recv	#
+wsrep_cert_deps_distance	#
+wsrep_apply_oooe	#
+wsrep_apply_oool	#
+wsrep_apply_window	#
+wsrep_commit_oooe	#
+wsrep_commit_oool	#
+wsrep_commit_window	#
+wsrep_local_state	#
+wsrep_local_state_comment	#
+wsrep_cert_index_size	#
+wsrep_causal_reads	#
+wsrep_cert_interval	#
+wsrep_incoming_addresses	#
+wsrep_cluster_conf_id	#
+wsrep_cluster_size	#
+wsrep_cluster_state_uuid	#
+wsrep_cluster_status	#
+wsrep_connected	#
+wsrep_local_bf_aborts	#
+wsrep_local_index	#
+wsrep_provider_name	#
+wsrep_provider_vendor	#
+wsrep_provider_version	#
+wsrep_ready	#
+wsrep_thread_count	#
+
+SHOW GLOBAL STATUS LIKE 'wsrep_%';
+Variable_name	Value
+wsrep_local_state_uuid	#
+wsrep_protocol_version	#
+wsrep_last_committed	#
+wsrep_replicated	#
+wsrep_replicated_bytes	#
+wsrep_repl_keys	#
+wsrep_repl_keys_bytes	#
+wsrep_repl_data_bytes	#
+wsrep_repl_other_bytes	#
+wsrep_received	#
+wsrep_received_bytes	#
+wsrep_local_commits	#
+wsrep_local_cert_failures	#
+wsrep_local_replays	#
+wsrep_local_send_queue	#
+wsrep_local_send_queue_avg	#
+wsrep_local_recv_queue	#
+wsrep_local_recv_queue_avg	#
+wsrep_local_cached_downto	#
+wsrep_flow_control_paused_ns	#
+wsrep_flow_control_paused	#
+wsrep_flow_control_sent	#
+wsrep_flow_control_recv	#
+wsrep_cert_deps_distance	#
+wsrep_apply_oooe	#
+wsrep_apply_oool	#
+wsrep_apply_window	#
+wsrep_commit_oooe	#
+wsrep_commit_oool	#
+wsrep_commit_window	#
+wsrep_local_state	#
+wsrep_local_state_comment	#
+wsrep_cert_index_size	#
+wsrep_causal_reads	#
+wsrep_cert_interval	#
+wsrep_incoming_addresses	#
+wsrep_cluster_conf_id	#
+wsrep_cluster_size	#
+wsrep_cluster_state_uuid	#
+wsrep_cluster_status	#
+wsrep_connected	#
+wsrep_local_bf_aborts	#
+wsrep_local_index	#
+wsrep_provider_name	#
+wsrep_provider_vendor	#
+wsrep_provider_version	#
+wsrep_ready	#
+wsrep_thread_count	#
+SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';
+Variable_name	Value
+wsrep_local_state_comment	#
+# Should show nothing.
+SHOW STATUS LIKE 'x';
+Variable_name	Value
+SET GLOBAL wsrep_provider=none;
+#
+# MDEV#6079: xtrabackup SST failing with maria-10.0-galera
+#
+
+SHOW STATUS LIKE 'wsrep_local_state_uuid';
+Variable_name	Value
+wsrep_local_state_uuid	#
+
+SHOW STATUS LIKE 'wsrep_last_committed';
+Variable_name	Value
+wsrep_last_committed	#
+SET GLOBAL wsrep_provider=none;
+
+#
+# MDEV#6206: wsrep_slave_threads subtracts from max_connections
+#
+call mtr.add_suppression("safe_mutex: Found wrong usage of mutex 'LOCK_wsrep_slave_threads' and 'LOCK_global_system_variables'");
+call mtr.add_suppression("WSREP: Failed to get provider options");
+SELECT @@global.wsrep_provider;
+@@global.wsrep_provider
+libgalera_smm.so
+SELECT @@global.wsrep_slave_threads;
+@@global.wsrep_slave_threads
+1
+SELECT @@global.wsrep_cluster_address;
+@@global.wsrep_cluster_address
+NULL
+SHOW STATUS LIKE 'threads_connected';
+Variable_name	Value
+Threads_connected	1
+SHOW STATUS LIKE 'wsrep_thread_count';
+Variable_name	Value
+wsrep_thread_count	0
+
+SELECT @@global.wsrep_provider;
+@@global.wsrep_provider
+libgalera_smm.so
+SELECT @@global.wsrep_cluster_address;
+@@global.wsrep_cluster_address
+NULL
+SHOW STATUS LIKE 'threads_connected';
+Variable_name	Value
+Threads_connected	1
+SHOW STATUS LIKE 'wsrep_thread_count';
+Variable_name	Value
+wsrep_thread_count	0
+
+# Setting wsrep_cluster_address triggers the creation of
+# applier/rollbacker threads.
+SET GLOBAL wsrep_cluster_address= 'gcomm://';
+# Wait for applier threads to get created.
+SELECT @@global.wsrep_provider;
+@@global.wsrep_provider
+libgalera_smm.so
+SELECT @@global.wsrep_cluster_address;
+@@global.wsrep_cluster_address
+gcomm://
+SHOW STATUS LIKE 'threads_connected';
+Variable_name	Value
+Threads_connected	1
+SHOW STATUS LIKE 'wsrep_thread_count';
+Variable_name	Value
+wsrep_thread_count	2
+
+SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads;
+SET GLOBAL wsrep_slave_threads= 10;
+# Wait for applier threads to get created.
+SHOW STATUS LIKE 'threads_connected';
+Variable_name	Value
+Threads_connected	1
+SHOW STATUS LIKE 'wsrep_thread_count';
+Variable_name	Value
+wsrep_thread_count	11
+SET GLOBAL wsrep_slave_threads= @wsrep_slave_threads_saved;
+SET GLOBAL wsrep_provider= none;
+SET GLOBAL wsrep_cluster_address= '';
+SET GLOBAL wsrep_provider_options= '';
+#
+# MDEV#6411: Setting set @@global.wsrep_sst_auth=NULL causes crash
+#
+SET @wsrep_sst_auth_saved= @@global.wsrep_sst_auth;
+SET @@global.wsrep_sst_auth= 'user:pass';
+SELECT @@global.wsrep_sst_auth;
+@@global.wsrep_sst_auth
+********
+SET @@global.wsrep_sst_auth= '';
+SELECT @@global.wsrep_sst_auth;
+@@global.wsrep_sst_auth
+
+SET @@global.wsrep_sst_auth= NULL;
+SELECT @@global.wsrep_sst_auth;
+@@global.wsrep_sst_auth
+NULL
+SET @@global.wsrep_sst_auth= @wsrep_sst_auth_saved;
+# End of test.
diff --git a/mysql-test/suite/wsrep/t/binlog_format.opt b/mysql-test/suite/wsrep/t/binlog_format.opt
new file mode 100644
index 00000000000..1b8937b91f3
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/binlog_format.opt
@@ -0,0 +1 @@
+--binlog-format=row --innodb_autoinc_lock_mode=2 --innodb_locks_unsafe_for_binlog=1 --wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --wsrep_provider_options='base_port=$GALERA_BASE_PORT' --wsrep-on=1 --log-bin
diff --git a/mysql-test/suite/wsrep/t/binlog_format.test b/mysql-test/suite/wsrep/t/binlog_format.test
new file mode 100644
index 00000000000..99d34873512
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/binlog_format.test
@@ -0,0 +1,27 @@
+--source include/have_wsrep_enabled.inc
+--source include/have_binlog_format_row.inc
+#
+# MDEV-4227: Galera server should stop crashing on setting binlog_format STATEMENT
+#
+call mtr.add_suppression("WSREP: cannot get fake InnoDB transaction ID");
+call mtr.add_suppression("WSREP: Could not open saved state file for reading:.*");
+
+SHOW VARIABLES LIKE 'binlog_format';
+-- error ER_WRONG_VALUE_FOR_VAR
+SET binlog_format=STATEMENT;
+SHOW WARNINGS;
+SHOW VARIABLES LIKE 'binlog_format';
+CREATE TABLE IF NOT EXISTS test.t1 AS SELECT * FROM information_schema.routines WHERE 1 = 0;
+-- error ER_WRONG_VALUE_FOR_VAR
+SET binlog_format=MIXED;
+SHOW WARNINGS;
+SHOW VARIABLES LIKE 'binlog_format';
+CREATE TABLE IF NOT EXISTS test.t2 AS SELECT * FROM information_schema.routines WHERE 1 = 0;
+SET binlog_format=ROW;
+SHOW WARNINGS;
+SHOW VARIABLES LIKE 'binlog_format';
+CREATE TABLE IF NOT EXISTS test.t3 AS SELECT * FROM information_schema.routines WHERE 1 = 0;
+DROP TABLE IF EXISTS test.t1;
+DROP TABLE IF EXISTS test.t2;
+DROP TABLE IF EXISTS test.t3;
+
diff --git a/mysql-test/suite/wsrep/t/innodb_load_xa_with_wsrep.opt b/mysql-test/suite/wsrep/t/innodb_load_xa_with_wsrep.opt
new file mode 100644
index 00000000000..4ff27e659ce
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/innodb_load_xa_with_wsrep.opt
@@ -0,0 +1 @@
+--ignore-builtin-innodb --loose-innodb --log-bin
diff --git a/mysql-test/suite/wsrep/t/innodb_load_xa_with_wsrep.test b/mysql-test/suite/wsrep/t/innodb_load_xa_with_wsrep.test
new file mode 100644
index 00000000000..413faf54864
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/innodb_load_xa_with_wsrep.test
@@ -0,0 +1,19 @@
+#
+# MDEV-6082 Assertion `0' fails in TC_LOG_DUMMY::log_and_order on DML after installing TokuDB at runtime on server with disabled InnoDB
+#
+--source include/not_embedded.inc
+--source include/have_wsrep.inc
+
+if (!$HA_INNODB_SO) {
+  --skip Need InnoDB plugin
+}
+install plugin innodb soname 'ha_innodb';
+select engine,support,transactions,xa from information_schema.engines where engine='innodb';
+create table t1 (a int) engine=innodb;
+start transaction;
+insert t1 values (1);
+insert t1 values (2);
+commit;
+--source include/show_binlog_events.inc
+drop table t1;
+uninstall plugin innodb;
diff --git a/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.test b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.test
new file mode 100644
index 00000000000..100e09d3afb
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/mysql_tzinfo_to_sql_symlink.test
@@ -0,0 +1,40 @@
+--source include/have_wsrep.inc
+--source include/have_symlink.inc
+--source include/not_windows.inc
+
+--echo #
+--echo # MDEV-5226 mysql_tzinfo_to_sql errors with tzdata 2013f and above
+--echo #
+
+--exec mkdir $MYSQLTEST_VARDIR/zoneinfo
+--exec ln -s $MYSQLTEST_VARDIR/zoneinfo $MYSQLTEST_VARDIR/zoneinfo/posix
+--copy_file std_data/zoneinfo/GMT $MYSQLTEST_VARDIR/zoneinfo/GMT
+--copy_file std_data/words.dat $MYSQLTEST_VARDIR/zoneinfo/garbage
+--copy_file std_data/words.dat $MYSQLTEST_VARDIR/zoneinfo/ignored.tab
+
+--echo # Verbose run
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--exec $MYSQL_TZINFO_TO_SQL --verbose $MYSQLTEST_VARDIR/zoneinfo 2>&1
+
+--echo # Silent run
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--exec $MYSQL_TZINFO_TO_SQL $MYSQLTEST_VARDIR/zoneinfo 2>&1
+
+--echo #
+--echo # Testing with explicit timezonefile
+--echo #
+
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
+--exec $MYSQL_TZINFO_TO_SQL $MYSQLTEST_VARDIR/zoneinfo/GMT XXX 2>&1
+
+--echo #
+--echo # Testing --leap
+--echo #
+
+--exec $MYSQL_TZINFO_TO_SQL --leap $MYSQLTEST_VARDIR/zoneinfo/GMT 2>&1
+
+#
+# Cleanup
+#
+
+--exec rm -rf $MYSQLTEST_VARDIR/zoneinfo
diff --git a/mysql-test/suite/wsrep/t/pool_of_threads.opt b/mysql-test/suite/wsrep/t/pool_of_threads.opt
new file mode 100644
index 00000000000..5ee38531a4a
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/pool_of_threads.opt
@@ -0,0 +1 @@
+--binlog-format=row --innodb_autoinc_lock_mode=2 --innodb_locks_unsafe_for_binlog=1 --wsrep-provider=$WSREP_PROVIDER --wsrep-cluster-address=gcomm:// --wsrep_provider_options='base_port=$GALERA_BASE_PORT' --thread_handling=pool-of-threads
diff --git a/mysql-test/suite/wsrep/t/pool_of_threads.test b/mysql-test/suite/wsrep/t/pool_of_threads.test
new file mode 100644
index 00000000000..f4fffaf4f9a
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/pool_of_threads.test
@@ -0,0 +1,11 @@
+--source include/have_wsrep.inc
+
+--echo
+--echo #
+--echo # MDEV#5687: Maria doesn't shutdown following upgrade to 5.5.35-galera
+--echo #
+
+# Note: This test is to ensure that server shuts down properly.
+SELECT @@GLOBAL.thread_handling;
+
+--echo # End of test.
diff --git a/mysql-test/suite/wsrep/t/trans.test b/mysql-test/suite/wsrep/t/trans.test
new file mode 100644
index 00000000000..d8c4a4722a0
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/trans.test
@@ -0,0 +1,14 @@
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+
+--echo #
+--echo # MDEV-4222 : Assertion `( ((global_system_variables.wsrep_on) &&
+--echo #            (thd && thd->variables.wsrep_on)) && srep_emulate_bin_log)
+--echo #            || mysql_bin_log .is_open()' fails on SAVEPOINT with
+--echo #            disabled wsrep_provider
+--echo #
+
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+SAVEPOINT A;
+
+--echo End of test.
diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test
new file mode 100644
index 00000000000..6cc895a75d9
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/variables.test
@@ -0,0 +1,146 @@
+--source include/have_wsrep.inc
+
+# Set galera's base_port so that test can run in parallel with other galera
+# tests.
+--disable_query_log
+eval SET GLOBAL wsrep_provider_options='base_port=$GALERA_BASE_PORT';
+--enable_query_log
+
+--echo
+--echo # MDEV#5534: mysql_tzinfo_to_sql generates wrong query
+--echo #
+--echo # Testing wsrep_replicate_myisam variable.
+
+--disable_query_log
+eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
+--enable_query_log
+
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@session.wsrep_replicate_myisam;
+SELECT @@global.wsrep_replicate_myisam;
+
+--error ER_GLOBAL_VARIABLE
+SET SESSION wsrep_replicate_myisam= ON;
+SET GLOBAL wsrep_replicate_myisam= ON;
+
+# Reset it back.
+SET GLOBAL wsrep_replicate_myisam= OFF;
+SET GLOBAL wsrep_provider=none;
+
+--echo #
+--echo # MDEV#5790: SHOW GLOBAL STATUS LIKE does not show the correct list of
+--echo # variables when using "_"
+--echo #
+
+CALL mtr.add_suppression("WSREP: Could not open saved state file for reading.*");
+
+--disable_query_log
+eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
+--enable_query_log
+
+--replace_column 2 #
+SHOW GLOBAL STATUS LIKE 'wsrep%';
+
+--echo
+--replace_column 2 #
+SHOW GLOBAL STATUS LIKE 'wsrep_%';
+
+--replace_column 2 #
+SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';
+
+--echo # Should show nothing.
+SHOW STATUS LIKE 'x';
+
+# Reset it back.
+SET GLOBAL wsrep_provider=none;
+
+--echo #
+--echo # MDEV#6079: xtrabackup SST failing with maria-10.0-galera
+--echo #
+
+--disable_query_log
+eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
+--enable_query_log
+
+# The following 2 variables are used in innobackupex during xtrabackup-based
+# SST.
+--echo
+--replace_column 2 #
+SHOW STATUS LIKE 'wsrep_local_state_uuid';
+--echo
+--replace_column 2 #
+SHOW STATUS LIKE 'wsrep_last_committed';
+
+# Reset it back.
+SET GLOBAL wsrep_provider=none;
+
+--echo
+--echo #
+--echo # MDEV#6206: wsrep_slave_threads subtracts from max_connections
+--echo #
+call mtr.add_suppression("safe_mutex: Found wrong usage of mutex 'LOCK_wsrep_slave_threads' and 'LOCK_global_system_variables'");
+call mtr.add_suppression("WSREP: Failed to get provider options");
+
+--disable_query_log
+eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
+--enable_query_log
+
+--replace_regex /.*libgalera_smm.*/libgalera_smm.so/
+SELECT @@global.wsrep_provider;
+SELECT @@global.wsrep_slave_threads;
+SELECT @@global.wsrep_cluster_address;
+SHOW STATUS LIKE 'threads_connected';
+SHOW STATUS LIKE 'wsrep_thread_count';
+--echo
+
+--disable_query_log
+eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
+--enable_query_log
+
+--replace_regex /.*libgalera_smm.*/libgalera_smm.so/
+SELECT @@global.wsrep_provider;
+SELECT @@global.wsrep_cluster_address;
+SHOW STATUS LIKE 'threads_connected';
+SHOW STATUS LIKE 'wsrep_thread_count';
+--echo
+
+--echo # Setting wsrep_cluster_address triggers the creation of
+--echo # applier/rollbacker threads.
+SET GLOBAL wsrep_cluster_address= 'gcomm://';
+--echo # Wait for applier threads to get created.
+sleep 3; 
+
+--replace_regex /.*libgalera_smm.*/libgalera_smm.so/
+SELECT @@global.wsrep_provider;
+SELECT @@global.wsrep_cluster_address;
+SHOW STATUS LIKE 'threads_connected';
+SHOW STATUS LIKE 'wsrep_thread_count';
+--echo
+
+SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads;
+SET GLOBAL wsrep_slave_threads= 10;
+--echo # Wait for applier threads to get created.
+sleep 3; 
+SHOW STATUS LIKE 'threads_connected';
+SHOW STATUS LIKE 'wsrep_thread_count';
+
+# reset (for mtr internal checks)
+SET GLOBAL wsrep_slave_threads= @wsrep_slave_threads_saved;
+SET GLOBAL wsrep_provider= none;
+SET GLOBAL wsrep_cluster_address= '';
+SET GLOBAL wsrep_provider_options= '';
+
+--echo #
+--echo # MDEV#6411: Setting set @@global.wsrep_sst_auth=NULL causes crash
+--echo #
+SET @wsrep_sst_auth_saved= @@global.wsrep_sst_auth;
+SET @@global.wsrep_sst_auth= 'user:pass';
+SELECT @@global.wsrep_sst_auth;
+SET @@global.wsrep_sst_auth= '';
+SELECT @@global.wsrep_sst_auth;
+SET @@global.wsrep_sst_auth= NULL;
+SELECT @@global.wsrep_sst_auth;
+SET @@global.wsrep_sst_auth= @wsrep_sst_auth_saved;
+
+--echo # End of test.
+
diff --git a/mysql-test/t/mysql_tzinfo_to_sql_symlink.test b/mysql-test/t/mysql_tzinfo_to_sql_symlink.test
index 1ba4e91be3c..0c70315c93b 100644
--- a/mysql-test/t/mysql_tzinfo_to_sql_symlink.test
+++ b/mysql-test/t/mysql_tzinfo_to_sql_symlink.test
@@ -1,5 +1,16 @@
 --source include/have_symlink.inc
 --source include/not_windows.inc
+--source include/not_wsrep.inc
+
+# Note: The output of mysql_tzinfo_to_sql is different if server is compiled
+# with wsrep. Hence a copy of this test has been placed under wsrep suite with
+# the updated result. (lp:1161432)
+--source include/not_wsrep.inc
+
+# Note: The output of mysql_tzinfo_to_sql is different if server is compiled
+# with wsrep. Hence a copy of this test has been placed under wsrep suite with
+# the updated result. (lp:1161432)
+--source include/not_wsrep.inc
 
 --echo #
 --echo # MDEV-5226 mysql_tzinfo_to_sql errors with tzdata 2013f and above
diff --git a/mysql-test/t/mysqld--help.test b/mysql-test/t/mysqld--help.test
index c27fa58f935..a8cafed9efb 100644
--- a/mysql-test/t/mysqld--help.test
+++ b/mysql-test/t/mysqld--help.test
@@ -4,6 +4,7 @@
 --source include/not_embedded.inc
 --source include/have_perfschema.inc
 --source include/platform.inc
+--source include/not_wsrep.inc
 
 #
 # force lower-case-table-names=1 (linux/macosx have different defaults)
@@ -22,7 +23,7 @@ perl;
                log-slow-queries pid-file slow-query-log-file log-basename
                datadir slave-load-tmpdir tmpdir socket thread-pool-size
                large-files-support lower-case-file-system system-time-zone
-               version.*/;
+               wsrep-node-name version.*/;
 
   # Plugins which may or may not be there:
   @plugins=qw/innodb ndb archive blackhole federated partition ndbcluster
diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt
index f0d25dae6b9..e342427c737 100644
--- a/mysys/CMakeLists.txt
+++ b/mysys/CMakeLists.txt
@@ -15,6 +15,10 @@
 
 INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/mysys)
 
+IF(WITH_WSREP)
+  BUILD_WITH_WSREP()
+ENDIF()
+
 SET(MYSYS_SOURCES  array.c charset-def.c charset.c checksum.c my_default.c
 				errors.c hash.c list.c
                                 mf_cache.c mf_dirname.c mf_fn_ext.c
diff --git a/mysys/my_default.c b/mysys/my_default.c
index 1e4038d17fb..1a29fd87a34 100644
--- a/mysys/my_default.c
+++ b/mysys/my_default.c
@@ -88,6 +88,12 @@ static char my_defaults_extra_file_buffer[FN_REFLEN];
 
 static my_bool defaults_already_read= FALSE;
 
+#ifdef WITH_WSREP
+/* The only purpose of this global array is to hold full name of my.cnf
+ * which seems to be otherwise unavailable */
+char wsrep_defaults_file[FN_REFLEN + 10]={0,};
+#endif /* WITH_WREP */
+
 /* Which directories are searched for options (and in which order) */
 
 #define MAX_DEFAULT_DIRS 6
@@ -804,6 +810,12 @@ static int search_default_file_with_ext(Process_option_func opt_handler,
   if (!(fp= mysql_file_fopen(key_file_cnf, name, O_RDONLY, MYF(0))))
     return 1;					/* Ignore wrong files */
 
+#ifdef WITH_WSREP
+  /* make sure we do this only once - for top-level file */
+  if ('\0' == wsrep_defaults_file[0])
+    strmake(wsrep_defaults_file, name, sizeof(wsrep_defaults_file) - 1);
+#endif /* WITH_WSREP */
+
   while (mysql_file_fgets(buff, sizeof(buff) - 1, fp))
   {
     line++;
diff --git a/mysys/my_static.c b/mysys/my_static.c
index 84d2dc64fc6..4c977837e05 100644
--- a/mysys/my_static.c
+++ b/mysys/my_static.c
@@ -68,6 +68,10 @@ uint    my_large_page_size= 0;
 int volatile my_have_got_alarm=0;	/* declare variable to reset */
 ulong my_time_to_wait_for_lock=2;	/* In seconds */
 
+#ifdef WITH_WSREP
+my_bool mysys_wsrep= 0;
+#endif
+
 	/* from errors.c */
 #ifdef SHARED_LIBRARY
 const char *globerrs[GLOBERRS];		/* my_error_messages is here */
diff --git a/mysys/mysys_priv.h b/mysys/mysys_priv.h
index 9c6855bb92f..80c6a981db2 100644
--- a/mysys/mysys_priv.h
+++ b/mysys/mysys_priv.h
@@ -62,6 +62,8 @@ extern mysql_mutex_t THR_LOCK_malloc, THR_LOCK_open, THR_LOCK_keycache;
 extern mysql_mutex_t THR_LOCK_lock, THR_LOCK_net;
 extern mysql_mutex_t THR_LOCK_charset;
 
+extern my_bool mysys_wsrep;
+
 #include <mysql/psi/mysql_file.h>
 
 #ifdef HAVE_PSI_INTERFACE
diff --git a/mysys/thr_lock.c b/mysys/thr_lock.c
index 60e01974320..acc7d718546 100644
--- a/mysys/thr_lock.c
+++ b/mysys/thr_lock.c
@@ -95,6 +95,24 @@ my_bool thr_lock_inited=0;
 ulong locks_immediate = 0L, locks_waited = 0L;
 enum thr_lock_type thr_upgraded_concurrent_insert_lock = TL_WRITE;
 
+#ifdef WITH_WSREP
+static wsrep_thd_is_brute_force_fun wsrep_thd_is_brute_force= NULL;
+static wsrep_abort_thd_fun wsrep_abort_thd= NULL;
+static my_bool wsrep_debug;
+static my_bool wsrep_convert_LOCK_to_trx;
+static wsrep_on_fun wsrep_on = NULL;
+
+void wsrep_thr_lock_init(
+    wsrep_thd_is_brute_force_fun bf_fun, wsrep_abort_thd_fun abort_fun,
+    my_bool debug, my_bool convert_LOCK_to_trx, wsrep_on_fun on_fun
+) {
+  wsrep_thd_is_brute_force = bf_fun;
+  wsrep_abort_thd          = abort_fun;
+  wsrep_debug              = debug;
+  wsrep_convert_LOCK_to_trx= convert_LOCK_to_trx;
+  wsrep_on                 = on_fun;
+}
+#endif
 /* The following constants are only for debug output */
 #define MAX_THREADS 1000
 #define MAX_LOCKS   1000
@@ -646,6 +664,92 @@ wait_for_lock(struct st_lock_list *wait, THR_LOCK_DATA *data,
   DBUG_RETURN(result);
 }
 
+#ifdef WITH_WSREP
+/*
+ * If brute force applier would need to wait for a thr lock,
+ * it needs to make sure that it will get the lock without (too much) 
+ * delay. 
+ * We identify here the owners of blocking locks and ask them to
+ * abort. We then put our lock request in the first place in the
+ * wait queue. When lock holders abort (one by one) the lock release
+ * algorithm should grant the lock to us. We rely on this and proceed
+ * to wait_for_locks().
+ * wsrep_break_locks() should be called in all the cases, where lock
+ * wait would happen.
+ *
+ * TODO: current implementation might not cover all possible lock wait
+ *       situations. This needs an review still.
+ * TODO: lock release, might favor some other lock (instead our bf).
+ *       This needs an condition to check for bf locks first.
+ * TODO: we still have a debug fprintf, this should be removed
+ */
+static my_bool 
+wsrep_break_lock(
+    THR_LOCK_DATA *data, struct st_lock_list *lock_queue1, 
+    struct st_lock_list *wait_queue)
+{
+  if (wsrep_on(data->owner->mysql_thd) &&
+      wsrep_thd_is_brute_force          &&
+      wsrep_thd_is_brute_force(data->owner->mysql_thd, TRUE))
+  {
+    THR_LOCK_DATA *holder;
+
+    /* if locking session conversion to transaction has been enabled,
+       we know that this conflicting lock must be read lock and furthermore,
+       lock holder is read-only. It is safe to wait for him.
+    */
+#ifdef TODO_WHEN_LOCK_TABLES_IS_A_TRANSACTION
+    if (wsrep_convert_LOCK_to_trx && 
+	(THD*)(data->owner->mysql_thd)->in_lock_tables)
+    {
+      if (wsrep_debug) 
+        fprintf(stderr,"WSREP wsrep_break_lock read lock untouched\n");
+      return FALSE;
+    }
+#endif
+    if (wsrep_debug) 
+      fprintf(stderr,"WSREP wsrep_break_lock aborting locks\n");
+
+    /* aborting lock holder(s) here */
+    for (holder=(lock_queue1) ? lock_queue1->data : NULL; 
+	 holder; 
+	 holder=holder->next) 
+    {
+      if (!wsrep_thd_is_brute_force(holder->owner->mysql_thd, TRUE))
+      {
+        wsrep_abort_thd(data->owner->mysql_thd, 
+                        holder->owner->mysql_thd, FALSE);
+      }
+      else
+      {
+        if (wsrep_debug) 
+          fprintf(stderr,"WSREP wsrep_break_lock skipping BF lock conflict\n");
+         return FALSE;
+      }
+    }
+        
+    /* Add our lock to the head of the wait queue */
+    if (*(wait_queue->last)==wait_queue->data)
+    {
+      wait_queue->last=&data->next;
+      assert(wait_queue->data==0);
+    }
+    else
+    {
+      assert(wait_queue->data!=0);
+      wait_queue->data->prev=&data->next;
+    }
+    data->next=wait_queue->data;
+    data->prev=&wait_queue->data;
+    wait_queue->data=data;
+    data->cond=get_cond();
+
+    statistic_increment(locks_immediate,&THR_LOCK_lock);
+    return TRUE;
+  }
+  return FALSE;
+}
+#endif
 
 static enum enum_thr_lock_result
 thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout)
@@ -654,6 +758,9 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout)
   enum enum_thr_lock_result result= THR_LOCK_SUCCESS;
   struct st_lock_list *wait_queue;
   enum thr_lock_type lock_type= data->type;
+#ifdef WITH_WSREP
+  my_bool wsrep_lock_inserted= FALSE;
+#endif
   MYSQL_TABLE_WAIT_VARIABLES(locker, state) /* no ';' */
   DBUG_ENTER("thr_lock");
 
@@ -750,6 +857,14 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout)
       lock but a high priority write waiting in the write_wait queue.
       In the latter case we should yield the lock to the writer.
     */
+#ifdef WITH_WSREP
+    if (mysys_wsrep && wsrep_break_lock(data, &lock->write, &lock->read_wait))
+    {
+      wsrep_lock_inserted= TRUE;
+    }
+  wsrep_read_wait:
+#endif
+
     wait_queue= &lock->read_wait;
   }
   else						/* Request for WRITE lock */
@@ -891,9 +1006,22 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout)
       DBUG_PRINT("lock",("write locked 3 by thread: 0x%lx  type: %d",
 			 lock->read.data->owner->thread_id, data->type));
     }
+#ifdef WITH_WSREP
+    if (mysys_wsrep && wsrep_break_lock(data, &lock->write, &lock->write_wait))
+    {
+      wsrep_lock_inserted= TRUE;
+    }
+  wsrep_write_wait:
+
+#endif
+
     wait_queue= &lock->write_wait;
   }
   /* Can't get lock yet;  Wait for it */
+#ifdef WITH_WSREP
+  if (mysys_wsrep && wsrep_lock_inserted && wsrep_on(data->owner->mysql_thd))
+    DBUG_RETURN(wait_for_lock(wait_queue, data, 1, lock_wait_timeout));
+#endif
   result= wait_for_lock(wait_queue, data, 0, lock_wait_timeout);
   MYSQL_END_TABLE_LOCK_WAIT(locker);
   DBUG_RETURN(result);
diff --git a/plugin/feedback/CMakeLists.txt b/plugin/feedback/CMakeLists.txt
index a243ba07751..9070de26ccb 100644
--- a/plugin/feedback/CMakeLists.txt
+++ b/plugin/feedback/CMakeLists.txt
@@ -18,6 +18,7 @@ IF(WIN32)
 ENDIF(WIN32)
 
 MYSQL_ADD_PLUGIN(FEEDBACK ${FEEDBACK_SOURCES}
+  RECOMPILE_FOR_EMBEDDED
   LINK_LIBRARIES ${SSL_LIBRARIES}
   ${MAYBE_STATIC_ONLY} DEFAULT)
 
diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt
index d3a7861269c..1f0f28061ed 100644
--- a/scripts/CMakeLists.txt
+++ b/scripts/CMakeLists.txt
@@ -327,6 +327,15 @@ IF(WIN32)
     INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/${file}.pl COMPONENT Server_Scripts)
   ENDFOREACH()
 ELSE()
+  IF(WITH_WSREP)
+    SET(WSREP_BINARIES
+      wsrep_sst_common
+      wsrep_sst_mysqldump
+      wsrep_sst_rsync
+      wsrep_sst_xtrabackup
+      wsrep_sst_xtrabackup-v2
+    )
+  ENDIF()
   # Configure this one, for testing, but do not install it.
   CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/mysql_config.pl.in
     ${CMAKE_CURRENT_BINARY_DIR}/mysql_config.pl ESCAPE_QUOTES @ONLY)
@@ -346,6 +355,7 @@ ELSE()
     mysqldumpslow
     mysqld_multi
     mysqld_safe
+    ${WSREP_BINARIES}
   )
   FOREACH(file ${BIN_SCRIPTS})
     IF(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${file}.sh)
diff --git a/scripts/mysqld_multi.sh b/scripts/mysqld_multi.sh
index cd1b6fc18b7..93ab717c8ae 100644
--- a/scripts/mysqld_multi.sh
+++ b/scripts/mysqld_multi.sh
@@ -38,7 +38,7 @@ use Getopt::Long;
 use POSIX qw(strftime getcwd);
 
 $|=1;
-$VER="2.16";
+$VER="2.20";
 
 my @defaults_options;   #  Leading --no-defaults, --defaults-file, etc.
 
@@ -138,6 +138,7 @@ sub main
     print "will be disabled\nand some will be enabled.\n\n";
   }
 
+  init_log() if (!defined($opt_log));
   $groupids = $ARGV[1];
   if ($opt_version)
   {
@@ -163,7 +164,6 @@ sub main
 	       !($ARGV[0] =~ m/^stop$/i) &&
 	       !($ARGV[0] =~ m/^report$/i)));
 
-  init_log() if (!defined($opt_log));
   if (!$opt_no_log)
   {
     w2log("$my_progname log file version $VER; run: ",
@@ -218,6 +218,10 @@ sub quote_shell_word
   return $option;
 }
 
+####
+#### get options for a group
+####
+
 sub defaults_for_group
 {
   my ($group) = @_;
diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh
index 9d0dd0e68ac..4e7b098c8d8 100644
--- a/scripts/mysqld_safe.sh
+++ b/scripts/mysqld_safe.sh
@@ -189,6 +189,89 @@ shell_quote_string() {
   echo "$1" | sed -e 's,\([^a-zA-Z0-9/_.=-]\),\\\1,g'
 }
 
+wsrep_pick_url() {
+  [ $# -eq 0 ] && return 0
+
+  log_error "WSREP: 'wsrep_urls' is DEPRECATED! Use wsrep_cluster_address to specify multiple addresses instead."
+
+  if ! which nc >/dev/null; then
+    log_error "ERROR: nc tool not found in PATH! Make sure you have it installed."
+    return 1
+  fi
+
+  local url
+  # Assuming URL in the form scheme://host:port
+  # If host and port are not NULL, the liveness of URL is assumed to be tested
+  # If port part is absent, the url is returned literally and unconditionally
+  # If every URL has port but none is reachable, nothing is returned
+  for url in `echo $@ | sed s/,/\ /g` 0; do
+    local host=`echo $url | cut -d \: -f 2 | sed s/^\\\/\\\///`
+    local port=`echo $url | cut -d \: -f 3`
+    [ -z "$port" ] && break
+    nc -z "$host" $port >/dev/null && break
+  done
+
+  if [ "$url" == "0" ]; then
+    log_error "ERROR: none of the URLs in '$@' is reachable."
+    return 1
+  fi
+
+  echo $url
+}
+
+# Run mysqld with --wsrep-recover and parse recovered position from log.
+# Position will be stored in wsrep_start_position_opt global.
+wsrep_start_position_opt=""
+wsrep_recover_position() {
+  local mysqld_cmd="$@"
+  local euid=$(id -u)
+  local ret=0
+
+  local wr_logfile=$(mktemp $DATADIR/wsrep_recovery.XXXXXX)
+
+  # safety checks
+  if [ -z $wr_logfile ]; then
+    log_error "WSREP: mktemp failed"
+    return 1
+  fi
+
+  if [ -f $wr_logfile ]; then
+    [ "$euid" = "0" ] && chown $user $wr_logfile
+    chmod 600 $wr_logfile
+  else
+    log_error "WSREP: mktemp failed"
+    return 1
+  fi
+
+  local wr_pidfile="$DATADIR/"`@HOSTNAME@`"-recover.pid"
+
+  local wr_options="--log_error='$wr_logfile' --pid-file='$wr_pidfile'"
+
+  log_notice "WSREP: Running position recovery with $wr_options"
+
+  eval_log_error "$mysqld_cmd --wsrep_recover $wr_options"
+
+  local rp="$(grep 'WSREP: Recovered position:' $wr_logfile)"
+  if [ -z "$rp" ]; then
+    local skipped="$(grep WSREP $wr_logfile | grep 'skipping position recovery')"
+    if [ -z "$skipped" ]; then
+      log_error "WSREP: Failed to recover position: '`cat $wr_logfile`'"
+      ret=1
+    else
+      log_notice "WSREP: Position recovery skipped"
+    fi
+  else
+    local start_pos="$(echo $rp | sed 's/.*WSREP\:\ Recovered\ position://' \
+        | sed 's/^[ \t]*//')"
+    log_notice "WSREP: Recovered position $start_pos"
+    wsrep_start_position_opt="--wsrep_start_position=$start_pos"
+  fi
+
+  [ $ret -eq 0 ] && rm $wr_logfile
+
+  return $ret
+}
+
 parse_arguments() {
   # We only need to pass arguments through to the server if we don't
   # handle them here.  So, we collect unrecognized options (passed on
@@ -245,6 +328,14 @@ parse_arguments() {
       --timezone=*) TZ="$val"; export TZ; ;;
       --flush[-_]caches) flush_caches=1 ;;
       --numa[-_]interleave) numa_interleave=1 ;;
+      --wsrep[-_]urls=*) wsrep_urls="$val"; ;;
+      --wsrep[-_]provider=*)
+        if test -n "$val" && test "$val" != "none"
+        then
+          wsrep_restart=1
+        fi
+        append_arg_to_args "$arg"
+        ;;
 
       --help) usage ;;
 
@@ -860,13 +951,28 @@ max_fast_restarts=5
 # flag whether a usable sleep command exists
 have_sleep=1
 
+# maximum number of wsrep restarts
+max_wsrep_restarts=0
+
 while true
 do
   rm -f "$pid_file"	# Some extra safety
 
   start_time=`date +%M%S`
 
-  eval_log_error "$cmd"
+  # this sets wsrep_start_position_opt
+  wsrep_recover_position "$cmd"
+
+  [ $? -ne 0 ] && exit 1 #
+
+  [ -n "$wsrep_urls" ] && url=`wsrep_pick_url $wsrep_urls` # check connect address
+
+  if [ -z "$url" ]
+  then
+    eval_log_error "$cmd $wsrep_start_position_opt $nohup_redir"
+  else
+    eval_log_error "$cmd $wsrep_start_position_opt --wsrep_cluster_address=$url $nohup_redir"
+  fi
 
   if [ $want_syslog -eq 0 -a ! -f "$err_log" ]; then
     touch "$err_log"                    # hypothetical: log was renamed but not
@@ -936,6 +1042,20 @@ do
       I=`expr $I + 1`
     done
   fi
+
+  if [ -n "$wsrep_restart" ]
+  then
+    if [ $wsrep_restart -le $max_wsrep_restarts ]
+    then
+      wsrep_restart=`expr $wsrep_restart + 1`
+      log_notice "WSREP: sleeping 15 seconds before restart"
+      sleep 15
+    else
+      log_notice "WSREP: not restarting wsrep node automatically"
+      break
+    fi
+  fi
+
   log_notice "mysqld restarted"
   if test -n "$CRASH_SCRIPT"
   then
diff --git a/scripts/wsrep_sst_common b/scripts/wsrep_sst_common
new file mode 100755
index 00000000000..88f5d80f53a
--- /dev/null
+++ b/scripts/wsrep_sst_common
@@ -0,0 +1,157 @@
+# Copyright (C) 2012-2014 Codership Oy
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# This is a common command line parser to be sourced by other SST scripts
+
+set -u
+
+WSREP_SST_OPT_BYPASS=0
+WSREP_SST_OPT_BINLOG=""
+WSREP_SST_OPT_DATA=""
+WSREP_SST_OPT_AUTH=""
+
+while [ $# -gt 0 ]; do
+case "$1" in
+    '--address')
+        readonly WSREP_SST_OPT_ADDR="$2"
+        shift
+        ;;
+    '--auth')
+        WSREP_SST_OPT_AUTH="$2"
+        shift
+        ;;
+    '--bypass')
+        WSREP_SST_OPT_BYPASS=1
+        ;;
+    '--datadir')
+        readonly WSREP_SST_OPT_DATA="$2"
+        shift
+        ;;
+    '--defaults-file')
+        readonly WSREP_SST_OPT_CONF="$2"
+        shift
+        ;;
+    '--host')
+        readonly WSREP_SST_OPT_HOST="$2"
+        shift
+        ;;
+    '--local-port')
+        readonly WSREP_SST_OPT_LPORT="$2"
+        shift
+        ;;
+    '--parent')
+        readonly WSREP_SST_OPT_PARENT="$2"
+        shift
+        ;;
+    '--password')
+        WSREP_SST_OPT_PSWD="$2"
+        shift
+        ;;
+    '--port')
+        readonly WSREP_SST_OPT_PORT="$2"
+        shift
+        ;;
+    '--role')
+        readonly WSREP_SST_OPT_ROLE="$2"
+        shift
+        ;;
+    '--socket')
+        readonly WSREP_SST_OPT_SOCKET="$2"
+        shift
+        ;;
+    '--user')
+        WSREP_SST_OPT_USER="$2"
+        shift
+        ;;
+    '--gtid')
+        readonly WSREP_SST_OPT_GTID="$2"
+        shift
+        ;;
+    '--binlog')
+        WSREP_SST_OPT_BINLOG="$2"
+        shift
+        ;;
+    *) # must be command
+       # usage
+       # exit 1
+       ;;
+esac
+shift
+done
+readonly WSREP_SST_OPT_BYPASS
+readonly WSREP_SST_OPT_BINLOG
+
+# State Snapshot Transfer authentication password was displayed in the ps output. Bug fixed #1200727.
+if my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -q "wsrep_sst_auth";then
+    if [ -z "$WSREP_SST_OPT_AUTH" -o "$WSREP_SST_OPT_AUTH" = "(null)" ];then
+            WSREP_SST_OPT_AUTH=$(my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -- "--wsrep_sst_auth" | cut -d= -f2)
+    fi
+fi
+
+if [ -n "${WSREP_SST_OPT_DATA:-}" ]
+then
+    SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress"
+else
+    SST_PROGRESS_FILE=""
+fi
+
+
+wsrep_log()
+{
+    # echo everything to stderr so that it gets into common error log
+    # deliberately made to look different from the rest of the log
+    local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)"
+    echo "$tst WSREP_SST: " >&2
+}
+
+wsrep_log_error()
+{
+    wsrep_log "[ERROR] $*"
+}
+
+wsrep_log_info()
+{
+    wsrep_log "[INFO] $*"
+}
+
+wsrep_cleanup_progress_file()
+{
+    [ -n "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null
+}
+
+wsrep_check_program()
+{
+    local prog=$1
+
+    if ! which $prog >/dev/null
+    then
+        echo "'$prog' not found in PATH"
+        return 2 # no such file or directory
+    fi
+}
+
+wsrep_check_programs()
+{
+    local ret=0
+
+    while [ $# -gt 0 ]
+    do
+        wsrep_check_program $1 || ret=$?
+        shift
+    done
+
+    return $ret
+}
diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh
new file mode 100644
index 00000000000..88f5d80f53a
--- /dev/null
+++ b/scripts/wsrep_sst_common.sh
@@ -0,0 +1,157 @@
+# Copyright (C) 2012-2014 Codership Oy
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# This is a common command line parser to be sourced by other SST scripts
+
+set -u
+
+WSREP_SST_OPT_BYPASS=0
+WSREP_SST_OPT_BINLOG=""
+WSREP_SST_OPT_DATA=""
+WSREP_SST_OPT_AUTH=""
+
+while [ $# -gt 0 ]; do
+case "$1" in
+    '--address')
+        readonly WSREP_SST_OPT_ADDR="$2"
+        shift
+        ;;
+    '--auth')
+        WSREP_SST_OPT_AUTH="$2"
+        shift
+        ;;
+    '--bypass')
+        WSREP_SST_OPT_BYPASS=1
+        ;;
+    '--datadir')
+        readonly WSREP_SST_OPT_DATA="$2"
+        shift
+        ;;
+    '--defaults-file')
+        readonly WSREP_SST_OPT_CONF="$2"
+        shift
+        ;;
+    '--host')
+        readonly WSREP_SST_OPT_HOST="$2"
+        shift
+        ;;
+    '--local-port')
+        readonly WSREP_SST_OPT_LPORT="$2"
+        shift
+        ;;
+    '--parent')
+        readonly WSREP_SST_OPT_PARENT="$2"
+        shift
+        ;;
+    '--password')
+        WSREP_SST_OPT_PSWD="$2"
+        shift
+        ;;
+    '--port')
+        readonly WSREP_SST_OPT_PORT="$2"
+        shift
+        ;;
+    '--role')
+        readonly WSREP_SST_OPT_ROLE="$2"
+        shift
+        ;;
+    '--socket')
+        readonly WSREP_SST_OPT_SOCKET="$2"
+        shift
+        ;;
+    '--user')
+        WSREP_SST_OPT_USER="$2"
+        shift
+        ;;
+    '--gtid')
+        readonly WSREP_SST_OPT_GTID="$2"
+        shift
+        ;;
+    '--binlog')
+        WSREP_SST_OPT_BINLOG="$2"
+        shift
+        ;;
+    *) # must be command
+       # usage
+       # exit 1
+       ;;
+esac
+shift
+done
+readonly WSREP_SST_OPT_BYPASS
+readonly WSREP_SST_OPT_BINLOG
+
+# State Snapshot Transfer authentication password was displayed in the ps output. Bug fixed #1200727.
+if my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -q "wsrep_sst_auth";then
+    if [ -z "$WSREP_SST_OPT_AUTH" -o "$WSREP_SST_OPT_AUTH" = "(null)" ];then
+            WSREP_SST_OPT_AUTH=$(my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -- "--wsrep_sst_auth" | cut -d= -f2)
+    fi
+fi
+
+if [ -n "${WSREP_SST_OPT_DATA:-}" ]
+then
+    SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress"
+else
+    SST_PROGRESS_FILE=""
+fi
+
+
+wsrep_log()
+{
+    # echo everything to stderr so that it gets into common error log
+    # deliberately made to look different from the rest of the log
+    local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)"
+    echo "$tst WSREP_SST: " >&2
+}
+
+wsrep_log_error()
+{
+    wsrep_log "[ERROR] $*"
+}
+
+wsrep_log_info()
+{
+    wsrep_log "[INFO] $*"
+}
+
+wsrep_cleanup_progress_file()
+{
+    [ -n "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null
+}
+
+wsrep_check_program()
+{
+    local prog=$1
+
+    if ! which $prog >/dev/null
+    then
+        echo "'$prog' not found in PATH"
+        return 2 # no such file or directory
+    fi
+}
+
+wsrep_check_programs()
+{
+    local ret=0
+
+    while [ $# -gt 0 ]
+    do
+        wsrep_check_program $1 || ret=$?
+        shift
+    done
+
+    return $ret
+}
diff --git a/scripts/wsrep_sst_mysqldump b/scripts/wsrep_sst_mysqldump
new file mode 100755
index 00000000000..d4093dbcaef
--- /dev/null
+++ b/scripts/wsrep_sst_mysqldump
@@ -0,0 +1,131 @@
+#!/bin/bash -e
+# Copyright (C) 2009 Codership Oy
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# This is a reference script for mysqldump-based state snapshot tansfer
+
+# This variable is not used in mysqldump sst, so better initialize it
+# to avoid shell's "parameter not set" message.
+WSREP_SST_OPT_CONF=""
+
+. $(dirname $0)/wsrep_sst_common
+
+EINVAL=22
+PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin
+
+local_ip()
+{
+    [ "$1" = "127.0.0.1" ]      && return 0
+    [ "$1" = "localhost" ]      && return 0
+    [ "$1" = "$(hostname -s)" ] && return 0
+    [ "$1" = "$(hostname -f)" ] && return 0
+    [ "$1" = "$(hostname -d)" ] && return 0
+
+    # Now if ip program is not found in the path, we can't return 0 since
+    # it would block any address. Thankfully grep should fail in this case
+    ip route get "$1" | grep local >/dev/null && return 0
+
+    return 1
+}
+
+if test -z "$WSREP_SST_OPT_USER";  then wsrep_log_error "USER cannot be nil";  exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_HOST";  then wsrep_log_error "HOST cannot be nil";  exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_PORT";  then wsrep_log_error "PORT cannot be nil";  exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_LPORT"; then wsrep_log_error "LPORT cannot be nil"; exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_SOCKET";then wsrep_log_error "SOCKET cannot be nil";exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_GTID";  then wsrep_log_error "GTID cannot be nil";  exit $EINVAL; fi
+
+if local_ip $WSREP_SST_OPT_HOST && \
+   [ "$WSREP_SST_OPT_PORT" = "$WSREP_SST_OPT_LPORT" ]
+then
+    wsrep_log_error \
+    "destination address '$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT' matches source address."
+    exit $EINVAL
+fi
+
+# Check client version
+if ! mysql --version | grep 'Distrib 10' >/dev/null
+then
+    mysql --version >&2
+    wsrep_log_error "this operation requires MySQL client version 10 or newer"
+    exit $EINVAL
+fi
+
+# For Bug:1293798
+if [ -z "$WSREP_SST_OPT_PSWD" -a -n "$WSREP_SST_OPT_AUTH" ]; then
+    WSREP_SST_OPT_USER=$(echo $WSREP_SST_OPT_AUTH | cut -d: -f1)
+    WSREP_SST_OPT_PSWD=$(echo $WSREP_SST_OPT_AUTH | cut -d: -f2)
+fi
+AUTH="-u$WSREP_SST_OPT_USER"
+if test -n "$WSREP_SST_OPT_PSWD"; then AUTH="$AUTH -p$WSREP_SST_OPT_PSWD"; fi
+
+STOP_WSREP="SET wsrep_on=OFF;"
+
+# NOTE: we don't use --routines here because we're dumping mysql.proc table
+MYSQLDUMP="mysqldump $AUTH -S$WSREP_SST_OPT_SOCKET \
+--add-drop-database --add-drop-table --skip-add-locks --create-options \
+--disable-keys --extended-insert --skip-lock-tables --quick --set-charset \
+--skip-comments --flush-privileges --all-databases"
+
+# mysqldump cannot restore CSV tables, fix this issue
+CSV_TABLES_FIX="
+set sql_mode='';
+
+USE mysql;
+
+SET @cond = (SELECT (SUPPORT = 'YES' or SUPPORT = 'DEFAULT') FROM INFORMATION_SCHEMA.ENGINES WHERE ENGINE = 'csv');
+
+SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS general_log ( event_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL, server_id int(10) unsigned NOT NULL, command_type varchar(64) NOT NULL, argument mediumtext NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"General log\"', 'SET @dummy = 0');
+
+PREPARE stmt FROM @stmt;
+EXECUTE stmt;
+DROP PREPARE stmt;
+
+SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS slow_log ( start_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, query_time time(6) NOT NULL, lock_time time(6) NOT NULL, rows_sent int(11) NOT NULL, rows_examined int(11) NOT NULL, db varchar(512) NOT NULL, last_insert_id int(11) NOT NULL, insert_id int(11) NOT NULL, server_id int(10) unsigned NOT NULL, sql_text mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"Slow log\"', 'SET @dummy = 0');
+
+PREPARE stmt FROM @stmt;
+EXECUTE stmt;
+DROP PREPARE stmt;"
+
+SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';"
+
+MYSQL="mysql $AUTH -h$WSREP_SST_OPT_HOST -P$WSREP_SST_OPT_PORT "\
+"--disable-reconnect --connect_timeout=10"
+
+# need to disable logging when loading the dump
+# reason is that dump contains ALTER TABLE for log tables, and
+# this causes an error if logging is enabled
+GENERAL_LOG_OPT=`$MYSQL --skip-column-names -e"$STOP_WSREP SELECT @@GENERAL_LOG"`
+SLOW_LOG_OPT=`$MYSQL --skip-column-names -e"$STOP_WSREP SELECT @@SLOW_QUERY_LOG"`
+$MYSQL -e"$STOP_WSREP SET GLOBAL GENERAL_LOG=OFF"
+$MYSQL -e"$STOP_WSREP SET GLOBAL SLOW_QUERY_LOG=OFF"
+
+# commands to restore log settings
+RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;"
+RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;"
+
+if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
+then
+    (echo $STOP_WSREP && $MYSQLDUMP && echo $CSV_TABLES_FIX \
+    && echo $RESTORE_GENERAL_LOG && echo $RESTORE_SLOW_QUERY_LOG \
+    && echo $SET_START_POSITION \
+    || echo "SST failed to complete;") | $MYSQL
+else
+    wsrep_log_info "Bypassing state dump."
+    echo $SET_START_POSITION | $MYSQL
+fi
+
+#
diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh
new file mode 100644
index 00000000000..d4093dbcaef
--- /dev/null
+++ b/scripts/wsrep_sst_mysqldump.sh
@@ -0,0 +1,131 @@
+#!/bin/bash -e
+# Copyright (C) 2009 Codership Oy
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# This is a reference script for mysqldump-based state snapshot tansfer
+
+# This variable is not used in mysqldump sst, so better initialize it
+# to avoid shell's "parameter not set" message.
+WSREP_SST_OPT_CONF=""
+
+. $(dirname $0)/wsrep_sst_common
+
+EINVAL=22
+PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin
+
+local_ip()
+{
+    [ "$1" = "127.0.0.1" ]      && return 0
+    [ "$1" = "localhost" ]      && return 0
+    [ "$1" = "$(hostname -s)" ] && return 0
+    [ "$1" = "$(hostname -f)" ] && return 0
+    [ "$1" = "$(hostname -d)" ] && return 0
+
+    # Now if ip program is not found in the path, we can't return 0 since
+    # it would block any address. Thankfully grep should fail in this case
+    ip route get "$1" | grep local >/dev/null && return 0
+
+    return 1
+}
+
+if test -z "$WSREP_SST_OPT_USER";  then wsrep_log_error "USER cannot be nil";  exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_HOST";  then wsrep_log_error "HOST cannot be nil";  exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_PORT";  then wsrep_log_error "PORT cannot be nil";  exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_LPORT"; then wsrep_log_error "LPORT cannot be nil"; exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_SOCKET";then wsrep_log_error "SOCKET cannot be nil";exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_GTID";  then wsrep_log_error "GTID cannot be nil";  exit $EINVAL; fi
+
+if local_ip $WSREP_SST_OPT_HOST && \
+   [ "$WSREP_SST_OPT_PORT" = "$WSREP_SST_OPT_LPORT" ]
+then
+    wsrep_log_error \
+    "destination address '$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT' matches source address."
+    exit $EINVAL
+fi
+
+# Check client version
+if ! mysql --version | grep 'Distrib 10' >/dev/null
+then
+    mysql --version >&2
+    wsrep_log_error "this operation requires MySQL client version 10 or newer"
+    exit $EINVAL
+fi
+
+# For Bug:1293798
+if [ -z "$WSREP_SST_OPT_PSWD" -a -n "$WSREP_SST_OPT_AUTH" ]; then
+    WSREP_SST_OPT_USER=$(echo $WSREP_SST_OPT_AUTH | cut -d: -f1)
+    WSREP_SST_OPT_PSWD=$(echo $WSREP_SST_OPT_AUTH | cut -d: -f2)
+fi
+AUTH="-u$WSREP_SST_OPT_USER"
+if test -n "$WSREP_SST_OPT_PSWD"; then AUTH="$AUTH -p$WSREP_SST_OPT_PSWD"; fi
+
+STOP_WSREP="SET wsrep_on=OFF;"
+
+# NOTE: we don't use --routines here because we're dumping mysql.proc table
+MYSQLDUMP="mysqldump $AUTH -S$WSREP_SST_OPT_SOCKET \
+--add-drop-database --add-drop-table --skip-add-locks --create-options \
+--disable-keys --extended-insert --skip-lock-tables --quick --set-charset \
+--skip-comments --flush-privileges --all-databases"
+
+# mysqldump cannot restore CSV tables, fix this issue
+CSV_TABLES_FIX="
+set sql_mode='';
+
+USE mysql;
+
+SET @cond = (SELECT (SUPPORT = 'YES' or SUPPORT = 'DEFAULT') FROM INFORMATION_SCHEMA.ENGINES WHERE ENGINE = 'csv');
+
+SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS general_log ( event_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL, server_id int(10) unsigned NOT NULL, command_type varchar(64) NOT NULL, argument mediumtext NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"General log\"', 'SET @dummy = 0');
+
+PREPARE stmt FROM @stmt;
+EXECUTE stmt;
+DROP PREPARE stmt;
+
+SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS slow_log ( start_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, query_time time(6) NOT NULL, lock_time time(6) NOT NULL, rows_sent int(11) NOT NULL, rows_examined int(11) NOT NULL, db varchar(512) NOT NULL, last_insert_id int(11) NOT NULL, insert_id int(11) NOT NULL, server_id int(10) unsigned NOT NULL, sql_text mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"Slow log\"', 'SET @dummy = 0');
+
+PREPARE stmt FROM @stmt;
+EXECUTE stmt;
+DROP PREPARE stmt;"
+
+SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';"
+
+MYSQL="mysql $AUTH -h$WSREP_SST_OPT_HOST -P$WSREP_SST_OPT_PORT "\
+"--disable-reconnect --connect_timeout=10"
+
+# need to disable logging when loading the dump
+# reason is that dump contains ALTER TABLE for log tables, and
+# this causes an error if logging is enabled
+GENERAL_LOG_OPT=`$MYSQL --skip-column-names -e"$STOP_WSREP SELECT @@GENERAL_LOG"`
+SLOW_LOG_OPT=`$MYSQL --skip-column-names -e"$STOP_WSREP SELECT @@SLOW_QUERY_LOG"`
+$MYSQL -e"$STOP_WSREP SET GLOBAL GENERAL_LOG=OFF"
+$MYSQL -e"$STOP_WSREP SET GLOBAL SLOW_QUERY_LOG=OFF"
+
+# commands to restore log settings
+RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;"
+RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;"
+
+if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
+then
+    (echo $STOP_WSREP && $MYSQLDUMP && echo $CSV_TABLES_FIX \
+    && echo $RESTORE_GENERAL_LOG && echo $RESTORE_SLOW_QUERY_LOG \
+    && echo $SET_START_POSITION \
+    || echo "SST failed to complete;") | $MYSQL
+else
+    wsrep_log_info "Bypassing state dump."
+    echo $SET_START_POSITION | $MYSQL
+fi
+
+#
diff --git a/scripts/wsrep_sst_rsync b/scripts/wsrep_sst_rsync
new file mode 100755
index 00000000000..86bf557662d
--- /dev/null
+++ b/scripts/wsrep_sst_rsync
@@ -0,0 +1,335 @@
+#!/bin/bash -ue
+
+# Copyright (C) 2010-2014 Codership Oy
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# This is a reference script for rsync-based state snapshot tansfer
+
+RSYNC_PID=
+RSYNC_CONF=
+OS=$(uname)
+[ "$OS" == "Darwin" ] && export -n LD_LIBRARY_PATH
+
+# Setting the path for lsof on CentOS
+export PATH="/usr/sbin:/sbin:$PATH"
+
+. $(dirname $0)/wsrep_sst_common
+
+wsrep_check_programs rsync
+
+cleanup_joiner()
+{
+    wsrep_log_info "Joiner cleanup."
+    local PID=$(cat "$RSYNC_PID" 2>/dev/null || echo 0)
+    [ "0" != "$PID" ] && kill $PID && sleep 0.5 && kill -9 $PID >/dev/null 2>&1 \
+    || :
+    rm -rf "$RSYNC_CONF"
+    rm -rf "$MAGIC_FILE"
+    rm -rf "$RSYNC_PID"
+    wsrep_log_info "Joiner cleanup done."
+    if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then
+        wsrep_cleanup_progress_file
+    fi
+}
+
+check_pid()
+{
+    local pid_file=$1
+    [ -r "$pid_file" ] && ps -p $(cat $pid_file) >/dev/null 2>&1
+}
+
+check_pid_and_port()
+{
+    local pid_file=$1
+    local rsync_pid=$2
+    local rsync_port=$3
+
+    if ! which lsof > /dev/null; then
+      wsrep_log_error "lsof tool not found in PATH! Make sure you have it installed."
+      exit 2 # ENOENT
+    fi
+
+    local port_info=$(lsof -i :$rsync_port -Pn 2>/dev/null | \
+        grep "(LISTEN)")
+    local is_rsync=$(echo $port_info | \
+        grep -w '^rsync[[:space:]]\+'"$rsync_pid" 2>/dev/null)
+
+    if [ -n "$port_info" -a -z "$is_rsync" ]; then
+        wsrep_log_error "rsync daemon port '$rsync_port' has been taken"
+        exit 16 # EBUSY
+    fi
+    check_pid $pid_file && \
+        [ -n "$port_info" ] && [ -n "$is_rsync" ] && \
+        [ $(cat $pid_file) -eq $rsync_pid ]
+}
+
+MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete"
+rm -rf "$MAGIC_FILE"
+
+BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar"
+BINLOG_N_FILES=1
+rm -f "$BINLOG_TAR_FILE" || :
+
+if ! [ -z $WSREP_SST_OPT_BINLOG ]
+then
+    BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG)
+    BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG)
+fi
+
+WSREP_LOG_DIR=${WSREP_LOG_DIR:-""}
+# if WSREP_LOG_DIR env. variable is not set, try to get it from my.cnf
+if [ -z "$WSREP_LOG_DIR" ]; then
+    SCRIPT_DIR="$(cd $(dirname "$0"); pwd -P)"
+    WSREP_LOG_DIR=$($SCRIPT_DIR/my_print_defaults --defaults-file \
+                   "$WSREP_SST_OPT_CONF" mysqld server mysqld-10.0 mariadb mariadb-10.0 \
+                    | grep -- '--innodb[-_]log[-_]group[-_]home[-_]dir=' \
+                    | cut -b 29- )
+fi
+
+if [ -n "$WSREP_LOG_DIR" ]; then
+    # handle both relative and absolute paths
+    WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; mkdir -p "$WSREP_LOG_DIR"; cd $WSREP_LOG_DIR; pwd -P)
+else
+    # default to datadir
+    WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P)
+fi
+
+# Old filter - include everything except selected
+# FILTER=(--exclude '*.err' --exclude '*.pid' --exclude '*.sock' \
+#         --exclude '*.conf' --exclude core --exclude 'galera.*' \
+#         --exclude grastate.txt --exclude '*.pem' \
+#         --exclude '*.[0-9][0-9][0-9][0-9][0-9][0-9]' --exclude '*.index')
+
+# New filter - exclude everything except dirs (schemas) and innodb files
+FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes'
+        -f '+ /wsrep_sst_binlog.tar' -f '+ /ib_lru_dump' -f '+ /ibdata*' -f '+ /*/' -f '- /*')
+
+if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
+then
+
+    if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
+    then
+
+        FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed"
+        rm -rf "$FLUSHED"
+
+        # Use deltaxfer only for WAN
+        inv=$(basename $0)
+        [ "$inv" = "wsrep_sst_rsync_wan" ] && WHOLE_FILE_OPT="" \
+                                           || WHOLE_FILE_OPT="--whole-file"
+
+        echo "flush tables"
+
+        # wait for tables flushed and state ID written to the file
+        while [ ! -r "$FLUSHED" ] && ! grep -q ':' "$FLUSHED" >/dev/null 2>&1
+        do
+            sleep 0.2
+        done
+
+        STATE="$(cat $FLUSHED)"
+        rm -rf "$FLUSHED"
+
+        sync
+
+        if ! [ -z $WSREP_SST_OPT_BINLOG ]
+        then
+            # Prepare binlog files
+            pushd $BINLOG_DIRNAME &> /dev/null
+            binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_FILENAME}.index)
+            binlog_files=""
+            for ii in $binlog_files_full
+            do
+                binlog_files="$binlog_files $(basename $ii)"
+            done
+            if ! [ -z "$binlog_files" ]
+            then
+                wsrep_log_info "Preparing binlog files for transfer:"
+                tar -cvf $BINLOG_TAR_FILE $binlog_files >&2
+            fi
+            popd &> /dev/null
+        fi
+
+        # first, the normal directories, so that we can detect incompatible protocol
+        RC=0
+        rsync --owner --group --perms --links --specials \
+              --ignore-times --inplace --dirs --delete --quiet \
+              $WHOLE_FILE_OPT "${FILTER[@]}" "$WSREP_SST_OPT_DATA/" \
+              rsync://$WSREP_SST_OPT_ADDR >&2 || RC=$?
+
+        if [ "$RC" -ne 0 ]; then
+            wsrep_log_error "rsync returned code $RC:"
+
+            case $RC in
+            12) RC=71  # EPROTO
+                wsrep_log_error \
+                "rsync server on the other end has incompatible protocol. " \
+                "Make sure you have the same version of rsync on all nodes."
+                ;;
+            22) RC=12  # ENOMEM
+                ;;
+            *)  RC=255 # unknown error
+                ;;
+            esac
+            exit $RC
+        fi
+
+        # second, we transfer InnoDB log files
+        rsync --owner --group --perms --links --specials \
+              --ignore-times --inplace --dirs --delete --quiet \
+              $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '- **' "$WSREP_LOG_DIR/" \
+              rsync://$WSREP_SST_OPT_ADDR-log_dir >&2 || RC=$?
+
+        if [ $RC -ne 0 ]; then
+            wsrep_log_error "rsync innodb_log_group_home_dir returned code $RC:"
+            exit 255 # unknown error
+        fi
+
+        # then, we parallelize the transfer of database directories, use . so that pathconcatenation works
+        pushd "$WSREP_SST_OPT_DATA" >/dev/null
+
+        count=1
+        [ "$OS" == "Linux" ] && count=$(grep -c processor /proc/cpuinfo)
+        [ "$OS" == "Darwin" -o "$OS" == "FreeBSD" ] && count=$(sysctl -n hw.ncpu)
+
+        find . -maxdepth 1 -mindepth 1 -type d -print0 | xargs -I{} -0 -P $count \
+             rsync --owner --group --perms --links --specials \
+             --ignore-times --inplace --recursive --delete --quiet \
+             $WHOLE_FILE_OPT --exclude '*/ib_logfile*' "$WSREP_SST_OPT_DATA"/{}/ \
+             rsync://$WSREP_SST_OPT_ADDR/{} >&2 || RC=$?
+
+        popd >/dev/null
+
+        if [ $RC -ne 0 ]; then
+            wsrep_log_error "find/rsync returned code $RC:"
+            exit 255 # unknown error
+        fi
+
+    else # BYPASS
+        wsrep_log_info "Bypassing state dump."
+        STATE="$WSREP_SST_OPT_GTID"
+    fi
+
+    echo "continue" # now server can resume updating data
+
+    echo "$STATE" > "$MAGIC_FILE"
+    rsync --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR
+
+    echo "done $STATE"
+
+elif [ "$WSREP_SST_OPT_ROLE" = "joiner" ]
+then
+    wsrep_check_programs lsof
+
+    touch $SST_PROGRESS_FILE
+    MYSQLD_PID=$WSREP_SST_OPT_PARENT
+
+    MODULE="rsync_sst"
+
+    RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid"
+
+    if check_pid $RSYNC_PID
+    then
+        wsrep_log_error "rsync daemon already running."
+        exit 114 # EALREADY
+    fi
+    rm -rf "$RSYNC_PID"
+
+    ADDR=$WSREP_SST_OPT_ADDR
+    RSYNC_PORT=$(echo $ADDR | awk -F ':' '{ print $2 }')
+    if [ -z "$RSYNC_PORT" ]
+    then
+        RSYNC_PORT=4444
+        ADDR="$(echo $ADDR | awk -F ':' '{ print $1 }'):$RSYNC_PORT"
+    fi
+
+    trap "exit 32" HUP PIPE
+    trap "exit 3"  INT TERM ABRT
+    trap cleanup_joiner EXIT
+
+    RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
+
+cat << EOF > "$RSYNC_CONF"
+pid file = $RSYNC_PID
+use chroot = no
+read only = no
+timeout = 300
+[$MODULE]
+    path = $WSREP_SST_OPT_DATA
+[$MODULE-log_dir]
+    path = $WSREP_LOG_DIR
+EOF
+
+#    rm -rf "$DATA"/ib_logfile* # we don't want old logs around
+
+    # listen at all interfaces (for firewalled setups)
+    rsync --daemon --no-detach --port $RSYNC_PORT --config "$RSYNC_CONF" &
+    RSYNC_REAL_PID=$!
+
+    until check_pid_and_port $RSYNC_PID $RSYNC_REAL_PID $RSYNC_PORT
+    do
+        sleep 0.2
+    done
+
+    echo "ready $ADDR/$MODULE"
+
+    # wait for SST to complete by monitoring magic file
+    while [ ! -r "$MAGIC_FILE" ] && check_pid "$RSYNC_PID" && \
+          ps -p $MYSQLD_PID >/dev/null
+    do
+        sleep 1
+    done
+
+    if ! ps -p $MYSQLD_PID >/dev/null
+    then
+        wsrep_log_error \
+        "Parent mysqld process (PID:$MYSQLD_PID) terminated unexpectedly."
+        exit 32
+    fi
+
+    if ! [ -z $WSREP_SST_OPT_BINLOG ]
+    then
+
+        pushd $BINLOG_DIRNAME &> /dev/null
+        if [ -f $BINLOG_TAR_FILE ]
+        then
+            # Clean up old binlog files first
+            rm -f ${BINLOG_FILENAME}.*
+            wsrep_log_info "Extracting binlog files:"
+            tar -xvf $BINLOG_TAR_FILE >&2
+            for ii in $(ls -1 ${BINLOG_FILENAME}.*)
+            do
+                echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_FILENAME}.index
+            done
+        fi
+        popd &> /dev/null
+    fi
+    if [ -r "$MAGIC_FILE" ]
+    then
+        cat "$MAGIC_FILE" # output UUID:seqno
+    else
+        # this message should cause joiner to abort
+        echo "rsync process ended without creating '$MAGIC_FILE'"
+    fi
+    wsrep_cleanup_progress_file
+#    cleanup_joiner
+else
+    wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'"
+    exit 22 # EINVAL
+fi
+
+rm -f $BINLOG_TAR_FILE || :
+
+exit 0
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
new file mode 100644
index 00000000000..86bf557662d
--- /dev/null
+++ b/scripts/wsrep_sst_rsync.sh
@@ -0,0 +1,335 @@
+#!/bin/bash -ue
+
+# Copyright (C) 2010-2014 Codership Oy
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# This is a reference script for rsync-based state snapshot tansfer
+
+RSYNC_PID=
+RSYNC_CONF=
+OS=$(uname)
+[ "$OS" == "Darwin" ] && export -n LD_LIBRARY_PATH
+
+# Setting the path for lsof on CentOS
+export PATH="/usr/sbin:/sbin:$PATH"
+
+. $(dirname $0)/wsrep_sst_common
+
+wsrep_check_programs rsync
+
+cleanup_joiner()
+{
+    wsrep_log_info "Joiner cleanup."
+    local PID=$(cat "$RSYNC_PID" 2>/dev/null || echo 0)
+    [ "0" != "$PID" ] && kill $PID && sleep 0.5 && kill -9 $PID >/dev/null 2>&1 \
+    || :
+    rm -rf "$RSYNC_CONF"
+    rm -rf "$MAGIC_FILE"
+    rm -rf "$RSYNC_PID"
+    wsrep_log_info "Joiner cleanup done."
+    if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then
+        wsrep_cleanup_progress_file
+    fi
+}
+
+check_pid()
+{
+    local pid_file=$1
+    [ -r "$pid_file" ] && ps -p $(cat $pid_file) >/dev/null 2>&1
+}
+
+check_pid_and_port()
+{
+    local pid_file=$1
+    local rsync_pid=$2
+    local rsync_port=$3
+
+    if ! which lsof > /dev/null; then
+      wsrep_log_error "lsof tool not found in PATH! Make sure you have it installed."
+      exit 2 # ENOENT
+    fi
+
+    local port_info=$(lsof -i :$rsync_port -Pn 2>/dev/null | \
+        grep "(LISTEN)")
+    local is_rsync=$(echo $port_info | \
+        grep -w '^rsync[[:space:]]\+'"$rsync_pid" 2>/dev/null)
+
+    if [ -n "$port_info" -a -z "$is_rsync" ]; then
+        wsrep_log_error "rsync daemon port '$rsync_port' has been taken"
+        exit 16 # EBUSY
+    fi
+    check_pid $pid_file && \
+        [ -n "$port_info" ] && [ -n "$is_rsync" ] && \
+        [ $(cat $pid_file) -eq $rsync_pid ]
+}
+
+MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete"
+rm -rf "$MAGIC_FILE"
+
+BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar"
+BINLOG_N_FILES=1
+rm -f "$BINLOG_TAR_FILE" || :
+
+if ! [ -z $WSREP_SST_OPT_BINLOG ]
+then
+    BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG)
+    BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG)
+fi
+
+WSREP_LOG_DIR=${WSREP_LOG_DIR:-""}
+# if WSREP_LOG_DIR env. variable is not set, try to get it from my.cnf
+if [ -z "$WSREP_LOG_DIR" ]; then
+    SCRIPT_DIR="$(cd $(dirname "$0"); pwd -P)"
+    WSREP_LOG_DIR=$($SCRIPT_DIR/my_print_defaults --defaults-file \
+                   "$WSREP_SST_OPT_CONF" mysqld server mysqld-10.0 mariadb mariadb-10.0 \
+                    | grep -- '--innodb[-_]log[-_]group[-_]home[-_]dir=' \
+                    | cut -b 29- )
+fi
+
+if [ -n "$WSREP_LOG_DIR" ]; then
+    # handle both relative and absolute paths
+    WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; mkdir -p "$WSREP_LOG_DIR"; cd $WSREP_LOG_DIR; pwd -P)
+else
+    # default to datadir
+    WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P)
+fi
+
+# Old filter - include everything except selected
+# FILTER=(--exclude '*.err' --exclude '*.pid' --exclude '*.sock' \
+#         --exclude '*.conf' --exclude core --exclude 'galera.*' \
+#         --exclude grastate.txt --exclude '*.pem' \
+#         --exclude '*.[0-9][0-9][0-9][0-9][0-9][0-9]' --exclude '*.index')
+
+# New filter - exclude everything except dirs (schemas) and innodb files
+FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes'
+        -f '+ /wsrep_sst_binlog.tar' -f '+ /ib_lru_dump' -f '+ /ibdata*' -f '+ /*/' -f '- /*')
+
+if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
+then
+
+    if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
+    then
+
+        FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed"
+        rm -rf "$FLUSHED"
+
+        # Use deltaxfer only for WAN
+        inv=$(basename $0)
+        [ "$inv" = "wsrep_sst_rsync_wan" ] && WHOLE_FILE_OPT="" \
+                                           || WHOLE_FILE_OPT="--whole-file"
+
+        echo "flush tables"
+
+        # wait for tables flushed and state ID written to the file
+        while [ ! -r "$FLUSHED" ] && ! grep -q ':' "$FLUSHED" >/dev/null 2>&1
+        do
+            sleep 0.2
+        done
+
+        STATE="$(cat $FLUSHED)"
+        rm -rf "$FLUSHED"
+
+        sync
+
+        if ! [ -z $WSREP_SST_OPT_BINLOG ]
+        then
+            # Prepare binlog files
+            pushd $BINLOG_DIRNAME &> /dev/null
+            binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_FILENAME}.index)
+            binlog_files=""
+            for ii in $binlog_files_full
+            do
+                binlog_files="$binlog_files $(basename $ii)"
+            done
+            if ! [ -z "$binlog_files" ]
+            then
+                wsrep_log_info "Preparing binlog files for transfer:"
+                tar -cvf $BINLOG_TAR_FILE $binlog_files >&2
+            fi
+            popd &> /dev/null
+        fi
+
+        # first, the normal directories, so that we can detect incompatible protocol
+        RC=0
+        rsync --owner --group --perms --links --specials \
+              --ignore-times --inplace --dirs --delete --quiet \
+              $WHOLE_FILE_OPT "${FILTER[@]}" "$WSREP_SST_OPT_DATA/" \
+              rsync://$WSREP_SST_OPT_ADDR >&2 || RC=$?
+
+        if [ "$RC" -ne 0 ]; then
+            wsrep_log_error "rsync returned code $RC:"
+
+            case $RC in
+            12) RC=71  # EPROTO
+                wsrep_log_error \
+                "rsync server on the other end has incompatible protocol. " \
+                "Make sure you have the same version of rsync on all nodes."
+                ;;
+            22) RC=12  # ENOMEM
+                ;;
+            *)  RC=255 # unknown error
+                ;;
+            esac
+            exit $RC
+        fi
+
+        # second, we transfer InnoDB log files
+        rsync --owner --group --perms --links --specials \
+              --ignore-times --inplace --dirs --delete --quiet \
+              $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '- **' "$WSREP_LOG_DIR/" \
+              rsync://$WSREP_SST_OPT_ADDR-log_dir >&2 || RC=$?
+
+        if [ $RC -ne 0 ]; then
+            wsrep_log_error "rsync innodb_log_group_home_dir returned code $RC:"
+            exit 255 # unknown error
+        fi
+
+        # then, we parallelize the transfer of database directories, use . so that pathconcatenation works
+        pushd "$WSREP_SST_OPT_DATA" >/dev/null
+
+        count=1
+        [ "$OS" == "Linux" ] && count=$(grep -c processor /proc/cpuinfo)
+        [ "$OS" == "Darwin" -o "$OS" == "FreeBSD" ] && count=$(sysctl -n hw.ncpu)
+
+        find . -maxdepth 1 -mindepth 1 -type d -print0 | xargs -I{} -0 -P $count \
+             rsync --owner --group --perms --links --specials \
+             --ignore-times --inplace --recursive --delete --quiet \
+             $WHOLE_FILE_OPT --exclude '*/ib_logfile*' "$WSREP_SST_OPT_DATA"/{}/ \
+             rsync://$WSREP_SST_OPT_ADDR/{} >&2 || RC=$?
+
+        popd >/dev/null
+
+        if [ $RC -ne 0 ]; then
+            wsrep_log_error "find/rsync returned code $RC:"
+            exit 255 # unknown error
+        fi
+
+    else # BYPASS
+        wsrep_log_info "Bypassing state dump."
+        STATE="$WSREP_SST_OPT_GTID"
+    fi
+
+    echo "continue" # now server can resume updating data
+
+    echo "$STATE" > "$MAGIC_FILE"
+    rsync --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR
+
+    echo "done $STATE"
+
+elif [ "$WSREP_SST_OPT_ROLE" = "joiner" ]
+then
+    wsrep_check_programs lsof
+
+    touch $SST_PROGRESS_FILE
+    MYSQLD_PID=$WSREP_SST_OPT_PARENT
+
+    MODULE="rsync_sst"
+
+    RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid"
+
+    if check_pid $RSYNC_PID
+    then
+        wsrep_log_error "rsync daemon already running."
+        exit 114 # EALREADY
+    fi
+    rm -rf "$RSYNC_PID"
+
+    ADDR=$WSREP_SST_OPT_ADDR
+    RSYNC_PORT=$(echo $ADDR | awk -F ':' '{ print $2 }')
+    if [ -z "$RSYNC_PORT" ]
+    then
+        RSYNC_PORT=4444
+        ADDR="$(echo $ADDR | awk -F ':' '{ print $1 }'):$RSYNC_PORT"
+    fi
+
+    trap "exit 32" HUP PIPE
+    trap "exit 3"  INT TERM ABRT
+    trap cleanup_joiner EXIT
+
+    RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
+
+cat << EOF > "$RSYNC_CONF"
+pid file = $RSYNC_PID
+use chroot = no
+read only = no
+timeout = 300
+[$MODULE]
+    path = $WSREP_SST_OPT_DATA
+[$MODULE-log_dir]
+    path = $WSREP_LOG_DIR
+EOF
+
+#    rm -rf "$DATA"/ib_logfile* # we don't want old logs around
+
+    # listen at all interfaces (for firewalled setups)
+    rsync --daemon --no-detach --port $RSYNC_PORT --config "$RSYNC_CONF" &
+    RSYNC_REAL_PID=$!
+
+    until check_pid_and_port $RSYNC_PID $RSYNC_REAL_PID $RSYNC_PORT
+    do
+        sleep 0.2
+    done
+
+    echo "ready $ADDR/$MODULE"
+
+    # wait for SST to complete by monitoring magic file
+    while [ ! -r "$MAGIC_FILE" ] && check_pid "$RSYNC_PID" && \
+          ps -p $MYSQLD_PID >/dev/null
+    do
+        sleep 1
+    done
+
+    if ! ps -p $MYSQLD_PID >/dev/null
+    then
+        wsrep_log_error \
+        "Parent mysqld process (PID:$MYSQLD_PID) terminated unexpectedly."
+        exit 32
+    fi
+
+    if ! [ -z $WSREP_SST_OPT_BINLOG ]
+    then
+
+        pushd $BINLOG_DIRNAME &> /dev/null
+        if [ -f $BINLOG_TAR_FILE ]
+        then
+            # Clean up old binlog files first
+            rm -f ${BINLOG_FILENAME}.*
+            wsrep_log_info "Extracting binlog files:"
+            tar -xvf $BINLOG_TAR_FILE >&2
+            for ii in $(ls -1 ${BINLOG_FILENAME}.*)
+            do
+                echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_FILENAME}.index
+            done
+        fi
+        popd &> /dev/null
+    fi
+    if [ -r "$MAGIC_FILE" ]
+    then
+        cat "$MAGIC_FILE" # output UUID:seqno
+    else
+        # this message should cause joiner to abort
+        echo "rsync process ended without creating '$MAGIC_FILE'"
+    fi
+    wsrep_cleanup_progress_file
+#    cleanup_joiner
+else
+    wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'"
+    exit 22 # EINVAL
+fi
+
+rm -f $BINLOG_TAR_FILE || :
+
+exit 0
diff --git a/scripts/wsrep_sst_xtrabackup b/scripts/wsrep_sst_xtrabackup
new file mode 100755
index 00000000000..6b33eabee23
--- /dev/null
+++ b/scripts/wsrep_sst_xtrabackup
@@ -0,0 +1,715 @@
+#!/bin/bash -ue
+# Copyright (C) 2013 Percona Inc
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# Optional dependencies and options documented here: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 
+# Make sure to read that before proceeding!
+
+
+
+
+. $(dirname $0)/wsrep_sst_common
+
+ealgo=""
+ekey=""
+ekeyfile=""
+encrypt=0
+nproc=1
+ecode=0
+XTRABACKUP_PID=""
+SST_PORT=""
+REMOTEIP=""
+tcert=""
+tpem=""
+sockopt=""
+progress=""
+ttime=0
+totime=0
+lsn=""
+incremental=0
+ecmd=""
+rlimit=""
+
+sfmt="tar"
+strmcmd=""
+tfmt=""
+tcmd=""
+rebuild=0
+rebuildcmd=""
+payload=0
+pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' "
+pvopts="-f  -i 10 -N $WSREP_SST_OPT_ROLE "
+uextra=0
+
+if which pv &>/dev/null && pv --help | grep -q FORMAT;then 
+    pvopts+=$pvformat
+fi
+pcmd="pv $pvopts"
+declare -a RC
+
+INNOBACKUPEX_BIN=innobackupex
+readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ })
+DATA="${WSREP_SST_OPT_DATA}"
+INFO_FILE="xtrabackup_galera_info"
+IST_FILE="xtrabackup_ist"
+MAGIC_FILE="${DATA}/${INFO_FILE}"
+
+# Setting the path for ss and ip
+export PATH="/usr/sbin:/sbin:$PATH"
+
+timeit(){
+    local stage=$1
+    shift
+    local cmd="$@"
+    local x1 x2 took extcode
+
+    if [[ $ttime -eq 1 ]];then 
+        x1=$(date +%s)
+        wsrep_log_info "Evaluating $cmd"
+        eval "$cmd"
+        extcode=$?
+        x2=$(date +%s)
+        took=$(( x2-x1 ))
+        wsrep_log_info "NOTE: $stage took $took seconds"
+        totime=$(( totime+took ))
+    else 
+        wsrep_log_info "Evaluating $cmd"
+        eval "$cmd"
+        extcode=$?
+    fi
+    return $extcode
+}
+
+get_keys()
+{
+    if [[ $encrypt -eq 2 ]];then 
+        return 
+    fi
+
+    if [[ $encrypt -eq 0 ]];then 
+        if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then
+            wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
+        fi
+        return
+    fi
+
+    if [[ $sfmt == 'tar' ]];then
+        wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format"
+        encrypt=0
+        return
+    fi
+
+    wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4"
+
+    if [[ -z $ealgo ]];then
+        wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out"
+        exit 3
+    fi
+
+    if [[ -z $ekey && ! -r $ekeyfile ]];then
+        wsrep_log_error "FATAL: Either key or keyfile must be readable"
+        exit 3
+    fi
+
+    if [[ -z $ekey ]];then
+        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile"
+    else
+        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey"
+    fi
+
+    if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
+        ecmd+=" -d"
+    fi
+}
+
+get_transfer()
+{
+    if [[ -z $SST_PORT ]];then 
+        TSST_PORT=4444
+    else 
+        TSST_PORT=$SST_PORT
+    fi
+
+    if [[ $tfmt == 'nc' ]];then
+        if [[ ! -x `which nc` ]];then 
+            wsrep_log_error "nc(netcat) not found in path: $PATH"
+            exit 2
+        fi
+        wsrep_log_info "Using netcat as streamer"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            tcmd="nc -dl ${TSST_PORT}"
+        else
+            tcmd="nc ${REMOTEIP} ${TSST_PORT}"
+        fi
+    else
+        tfmt='socat'
+        wsrep_log_info "Using socat as streamer"
+        if [[ ! -x `which socat` ]];then 
+            wsrep_log_error "socat not found in path: $PATH"
+            exit 2
+        fi
+
+        if [[ $encrypt -eq 2 ]] && ! socat -V | grep -q OPENSSL;then 
+            wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer"
+            encrypt=0
+        fi
+
+        if [[ $encrypt -eq 2 ]];then 
+            wsrep_log_info "Using openssl based encryption with socat"
+            if [[ -z $tpem || -z $tcert ]];then 
+                wsrep_log_error "Both PEM and CRT files required"
+                exit 22
+            fi
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert"
+                tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio"
+            else
+                wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert"
+                tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}"
+            fi
+        else 
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio"
+            else
+                tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}"
+            fi
+        fi
+    fi
+
+}
+
+parse_cnf()
+{
+    local group=$1
+    local var=$2
+    reval=$(my_print_defaults -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-)
+    if [[ -z $reval ]];then 
+        [[ -n $3 ]] && reval=$3
+    fi
+    echo $reval
+}
+
+get_footprint()
+{
+    pushd $WSREP_SST_OPT_DATA 1>/dev/null
+    payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }')
+    if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 
+        # QuickLZ has around 50% compression ratio
+        # When compression/compaction used, the progress is only an approximate.
+        payload=$(( payload*1/2 ))
+    fi
+    popd 1>/dev/null
+    pcmd+=" -s $payload"
+    adjust_progress
+}
+
+adjust_progress()
+{
+    if [[ -n $progress && $progress != '1' ]];then 
+        if [[ -e $progress ]];then 
+            pcmd+=" 2>>$progress"
+        else 
+            pcmd+=" 2>$progress"
+        fi
+    elif [[ -z $progress && -n $rlimit  ]];then 
+            # When rlimit is non-zero
+            pcmd="pv -q"
+    fi 
+
+    if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
+        wsrep_log_info "Rate-limiting SST to $rlimit"
+        pcmd+=" -L \$rlimit"
+    fi
+}
+
+read_cnf()
+{
+    sfmt=$(parse_cnf sst streamfmt "tar")
+    tfmt=$(parse_cnf sst transferfmt "socat")
+    tcert=$(parse_cnf sst tca "")
+    tpem=$(parse_cnf sst tcert "")
+    encrypt=$(parse_cnf sst encrypt 0)
+    sockopt=$(parse_cnf sst sockopt "")
+    progress=$(parse_cnf sst progress "")
+    rebuild=$(parse_cnf sst rebuild 0)
+    ttime=$(parse_cnf sst time 0)
+    incremental=$(parse_cnf sst incremental 0)
+    ealgo=$(parse_cnf xtrabackup encrypt "")
+    ekey=$(parse_cnf xtrabackup encrypt-key "")
+    ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "")
+
+    # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 
+    if [[ -z $ealgo ]];then
+        ealgo=$(parse_cnf sst encrypt-algo "")
+        ekey=$(parse_cnf sst encrypt-key "")
+        ekeyfile=$(parse_cnf sst encrypt-key-file "")
+    fi
+    rlimit=$(parse_cnf sst rlimit "")
+    uextra=$(parse_cnf sst use_extra 0)
+}
+
+get_stream()
+{
+    if [[ $sfmt == 'xbstream' ]];then 
+        wsrep_log_info "Streaming with xbstream"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            strmcmd="xbstream -x"
+        else
+            strmcmd="xbstream -c \${INFO_FILE} \${IST_FILE}"
+        fi
+    else
+        sfmt="tar"
+        wsrep_log_info "Streaming with tar"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            strmcmd="tar xfi - --recursive-unlink -h"
+        else
+            strmcmd="tar cf - \${INFO_FILE} \${IST_FILE}"
+        fi
+
+    fi
+}
+
+get_proc()
+{
+    set +e
+    nproc=$(grep -c processor /proc/cpuinfo)
+    [[ -z $nproc || $nproc -eq 0 ]] && nproc=1
+    set -e
+}
+
+sig_joiner_cleanup()
+{
+    wsrep_log_error "Removing $MAGIC_FILE file due to signal"
+    rm -f "$MAGIC_FILE"
+}
+
+cleanup_joiner()
+{
+    # Since this is invoked just after exit NNN
+    local estatus=$?
+    if [[ $estatus -ne 0 ]];then 
+        wsrep_log_error "Cleanup after exit with status:$estatus"
+    fi
+    if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then
+        wsrep_log_info "Removing the sst_in_progress file"
+        wsrep_cleanup_progress_file
+    fi
+    if [[ -n $progress && -p $progress ]];then 
+        wsrep_log_info "Cleaning up fifo file $progress"
+        rm $progress
+    fi
+}
+
+check_pid()
+{
+    local pid_file="$1"
+    [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1
+}
+
+cleanup_donor()
+{
+    # Since this is invoked just after exit NNN
+    local estatus=$?
+    if [[ $estatus -ne 0 ]];then 
+        wsrep_log_error "Cleanup after exit with status:$estatus"
+    fi
+
+    if [[ -n $XTRABACKUP_PID ]];then 
+        if check_pid $XTRABACKUP_PID
+        then
+            wsrep_log_error "xtrabackup process is still running. Killing... "
+            kill_xtrabackup
+        fi
+
+        rm -f $XTRABACKUP_PID 
+    fi
+    rm -f ${DATA}/${IST_FILE}
+
+    if [[ -n $progress && -p $progress ]];then 
+        wsrep_log_info "Cleaning up fifo file $progress"
+        rm $progress
+    fi
+}
+
+kill_xtrabackup()
+{
+    local PID=$(cat $XTRABACKUP_PID)
+    [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
+    rm -f "$XTRABACKUP_PID"
+}
+
+setup_ports()
+{
+    if [[ "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
+        SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }')
+        REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }')
+        lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }')
+    else
+        SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }')
+    fi
+}
+
+# waits ~10 seconds for nc to open the port and then reports ready
+# (regardless of timeout)
+wait_for_listen()
+{
+    local PORT=$1
+    local ADDR=$2
+    local MODULE=$3
+    for i in {1..50}
+    do
+        ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break
+        sleep 0.2
+    done
+    if [[ $incremental -eq 1 ]];then 
+        echo "ready ${ADDR}/${MODULE}/$lsn"
+    else 
+    echo "ready ${ADDR}/${MODULE}"
+    fi
+}
+
+check_extra()
+{
+    local use_socket=1
+    if [[ $uextra -eq 1 ]];then 
+        if my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 
+            local eport=$(my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2)
+            if [[ -n $eport ]];then 
+                # Xtrabackup works only locally.
+                # Hence, setting host to 127.0.0.1 unconditionally. 
+                wsrep_log_info "SST through extra_port $eport"
+                INNOEXTRA+=" --host=127.0.0.1 --port=$eport "
+                use_socket=0
+            else 
+                wsrep_log_error "Extra port $eport null, failing"
+                exit 1
+            fi
+        else 
+            wsrep_log_info "Thread pool not set, ignore the option use_extra"
+        fi
+    fi
+    if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then
+        INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}"
+    fi
+}
+
+if [[ ! -x `which innobackupex` ]];then 
+    wsrep_log_error "innobackupex not in path: $PATH"
+    exit 2
+fi
+
+rm -f "${MAGIC_FILE}"
+
+if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then 
+    wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}"
+    exit 22
+fi
+
+read_cnf
+setup_ports
+get_stream
+get_transfer
+
+INNOEXTRA=""
+INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log"
+INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \$INNOEXTRA --galera-info --stream=\$sfmt \${TMPDIR} 2>\${DATA}/innobackup.backup.log"
+
+if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
+then
+    trap cleanup_donor EXIT
+
+    if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
+    then
+        TMPDIR="${TMPDIR:-/tmp}"
+
+        if [ "${AUTH[0]}" != "(null)" ]; then
+           INNOEXTRA+=" --user=${AUTH[0]}"
+       fi
+
+        if [ ${#AUTH[*]} -eq 2 ]; then
+           INNOEXTRA+=" --password=${AUTH[1]}"
+        elif [ "${AUTH[0]}" != "(null)" ]; then
+           # Empty password, used for testing, debugging etc.
+           INNOEXTRA+=" --password="
+        fi
+
+        get_keys
+        if [[ $encrypt -eq 1 ]];then
+            if [[ -n $ekey ]];then
+                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey "
+            else 
+                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile "
+            fi
+        fi
+
+        if [[ -n $lsn ]];then 
+                INNOEXTRA+=" --incremental --incremental-lsn=$lsn "
+        fi
+
+        check_extra
+
+        wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT}"
+
+        if [[ -n $progress ]];then 
+            get_footprint
+            tcmd="$pcmd | $tcmd"
+        elif [[ -n $rlimit ]];then 
+            adjust_progress
+            tcmd="$pcmd | $tcmd"
+        fi
+
+        set +e
+        timeit "Donor-Transfer" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+        set -e
+
+        if [ ${RC[0]} -ne 0 ]; then
+          wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \
+                          "Check ${DATA}/innobackup.backup.log"
+          exit 22
+        elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then 
+          wsrep_log_error "$tcmd finished with error: ${RC[1]}"
+          exit 22
+        fi
+
+        # innobackupex implicitly writes PID to fixed location in ${TMPDIR}
+        XTRABACKUP_PID="${TMPDIR}/xtrabackup_pid"
+
+
+    else # BYPASS FOR IST
+
+        wsrep_log_info "Bypassing the SST for IST"
+        STATE="${WSREP_SST_OPT_GTID}"
+        echo "continue" # now server can resume updating data
+        echo "${STATE}" > "${MAGIC_FILE}"
+        echo "1" > "${DATA}/${IST_FILE}"
+        get_keys
+        pushd ${DATA} 1>/dev/null
+        set +e
+        if [[ $encrypt -eq 1 ]];then
+            tcmd=" $ecmd | $tcmd"
+        fi
+        timeit "Donor-IST-Unencrypted-transfer" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+        set -e
+        popd 1>/dev/null
+
+        for ecode in "${RC[@]}";do 
+            if [[ $ecode -ne 0 ]];then 
+                wsrep_log_error "Error while streaming data to joiner node: " \
+                                "exit codes: ${RC[@]}"
+                exit 1
+            fi
+        done
+    fi
+
+    echo "done ${WSREP_SST_OPT_GTID}"
+    wsrep_log_info "Total time on donor: $totime seconds"
+
+elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ]
+then
+    [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
+    touch $SST_PROGRESS_FILE
+
+    if [[ ! -e ${DATA}/ibdata1 ]];then 
+        incremental=0
+    fi
+
+    if [[ $incremental -eq 1 ]];then 
+        wsrep_log_info "Incremental SST enabled"
+        #lsn=$(/pxc/bin/mysqld --defaults-file=$WSREP_SST_OPT_CONF  --basedir=/pxc  --wsrep-recover 2>&1 | grep -o 'log sequence number .*' | cut -d " " -f 4 | head -1)
+        lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ')
+        wsrep_log_info "Recovered LSN: $lsn"
+    fi
+
+    sencrypted=1
+    nthreads=1
+
+    MODULE="xtrabackup_sst"
+
+    # May need xtrabackup_checkpoints later on
+    rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info  ${DATA}/xtrabackup_logfile
+
+    ADDR=${WSREP_SST_OPT_ADDR}
+    if [ -z "${SST_PORT}" ]
+    then
+        SST_PORT=4444
+        ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}"
+    fi
+
+    wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} &
+
+    trap sig_joiner_cleanup HUP PIPE INT TERM
+    trap cleanup_joiner EXIT
+
+    if [[ -n $progress ]];then 
+        adjust_progress
+        tcmd+=" | $pcmd"
+    fi
+
+    if [[ $incremental -eq 1 ]];then 
+        BDATA=$DATA
+        DATA=$(mktemp -d)
+        MAGIC_FILE="${DATA}/${INFO_FILE}"
+    fi
+
+    get_keys
+    set +e
+    if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then
+        strmcmd=" $ecmd | $strmcmd"
+    fi
+
+    pushd ${DATA} 1>/dev/null
+    timeit "Joiner-Recv-Unencrypted" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+    popd 1>/dev/null 
+
+    set -e
+
+    if [[ $sfmt == 'xbstream' ]];then 
+        # Special handling till lp:1193240 is fixed"
+        if [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then 
+            wsrep_log_error "Xbstream failed"
+            wsrep_log_error "Data directory ${DATA} may not be empty: lp:1193240" \
+                            "Manual intervention required in that case"
+            exit 32
+        fi
+    fi
+
+    wait %% # join for wait_for_listen thread
+
+    for ecode in "${RC[@]}";do 
+        if [[ $ecode -ne 0 ]];then 
+            wsrep_log_error "Error while getting data from donor node: " \
+                            "exit codes: ${RC[@]}"
+            exit 32
+        fi
+    done
+
+    if [ ! -r "${MAGIC_FILE}" ]
+    then
+        # this message should cause joiner to abort
+        wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'"
+        wsrep_log_info "Contents of datadir" 
+        wsrep_log_info "$(ls -l ${DATA}/**/*)"
+        exit 32
+    fi
+
+    if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null
+    then
+        wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." 
+        exit 32
+    fi
+
+    if [ ! -r "${DATA}/${IST_FILE}" ]
+    then
+        wsrep_log_info "Proceeding with SST"
+        wsrep_log_info "Removing existing ib_logfile files"
+        if [[ $incremental -ne 1 ]];then 
+            rm -f ${DATA}/ib_logfile*
+        else
+            rm -f ${BDATA}/ib_logfile*
+        fi
+
+        get_proc
+
+        # Rebuild indexes for compact backups
+        if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then 
+            wsrep_log_info "Index compaction detected"
+            rebuild=1
+        fi
+
+        if [[ $rebuild -eq 1 ]];then 
+            nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc)
+            wsrep_log_info "Rebuilding during prepare with $nthreads threads"
+            rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads"
+        fi
+
+        if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then
+
+            wsrep_log_info "Compressed qpress files found"
+
+            if [[ ! -x `which qpress` ]];then 
+                wsrep_log_error "qpress not found in path: $PATH"
+                exit 22
+            fi
+
+            if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then
+                count=$(find ${DATA} -type f -name '*.qp' | wc -l)
+                count=$(( count*2 ))
+                if pv --help | grep -q FORMAT;then 
+                    pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
+                else 
+                    pvopts="-f -s $count -l -N Decompression"
+                fi
+                pcmd="pv $pvopts"
+                adjust_progress
+                dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d"
+            else 
+                dcmd="xargs -n 2 qpress -T${nproc}d"
+            fi
+
+            wsrep_log_info "Removing existing ibdata1 file"
+            rm -f ${DATA}/ibdata1
+
+            # Decompress the qpress files 
+            wsrep_log_info "Decompression with $nproc threads"
+            timeit "Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
+            extcode=$?
+
+            if [[ $extcode -eq 0 ]];then
+                wsrep_log_info "Removing qpress files after decompression"
+                find ${DATA} -type f -name '*.qp' -delete 
+                if [[ $? -ne 0 ]];then 
+                    wsrep_log_error "Something went wrong with deletion of qpress files. Investigate"
+                fi
+            else
+                wsrep_log_error "Decompression failed. Exit code: $extcode"
+                exit 22
+            fi
+        fi
+
+        if [[ $incremental -eq 1 ]];then 
+            # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues.
+            INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \
+                --ibbackup=xtrabackup_55 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log"
+        fi
+
+        wsrep_log_info "Preparing the backup at ${DATA}"
+        timeit "Xtrabackup prepare stage" "$INNOAPPLY"
+
+        if [[ $incremental -eq 1 ]];then 
+            wsrep_log_info "Cleaning up ${DATA} after incremental SST"
+            [[ -d ${DATA} ]] && rm -rf ${DATA}
+            DATA=$BDATA
+        fi
+
+        if [ $? -ne 0 ];
+        then
+            wsrep_log_error "${INNOBACKUPEX_BIN} finished with errors. Check ${DATA}/innobackup.prepare.log" 
+            exit 22
+        fi
+    else 
+        wsrep_log_info "${IST_FILE} received from donor: Running IST"
+    fi
+
+    if [[ ! -r ${MAGIC_FILE} ]];then
+        wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable"
+        exit 2
+    fi
+
+    cat "${MAGIC_FILE}" # output UUID:seqno
+    wsrep_log_info "Total time on joiner: $totime seconds"
+fi
+
+exit 0
diff --git a/scripts/wsrep_sst_xtrabackup-v2 b/scripts/wsrep_sst_xtrabackup-v2
new file mode 100755
index 00000000000..f4d133d4f8e
--- /dev/null
+++ b/scripts/wsrep_sst_xtrabackup-v2
@@ -0,0 +1,930 @@
+#!/bin/bash -ue
+# Copyright (C) 2013 Percona Inc
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# Documentation: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 
+# Make sure to read that before proceeding!
+
+
+
+
+. $(dirname $0)/wsrep_sst_common
+
+ealgo=""
+ekey=""
+ekeyfile=""
+encrypt=0
+nproc=1
+ecode=0
+XTRABACKUP_PID=""
+SST_PORT=""
+REMOTEIP=""
+tcert=""
+tpem=""
+tkey=""
+sockopt=""
+progress=""
+ttime=0
+totime=0
+lsn=""
+incremental=0
+ecmd=""
+rlimit=""
+# Initially
+stagemsg="${WSREP_SST_OPT_ROLE}"
+cpat=""
+speciald=0
+ib_home_dir=""
+ib_log_dir=""
+
+sfmt="tar"
+strmcmd=""
+tfmt=""
+tcmd=""
+rebuild=0
+rebuildcmd=""
+payload=0
+pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' "
+pvopts="-f  -i 10 -N $WSREP_SST_OPT_ROLE "
+STATDIR=""
+uextra=0
+disver=""
+
+tmpopts=""
+itmpdir=""
+xtmpdir=""
+
+scomp=""
+sdecomp=""
+
+if which pv &>/dev/null && pv --help | grep -q FORMAT;then 
+    pvopts+=$pvformat
+fi
+pcmd="pv $pvopts"
+declare -a RC
+
+INNOBACKUPEX_BIN=innobackupex
+readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ })
+DATA="${WSREP_SST_OPT_DATA}"
+INFO_FILE="xtrabackup_galera_info"
+IST_FILE="xtrabackup_ist"
+MAGIC_FILE="${DATA}/${INFO_FILE}"
+
+# Setting the path for ss and ip
+export PATH="/usr/sbin:/sbin:$PATH"
+
+timeit(){
+    local stage=$1
+    shift
+    local cmd="$@"
+    local x1 x2 took extcode
+
+    if [[ $ttime -eq 1 ]];then 
+        x1=$(date +%s)
+        wsrep_log_info "Evaluating $cmd"
+        eval "$cmd"
+        extcode=$?
+        x2=$(date +%s)
+        took=$(( x2-x1 ))
+        wsrep_log_info "NOTE: $stage took $took seconds"
+        totime=$(( totime+took ))
+    else 
+        wsrep_log_info "Evaluating $cmd"
+        eval "$cmd"
+        extcode=$?
+    fi
+    return $extcode
+}
+
+get_keys()
+{
+    # $encrypt -eq 1 is for internal purposes only
+    if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then 
+        return 
+    fi
+
+    if [[ $encrypt -eq 0 ]];then 
+        if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then
+            wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
+        fi
+        return
+    fi
+
+    if [[ $sfmt == 'tar' ]];then
+        wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format"
+        encrypt=-1
+        return
+    fi
+
+    wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4"
+
+    if [[ -z $ealgo ]];then
+        wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out"
+        exit 3
+    fi
+
+    if [[ -z $ekey && ! -r $ekeyfile ]];then
+        wsrep_log_error "FATAL: Either key or keyfile must be readable"
+        exit 3
+    fi
+
+    if [[ -z $ekey ]];then
+        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile"
+    else
+        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey"
+    fi
+
+    if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
+        ecmd+=" -d"
+    fi
+
+    stagemsg+="-XB-Encrypted"
+}
+
+get_transfer()
+{
+    if [[ -z $SST_PORT ]];then 
+        TSST_PORT=4444
+    else 
+        TSST_PORT=$SST_PORT
+    fi
+
+    if [[ $tfmt == 'nc' ]];then
+        if [[ ! -x `which nc` ]];then 
+            wsrep_log_error "nc(netcat) not found in path: $PATH"
+            exit 2
+        fi
+        wsrep_log_info "Using netcat as streamer"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            tcmd="nc -dl ${TSST_PORT}"
+        else
+            tcmd="nc ${REMOTEIP} ${TSST_PORT}"
+        fi
+    else
+        tfmt='socat'
+        wsrep_log_info "Using socat as streamer"
+        if [[ ! -x `which socat` ]];then 
+            wsrep_log_error "socat not found in path: $PATH"
+            exit 2
+        fi
+
+        if [[ $encrypt -eq 2 || $encrypt -eq 3 ]] && ! socat -V | grep -q WITH_OPENSSL;then 
+            wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer"
+            encrypt=-1
+        fi
+
+        if [[ $encrypt -eq 2 ]];then 
+            wsrep_log_info "Using openssl based encryption with socat: with crt and pem"
+            if [[ -z $tpem || -z $tcert ]];then 
+                wsrep_log_error "Both PEM and CRT files required"
+                exit 22
+            fi
+            stagemsg+="-OpenSSL-Encrypted-2"
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert"
+                tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio"
+            else
+                wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert"
+                tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}"
+            fi
+        elif [[ $encrypt -eq 3 ]];then
+            wsrep_log_info "Using openssl based encryption with socat: with key and crt"
+            if [[ -z $tpem || -z $tkey ]];then 
+                wsrep_log_error "Both certificate and key files required"
+                exit 22
+            fi
+            stagemsg+="-OpenSSL-Encrypted-3"
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                wsrep_log_info "Decrypting with certificate $tpem, key $tkey"
+                tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,key=${tkey},verify=0${sockopt} stdio"
+            else
+                wsrep_log_info "Encrypting with certificate $tpem, key $tkey"
+                tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,key=${tkey},verify=0${sockopt}"
+            fi
+
+        else 
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio"
+            else
+                tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}"
+            fi
+        fi
+    fi
+
+}
+
+parse_cnf()
+{
+    local group=$1
+    local var=$2
+    reval=$(my_print_defaults -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-)
+    if [[ -z $reval ]];then 
+        [[ -n $3 ]] && reval=$3
+    fi
+    echo $reval
+}
+
+get_footprint()
+{
+    pushd $WSREP_SST_OPT_DATA 1>/dev/null
+    payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }')
+    if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 
+        # QuickLZ has around 50% compression ratio
+        # When compression/compaction used, the progress is only an approximate.
+        payload=$(( payload*1/2 ))
+    fi
+    popd 1>/dev/null
+    pcmd+=" -s $payload"
+    adjust_progress
+}
+
+adjust_progress()
+{
+    if [[ -n $progress && $progress != '1' ]];then 
+        if [[ -e $progress ]];then 
+            pcmd+=" 2>>$progress"
+        else 
+            pcmd+=" 2>$progress"
+        fi
+    elif [[ -z $progress && -n $rlimit  ]];then 
+            # When rlimit is non-zero
+            pcmd="pv -q"
+    fi 
+
+    if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
+        wsrep_log_info "Rate-limiting SST to $rlimit"
+        pcmd+=" -L \$rlimit"
+    fi
+}
+
+read_cnf()
+{
+    sfmt=$(parse_cnf sst streamfmt "xbstream")
+    tfmt=$(parse_cnf sst transferfmt "socat")
+    tcert=$(parse_cnf sst tca "")
+    tpem=$(parse_cnf sst tcert "")
+    tkey=$(parse_cnf sst tkey "")
+    encrypt=$(parse_cnf sst encrypt 0)
+    sockopt=$(parse_cnf sst sockopt "")
+    progress=$(parse_cnf sst progress "")
+    rebuild=$(parse_cnf sst rebuild 0)
+    ttime=$(parse_cnf sst time 0)
+    cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$')
+    incremental=$(parse_cnf sst incremental 0)
+    ealgo=$(parse_cnf xtrabackup encrypt "")
+    ekey=$(parse_cnf xtrabackup encrypt-key "")
+    ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "")
+    scomp=$(parse_cnf sst compressor "")
+    sdecomp=$(parse_cnf sst decompressor "")
+
+
+    # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 
+    if [[ -z $ealgo ]];then
+        ealgo=$(parse_cnf sst encrypt-algo "")
+        ekey=$(parse_cnf sst encrypt-key "")
+        ekeyfile=$(parse_cnf sst encrypt-key-file "")
+    fi
+    rlimit=$(parse_cnf sst rlimit "")
+    uextra=$(parse_cnf sst use-extra 0)
+    speciald=$(parse_cnf sst sst-special-dirs 1)
+    iopts=$(parse_cnf sst inno-backup-opts "")
+    iapts=$(parse_cnf sst inno-apply-opts "")
+    impts=$(parse_cnf sst inno-move-opts "")
+    stimeout=$(parse_cnf sst sst-initial-timeout 100)
+}
+
+get_stream()
+{
+    if [[ $sfmt == 'xbstream' ]];then 
+        wsrep_log_info "Streaming with xbstream"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            strmcmd="xbstream -x"
+        else
+            strmcmd="xbstream -c \${INFO_FILE}"
+        fi
+    else
+        sfmt="tar"
+        wsrep_log_info "Streaming with tar"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            strmcmd="tar xfi - "
+        else
+            strmcmd="tar cf - \${INFO_FILE} "
+        fi
+
+    fi
+}
+
+get_proc()
+{
+    set +e
+    nproc=$(grep -c processor /proc/cpuinfo)
+    [[ -z $nproc || $nproc -eq 0 ]] && nproc=1
+    set -e
+}
+
+sig_joiner_cleanup()
+{
+    wsrep_log_error "Removing $MAGIC_FILE file due to signal"
+    rm -f "$MAGIC_FILE"
+}
+
+cleanup_joiner()
+{
+    # Since this is invoked just after exit NNN
+    local estatus=$?
+    if [[ $estatus -ne 0 ]];then 
+        wsrep_log_error "Cleanup after exit with status:$estatus"
+    fi
+    if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then
+        wsrep_log_info "Removing the sst_in_progress file"
+        wsrep_cleanup_progress_file
+    fi
+    if [[ -n $progress && -p $progress ]];then 
+        wsrep_log_info "Cleaning up fifo file $progress"
+        rm $progress
+    fi
+    if [[ -n ${STATDIR:-} ]];then 
+       [[ -d $STATDIR ]] && rm -rf $STATDIR
+    fi
+}
+
+check_pid()
+{
+    local pid_file="$1"
+    [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1
+}
+
+cleanup_donor()
+{
+    # Since this is invoked just after exit NNN
+    local estatus=$?
+    if [[ $estatus -ne 0 ]];then 
+        wsrep_log_error "Cleanup after exit with status:$estatus"
+    fi
+
+    if [[ -n ${XTRABACKUP_PID:-} ]];then 
+        if check_pid $XTRABACKUP_PID
+        then
+            wsrep_log_error "xtrabackup process is still running. Killing... "
+            kill_xtrabackup
+        fi
+
+    fi
+    rm -f ${DATA}/${IST_FILE} || true
+
+    if [[ -n $progress && -p $progress ]];then 
+        wsrep_log_info "Cleaning up fifo file $progress"
+        rm -f $progress || true
+    fi
+
+    wsrep_log_info "Cleaning up temporary directories"
+
+    if [[ -n $xtmpdir ]];then 
+       [[ -d $xtmpdir ]] &&  rm -rf $xtmpdir || true
+    fi
+
+    if [[ -n $itmpdir ]];then 
+       [[ -d $itmpdir ]] &&  rm -rf $itmpdir || true
+    fi
+}
+
+kill_xtrabackup()
+{
+    local PID=$(cat $XTRABACKUP_PID)
+    [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
+    wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID"
+    rm -f "$XTRABACKUP_PID" || true
+}
+
+setup_ports()
+{
+    if [[ "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
+        SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }')
+        REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }')
+        lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }')
+    else
+        SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }')
+    fi
+}
+
+# waits ~10 seconds for nc to open the port and then reports ready
+# (regardless of timeout)
+wait_for_listen()
+{
+    local PORT=$1
+    local ADDR=$2
+    local MODULE=$3
+    for i in {1..50}
+    do
+        ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break
+        sleep 0.2
+    done
+    if [[ $incremental -eq 1 ]];then 
+        echo "ready ${ADDR}/${MODULE}/$lsn"
+    else 
+        echo "ready ${ADDR}/${MODULE}"
+    fi
+}
+
+check_extra()
+{
+    local use_socket=1
+    if [[ $uextra -eq 1 ]];then 
+        if my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 
+            local eport=$(my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2)
+            if [[ -n $eport ]];then 
+                # Xtrabackup works only locally.
+                # Hence, setting host to 127.0.0.1 unconditionally. 
+                wsrep_log_info "SST through extra_port $eport"
+                INNOEXTRA+=" --host=127.0.0.1 --port=$eport "
+                use_socket=0
+            else 
+                wsrep_log_error "Extra port $eport null, failing"
+                exit 1
+            fi
+        else 
+            wsrep_log_info "Thread pool not set, ignore the option use_extra"
+        fi
+    fi
+    if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then
+        INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}"
+    fi
+}
+
+recv_joiner()
+{
+    local dir=$1
+    local msg=$2 
+    local tmt=$3
+    local ltcmd
+
+    pushd ${dir} 1>/dev/null
+    set +e
+
+    if [[ $tmt -gt 0 && -x `which timeout` ]];then 
+        if timeout --help | grep -q -- '-k';then 
+            ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
+        else 
+            ltcmd="timeout $tmt $tcmd"
+        fi
+        timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+    else 
+        timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+    fi
+
+    set -e
+    popd 1>/dev/null 
+
+    if [[ ${RC[0]} -eq 124 ]];then 
+        wsrep_log_error "Possible timeout in receving first data from donor in gtid stage"
+        exit 32
+    fi
+
+    for ecode in "${RC[@]}";do 
+        if [[ $ecode -ne 0 ]];then 
+            wsrep_log_error "Error while getting data from donor node: " \
+                            "exit codes: ${RC[@]}"
+            exit 32
+        fi
+    done
+
+    if [ ! -r "${MAGIC_FILE}" ];then
+        # this message should cause joiner to abort
+        wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'"
+        wsrep_log_info "Contents of datadir" 
+        wsrep_log_info "$(ls -l ${dir}/*)"
+        exit 32
+    fi
+}
+
+
+send_donor()
+{
+    local dir=$1
+    local msg=$2 
+
+    pushd ${dir} 1>/dev/null
+    set +e
+    timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+    set -e
+    popd 1>/dev/null 
+
+
+    for ecode in "${RC[@]}";do 
+        if [[ $ecode -ne 0 ]];then 
+            wsrep_log_error "Error while getting data from donor node: " \
+                            "exit codes: ${RC[@]}"
+            exit 32
+        fi
+    done
+
+}
+
+if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then 
+    wsrep_log_error "innobackupex not in path: $PATH"
+    exit 2
+fi
+
+rm -f "${MAGIC_FILE}"
+
+if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then 
+    wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}"
+    exit 22
+fi
+
+read_cnf
+setup_ports
+get_stream
+get_transfer
+
+if ${INNOBACKUPEX_BIN} /tmp --help  | grep -q -- '--version-check'; then 
+    disver="--no-version-check"
+fi
+
+
+INNOEXTRA=""
+INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log"
+INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts  --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log"
+INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log"
+
+if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
+then
+    trap cleanup_donor EXIT
+
+    if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
+    then
+
+        if [[ -z $(parse_cnf mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then 
+            xtmpdir=$(mktemp -d)
+            tmpopts=" --tmpdir=$xtmpdir "
+            wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory"
+        fi
+
+        itmpdir=$(mktemp -d)
+        wsrep_log_info "Using $itmpdir as innobackupex temporary directory"
+
+        if [ "${AUTH[0]}" != "(null)" ]; then
+           INNOEXTRA+=" --user=${AUTH[0]}"
+       fi
+
+        if [ ${#AUTH[*]} -eq 2 ]; then
+           INNOEXTRA+=" --password=${AUTH[1]}"
+        elif [ "${AUTH[0]}" != "(null)" ]; then
+           # Empty password, used for testing, debugging etc.
+           INNOEXTRA+=" --password="
+        fi
+
+        get_keys
+        if [[ $encrypt -eq 1 ]];then
+            if [[ -n $ekey ]];then
+                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey "
+            else 
+                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile "
+            fi
+        fi
+
+        if [[ -n $lsn ]];then 
+                INNOEXTRA+=" --incremental --incremental-lsn=$lsn "
+        fi
+
+        check_extra
+
+        wsrep_log_info "Streaming GTID file before SST"
+
+        echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}"
+
+        ttcmd="$tcmd"
+
+        if [[ $encrypt -eq 1 ]];then
+            if [[ -n $scomp ]];then 
+                tcmd=" $ecmd | $scomp | $tcmd "
+            else 
+                tcmd=" $ecmd | $tcmd "
+            fi
+        elif [[ -n $scomp ]];then 
+            tcmd=" $scomp | $tcmd "
+        fi
+
+
+        send_donor $DATA "${stagemsg}-gtid"
+
+        tcmd="$ttcmd"
+        if [[ -n $progress ]];then 
+            get_footprint
+            tcmd="$pcmd | $tcmd"
+        elif [[ -n $rlimit ]];then 
+            adjust_progress
+            tcmd="$pcmd | $tcmd"
+        fi
+
+        wsrep_log_info "Sleeping before data transfer for SST"
+        sleep 10
+
+        wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}"
+
+        if [[ -n $scomp ]];then 
+            tcmd="$scomp | $tcmd"
+        fi
+
+        set +e
+        timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+        set -e
+
+        if [ ${RC[0]} -ne 0 ]; then
+          wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \
+                          "Check ${DATA}/innobackup.backup.log"
+          exit 22
+        elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then 
+          wsrep_log_error "$tcmd finished with error: ${RC[1]}"
+          exit 22
+        fi
+
+        # innobackupex implicitly writes PID to fixed location in $xtmpdir
+        XTRABACKUP_PID="$xtmpdir/xtrabackup_pid"
+
+
+    else # BYPASS FOR IST
+
+        wsrep_log_info "Bypassing the SST for IST"
+        echo "continue" # now server can resume updating data
+        echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}"
+        echo "1" > "${DATA}/${IST_FILE}"
+        get_keys
+        if [[ $encrypt -eq 1 ]];then
+            if [[ -n $scomp ]];then 
+                tcmd=" $ecmd | $scomp | $tcmd "
+            else
+                tcmd=" $ecmd | $tcmd "
+            fi
+        elif [[ -n $scomp ]];then 
+            tcmd=" $scomp | $tcmd "
+        fi
+        strmcmd+=" \${IST_FILE}"
+
+        send_donor $DATA "${stagemsg}-IST"
+
+    fi
+
+    echo "done ${WSREP_SST_OPT_GTID}"
+    wsrep_log_info "Total time on donor: $totime seconds"
+
+elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ]
+then
+    [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
+    [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE
+
+    if [[ $speciald -eq 1 ]];then 
+        ib_home_dir=$(parse_cnf mysqld innodb-data-home-dir "")
+        ib_log_dir=$(parse_cnf mysqld innodb-log-group-home-dir "")
+        if [[ -z $ib_home_dir && -z $ib_log_dir ]];then 
+            speciald=0
+        fi
+    fi
+
+    stagemsg="Joiner-Recv"
+
+    if [[ ! -e ${DATA}/ibdata1 ]];then 
+        incremental=0
+    fi
+
+    if [[ $incremental -eq 1 ]];then 
+        wsrep_log_info "Incremental SST enabled: NOT SUPPORTED yet"
+        lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ')
+        wsrep_log_info "Recovered LSN: $lsn"
+    fi
+
+    sencrypted=1
+    nthreads=1
+
+    MODULE="xtrabackup_sst"
+
+    rm -f "${DATA}/${IST_FILE}"
+
+    # May need xtrabackup_checkpoints later on
+    rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info  ${DATA}/xtrabackup_logfile
+
+    ADDR=${WSREP_SST_OPT_ADDR}
+    if [ -z "${SST_PORT}" ]
+    then
+        SST_PORT=4444
+        ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}"
+    fi
+
+    wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} &
+
+    trap sig_joiner_cleanup HUP PIPE INT TERM
+    trap cleanup_joiner EXIT
+
+    if [[ -n $progress ]];then 
+        adjust_progress
+        tcmd+=" | $pcmd"
+    fi
+
+    if [[ $incremental -eq 1 ]];then 
+        BDATA=$DATA
+        DATA=$(mktemp -d)
+        MAGIC_FILE="${DATA}/${INFO_FILE}"
+    fi
+
+    get_keys
+    if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then
+        if [[ -n $sdecomp ]];then 
+            strmcmd=" $sdecomp | $ecmd | $strmcmd"
+        else 
+            strmcmd=" $ecmd | $strmcmd"
+        fi
+    elif [[ -n $sdecomp ]];then 
+            strmcmd=" $sdecomp | $strmcmd"
+    fi
+
+    STATDIR=$(mktemp -d)
+    MAGIC_FILE="${STATDIR}/${INFO_FILE}"
+    recv_joiner $STATDIR  "${stagemsg}-gtid" $stimeout
+
+    if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null
+    then
+        wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." 
+        exit 32
+    fi
+
+    if [ ! -r "${STATDIR}/${IST_FILE}" ]
+    then
+        wsrep_log_info "Proceeding with SST"
+
+        if [[ $speciald -eq 1 && -d ${DATA}/.sst ]];then 
+            wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous SST"
+        fi
+
+        if [[ $incremental -ne 1 ]];then 
+            if [[ $speciald -eq 1 ]];then 
+                wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories"
+                find $ib_home_dir $ib_log_dir $DATA -mindepth 1  -regex $cpat  -prune  -o -exec rm -rfv {} 1>&2 \+
+            else 
+                wsrep_log_info "Cleaning the existing datadir"
+                find $DATA -mindepth 1  -regex $cpat  -prune  -o -exec rm -rfv {} 1>&2 \+
+            fi
+            tempdir=$(parse_cnf mysqld log-bin "")
+            if [[ -n ${tempdir:-} ]];then
+                binlog_dir=$(dirname $tempdir)
+                binlog_file=$(basename $tempdir)
+                if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then
+                    pattern="$binlog_dir/$binlog_file\.[0-9]+$"
+                    wsrep_log_info "Cleaning the binlog directory $binlog_dir as well"
+                    find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+
+                    rm $binlog_dir/*.index || true
+                fi
+            fi
+
+        else
+            wsrep_log_info "Removing existing ib_logfile files"
+            rm -f ${BDATA}/ib_logfile*
+        fi
+
+
+        if [[ $speciald -eq 1 ]];then 
+            mkdir -p ${DATA}/.sst
+            TDATA=${DATA}
+            DATA="${DATA}/.sst"
+        fi
+
+
+        MAGIC_FILE="${DATA}/${INFO_FILE}"
+        recv_joiner $DATA "${stagemsg}-SST" 0
+
+        get_proc
+
+        # Rebuild indexes for compact backups
+        if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then 
+            wsrep_log_info "Index compaction detected"
+            rebuild=1
+        fi
+
+        if [[ $rebuild -eq 1 ]];then 
+            nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc)
+            wsrep_log_info "Rebuilding during prepare with $nthreads threads"
+            rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads"
+        fi
+
+        if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then
+
+            wsrep_log_info "Compressed qpress files found"
+
+            if [[ ! -x `which qpress` ]];then 
+                wsrep_log_error "qpress not found in path: $PATH"
+                exit 22
+            fi
+
+            if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then
+                count=$(find ${DATA} -type f -name '*.qp' | wc -l)
+                count=$(( count*2 ))
+                if pv --help | grep -q FORMAT;then 
+                    pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
+                else 
+                    pvopts="-f -s $count -l -N Decompression"
+                fi
+                pcmd="pv $pvopts"
+                adjust_progress
+                dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d"
+            else 
+                dcmd="xargs -n 2 qpress -T${nproc}d"
+            fi
+
+
+            # Decompress the qpress files 
+            wsrep_log_info "Decompression with $nproc threads"
+            timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
+            extcode=$?
+
+            if [[ $extcode -eq 0 ]];then
+                wsrep_log_info "Removing qpress files after decompression"
+                find ${DATA} -type f -name '*.qp' -delete 
+                if [[ $? -ne 0 ]];then 
+                    wsrep_log_error "Something went wrong with deletion of qpress files. Investigate"
+                fi
+            else
+                wsrep_log_error "Decompression failed. Exit code: $extcode"
+                exit 22
+            fi
+        fi
+
+
+        if  [[ ! -z $WSREP_SST_OPT_BINLOG ]];then
+
+            BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG)
+            BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG)
+
+            # To avoid comparing data directory and BINLOG_DIRNAME 
+            mv $DATA/${BINLOG_FILENAME}.* $BINLOG_DIRNAME/ 2>/dev/null || true
+
+            pushd $BINLOG_DIRNAME &>/dev/null
+            for bfiles in $(ls -1 ${BINLOG_FILENAME}.*);do
+                echo ${BINLOG_DIRNAME}/${bfiles} >> ${BINLOG_FILENAME}.index
+            done
+            popd &> /dev/null
+
+        fi
+
+        if [[ $incremental -eq 1 ]];then 
+            # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues.
+            INNOAPPLY="${INNOBACKUPEX_BIN} $disver --defaults-file=${WSREP_SST_OPT_CONF} \
+                --ibbackup=xtrabackup_56 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log"
+        fi
+
+        wsrep_log_info "Preparing the backup at ${DATA}"
+        timeit "Xtrabackup prepare stage" "$INNOAPPLY"
+
+        if [ $? -ne 0 ];
+        then
+            wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" 
+            exit 22
+        fi
+
+        if [[ $speciald -eq 1 ]];then 
+            MAGIC_FILE="${TDATA}/${INFO_FILE}"
+            set +e
+            rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log
+            set -e
+            wsrep_log_info "Moving the backup to ${TDATA}"
+            timeit "Xtrabackup move stage" "$INNOMOVE"
+            if [[ $? -eq 0 ]];then 
+                wsrep_log_info "Move successful, removing ${DATA}"
+                rm -rf $DATA
+                DATA=${TDATA}
+            else 
+                wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis"
+                wsrep_log_error "Check ${DATA}/innobackup.move.log for details"
+            fi
+        fi
+
+        if [[ $incremental -eq 1 ]];then 
+            wsrep_log_info "Cleaning up ${DATA} after incremental SST"
+            [[ -d ${DATA} ]] && rm -rf ${DATA}
+            DATA=$BDATA
+        fi
+
+    else 
+        wsrep_log_info "${IST_FILE} received from donor: Running IST"
+    fi
+
+    if [[ ! -r ${MAGIC_FILE} ]];then 
+        wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable"
+        exit 2
+    fi
+    cat "${MAGIC_FILE}" # output UUID:seqno
+    wsrep_log_info "Total time on joiner: $totime seconds"
+fi
+
+exit 0
diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh
new file mode 100644
index 00000000000..f4d133d4f8e
--- /dev/null
+++ b/scripts/wsrep_sst_xtrabackup-v2.sh
@@ -0,0 +1,930 @@
+#!/bin/bash -ue
+# Copyright (C) 2013 Percona Inc
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# Documentation: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 
+# Make sure to read that before proceeding!
+
+
+
+
+. $(dirname $0)/wsrep_sst_common
+
+ealgo=""
+ekey=""
+ekeyfile=""
+encrypt=0
+nproc=1
+ecode=0
+XTRABACKUP_PID=""
+SST_PORT=""
+REMOTEIP=""
+tcert=""
+tpem=""
+tkey=""
+sockopt=""
+progress=""
+ttime=0
+totime=0
+lsn=""
+incremental=0
+ecmd=""
+rlimit=""
+# Initially
+stagemsg="${WSREP_SST_OPT_ROLE}"
+cpat=""
+speciald=0
+ib_home_dir=""
+ib_log_dir=""
+
+sfmt="tar"
+strmcmd=""
+tfmt=""
+tcmd=""
+rebuild=0
+rebuildcmd=""
+payload=0
+pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' "
+pvopts="-f  -i 10 -N $WSREP_SST_OPT_ROLE "
+STATDIR=""
+uextra=0
+disver=""
+
+tmpopts=""
+itmpdir=""
+xtmpdir=""
+
+scomp=""
+sdecomp=""
+
+if which pv &>/dev/null && pv --help | grep -q FORMAT;then 
+    pvopts+=$pvformat
+fi
+pcmd="pv $pvopts"
+declare -a RC
+
+INNOBACKUPEX_BIN=innobackupex
+readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ })
+DATA="${WSREP_SST_OPT_DATA}"
+INFO_FILE="xtrabackup_galera_info"
+IST_FILE="xtrabackup_ist"
+MAGIC_FILE="${DATA}/${INFO_FILE}"
+
+# Setting the path for ss and ip
+export PATH="/usr/sbin:/sbin:$PATH"
+
+timeit(){
+    local stage=$1
+    shift
+    local cmd="$@"
+    local x1 x2 took extcode
+
+    if [[ $ttime -eq 1 ]];then 
+        x1=$(date +%s)
+        wsrep_log_info "Evaluating $cmd"
+        eval "$cmd"
+        extcode=$?
+        x2=$(date +%s)
+        took=$(( x2-x1 ))
+        wsrep_log_info "NOTE: $stage took $took seconds"
+        totime=$(( totime+took ))
+    else 
+        wsrep_log_info "Evaluating $cmd"
+        eval "$cmd"
+        extcode=$?
+    fi
+    return $extcode
+}
+
+get_keys()
+{
+    # $encrypt -eq 1 is for internal purposes only
+    if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then 
+        return 
+    fi
+
+    if [[ $encrypt -eq 0 ]];then 
+        if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then
+            wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
+        fi
+        return
+    fi
+
+    if [[ $sfmt == 'tar' ]];then
+        wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format"
+        encrypt=-1
+        return
+    fi
+
+    wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4"
+
+    if [[ -z $ealgo ]];then
+        wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out"
+        exit 3
+    fi
+
+    if [[ -z $ekey && ! -r $ekeyfile ]];then
+        wsrep_log_error "FATAL: Either key or keyfile must be readable"
+        exit 3
+    fi
+
+    if [[ -z $ekey ]];then
+        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile"
+    else
+        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey"
+    fi
+
+    if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
+        ecmd+=" -d"
+    fi
+
+    stagemsg+="-XB-Encrypted"
+}
+
+get_transfer()
+{
+    if [[ -z $SST_PORT ]];then 
+        TSST_PORT=4444
+    else 
+        TSST_PORT=$SST_PORT
+    fi
+
+    if [[ $tfmt == 'nc' ]];then
+        if [[ ! -x `which nc` ]];then 
+            wsrep_log_error "nc(netcat) not found in path: $PATH"
+            exit 2
+        fi
+        wsrep_log_info "Using netcat as streamer"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            tcmd="nc -dl ${TSST_PORT}"
+        else
+            tcmd="nc ${REMOTEIP} ${TSST_PORT}"
+        fi
+    else
+        tfmt='socat'
+        wsrep_log_info "Using socat as streamer"
+        if [[ ! -x `which socat` ]];then 
+            wsrep_log_error "socat not found in path: $PATH"
+            exit 2
+        fi
+
+        if [[ $encrypt -eq 2 || $encrypt -eq 3 ]] && ! socat -V | grep -q WITH_OPENSSL;then 
+            wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer"
+            encrypt=-1
+        fi
+
+        if [[ $encrypt -eq 2 ]];then 
+            wsrep_log_info "Using openssl based encryption with socat: with crt and pem"
+            if [[ -z $tpem || -z $tcert ]];then 
+                wsrep_log_error "Both PEM and CRT files required"
+                exit 22
+            fi
+            stagemsg+="-OpenSSL-Encrypted-2"
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert"
+                tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio"
+            else
+                wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert"
+                tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}"
+            fi
+        elif [[ $encrypt -eq 3 ]];then
+            wsrep_log_info "Using openssl based encryption with socat: with key and crt"
+            if [[ -z $tpem || -z $tkey ]];then 
+                wsrep_log_error "Both certificate and key files required"
+                exit 22
+            fi
+            stagemsg+="-OpenSSL-Encrypted-3"
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                wsrep_log_info "Decrypting with certificate $tpem, key $tkey"
+                tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,key=${tkey},verify=0${sockopt} stdio"
+            else
+                wsrep_log_info "Encrypting with certificate $tpem, key $tkey"
+                tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,key=${tkey},verify=0${sockopt}"
+            fi
+
+        else 
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio"
+            else
+                tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}"
+            fi
+        fi
+    fi
+
+}
+
+parse_cnf()
+{
+    local group=$1
+    local var=$2
+    reval=$(my_print_defaults -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-)
+    if [[ -z $reval ]];then 
+        [[ -n $3 ]] && reval=$3
+    fi
+    echo $reval
+}
+
+get_footprint()
+{
+    pushd $WSREP_SST_OPT_DATA 1>/dev/null
+    payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }')
+    if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 
+        # QuickLZ has around 50% compression ratio
+        # When compression/compaction used, the progress is only an approximate.
+        payload=$(( payload*1/2 ))
+    fi
+    popd 1>/dev/null
+    pcmd+=" -s $payload"
+    adjust_progress
+}
+
+adjust_progress()
+{
+    if [[ -n $progress && $progress != '1' ]];then 
+        if [[ -e $progress ]];then 
+            pcmd+=" 2>>$progress"
+        else 
+            pcmd+=" 2>$progress"
+        fi
+    elif [[ -z $progress && -n $rlimit  ]];then 
+            # When rlimit is non-zero
+            pcmd="pv -q"
+    fi 
+
+    if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
+        wsrep_log_info "Rate-limiting SST to $rlimit"
+        pcmd+=" -L \$rlimit"
+    fi
+}
+
+read_cnf()
+{
+    sfmt=$(parse_cnf sst streamfmt "xbstream")
+    tfmt=$(parse_cnf sst transferfmt "socat")
+    tcert=$(parse_cnf sst tca "")
+    tpem=$(parse_cnf sst tcert "")
+    tkey=$(parse_cnf sst tkey "")
+    encrypt=$(parse_cnf sst encrypt 0)
+    sockopt=$(parse_cnf sst sockopt "")
+    progress=$(parse_cnf sst progress "")
+    rebuild=$(parse_cnf sst rebuild 0)
+    ttime=$(parse_cnf sst time 0)
+    cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$')
+    incremental=$(parse_cnf sst incremental 0)
+    ealgo=$(parse_cnf xtrabackup encrypt "")
+    ekey=$(parse_cnf xtrabackup encrypt-key "")
+    ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "")
+    scomp=$(parse_cnf sst compressor "")
+    sdecomp=$(parse_cnf sst decompressor "")
+
+
+    # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 
+    if [[ -z $ealgo ]];then
+        ealgo=$(parse_cnf sst encrypt-algo "")
+        ekey=$(parse_cnf sst encrypt-key "")
+        ekeyfile=$(parse_cnf sst encrypt-key-file "")
+    fi
+    rlimit=$(parse_cnf sst rlimit "")
+    uextra=$(parse_cnf sst use-extra 0)
+    speciald=$(parse_cnf sst sst-special-dirs 1)
+    iopts=$(parse_cnf sst inno-backup-opts "")
+    iapts=$(parse_cnf sst inno-apply-opts "")
+    impts=$(parse_cnf sst inno-move-opts "")
+    stimeout=$(parse_cnf sst sst-initial-timeout 100)
+}
+
+get_stream()
+{
+    if [[ $sfmt == 'xbstream' ]];then 
+        wsrep_log_info "Streaming with xbstream"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            strmcmd="xbstream -x"
+        else
+            strmcmd="xbstream -c \${INFO_FILE}"
+        fi
+    else
+        sfmt="tar"
+        wsrep_log_info "Streaming with tar"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            strmcmd="tar xfi - "
+        else
+            strmcmd="tar cf - \${INFO_FILE} "
+        fi
+
+    fi
+}
+
+get_proc()
+{
+    set +e
+    nproc=$(grep -c processor /proc/cpuinfo)
+    [[ -z $nproc || $nproc -eq 0 ]] && nproc=1
+    set -e
+}
+
+sig_joiner_cleanup()
+{
+    wsrep_log_error "Removing $MAGIC_FILE file due to signal"
+    rm -f "$MAGIC_FILE"
+}
+
+cleanup_joiner()
+{
+    # Since this is invoked just after exit NNN
+    local estatus=$?
+    if [[ $estatus -ne 0 ]];then 
+        wsrep_log_error "Cleanup after exit with status:$estatus"
+    fi
+    if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then
+        wsrep_log_info "Removing the sst_in_progress file"
+        wsrep_cleanup_progress_file
+    fi
+    if [[ -n $progress && -p $progress ]];then 
+        wsrep_log_info "Cleaning up fifo file $progress"
+        rm $progress
+    fi
+    if [[ -n ${STATDIR:-} ]];then 
+       [[ -d $STATDIR ]] && rm -rf $STATDIR
+    fi
+}
+
+check_pid()
+{
+    local pid_file="$1"
+    [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1
+}
+
+cleanup_donor()
+{
+    # Since this is invoked just after exit NNN
+    local estatus=$?
+    if [[ $estatus -ne 0 ]];then 
+        wsrep_log_error "Cleanup after exit with status:$estatus"
+    fi
+
+    if [[ -n ${XTRABACKUP_PID:-} ]];then 
+        if check_pid $XTRABACKUP_PID
+        then
+            wsrep_log_error "xtrabackup process is still running. Killing... "
+            kill_xtrabackup
+        fi
+
+    fi
+    rm -f ${DATA}/${IST_FILE} || true
+
+    if [[ -n $progress && -p $progress ]];then 
+        wsrep_log_info "Cleaning up fifo file $progress"
+        rm -f $progress || true
+    fi
+
+    wsrep_log_info "Cleaning up temporary directories"
+
+    if [[ -n $xtmpdir ]];then 
+       [[ -d $xtmpdir ]] &&  rm -rf $xtmpdir || true
+    fi
+
+    if [[ -n $itmpdir ]];then 
+       [[ -d $itmpdir ]] &&  rm -rf $itmpdir || true
+    fi
+}
+
+kill_xtrabackup()
+{
+    local PID=$(cat $XTRABACKUP_PID)
+    [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
+    wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID"
+    rm -f "$XTRABACKUP_PID" || true
+}
+
+setup_ports()
+{
+    if [[ "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
+        SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }')
+        REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }')
+        lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }')
+    else
+        SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }')
+    fi
+}
+
+# waits ~10 seconds for nc to open the port and then reports ready
+# (regardless of timeout)
+wait_for_listen()
+{
+    local PORT=$1
+    local ADDR=$2
+    local MODULE=$3
+    for i in {1..50}
+    do
+        ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break
+        sleep 0.2
+    done
+    if [[ $incremental -eq 1 ]];then 
+        echo "ready ${ADDR}/${MODULE}/$lsn"
+    else 
+        echo "ready ${ADDR}/${MODULE}"
+    fi
+}
+
+check_extra()
+{
+    local use_socket=1
+    if [[ $uextra -eq 1 ]];then 
+        if my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 
+            local eport=$(my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2)
+            if [[ -n $eport ]];then 
+                # Xtrabackup works only locally.
+                # Hence, setting host to 127.0.0.1 unconditionally. 
+                wsrep_log_info "SST through extra_port $eport"
+                INNOEXTRA+=" --host=127.0.0.1 --port=$eport "
+                use_socket=0
+            else 
+                wsrep_log_error "Extra port $eport null, failing"
+                exit 1
+            fi
+        else 
+            wsrep_log_info "Thread pool not set, ignore the option use_extra"
+        fi
+    fi
+    if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then
+        INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}"
+    fi
+}
+
+recv_joiner()
+{
+    local dir=$1
+    local msg=$2 
+    local tmt=$3
+    local ltcmd
+
+    pushd ${dir} 1>/dev/null
+    set +e
+
+    if [[ $tmt -gt 0 && -x `which timeout` ]];then 
+        if timeout --help | grep -q -- '-k';then 
+            ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
+        else 
+            ltcmd="timeout $tmt $tcmd"
+        fi
+        timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+    else 
+        timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+    fi
+
+    set -e
+    popd 1>/dev/null 
+
+    if [[ ${RC[0]} -eq 124 ]];then 
+        wsrep_log_error "Possible timeout in receving first data from donor in gtid stage"
+        exit 32
+    fi
+
+    for ecode in "${RC[@]}";do 
+        if [[ $ecode -ne 0 ]];then 
+            wsrep_log_error "Error while getting data from donor node: " \
+                            "exit codes: ${RC[@]}"
+            exit 32
+        fi
+    done
+
+    if [ ! -r "${MAGIC_FILE}" ];then
+        # this message should cause joiner to abort
+        wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'"
+        wsrep_log_info "Contents of datadir" 
+        wsrep_log_info "$(ls -l ${dir}/*)"
+        exit 32
+    fi
+}
+
+
+send_donor()
+{
+    local dir=$1
+    local msg=$2 
+
+    pushd ${dir} 1>/dev/null
+    set +e
+    timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+    set -e
+    popd 1>/dev/null 
+
+
+    for ecode in "${RC[@]}";do 
+        if [[ $ecode -ne 0 ]];then 
+            wsrep_log_error "Error while getting data from donor node: " \
+                            "exit codes: ${RC[@]}"
+            exit 32
+        fi
+    done
+
+}
+
+if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then 
+    wsrep_log_error "innobackupex not in path: $PATH"
+    exit 2
+fi
+
+rm -f "${MAGIC_FILE}"
+
+if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then 
+    wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}"
+    exit 22
+fi
+
+read_cnf
+setup_ports
+get_stream
+get_transfer
+
+if ${INNOBACKUPEX_BIN} /tmp --help  | grep -q -- '--version-check'; then 
+    disver="--no-version-check"
+fi
+
+
+INNOEXTRA=""
+INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log"
+INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts  --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log"
+INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log"
+
+if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
+then
+    trap cleanup_donor EXIT
+
+    if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
+    then
+
+        if [[ -z $(parse_cnf mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then 
+            xtmpdir=$(mktemp -d)
+            tmpopts=" --tmpdir=$xtmpdir "
+            wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory"
+        fi
+
+        itmpdir=$(mktemp -d)
+        wsrep_log_info "Using $itmpdir as innobackupex temporary directory"
+
+        if [ "${AUTH[0]}" != "(null)" ]; then
+           INNOEXTRA+=" --user=${AUTH[0]}"
+       fi
+
+        if [ ${#AUTH[*]} -eq 2 ]; then
+           INNOEXTRA+=" --password=${AUTH[1]}"
+        elif [ "${AUTH[0]}" != "(null)" ]; then
+           # Empty password, used for testing, debugging etc.
+           INNOEXTRA+=" --password="
+        fi
+
+        get_keys
+        if [[ $encrypt -eq 1 ]];then
+            if [[ -n $ekey ]];then
+                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey "
+            else 
+                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile "
+            fi
+        fi
+
+        if [[ -n $lsn ]];then 
+                INNOEXTRA+=" --incremental --incremental-lsn=$lsn "
+        fi
+
+        check_extra
+
+        wsrep_log_info "Streaming GTID file before SST"
+
+        echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}"
+
+        ttcmd="$tcmd"
+
+        if [[ $encrypt -eq 1 ]];then
+            if [[ -n $scomp ]];then 
+                tcmd=" $ecmd | $scomp | $tcmd "
+            else 
+                tcmd=" $ecmd | $tcmd "
+            fi
+        elif [[ -n $scomp ]];then 
+            tcmd=" $scomp | $tcmd "
+        fi
+
+
+        send_donor $DATA "${stagemsg}-gtid"
+
+        tcmd="$ttcmd"
+        if [[ -n $progress ]];then 
+            get_footprint
+            tcmd="$pcmd | $tcmd"
+        elif [[ -n $rlimit ]];then 
+            adjust_progress
+            tcmd="$pcmd | $tcmd"
+        fi
+
+        wsrep_log_info "Sleeping before data transfer for SST"
+        sleep 10
+
+        wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}"
+
+        if [[ -n $scomp ]];then 
+            tcmd="$scomp | $tcmd"
+        fi
+
+        set +e
+        timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+        set -e
+
+        if [ ${RC[0]} -ne 0 ]; then
+          wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \
+                          "Check ${DATA}/innobackup.backup.log"
+          exit 22
+        elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then 
+          wsrep_log_error "$tcmd finished with error: ${RC[1]}"
+          exit 22
+        fi
+
+        # innobackupex implicitly writes PID to fixed location in $xtmpdir
+        XTRABACKUP_PID="$xtmpdir/xtrabackup_pid"
+
+
+    else # BYPASS FOR IST
+
+        wsrep_log_info "Bypassing the SST for IST"
+        echo "continue" # now server can resume updating data
+        echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}"
+        echo "1" > "${DATA}/${IST_FILE}"
+        get_keys
+        if [[ $encrypt -eq 1 ]];then
+            if [[ -n $scomp ]];then 
+                tcmd=" $ecmd | $scomp | $tcmd "
+            else
+                tcmd=" $ecmd | $tcmd "
+            fi
+        elif [[ -n $scomp ]];then 
+            tcmd=" $scomp | $tcmd "
+        fi
+        strmcmd+=" \${IST_FILE}"
+
+        send_donor $DATA "${stagemsg}-IST"
+
+    fi
+
+    echo "done ${WSREP_SST_OPT_GTID}"
+    wsrep_log_info "Total time on donor: $totime seconds"
+
+elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ]
+then
+    [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
+    [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE
+
+    if [[ $speciald -eq 1 ]];then 
+        ib_home_dir=$(parse_cnf mysqld innodb-data-home-dir "")
+        ib_log_dir=$(parse_cnf mysqld innodb-log-group-home-dir "")
+        if [[ -z $ib_home_dir && -z $ib_log_dir ]];then 
+            speciald=0
+        fi
+    fi
+
+    stagemsg="Joiner-Recv"
+
+    if [[ ! -e ${DATA}/ibdata1 ]];then 
+        incremental=0
+    fi
+
+    if [[ $incremental -eq 1 ]];then 
+        wsrep_log_info "Incremental SST enabled: NOT SUPPORTED yet"
+        lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ')
+        wsrep_log_info "Recovered LSN: $lsn"
+    fi
+
+    sencrypted=1
+    nthreads=1
+
+    MODULE="xtrabackup_sst"
+
+    rm -f "${DATA}/${IST_FILE}"
+
+    # May need xtrabackup_checkpoints later on
+    rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info  ${DATA}/xtrabackup_logfile
+
+    ADDR=${WSREP_SST_OPT_ADDR}
+    if [ -z "${SST_PORT}" ]
+    then
+        SST_PORT=4444
+        ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}"
+    fi
+
+    wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} &
+
+    trap sig_joiner_cleanup HUP PIPE INT TERM
+    trap cleanup_joiner EXIT
+
+    if [[ -n $progress ]];then 
+        adjust_progress
+        tcmd+=" | $pcmd"
+    fi
+
+    if [[ $incremental -eq 1 ]];then 
+        BDATA=$DATA
+        DATA=$(mktemp -d)
+        MAGIC_FILE="${DATA}/${INFO_FILE}"
+    fi
+
+    get_keys
+    if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then
+        if [[ -n $sdecomp ]];then 
+            strmcmd=" $sdecomp | $ecmd | $strmcmd"
+        else 
+            strmcmd=" $ecmd | $strmcmd"
+        fi
+    elif [[ -n $sdecomp ]];then 
+            strmcmd=" $sdecomp | $strmcmd"
+    fi
+
+    STATDIR=$(mktemp -d)
+    MAGIC_FILE="${STATDIR}/${INFO_FILE}"
+    recv_joiner $STATDIR  "${stagemsg}-gtid" $stimeout
+
+    if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null
+    then
+        wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." 
+        exit 32
+    fi
+
+    if [ ! -r "${STATDIR}/${IST_FILE}" ]
+    then
+        wsrep_log_info "Proceeding with SST"
+
+        if [[ $speciald -eq 1 && -d ${DATA}/.sst ]];then 
+            wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous SST"
+        fi
+
+        if [[ $incremental -ne 1 ]];then 
+            if [[ $speciald -eq 1 ]];then 
+                wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories"
+                find $ib_home_dir $ib_log_dir $DATA -mindepth 1  -regex $cpat  -prune  -o -exec rm -rfv {} 1>&2 \+
+            else 
+                wsrep_log_info "Cleaning the existing datadir"
+                find $DATA -mindepth 1  -regex $cpat  -prune  -o -exec rm -rfv {} 1>&2 \+
+            fi
+            tempdir=$(parse_cnf mysqld log-bin "")
+            if [[ -n ${tempdir:-} ]];then
+                binlog_dir=$(dirname $tempdir)
+                binlog_file=$(basename $tempdir)
+                if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then
+                    pattern="$binlog_dir/$binlog_file\.[0-9]+$"
+                    wsrep_log_info "Cleaning the binlog directory $binlog_dir as well"
+                    find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+
+                    rm $binlog_dir/*.index || true
+                fi
+            fi
+
+        else
+            wsrep_log_info "Removing existing ib_logfile files"
+            rm -f ${BDATA}/ib_logfile*
+        fi
+
+
+        if [[ $speciald -eq 1 ]];then 
+            mkdir -p ${DATA}/.sst
+            TDATA=${DATA}
+            DATA="${DATA}/.sst"
+        fi
+
+
+        MAGIC_FILE="${DATA}/${INFO_FILE}"
+        recv_joiner $DATA "${stagemsg}-SST" 0
+
+        get_proc
+
+        # Rebuild indexes for compact backups
+        if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then 
+            wsrep_log_info "Index compaction detected"
+            rebuild=1
+        fi
+
+        if [[ $rebuild -eq 1 ]];then 
+            nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc)
+            wsrep_log_info "Rebuilding during prepare with $nthreads threads"
+            rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads"
+        fi
+
+        if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then
+
+            wsrep_log_info "Compressed qpress files found"
+
+            if [[ ! -x `which qpress` ]];then 
+                wsrep_log_error "qpress not found in path: $PATH"
+                exit 22
+            fi
+
+            if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then
+                count=$(find ${DATA} -type f -name '*.qp' | wc -l)
+                count=$(( count*2 ))
+                if pv --help | grep -q FORMAT;then 
+                    pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
+                else 
+                    pvopts="-f -s $count -l -N Decompression"
+                fi
+                pcmd="pv $pvopts"
+                adjust_progress
+                dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d"
+            else 
+                dcmd="xargs -n 2 qpress -T${nproc}d"
+            fi
+
+
+            # Decompress the qpress files 
+            wsrep_log_info "Decompression with $nproc threads"
+            timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
+            extcode=$?
+
+            if [[ $extcode -eq 0 ]];then
+                wsrep_log_info "Removing qpress files after decompression"
+                find ${DATA} -type f -name '*.qp' -delete 
+                if [[ $? -ne 0 ]];then 
+                    wsrep_log_error "Something went wrong with deletion of qpress files. Investigate"
+                fi
+            else
+                wsrep_log_error "Decompression failed. Exit code: $extcode"
+                exit 22
+            fi
+        fi
+
+
+        if  [[ ! -z $WSREP_SST_OPT_BINLOG ]];then
+
+            BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG)
+            BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG)
+
+            # To avoid comparing data directory and BINLOG_DIRNAME 
+            mv $DATA/${BINLOG_FILENAME}.* $BINLOG_DIRNAME/ 2>/dev/null || true
+
+            pushd $BINLOG_DIRNAME &>/dev/null
+            for bfiles in $(ls -1 ${BINLOG_FILENAME}.*);do
+                echo ${BINLOG_DIRNAME}/${bfiles} >> ${BINLOG_FILENAME}.index
+            done
+            popd &> /dev/null
+
+        fi
+
+        if [[ $incremental -eq 1 ]];then 
+            # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues.
+            INNOAPPLY="${INNOBACKUPEX_BIN} $disver --defaults-file=${WSREP_SST_OPT_CONF} \
+                --ibbackup=xtrabackup_56 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log"
+        fi
+
+        wsrep_log_info "Preparing the backup at ${DATA}"
+        timeit "Xtrabackup prepare stage" "$INNOAPPLY"
+
+        if [ $? -ne 0 ];
+        then
+            wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" 
+            exit 22
+        fi
+
+        if [[ $speciald -eq 1 ]];then 
+            MAGIC_FILE="${TDATA}/${INFO_FILE}"
+            set +e
+            rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log
+            set -e
+            wsrep_log_info "Moving the backup to ${TDATA}"
+            timeit "Xtrabackup move stage" "$INNOMOVE"
+            if [[ $? -eq 0 ]];then 
+                wsrep_log_info "Move successful, removing ${DATA}"
+                rm -rf $DATA
+                DATA=${TDATA}
+            else 
+                wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis"
+                wsrep_log_error "Check ${DATA}/innobackup.move.log for details"
+            fi
+        fi
+
+        if [[ $incremental -eq 1 ]];then 
+            wsrep_log_info "Cleaning up ${DATA} after incremental SST"
+            [[ -d ${DATA} ]] && rm -rf ${DATA}
+            DATA=$BDATA
+        fi
+
+    else 
+        wsrep_log_info "${IST_FILE} received from donor: Running IST"
+    fi
+
+    if [[ ! -r ${MAGIC_FILE} ]];then 
+        wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable"
+        exit 2
+    fi
+    cat "${MAGIC_FILE}" # output UUID:seqno
+    wsrep_log_info "Total time on joiner: $totime seconds"
+fi
+
+exit 0
diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh
new file mode 100644
index 00000000000..6b33eabee23
--- /dev/null
+++ b/scripts/wsrep_sst_xtrabackup.sh
@@ -0,0 +1,715 @@
+#!/bin/bash -ue
+# Copyright (C) 2013 Percona Inc
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# Optional dependencies and options documented here: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 
+# Make sure to read that before proceeding!
+
+
+
+
+. $(dirname $0)/wsrep_sst_common
+
+ealgo=""
+ekey=""
+ekeyfile=""
+encrypt=0
+nproc=1
+ecode=0
+XTRABACKUP_PID=""
+SST_PORT=""
+REMOTEIP=""
+tcert=""
+tpem=""
+sockopt=""
+progress=""
+ttime=0
+totime=0
+lsn=""
+incremental=0
+ecmd=""
+rlimit=""
+
+sfmt="tar"
+strmcmd=""
+tfmt=""
+tcmd=""
+rebuild=0
+rebuildcmd=""
+payload=0
+pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' "
+pvopts="-f  -i 10 -N $WSREP_SST_OPT_ROLE "
+uextra=0
+
+if which pv &>/dev/null && pv --help | grep -q FORMAT;then 
+    pvopts+=$pvformat
+fi
+pcmd="pv $pvopts"
+declare -a RC
+
+INNOBACKUPEX_BIN=innobackupex
+readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ })
+DATA="${WSREP_SST_OPT_DATA}"
+INFO_FILE="xtrabackup_galera_info"
+IST_FILE="xtrabackup_ist"
+MAGIC_FILE="${DATA}/${INFO_FILE}"
+
+# Setting the path for ss and ip
+export PATH="/usr/sbin:/sbin:$PATH"
+
+timeit(){
+    local stage=$1
+    shift
+    local cmd="$@"
+    local x1 x2 took extcode
+
+    if [[ $ttime -eq 1 ]];then 
+        x1=$(date +%s)
+        wsrep_log_info "Evaluating $cmd"
+        eval "$cmd"
+        extcode=$?
+        x2=$(date +%s)
+        took=$(( x2-x1 ))
+        wsrep_log_info "NOTE: $stage took $took seconds"
+        totime=$(( totime+took ))
+    else 
+        wsrep_log_info "Evaluating $cmd"
+        eval "$cmd"
+        extcode=$?
+    fi
+    return $extcode
+}
+
+get_keys()
+{
+    if [[ $encrypt -eq 2 ]];then 
+        return 
+    fi
+
+    if [[ $encrypt -eq 0 ]];then 
+        if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then
+            wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
+        fi
+        return
+    fi
+
+    if [[ $sfmt == 'tar' ]];then
+        wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format"
+        encrypt=0
+        return
+    fi
+
+    wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4"
+
+    if [[ -z $ealgo ]];then
+        wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out"
+        exit 3
+    fi
+
+    if [[ -z $ekey && ! -r $ekeyfile ]];then
+        wsrep_log_error "FATAL: Either key or keyfile must be readable"
+        exit 3
+    fi
+
+    if [[ -z $ekey ]];then
+        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile"
+    else
+        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey"
+    fi
+
+    if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
+        ecmd+=" -d"
+    fi
+}
+
+get_transfer()
+{
+    if [[ -z $SST_PORT ]];then 
+        TSST_PORT=4444
+    else 
+        TSST_PORT=$SST_PORT
+    fi
+
+    if [[ $tfmt == 'nc' ]];then
+        if [[ ! -x `which nc` ]];then 
+            wsrep_log_error "nc(netcat) not found in path: $PATH"
+            exit 2
+        fi
+        wsrep_log_info "Using netcat as streamer"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            tcmd="nc -dl ${TSST_PORT}"
+        else
+            tcmd="nc ${REMOTEIP} ${TSST_PORT}"
+        fi
+    else
+        tfmt='socat'
+        wsrep_log_info "Using socat as streamer"
+        if [[ ! -x `which socat` ]];then 
+            wsrep_log_error "socat not found in path: $PATH"
+            exit 2
+        fi
+
+        if [[ $encrypt -eq 2 ]] && ! socat -V | grep -q OPENSSL;then 
+            wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer"
+            encrypt=0
+        fi
+
+        if [[ $encrypt -eq 2 ]];then 
+            wsrep_log_info "Using openssl based encryption with socat"
+            if [[ -z $tpem || -z $tcert ]];then 
+                wsrep_log_error "Both PEM and CRT files required"
+                exit 22
+            fi
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert"
+                tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio"
+            else
+                wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert"
+                tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}"
+            fi
+        else 
+            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio"
+            else
+                tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}"
+            fi
+        fi
+    fi
+
+}
+
+parse_cnf()
+{
+    local group=$1
+    local var=$2
+    reval=$(my_print_defaults -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-)
+    if [[ -z $reval ]];then 
+        [[ -n $3 ]] && reval=$3
+    fi
+    echo $reval
+}
+
+get_footprint()
+{
+    pushd $WSREP_SST_OPT_DATA 1>/dev/null
+    payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }')
+    if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 
+        # QuickLZ has around 50% compression ratio
+        # When compression/compaction used, the progress is only an approximate.
+        payload=$(( payload*1/2 ))
+    fi
+    popd 1>/dev/null
+    pcmd+=" -s $payload"
+    adjust_progress
+}
+
+adjust_progress()
+{
+    if [[ -n $progress && $progress != '1' ]];then 
+        if [[ -e $progress ]];then 
+            pcmd+=" 2>>$progress"
+        else 
+            pcmd+=" 2>$progress"
+        fi
+    elif [[ -z $progress && -n $rlimit  ]];then 
+            # When rlimit is non-zero
+            pcmd="pv -q"
+    fi 
+
+    if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
+        wsrep_log_info "Rate-limiting SST to $rlimit"
+        pcmd+=" -L \$rlimit"
+    fi
+}
+
+read_cnf()
+{
+    sfmt=$(parse_cnf sst streamfmt "tar")
+    tfmt=$(parse_cnf sst transferfmt "socat")
+    tcert=$(parse_cnf sst tca "")
+    tpem=$(parse_cnf sst tcert "")
+    encrypt=$(parse_cnf sst encrypt 0)
+    sockopt=$(parse_cnf sst sockopt "")
+    progress=$(parse_cnf sst progress "")
+    rebuild=$(parse_cnf sst rebuild 0)
+    ttime=$(parse_cnf sst time 0)
+    incremental=$(parse_cnf sst incremental 0)
+    ealgo=$(parse_cnf xtrabackup encrypt "")
+    ekey=$(parse_cnf xtrabackup encrypt-key "")
+    ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "")
+
+    # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 
+    if [[ -z $ealgo ]];then
+        ealgo=$(parse_cnf sst encrypt-algo "")
+        ekey=$(parse_cnf sst encrypt-key "")
+        ekeyfile=$(parse_cnf sst encrypt-key-file "")
+    fi
+    rlimit=$(parse_cnf sst rlimit "")
+    uextra=$(parse_cnf sst use_extra 0)
+}
+
+get_stream()
+{
+    if [[ $sfmt == 'xbstream' ]];then 
+        wsrep_log_info "Streaming with xbstream"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            strmcmd="xbstream -x"
+        else
+            strmcmd="xbstream -c \${INFO_FILE} \${IST_FILE}"
+        fi
+    else
+        sfmt="tar"
+        wsrep_log_info "Streaming with tar"
+        if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+            strmcmd="tar xfi - --recursive-unlink -h"
+        else
+            strmcmd="tar cf - \${INFO_FILE} \${IST_FILE}"
+        fi
+
+    fi
+}
+
+get_proc()
+{
+    set +e
+    nproc=$(grep -c processor /proc/cpuinfo)
+    [[ -z $nproc || $nproc -eq 0 ]] && nproc=1
+    set -e
+}
+
+sig_joiner_cleanup()
+{
+    wsrep_log_error "Removing $MAGIC_FILE file due to signal"
+    rm -f "$MAGIC_FILE"
+}
+
+cleanup_joiner()
+{
+    # Since this is invoked just after exit NNN
+    local estatus=$?
+    if [[ $estatus -ne 0 ]];then 
+        wsrep_log_error "Cleanup after exit with status:$estatus"
+    fi
+    if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then
+        wsrep_log_info "Removing the sst_in_progress file"
+        wsrep_cleanup_progress_file
+    fi
+    if [[ -n $progress && -p $progress ]];then 
+        wsrep_log_info "Cleaning up fifo file $progress"
+        rm $progress
+    fi
+}
+
+check_pid()
+{
+    local pid_file="$1"
+    [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1
+}
+
+cleanup_donor()
+{
+    # Since this is invoked just after exit NNN
+    local estatus=$?
+    if [[ $estatus -ne 0 ]];then 
+        wsrep_log_error "Cleanup after exit with status:$estatus"
+    fi
+
+    if [[ -n $XTRABACKUP_PID ]];then 
+        if check_pid $XTRABACKUP_PID
+        then
+            wsrep_log_error "xtrabackup process is still running. Killing... "
+            kill_xtrabackup
+        fi
+
+        rm -f $XTRABACKUP_PID 
+    fi
+    rm -f ${DATA}/${IST_FILE}
+
+    if [[ -n $progress && -p $progress ]];then 
+        wsrep_log_info "Cleaning up fifo file $progress"
+        rm $progress
+    fi
+}
+
+kill_xtrabackup()
+{
+    local PID=$(cat $XTRABACKUP_PID)
+    [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
+    rm -f "$XTRABACKUP_PID"
+}
+
+setup_ports()
+{
+    if [[ "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
+        SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }')
+        REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }')
+        lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }')
+    else
+        SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }')
+    fi
+}
+
+# waits ~10 seconds for nc to open the port and then reports ready
+# (regardless of timeout)
+wait_for_listen()
+{
+    local PORT=$1
+    local ADDR=$2
+    local MODULE=$3
+    for i in {1..50}
+    do
+        ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break
+        sleep 0.2
+    done
+    if [[ $incremental -eq 1 ]];then 
+        echo "ready ${ADDR}/${MODULE}/$lsn"
+    else 
+    echo "ready ${ADDR}/${MODULE}"
+    fi
+}
+
+check_extra()
+{
+    local use_socket=1
+    if [[ $uextra -eq 1 ]];then 
+        if my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 
+            local eport=$(my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2)
+            if [[ -n $eport ]];then 
+                # Xtrabackup works only locally.
+                # Hence, setting host to 127.0.0.1 unconditionally. 
+                wsrep_log_info "SST through extra_port $eport"
+                INNOEXTRA+=" --host=127.0.0.1 --port=$eport "
+                use_socket=0
+            else 
+                wsrep_log_error "Extra port $eport null, failing"
+                exit 1
+            fi
+        else 
+            wsrep_log_info "Thread pool not set, ignore the option use_extra"
+        fi
+    fi
+    if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then
+        INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}"
+    fi
+}
+
+if [[ ! -x `which innobackupex` ]];then 
+    wsrep_log_error "innobackupex not in path: $PATH"
+    exit 2
+fi
+
+rm -f "${MAGIC_FILE}"
+
+if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then 
+    wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}"
+    exit 22
+fi
+
+read_cnf
+setup_ports
+get_stream
+get_transfer
+
+INNOEXTRA=""
+INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log"
+INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \$INNOEXTRA --galera-info --stream=\$sfmt \${TMPDIR} 2>\${DATA}/innobackup.backup.log"
+
+if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
+then
+    trap cleanup_donor EXIT
+
+    if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
+    then
+        TMPDIR="${TMPDIR:-/tmp}"
+
+        if [ "${AUTH[0]}" != "(null)" ]; then
+           INNOEXTRA+=" --user=${AUTH[0]}"
+       fi
+
+        if [ ${#AUTH[*]} -eq 2 ]; then
+           INNOEXTRA+=" --password=${AUTH[1]}"
+        elif [ "${AUTH[0]}" != "(null)" ]; then
+           # Empty password, used for testing, debugging etc.
+           INNOEXTRA+=" --password="
+        fi
+
+        get_keys
+        if [[ $encrypt -eq 1 ]];then
+            if [[ -n $ekey ]];then
+                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey "
+            else 
+                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile "
+            fi
+        fi
+
+        if [[ -n $lsn ]];then 
+                INNOEXTRA+=" --incremental --incremental-lsn=$lsn "
+        fi
+
+        check_extra
+
+        wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT}"
+
+        if [[ -n $progress ]];then 
+            get_footprint
+            tcmd="$pcmd | $tcmd"
+        elif [[ -n $rlimit ]];then 
+            adjust_progress
+            tcmd="$pcmd | $tcmd"
+        fi
+
+        set +e
+        timeit "Donor-Transfer" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+        set -e
+
+        if [ ${RC[0]} -ne 0 ]; then
+          wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \
+                          "Check ${DATA}/innobackup.backup.log"
+          exit 22
+        elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then 
+          wsrep_log_error "$tcmd finished with error: ${RC[1]}"
+          exit 22
+        fi
+
+        # innobackupex implicitly writes PID to fixed location in ${TMPDIR}
+        XTRABACKUP_PID="${TMPDIR}/xtrabackup_pid"
+
+
+    else # BYPASS FOR IST
+
+        wsrep_log_info "Bypassing the SST for IST"
+        STATE="${WSREP_SST_OPT_GTID}"
+        echo "continue" # now server can resume updating data
+        echo "${STATE}" > "${MAGIC_FILE}"
+        echo "1" > "${DATA}/${IST_FILE}"
+        get_keys
+        pushd ${DATA} 1>/dev/null
+        set +e
+        if [[ $encrypt -eq 1 ]];then
+            tcmd=" $ecmd | $tcmd"
+        fi
+        timeit "Donor-IST-Unencrypted-transfer" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+        set -e
+        popd 1>/dev/null
+
+        for ecode in "${RC[@]}";do 
+            if [[ $ecode -ne 0 ]];then 
+                wsrep_log_error "Error while streaming data to joiner node: " \
+                                "exit codes: ${RC[@]}"
+                exit 1
+            fi
+        done
+    fi
+
+    echo "done ${WSREP_SST_OPT_GTID}"
+    wsrep_log_info "Total time on donor: $totime seconds"
+
+elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ]
+then
+    [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
+    touch $SST_PROGRESS_FILE
+
+    if [[ ! -e ${DATA}/ibdata1 ]];then 
+        incremental=0
+    fi
+
+    if [[ $incremental -eq 1 ]];then 
+        wsrep_log_info "Incremental SST enabled"
+        #lsn=$(/pxc/bin/mysqld --defaults-file=$WSREP_SST_OPT_CONF  --basedir=/pxc  --wsrep-recover 2>&1 | grep -o 'log sequence number .*' | cut -d " " -f 4 | head -1)
+        lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ')
+        wsrep_log_info "Recovered LSN: $lsn"
+    fi
+
+    sencrypted=1
+    nthreads=1
+
+    MODULE="xtrabackup_sst"
+
+    # May need xtrabackup_checkpoints later on
+    rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info  ${DATA}/xtrabackup_logfile
+
+    ADDR=${WSREP_SST_OPT_ADDR}
+    if [ -z "${SST_PORT}" ]
+    then
+        SST_PORT=4444
+        ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}"
+    fi
+
+    wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} &
+
+    trap sig_joiner_cleanup HUP PIPE INT TERM
+    trap cleanup_joiner EXIT
+
+    if [[ -n $progress ]];then 
+        adjust_progress
+        tcmd+=" | $pcmd"
+    fi
+
+    if [[ $incremental -eq 1 ]];then 
+        BDATA=$DATA
+        DATA=$(mktemp -d)
+        MAGIC_FILE="${DATA}/${INFO_FILE}"
+    fi
+
+    get_keys
+    set +e
+    if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then
+        strmcmd=" $ecmd | $strmcmd"
+    fi
+
+    pushd ${DATA} 1>/dev/null
+    timeit "Joiner-Recv-Unencrypted" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+    popd 1>/dev/null 
+
+    set -e
+
+    if [[ $sfmt == 'xbstream' ]];then 
+        # Special handling till lp:1193240 is fixed"
+        if [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then 
+            wsrep_log_error "Xbstream failed"
+            wsrep_log_error "Data directory ${DATA} may not be empty: lp:1193240" \
+                            "Manual intervention required in that case"
+            exit 32
+        fi
+    fi
+
+    wait %% # join for wait_for_listen thread
+
+    for ecode in "${RC[@]}";do 
+        if [[ $ecode -ne 0 ]];then 
+            wsrep_log_error "Error while getting data from donor node: " \
+                            "exit codes: ${RC[@]}"
+            exit 32
+        fi
+    done
+
+    if [ ! -r "${MAGIC_FILE}" ]
+    then
+        # this message should cause joiner to abort
+        wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'"
+        wsrep_log_info "Contents of datadir" 
+        wsrep_log_info "$(ls -l ${DATA}/**/*)"
+        exit 32
+    fi
+
+    if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null
+    then
+        wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." 
+        exit 32
+    fi
+
+    if [ ! -r "${DATA}/${IST_FILE}" ]
+    then
+        wsrep_log_info "Proceeding with SST"
+        wsrep_log_info "Removing existing ib_logfile files"
+        if [[ $incremental -ne 1 ]];then 
+            rm -f ${DATA}/ib_logfile*
+        else
+            rm -f ${BDATA}/ib_logfile*
+        fi
+
+        get_proc
+
+        # Rebuild indexes for compact backups
+        if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then 
+            wsrep_log_info "Index compaction detected"
+            rebuild=1
+        fi
+
+        if [[ $rebuild -eq 1 ]];then 
+            nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc)
+            wsrep_log_info "Rebuilding during prepare with $nthreads threads"
+            rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads"
+        fi
+
+        if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then
+
+            wsrep_log_info "Compressed qpress files found"
+
+            if [[ ! -x `which qpress` ]];then 
+                wsrep_log_error "qpress not found in path: $PATH"
+                exit 22
+            fi
+
+            if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then
+                count=$(find ${DATA} -type f -name '*.qp' | wc -l)
+                count=$(( count*2 ))
+                if pv --help | grep -q FORMAT;then 
+                    pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
+                else 
+                    pvopts="-f -s $count -l -N Decompression"
+                fi
+                pcmd="pv $pvopts"
+                adjust_progress
+                dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d"
+            else 
+                dcmd="xargs -n 2 qpress -T${nproc}d"
+            fi
+
+            wsrep_log_info "Removing existing ibdata1 file"
+            rm -f ${DATA}/ibdata1
+
+            # Decompress the qpress files 
+            wsrep_log_info "Decompression with $nproc threads"
+            timeit "Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
+            extcode=$?
+
+            if [[ $extcode -eq 0 ]];then
+                wsrep_log_info "Removing qpress files after decompression"
+                find ${DATA} -type f -name '*.qp' -delete 
+                if [[ $? -ne 0 ]];then 
+                    wsrep_log_error "Something went wrong with deletion of qpress files. Investigate"
+                fi
+            else
+                wsrep_log_error "Decompression failed. Exit code: $extcode"
+                exit 22
+            fi
+        fi
+
+        if [[ $incremental -eq 1 ]];then 
+            # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues.
+            INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \
+                --ibbackup=xtrabackup_55 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log"
+        fi
+
+        wsrep_log_info "Preparing the backup at ${DATA}"
+        timeit "Xtrabackup prepare stage" "$INNOAPPLY"
+
+        if [[ $incremental -eq 1 ]];then 
+            wsrep_log_info "Cleaning up ${DATA} after incremental SST"
+            [[ -d ${DATA} ]] && rm -rf ${DATA}
+            DATA=$BDATA
+        fi
+
+        if [ $? -ne 0 ];
+        then
+            wsrep_log_error "${INNOBACKUPEX_BIN} finished with errors. Check ${DATA}/innobackup.prepare.log" 
+            exit 22
+        fi
+    else 
+        wsrep_log_info "${IST_FILE} received from donor: Running IST"
+    fi
+
+    if [[ ! -r ${MAGIC_FILE} ]];then
+        wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable"
+        exit 2
+    fi
+
+    cat "${MAGIC_FILE}" # output UUID:seqno
+    wsrep_log_info "Total time on joiner: $totime seconds"
+fi
+
+exit 0
diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
index 1eb8dc7cdd6..803e14988d2 100644
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@ -13,6 +13,25 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
+
+IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY)
+  BUILD_WITH_WSREP()
+  SET(WSREP_INCLUDES ${CMAKE_SOURCE_DIR}/wsrep)
+  SET(WSREP_SOURCES
+    wsrep_check_opts.cc
+    wsrep_hton.cc
+    wsrep_mysqld.cc 
+    wsrep_notify.cc
+    wsrep_sst.cc
+    wsrep_utils.cc
+    wsrep_var.cc
+    wsrep_binlog.cc
+    wsrep_applier.cc
+    wsrep_thd.cc
+  )
+  SET(WSREP_LIB wsrep)
+ENDIF()
+
 INCLUDE_DIRECTORIES(
 ${CMAKE_SOURCE_DIR}/include 
 ${CMAKE_SOURCE_DIR}/sql 
@@ -20,6 +39,7 @@ ${PCRE_INCLUDES}
 ${ZLIB_INCLUDE_DIR}
 ${SSL_INCLUDE_DIRS}
 ${CMAKE_BINARY_DIR}/sql
+${WSREP_INCLUDES}
 )
 
 SET(GEN_SOURCES
@@ -91,6 +111,7 @@ SET (SQL_SOURCE
 			   ../sql-common/mysql_async.c
                my_apc.cc my_apc.h
                rpl_gtid.cc rpl_parallel.cc
+	       ${WSREP_SOURCES}
                table_cache.cc
                ${CMAKE_CURRENT_BINARY_DIR}/sql_builtin.cc
                ${GEN_SOURCES}
@@ -112,6 +133,7 @@ DTRACE_INSTRUMENT(sql)
 TARGET_LINK_LIBRARIES(sql ${MYSQLD_STATIC_PLUGIN_LIBS} 
   mysys mysys_ssl dbug strings vio pcre ${LIBJEMALLOC}
   ${LIBWRAP} ${LIBCRYPT} ${LIBDL} ${CMAKE_THREAD_LIBS_INIT}
+  ${WSREP_LIB}
   ${SSL_LIBRARIES})
 
 IF(WIN32)
diff --git a/sql/event_data_objects.cc b/sql/event_data_objects.cc
index d65180a60be..26d2e1ef985 100644
--- a/sql/event_data_objects.cc
+++ b/sql/event_data_objects.cc
@@ -1472,8 +1472,25 @@ end:
       bool save_tx_read_only= thd->tx_read_only;
       thd->tx_read_only= false;
 
+#ifdef WITH_WSREP
+      if (WSREP(thd))
+      {
+        // sql_print_information("sizeof(LEX) = %d", sizeof(struct LEX));
+        // sizeof(LEX) = 4512, so it's relatively safe to allocate it on stack.
+        LEX lex;
+        LEX* saved = thd->lex;
+        lex.sql_command = SQLCOM_DROP_EVENT;
+        thd->lex = &lex;
+        WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL);
+        thd->lex = saved;
+      }
+#endif
+
       ret= Events::drop_event(thd, dbname, name, FALSE);
 
+      WSREP_TO_ISOLATION_END;
+
+  error:
       thd->tx_read_only= save_tx_read_only;
       thd->security_ctx->master_access= saved_master_access;
     }
diff --git a/sql/events.cc b/sql/events.cc
index 63627b21777..725a7613e69 100644
--- a/sql/events.cc
+++ b/sql/events.cc
@@ -1128,7 +1128,20 @@ Events::load_events_from_db(THD *thd)
       delete et;
       goto end;
     }
-
+#ifdef WITH_WSREP
+    // when SST from master node who initials event, the event status is ENABLED
+    // this is problematic because there are two nodes with same events and both enabled.
+    if (WSREP(thd) && et->originator != thd->variables.server_id)
+    {
+        store_record(table, record[1]);
+        table->field[ET_FIELD_STATUS]->
+                store((longlong) Event_parse_data::SLAVESIDE_DISABLED,
+                      TRUE);
+        (void) table->file->ha_update_row(table->record[1], table->record[0]);
+        delete et;
+        continue;
+    }
+#endif
     /**
       Since the Event_queue_element object could be deleted inside
       Event_queue::create_event we should save the value of dropped flag
@@ -1174,6 +1187,47 @@ end:
   DBUG_RETURN(ret);
 }
 
+#ifdef WITH_WSREP
+int wsrep_create_event_query(THD *thd, uchar** buf, size_t* buf_len)
+{
+  char buffer[1024];
+  String log_query(buffer, sizeof(buffer), &my_charset_bin);
+
+  if (create_query_string(thd, &log_query))
+  {
+    WSREP_WARN("events create string failed: %s", thd->query());
+    return 1;
+  }
+  return wsrep_to_buf_helper(thd, log_query.ptr(), log_query.length(), buf, buf_len);
+}
+static int
+wsrep_alter_query_string(THD *thd, String *buf)
+{
+  /* Append the "ALTER" part of the query */
+  if (buf->append(STRING_WITH_LEN("ALTER ")))
+    return 1;
+  /* Append definer */
+  append_definer(thd, buf, &(thd->lex->definer->user), &(thd->lex->definer->host));
+  /* Append the left part of thd->query after event name part */
+  if (buf->append(thd->lex->stmt_definition_begin,
+                  thd->lex->stmt_definition_end -
+                  thd->lex->stmt_definition_begin))
+    return 1;
+ 
+  return 0;
+}
+int wsrep_alter_event_query(THD *thd, uchar** buf, size_t* buf_len)
+{
+  String log_query;
+
+  if (wsrep_alter_query_string(thd, &log_query))
+  {
+    WSREP_WARN("events alter string failed: %s", thd->query());
+    return 1;
+  }
+  return wsrep_to_buf_helper(thd, log_query.ptr(), log_query.length(), buf, buf_len);
+}
+#endif /* WITH_WSREP */
 /**
   @} (End of group Event_Scheduler)
 */
diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc
index f503b2f54e5..23af3825756 100644
--- a/sql/ha_partition.cc
+++ b/sql/ha_partition.cc
@@ -388,6 +388,15 @@ const char *ha_partition::table_type() const
 }
 
 
+#if defined(WITH_WSREP) && !defined(EMBEDDED_LIBRARY)
+int ha_partition::wsrep_db_type() const
+{ 
+  // we can do this since we only support a single engine type
+  return partition_ht()->db_type;
+}
+#endif /* WITH_WSREP && !EMBEDDED_LIBRARY */
+
+
 /*
   Destructor method
 
diff --git a/sql/ha_partition.h b/sql/ha_partition.h
index 71ae84b06a0..c3c72394374 100644
--- a/sql/ha_partition.h
+++ b/sql/ha_partition.h
@@ -1282,7 +1282,9 @@ public:
       DBUG_ASSERT(h == m_file[i]->ht);
     return h;
   }
-
+#if defined(WITH_WSREP) && !defined(EMBEDDED_LIBRARY)
+  virtual int wsrep_db_type() const;
+#endif /* WITH_WSREP && !EMBEDDED_LIBRARY */
 
   friend int cmp_key_rowid_part_id(void *ptr, uchar *ref1, uchar *ref2);
 };
diff --git a/sql/handler.cc b/sql/handler.cc
index 56e7da6430d..63ddb9c6d6e 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -50,6 +50,9 @@
 #include "../storage/maria/ha_maria.h"
 #endif
 
+#include "wsrep_mysqld.h"
+#include "wsrep.h"
+
 /*
   While we have legacy_db_type, we have this array to
   check for dups and to find handlerton from legacy_db_type.
@@ -1165,10 +1168,25 @@ int ha_prepare(THD *thd)
       {
         if ((err= ht->prepare(ht, thd, all)))
         {
-          my_error(ER_ERROR_DURING_COMMIT, MYF(0), err);
-          ha_rollback_trans(thd, all);
-          error=1;
-          break;
+#ifdef WITH_WSREP
+          if (ht == wsrep_hton)
+          {
+            error= 1;
+            /* avoid sending error, if we need to replay */
+            if (thd->wsrep_conflict_state!= MUST_REPLAY)
+            {
+              my_error(ER_LOCK_DEADLOCK, MYF(0), err);
+            }
+          }
+          else
+#endif
+          {
+            /* not wsrep hton, bail to native mysql behavior */
+            my_error(ER_ERROR_DURING_COMMIT, MYF(0), err);
+            ha_rollback_trans(thd, all);
+            error=1;
+            break;
+          }
         }
       }
       else
@@ -1366,8 +1384,9 @@ int ha_commit_trans(THD *thd, bool all)
     mdl_request.init(MDL_key::COMMIT, "", "", MDL_INTENTION_EXCLUSIVE,
                      MDL_EXPLICIT);
 
-    if (thd->mdl_context.acquire_lock(&mdl_request,
-                                      thd->variables.lock_wait_timeout))
+    if (IF_WSREP(!WSREP(thd),1) &&
+      thd->mdl_context.acquire_lock(&mdl_request,
+                                    thd->variables.lock_wait_timeout))
     {
       ha_rollback_trans(thd, all);
       thd->wakeup_subsequent_commits(1);
@@ -1414,8 +1433,28 @@ int ha_commit_trans(THD *thd, bool all)
     err= ht->prepare(ht, thd, all);
     status_var_increment(thd->status_var.ha_prepare_count);
     if (err)
-      my_error(ER_ERROR_DURING_COMMIT, MYF(0), err);
-
+    {
+#ifdef WITH_WSREP
+      if (ht == wsrep_hton)
+      {
+        switch (err) {
+        case WSREP_TRX_SIZE_EXCEEDED:
+          /* give user size exeeded error from wsrep_api.h */
+          my_error(ER_ERROR_DURING_COMMIT, MYF(0), WSREP_SIZE_EXCEEDED);
+          break;
+        case WSREP_TRX_CERT_FAIL:
+        case WSREP_TRX_ERROR:
+          /* avoid sending error, if we need to replay */
+          if (thd->wsrep_conflict_state!= MUST_REPLAY)
+          {
+            my_error(ER_LOCK_DEADLOCK, MYF(0), err);
+          }
+        }
+        goto err;
+      }
+#endif /* WITH_WSREP */
+        my_error(ER_ERROR_DURING_COMMIT, MYF(0), err);
+    }
     if (err)
       goto err;
 
@@ -1425,6 +1464,14 @@ int ha_commit_trans(THD *thd, bool all)
   DEBUG_SYNC(thd, "ha_commit_trans_after_prepare");
   DBUG_EXECUTE_IF("crash_commit_after_prepare", DBUG_SUICIDE(););
 
+#ifdef WITH_WSREP
+  if (!error && WSREP_ON && wsrep_is_wsrep_xid(&thd->transaction.xid_state.xid))
+  {
+    // xid was rewritten by wsrep
+    xid= wsrep_xid_seqno(&thd->transaction.xid_state.xid);
+  }
+#endif // WITH_WSREP
+
   if (!is_real_trans)
   {
     error= commit_one_phase_2(thd, all, trans, is_real_trans);
@@ -1801,7 +1848,10 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin,
                             got, hton_name(hton)->str);
       for (int i=0; i < got; i ++)
       {
-        my_xid x=info->list[i].get_my_xid();
+        my_xid x= IF_WSREP(WSREP_ON && wsrep_is_wsrep_xid(&info->list[i]) ?
+                           wsrep_xid_seqno(&info->list[i]) :
+                           info->list[i].get_my_xid(),
+		           info->list[i].get_my_xid());
         if (!x) // not "mine" - that is generated by external TM
         {
 #ifndef DBUG_OFF
@@ -3086,10 +3136,12 @@ int handler::update_auto_increment()
                                           variables->auto_increment_increment);
     auto_inc_intervals_count++;
     /* Row-based replication does not need to store intervals in binlog */
-    if (mysql_bin_log.is_open() && !thd->is_current_stmt_binlog_format_row())
-        thd->auto_inc_intervals_in_cur_stmt_for_binlog.append(auto_inc_interval_for_cur_row.minimum(),
-                                                              auto_inc_interval_for_cur_row.values(),
-                                                              variables->auto_increment_increment);
+    if (IF_WSREP(((WSREP(thd) && wsrep_emulate_bin_log ) || mysql_bin_log.is_open()), mysql_bin_log.is_open())
+        && !thd->is_current_stmt_binlog_format_row())
+      thd->auto_inc_intervals_in_cur_stmt_for_binlog.
+        append(auto_inc_interval_for_cur_row.minimum(),
+               auto_inc_interval_for_cur_row.values(),
+               variables->auto_increment_increment);
   }
 
   /*
@@ -5707,7 +5759,10 @@ static bool check_table_binlog_row_based(THD *thd, TABLE *table)
   return (thd->is_current_stmt_binlog_format_row() &&
           table->s->cached_row_logging_check &&
           (thd->variables.option_bits & OPTION_BIN_LOG) &&
-          mysql_bin_log.is_open());
+          /* applier and replayer should not binlog */
+          (IF_WSREP((WSREP_EMULATE_BINLOG(thd) &&
+		    (thd->wsrep_exec_mode != REPL_RECV)) ||
+		     mysql_bin_log.is_open(), mysql_bin_log.is_open())));
 }
 
 
@@ -5807,6 +5862,12 @@ static int binlog_log_row(TABLE* table,
   bool error= 0;
   THD *const thd= table->in_use;
 
+#ifdef WITH_WSREP
+  /* only InnoDB tables will be replicated through binlog emulation */
+  if (WSREP_EMULATE_BINLOG(thd) &&
+      table->file->partition_ht()->db_type != DB_TYPE_INNODB)
+    return 0;
+#endif /* WITH_WSREP */
   if (check_table_binlog_row_based(thd, table))
   {
     MY_BITMAP cols;
@@ -6136,6 +6197,76 @@ void handler::set_lock_type(enum thr_lock_type lock)
   table->reginfo.lock_type= lock;
 }
 
+#ifdef WITH_WSREP
+/**
+  @details
+  This function makes the storage engine to force the victim transaction
+  to abort. Currently, only innodb has this functionality, but any SE
+  implementing the wsrep API should provide this service to support
+  multi-master operation.
+
+  @param bf_thd       brute force THD asking for the abort
+  @param victim_thd   victim THD to be aborted
+
+  @return
+    always 0
+*/
+
+int ha_abort_transaction(THD *bf_thd, THD *victim_thd, my_bool signal)
+{
+  DBUG_ENTER("ha_abort_transaction");
+  if (!WSREP(bf_thd) &&
+      !(wsrep_OSU_method_options == WSREP_OSU_RSU &&
+        bf_thd->wsrep_exec_mode == TOTAL_ORDER)) {
+    DBUG_RETURN(0);
+  }
+
+  THD_TRANS *trans= &victim_thd->transaction.all;
+  Ha_trx_info *ha_info= trans->ha_list, *ha_info_next;
+
+  for (; ha_info; ha_info= ha_info_next)
+  {
+    handlerton *hton= ha_info->ht();
+    if (!hton->abort_transaction)
+    {
+        WSREP_WARN("cannot abort transaction");
+    }
+    else
+	    hton->abort_transaction(hton, bf_thd, victim_thd, signal);
+    ha_info_next= ha_info->next();
+    ha_info->reset(); /* keep it conveniently zero-filled */
+  }
+  DBUG_RETURN(0);
+}
+
+void ha_fake_trx_id(THD *thd)
+{
+  DBUG_ENTER("ha_fake_trx_id");
+  if (!WSREP(thd))
+  {
+    DBUG_VOID_RETURN;
+  }
+
+  THD_TRANS *trans= &thd->transaction.all;
+  Ha_trx_info *ha_info= trans->ha_list, *ha_info_next;
+
+  for (; ha_info; ha_info= ha_info_next)
+  {
+    handlerton *hton= ha_info->ht();
+    if (!hton->fake_trx_id)
+    {
+      WSREP_WARN("cannot get fake InnoDB transaction ID");
+    }
+    else
+      hton->fake_trx_id(hton, thd);
+    ha_info_next= ha_info->next();
+    ha_info->reset(); /* keep it conveniently zero-filled */
+  }
+  DBUG_VOID_RETURN;
+}
+#endif /* WITH_WSREP */
+
+
 #ifdef TRANS_LOG_MGM_EXAMPLE_CODE
 /*
   Example of transaction log management functions based on assumption that logs
diff --git a/sql/handler.h b/sql/handler.h
index 1e4fe5557b6..169ed209403 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -1230,6 +1230,11 @@ struct handlerton
    enum handler_create_iterator_result
      (*create_iterator)(handlerton *hton, enum handler_iterator_type type,
                         struct handler_iterator *fill_this_in);
+   int (*abort_transaction)(handlerton *hton, THD *bf_thd,
+			    THD *victim_thd, my_bool signal);
+   int (*set_checkpoint)(handlerton *hton, const XID* xid);
+   int (*get_checkpoint)(handlerton *hton, XID* xid);
+   void (*fake_trx_id)(handlerton *hton, THD *thd);
    /*
      Optional clauses in the CREATE/ALTER TABLE
    */
@@ -3962,6 +3967,9 @@ bool key_uses_partial_cols(TABLE_SHARE *table, uint keyno);
 extern const char *ha_row_type[];
 extern MYSQL_PLUGIN_IMPORT const char *tx_isolation_names[];
 extern MYSQL_PLUGIN_IMPORT const char *binlog_format_names[];
+#ifdef WITH_WSREP
+extern MYSQL_PLUGIN_IMPORT const char *wsrep_binlog_format_names[];
+#endif /* WITH_WSREP */
 extern TYPELIB tx_isolation_typelib;
 extern const char *myisam_stats_method_names[];
 extern ulong total_ha, total_ha_2pc;
@@ -4080,6 +4088,10 @@ int ha_rollback_to_savepoint(THD *thd, SAVEPOINT *sv);
 bool ha_rollback_to_savepoint_can_release_mdl(THD *thd);
 int ha_savepoint(THD *thd, SAVEPOINT *sv);
 int ha_release_savepoint(THD *thd, SAVEPOINT *sv);
+#ifdef WITH_WSREP
+int ha_abort_transaction(THD *bf_thd, THD *victim_thd, my_bool signal);
+void ha_fake_trx_id(THD *thd);
+#endif /* WITH_WSREP */
 
 /* these are called by storage engines */
 void trans_register_ha(THD *thd, bool all, handlerton *ht);
@@ -4110,6 +4122,9 @@ int ha_binlog_end(THD *thd);
 #define ha_binlog_wait(a) do {} while (0)
 #define ha_binlog_end(a)  do {} while (0)
 #endif
+#ifdef WITH_WSREP
+void wsrep_brute_force_aborts();
+#endif
 
 const char *get_canonical_filename(handler *file, const char *path,
                                    char *tmp_path);
diff --git a/sql/item_func.cc b/sql/item_func.cc
index ccb7ec56021..13803cff2d6 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -2769,7 +2769,19 @@ void Item_func_rand::seed_random(Item *arg)
     TODO: do not do reinit 'rand' for every execute of PS/SP if
     args[0] is a constant.
   */
-  uint32 tmp= (uint32) arg->val_int();
+  uint32 tmp;
+#ifdef WITH_WSREP
+  if (WSREP(current_thd))
+  {
+    if (current_thd->wsrep_exec_mode==REPL_RECV)
+      tmp= current_thd->wsrep_rand;
+    else
+      tmp= current_thd->wsrep_rand= (uint32) arg->val_int();
+   }
+  else
+#endif /* WITH_WSREP */
+    tmp= (uint32) arg->val_int();
+
   my_rnd_init(rand, (uint32) (tmp*0x10001L+55555555L),
              (uint32) (tmp*0x10000001L));
 }
diff --git a/sql/lock.cc b/sql/lock.cc
index 54c7720e750..a74a12c41c3 100644
--- a/sql/lock.cc
+++ b/sql/lock.cc
@@ -83,6 +83,7 @@
 #include "sql_acl.h"                       // SUPER_ACL
 #include <hash.h>
 #include <assert.h>
+#include "wsrep_mysqld.h"
 
 /**
   @defgroup Locking Locking
@@ -314,6 +315,7 @@ bool mysql_lock_tables(THD *thd, MYSQL_LOCK *sql_lock, uint flags)
   /* Copy the lock data array. thr_multi_lock() reorders its contents. */
   memmove(sql_lock->locks + sql_lock->lock_count, sql_lock->locks,
           sql_lock->lock_count * sizeof(*sql_lock->locks));
+
   /* Lock on the copied half of the lock data array. */
   rc= thr_lock_errno_to_mysql[(int) thr_multi_lock(sql_lock->locks +
                                                    sql_lock->lock_count,
@@ -329,7 +331,10 @@ end:
   {
     thd->send_kill_message();
     if (!rc)
+    {
       mysql_unlock_tables(thd, sql_lock, 0);
+      THD_STAGE_INFO(thd, stage_after_table_lock);
+    }
     rc= 1;
   }
   else if (rc > 1)
@@ -380,6 +385,8 @@ static int lock_external(THD *thd, TABLE **tables, uint count)
 void mysql_unlock_tables(THD *thd, MYSQL_LOCK *sql_lock, bool free_lock)
 {
   DBUG_ENTER("mysql_unlock_tables");
+  THD_STAGE_INFO(thd, stage_unlocking_tables);
+
   if (sql_lock->table_count)
     unlock_external(thd, sql_lock->table, sql_lock->table_count);
   if (sql_lock->lock_count)
@@ -1052,6 +1059,13 @@ void Global_read_lock::unlock_global_read_lock(THD *thd)
   {
     thd->mdl_context.release_lock(m_mdl_blocks_commits_lock);
     m_mdl_blocks_commits_lock= NULL;
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+    {
+      wsrep_locked_seqno= WSREP_SEQNO_UNDEFINED;
+      wsrep->resume(wsrep);
+    }
+#endif /* WITH_WSREP */
   }
   thd->mdl_context.release_lock(m_mdl_global_shared_lock);
   m_mdl_global_shared_lock= NULL;
@@ -1084,9 +1098,22 @@ bool Global_read_lock::make_global_read_lock_block_commit(THD *thd)
     If we didn't succeed lock_global_read_lock(), or if we already suceeded
     make_global_read_lock_block_commit(), do nothing.
   */
+
   if (m_state != GRL_ACQUIRED)
     DBUG_RETURN(0);
 
+#ifdef WITH_WSREP
+  if (WSREP_ON && m_mdl_blocks_commits_lock)
+  {
+    WSREP_DEBUG("GRL was in block commit mode when entering "
+		"make_global_read_lock_block_commit");
+    thd->mdl_context.release_lock(m_mdl_blocks_commits_lock);
+    m_mdl_blocks_commits_lock= NULL;
+    wsrep_locked_seqno= WSREP_SEQNO_UNDEFINED;
+    wsrep->resume(wsrep);
+  }
+#endif /* WITH_WSREP */
+
   mdl_request.init(MDL_key::COMMIT, "", "", MDL_SHARED, MDL_EXPLICIT);
 
   if (thd->mdl_context.acquire_lock(&mdl_request,
@@ -1096,6 +1123,25 @@ bool Global_read_lock::make_global_read_lock_block_commit(THD *thd)
   m_mdl_blocks_commits_lock= mdl_request.ticket;
   m_state= GRL_ACQUIRED_AND_BLOCKS_COMMIT;
 
+#ifdef WITH_WSREP
+  if (WSREP_ON)
+  {
+    long long ret = wsrep->pause(wsrep);
+    if (ret >= 0)
+    {
+      wsrep_locked_seqno= ret;
+    }
+    else if (ret != -ENOSYS) /* -ENOSYS - no provider */
+    {
+      WSREP_ERROR("Failed to pause provider: %lld (%s)", -ret, strerror(-ret));
+
+      DBUG_ASSERT(m_mdl_blocks_commits_lock == NULL);
+      wsrep_locked_seqno= WSREP_SEQNO_UNDEFINED;
+      my_error(ER_LOCK_DEADLOCK, MYF(0));
+      DBUG_RETURN(TRUE);
+     }
+  }
+#endif /* WITH_WSREP */
   DBUG_RETURN(FALSE);
 }
 
diff --git a/sql/log.cc b/sql/log.cc
index 75a895e25f8..371f9c06cce 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -52,9 +52,11 @@
 
 #include "sql_plugin.h"
 #include "rpl_handler.h"
+#include "wsrep_mysqld.h"
 #include "debug_sync.h"
 #include "sql_show.h"
 #include "my_pthread.h"
+#include "wsrep_mysqld.h"
 
 /* max size of the log message */
 #define MAX_LOG_BUFFER_SIZE 1024
@@ -63,6 +65,7 @@
 
 #define FLAGSTR(V,F) ((V)&(F)?#F" ":"")
 
+handlerton *binlog_hton;
 LOGGER logger;
 
 MYSQL_BIN_LOG mysql_bin_log(&sync_binlog_period);
@@ -511,8 +514,6 @@ private:
   binlog_cache_mngr(const binlog_cache_mngr& info);
 };
 
-handlerton *binlog_hton;
-
 bool LOGGER::is_log_table_enabled(uint log_table_type)
 {
   switch (log_table_type) {
@@ -526,7 +527,6 @@ bool LOGGER::is_log_table_enabled(uint log_table_type)
   }
 }
 
-
 /**
    Check if a given table is opened log table
 
@@ -1577,7 +1577,8 @@ binlog_trans_log_savepos(THD *thd, my_off_t *pos)
   DBUG_ENTER("binlog_trans_log_savepos");
   DBUG_ASSERT(pos != NULL);
   binlog_cache_mngr *const cache_mngr= thd->binlog_setup_trx_data();
-  DBUG_ASSERT(mysql_bin_log.is_open());
+  DBUG_ASSERT_IF_WSREP(((WSREP(thd) && wsrep_emulate_bin_log)) || mysql_bin_log.is_open());
+  DBUG_ASSERT(IF_WSREP(1, mysql_bin_log.is_open()));
   *pos= cache_mngr->trx_cache.get_byte_position();
   DBUG_PRINT("return", ("*pos: %lu", (ulong) *pos));
   DBUG_VOID_RETURN;
@@ -1625,7 +1626,8 @@ binlog_trans_log_truncate(THD *thd, my_off_t pos)
 int binlog_init(void *p)
 {
   binlog_hton= (handlerton *)p;
-  binlog_hton->state=opt_bin_log ? SHOW_OPTION_YES : SHOW_OPTION_NO;
+  binlog_hton->state=IF_WSREP(WSREP_ON || opt_bin_log, opt_bin_log) ?
+    SHOW_OPTION_YES : SHOW_OPTION_NO;
   binlog_hton->db_type=DB_TYPE_BINLOG;
   binlog_hton->savepoint_offset= sizeof(my_off_t);
   binlog_hton->close_connection= binlog_close_connection;
@@ -1745,6 +1747,16 @@ binlog_commit_flush_stmt_cache(THD *thd, bool all,
                                binlog_cache_mngr *cache_mngr)
 {
   DBUG_ENTER("binlog_commit_flush_stmt_cache");
+#ifdef WITH_WSREP
+  if (thd->wsrep_mysql_replicated > 0)
+  {
+    DBUG_ASSERT(WSREP_ON);
+    WSREP_DEBUG("avoiding binlog_commit_flush_trx_cache: %d",
+                thd->wsrep_mysql_replicated);
+    return 0;
+  }
+#endif
+
   Query_log_event end_evt(thd, STRING_WITH_LEN("COMMIT"),
                           FALSE, TRUE, TRUE, 0);
   DBUG_RETURN(binlog_flush_cache(thd, cache_mngr, &end_evt, all, TRUE, FALSE));
@@ -1900,12 +1912,12 @@ static bool trans_cannot_safely_rollback(THD *thd, bool all)
 
   return ((thd->variables.option_bits & OPTION_KEEP_LOG) ||
           (trans_has_updated_non_trans_table(thd) &&
-           thd->variables.binlog_format == BINLOG_FORMAT_STMT) ||
+           WSREP_FORMAT(thd->variables.binlog_format) == BINLOG_FORMAT_STMT) ||
           (cache_mngr->trx_cache.changes_to_non_trans_temp_table() &&
-           thd->variables.binlog_format == BINLOG_FORMAT_MIXED) ||
+           WSREP_FORMAT(thd->variables.binlog_format) == BINLOG_FORMAT_MIXED) ||
           (trans_has_updated_non_trans_table(thd) &&
            ending_single_stmt_trans(thd,all) &&
-           thd->variables.binlog_format == BINLOG_FORMAT_MIXED));
+           WSREP_FORMAT(thd->variables.binlog_format) == BINLOG_FORMAT_MIXED));
 }
 
 
@@ -1927,6 +1939,13 @@ static int binlog_commit(handlerton *hton, THD *thd, bool all)
   DBUG_ENTER("binlog_commit");
   binlog_cache_mngr *const cache_mngr=
     (binlog_cache_mngr*) thd_get_ha_data(thd, binlog_hton);
+#ifdef WITH_WSREP
+  if (!cache_mngr)
+  {
+    DBUG_ASSERT(WSREP(thd));
+    DBUG_RETURN(0);
+  }
+#endif /* WITH_WSREP */
 
   DBUG_PRINT("debug",
              ("all: %d, in_transaction: %s, all.modified_non_trans_table: %s, stmt.modified_non_trans_table: %s",
@@ -1983,6 +2002,14 @@ static int binlog_rollback(handlerton *hton, THD *thd, bool all)
   int error= 0;
   binlog_cache_mngr *const cache_mngr=
     (binlog_cache_mngr*) thd_get_ha_data(thd, binlog_hton);
+#ifdef WITH_WSREP
+  if (!cache_mngr)
+  {
+    DBUG_ASSERT(WSREP(thd));
+    DBUG_RETURN(0);
+  }
+
+#endif /* WITH_WSREP */
 
   DBUG_PRINT("debug", ("all: %s, all.modified_non_trans_table: %s, stmt.modified_non_trans_table: %s",
                        YESNO(all),
@@ -2011,8 +2038,8 @@ static int binlog_rollback(handlerton *hton, THD *thd, bool all)
     cache_mngr->reset(false, true);
     DBUG_RETURN(error);
   }
-
-  if (mysql_bin_log.check_write_error(thd))
+  if (IF_WSREP(!wsrep_emulate_bin_log,1) &&
+      mysql_bin_log.check_write_error(thd))
   {
     /*
       "all == true" means that a "rollback statement" triggered the error and
@@ -2043,9 +2070,9 @@ static int binlog_rollback(handlerton *hton, THD *thd, bool all)
     else if (ending_trans(thd, all) ||
              (!(thd->variables.option_bits & OPTION_KEEP_LOG) &&
               (!stmt_has_updated_non_trans_table(thd) ||
-               thd->variables.binlog_format != BINLOG_FORMAT_STMT) &&
+               WSREP_FORMAT(thd->variables.binlog_format) != BINLOG_FORMAT_STMT) &&
               (!cache_mngr->trx_cache.changes_to_non_trans_temp_table() ||
-               thd->variables.binlog_format != BINLOG_FORMAT_MIXED)))
+               WSREP_FORMAT(thd->variables.binlog_format) != BINLOG_FORMAT_MIXED)))
       error= binlog_truncate_trx_cache(thd, cache_mngr, all);
   }
 
@@ -2150,8 +2177,13 @@ bool MYSQL_BIN_LOG::check_write_error(THD *thd)
 
 static int binlog_savepoint_set(handlerton *hton, THD *thd, void *sv)
 {
-  DBUG_ENTER("binlog_savepoint_set");
   int error= 1;
+  DBUG_ENTER("binlog_savepoint_set");
+
+#ifdef WITH_WSREP
+  if (wsrep_emulate_bin_log)
+    DBUG_RETURN(0);
+#endif /* WITH_WSREP */
 
   char buf[1024];
   String log_query(buf, sizeof(buf), &my_charset_bin);
@@ -2190,7 +2222,8 @@ static int binlog_savepoint_rollback(handlerton *hton, THD *thd, void *sv)
     non-transactional table. Otherwise, truncate the binlog cache starting
     from the SAVEPOINT command.
   */
-  if (unlikely(trans_has_updated_non_trans_table(thd) ||
+  if (IF_WSREP(!wsrep_emulate_bin_log, 1) &&
+      unlikely(trans_has_updated_non_trans_table(thd) ||
                (thd->variables.option_bits & OPTION_KEEP_LOG)))
   {
     char buf[1024];
@@ -2204,7 +2237,10 @@ static int binlog_savepoint_rollback(handlerton *hton, THD *thd, void *sv)
                           TRUE, FALSE, TRUE, errcode);
     DBUG_RETURN(mysql_bin_log.write(&qinfo));
   }
-  binlog_trans_log_truncate(thd, *(my_off_t*)sv);
+
+  if (IF_WSREP(!wsrep_emulate_bin_log, 1))
+    binlog_trans_log_truncate(thd, *(my_off_t*)sv);
+
   DBUG_RETURN(0);
 }
 
@@ -5332,7 +5368,8 @@ int THD::binlog_write_table_map(TABLE *table, bool is_transactional,
     is_transactional= 1;
   
   /* Pre-conditions */
-  DBUG_ASSERT(is_current_stmt_binlog_format_row() && mysql_bin_log.is_open());
+  DBUG_ASSERT(is_current_stmt_binlog_format_row());
+  DBUG_ASSERT(IF_WSREP(WSREP_EMULATE_BINLOG(this), 0) || mysql_bin_log.is_open());
   DBUG_ASSERT(table->s->table_map_id != ULONG_MAX);
 
   Table_map_log_event
@@ -5465,7 +5502,7 @@ MYSQL_BIN_LOG::flush_and_set_pending_rows_event(THD *thd,
                                                 bool is_transactional)
 {
   DBUG_ENTER("MYSQL_BIN_LOG::flush_and_set_pending_rows_event(event)");
-  DBUG_ASSERT(mysql_bin_log.is_open());
+  DBUG_ASSERT(IF_WSREP(WSREP_EMULATE_BINLOG(thd),0) || mysql_bin_log.is_open());
   DBUG_PRINT("enter", ("event: 0x%lx", (long) event));
 
   int error= 0;
@@ -5791,7 +5828,8 @@ bool MYSQL_BIN_LOG::write(Log_event *event_info, my_bool *with_annotate)
      mostly called if is_open() *was* true a few instructions before, but it
      could have changed since.
   */
-  if (likely(is_open()))
+  /* applier and replayer can skip writing binlog events */
+  if (IF_WSREP((WSREP_EMULATE_BINLOG(thd) && (thd->wsrep_exec_mode != REPL_RECV)) || is_open(), likely(is_open())))
   {
     my_off_t UNINIT_VAR(my_org_b_tell);
 #ifdef HAVE_REPLICATION
@@ -6130,6 +6168,17 @@ int MYSQL_BIN_LOG::rotate(bool force_rotate, bool* check_purge)
   int error= 0;
   DBUG_ENTER("MYSQL_BIN_LOG::rotate");
 
+#ifdef WITH_WSREP
+  if (wsrep_to_isolation)
+  {
+    DBUG_ASSERT(WSREP_ON);
+    *check_purge= false;
+    WSREP_DEBUG("avoiding binlog rotate due to TO isolation: %d", 
+                wsrep_to_isolation);
+    DBUG_RETURN(0);
+  }
+#endif
+
   //todo: fix the macro def and restore safe_mutex_assert_owner(&LOCK_log);
   *check_purge= false;
 
@@ -6675,6 +6724,11 @@ MYSQL_BIN_LOG::write_transaction_to_binlog(THD *thd,
   Ha_trx_info *ha_info;
   DBUG_ENTER("MYSQL_BIN_LOG::write_transaction_to_binlog");
 
+#ifdef WITH_WSREP
+  if (wsrep_emulate_bin_log)
+    DBUG_RETURN(0);
+#endif /* WITH_WSREP */
+
   entry.thd= thd;
   entry.cache_mngr= cache_mngr;
   entry.error= 0;
@@ -6683,6 +6737,7 @@ MYSQL_BIN_LOG::write_transaction_to_binlog(THD *thd,
   entry.using_trx_cache= using_trx_cache;
   entry.need_unlog= false;
   ha_info= all ? thd->transaction.all.ha_list : thd->transaction.stmt.ha_list;
+
   for (; ha_info; ha_info= ha_info->next())
   {
     if (ha_info->is_started() && ha_info->ht() != binlog_hton &&
@@ -8819,7 +8874,10 @@ TC_LOG_BINLOG::log_and_order(THD *thd, my_xid xid, bool all,
 
   binlog_cache_mngr *cache_mngr= thd->binlog_setup_trx_data();
   if (!cache_mngr)
+  {
+    WSREP_DEBUG("Skipping empty log_xid: %s", thd->query());
     DBUG_RETURN(0);
+  }
 
   cache_mngr->using_xa= TRUE;
   cache_mngr->xa_xid= xid;
@@ -9649,3 +9707,50 @@ maria_declare_plugin(binlog)
   MariaDB_PLUGIN_MATURITY_STABLE /* maturity */
 }
 maria_declare_plugin_end;
+
+#ifdef WITH_WSREP
+IO_CACHE * get_trans_log(THD * thd)
+{
+  binlog_cache_mngr *cache_mngr = (binlog_cache_mngr*)
+    thd_get_ha_data(thd, binlog_hton);
+  if (cache_mngr)
+    return cache_mngr->get_binlog_cache_log(true);
+
+  WSREP_DEBUG("binlog cache not initialized, conn :%ld", thd->thread_id);
+  return NULL;
+}
+
+
+bool wsrep_trans_cache_is_empty(THD *thd)
+{
+  binlog_cache_mngr *const cache_mngr=
+      (binlog_cache_mngr*) thd_get_ha_data(thd, binlog_hton);
+  return (!cache_mngr || cache_mngr->trx_cache.empty());
+}
+
+
+void thd_binlog_trx_reset(THD * thd)
+{
+  /*
+    todo: fix autocommit select to not call the caller
+  */
+  if (thd_get_ha_data(thd, binlog_hton) != NULL)
+  {
+    binlog_cache_mngr *const cache_mngr=
+      (binlog_cache_mngr*) thd_get_ha_data(thd, binlog_hton);
+    if (cache_mngr)
+      cache_mngr->reset(false, true);
+  }
+  thd->clear_binlog_table_maps();
+}
+
+
+void thd_binlog_rollback_stmt(THD * thd)
+{
+  WSREP_DEBUG("thd_binlog_rollback_stmt :%ld", thd->thread_id);
+  binlog_cache_mngr *const cache_mngr=
+    (binlog_cache_mngr*) thd_get_ha_data(thd, binlog_hton);
+  if (cache_mngr)
+    cache_mngr->trx_cache.set_prev_position(MY_OFF_T_UNDEF);
+}
+#endif /* WITH_WSREP */
diff --git a/sql/log.h b/sql/log.h
index 67fcf068ec4..3e9a11b8b94 100644
--- a/sql/log.h
+++ b/sql/log.h
@@ -19,6 +19,8 @@
 
 #include "unireg.h"                    // REQUIRED: for other includes
 #include "handler.h"                            /* my_xid */
+#include "wsrep.h"
+#include "wsrep_mysqld.h"
 
 class Relay_log_info;
 
@@ -106,7 +108,11 @@ public:
   int log_and_order(THD *thd, my_xid xid, bool all,
                     bool need_prepare_ordered, bool need_commit_ordered)
   {
-    DBUG_ASSERT(0 /* Internal error - TC_LOG_DUMMY::log_and_order() called */);
+    /*
+      If we are not using WSREP this is an Internal error
+      - TC_LOG_DUMMY::log_and_order() called
+    */
+    DBUG_ASSERT(IF_WSREP(1,0));
     return 1;
   }
   int unlog(ulong cookie, my_xid xid)  { return 0; }
diff --git a/sql/log_event.cc b/sql/log_event.cc
index 6a85893803e..8e5f7a1f23e 100644
--- a/sql/log_event.cc
+++ b/sql/log_event.cc
@@ -44,7 +44,7 @@
 #include <mysql/psi/mysql_statement.h>
 #include <strfunc.h>
 #include "compat56.h"
-
+#include "wsrep_mysqld.h"
 #endif /* MYSQL_CLIENT */
 
 #include <base64.h>
@@ -3092,6 +3092,15 @@ Query_log_event::Query_log_event(THD* thd_arg, const char* query_arg,
 {
   time_t end_time;
 
+#ifdef WITH_WSREP
+  /*
+    If Query_log_event will contain non trans keyword (not BEGIN, COMMIT,
+    SAVEPOINT or ROLLBACK) we disable PA for this transaction.
+   */
+  if (WSREP_ON && !is_trans_keyword())
+    thd->wsrep_PA_safe= false;
+#endif /* WITH_WSREP */
+
   memset(&user, 0, sizeof(user));
   memset(&host, 0, sizeof(host));
 
@@ -4519,6 +4528,22 @@ Query_log_event::do_shall_skip(rpl_group_info *rgi)
       DBUG_RETURN(Log_event::EVENT_SKIP_COUNT);
     }
   }
+#ifdef WITH_WSREP
+  else if (WSREP_ON && wsrep_mysql_replication_bundle && opt_slave_domain_parallel_threads == 0 &&
+           thd->wsrep_mysql_replicated > 0 &&
+           (is_begin() || is_commit()))
+  {
+    if (++thd->wsrep_mysql_replicated < (int)wsrep_mysql_replication_bundle)
+    {
+      WSREP_DEBUG("skipping wsrep commit %d", thd->wsrep_mysql_replicated);
+      DBUG_RETURN(Log_event::EVENT_SKIP_IGNORE);
+    }
+    else
+    {
+      thd->wsrep_mysql_replicated = 0;
+    }
+  }
+#endif
   DBUG_RETURN(Log_event::do_shall_skip(rgi));
 }
 
@@ -7348,6 +7373,21 @@ Xid_log_event::do_shall_skip(rpl_group_info *rgi)
     thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_GTID_BEGIN);
     DBUG_RETURN(Log_event::EVENT_SKIP_COUNT);
   }
+#ifdef WITH_WSREP
+  else if (wsrep_mysql_replication_bundle && WSREP_ON &&
+           opt_slave_domain_parallel_threads == 0)
+  {
+    if (++thd->wsrep_mysql_replicated < (int)wsrep_mysql_replication_bundle)
+    {
+      WSREP_DEBUG("skipping wsrep commit %d", thd->wsrep_mysql_replicated);
+      DBUG_RETURN(Log_event::EVENT_SKIP_IGNORE);
+    }
+    else
+    {
+      thd->wsrep_mysql_replicated = 0;
+    }
+  }
+#endif
   DBUG_RETURN(Log_event::do_shall_skip(rgi));
 }
 #endif /* !MYSQL_CLIENT */
@@ -9625,6 +9665,18 @@ int Rows_log_event::do_apply_event(rpl_group_info *rgi)
     if (open_and_lock_tables(thd, rgi->tables_to_lock, FALSE, 0))
     {
       uint actual_error= thd->get_stmt_da()->sql_errno();
+#ifdef WITH_WSREP
+      if (WSREP(thd))
+      {
+        WSREP_WARN("BF applier failed to open_and_lock_tables: %u, fatal: %d "
+                   "wsrep = (exec_mode: %d conflict_state: %d seqno: %lld)",
+		   thd->get_stmt_da()->sql_errno(),
+                   thd->is_fatal_error,
+                   thd->wsrep_exec_mode,
+                   thd->wsrep_conflict_state,
+                   (long long)wsrep_thd_trx_seqno(thd));
+      }
+#endif
       if (thd->is_slave_error || thd->is_fatal_error)
       {
         /*
@@ -10771,8 +10823,8 @@ check_table_map(rpl_group_info *rgi, RPL_TABLE_LIST *table_list)
   DBUG_ENTER("check_table_map");
   enum_tbl_map_status res= OK_TO_PROCESS;
   Relay_log_info *rli= rgi->rli;
-
-  if (rgi->thd->slave_thread /* filtering is for slave only */ &&
+   if ((rgi->thd->slave_thread /* filtering is for slave only */ ||
+        IF_WSREP((WSREP(rgi->thd) && rgi->thd->wsrep_applier), 0)) &&
       (!rli->mi->rpl_filter->db_ok(table_list->db) ||
        (rli->mi->rpl_filter->is_on() && !rli->mi->rpl_filter->tables_ok("", table_list))))
     res= FILTERED_OUT;
@@ -11520,7 +11572,19 @@ int
 Write_rows_log_event::do_exec_row(rpl_group_info *rgi)
 {
   DBUG_ASSERT(m_table != NULL);
+  const char *tmp= thd->get_proc_info();
+  const char *message= "Write_rows_log_event::write_row()";
+
+#ifdef WSREP_PROC_INFO
+  my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+              "Write_rows_log_event::write_row(%lld)",
+              (long long) wsrep_thd_trx_seqno(thd));
+  message= thd->wsrep_info;
+#endif /* WSREP_PROC_INFO */
+
+  thd_proc_info(thd, message);
   int error= write_row(rgi, slave_exec_mode == SLAVE_EXEC_MODE_IDEMPOTENT);
+  thd_proc_info(thd, tmp);
 
   if (error && !thd->is_error())
   {
@@ -12198,15 +12262,34 @@ Delete_rows_log_event::do_after_row_operations(const Slave_reporting_capability
 int Delete_rows_log_event::do_exec_row(rpl_group_info *rgi)
 {
   int error;
+  const char *tmp= thd->get_proc_info();
+  const char *message= "Delete_rows_log_event::find_row()";
   const bool invoke_triggers=
     slave_run_triggers_for_rbr && !master_had_triggers && m_table->triggers;
   DBUG_ASSERT(m_table != NULL);
 
+#ifdef WSREP_PROC_INFO
+  my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+              "Delete_rows_log_event::find_row(%lld)",
+              (long long) wsrep_thd_trx_seqno(thd));
+  message= thd->wsrep_info;
+#endif /* WSREP_PROC_INFO */
+
+  thd_proc_info(thd, message);
   if (!(error= find_row(rgi))) 
   { 
     /*
       Delete the record found, located in record[0]
     */
+    message= "Delete_rows_log_event::ha_delete_row()";
+#ifdef WSREP_PROC_INFO
+    snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+             "Delete_rows_log_event::ha_delete_row(%lld)",
+             (long long) wsrep_thd_trx_seqno(thd));
+    message= thd->wsrep_info;
+#endif
+    thd_proc_info(thd, message);
+
     if (invoke_triggers &&
         process_triggers(TRG_EVENT_DELETE, TRG_ACTION_BEFORE, FALSE))
       error= HA_ERR_GENERIC; // in case if error is not set yet
@@ -12217,6 +12300,7 @@ int Delete_rows_log_event::do_exec_row(rpl_group_info *rgi)
       error= HA_ERR_GENERIC; // in case if error is not set yet
     m_table->file->ha_index_or_rnd_end();
   }
+  thd_proc_info(thd, tmp);
   return error;
 }
 
@@ -12344,8 +12428,18 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi)
 {
   const bool invoke_triggers=
     slave_run_triggers_for_rbr && !master_had_triggers && m_table->triggers;
+  const char *tmp= thd->get_proc_info();
+  const char *message= "Update_rows_log_event::find_row()";
   DBUG_ASSERT(m_table != NULL);
 
+#ifdef WSREP_PROC_INFO
+  my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+              "Update_rows_log_event::find_row(%lld)",
+              (long long) wsrep_thd_trx_seqno(thd));
+  message= thd->wsrep_info;
+#endif /* WSREP_PROC_INFO */
+
+  thd_proc_info(thd, message);
   int error= find_row(rgi); 
   if (error)
   {
@@ -12355,6 +12449,7 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi)
     */
     m_curr_row= m_curr_row_end;
     unpack_current_row(rgi);
+    thd_proc_info(thd, tmp);
     return error;
   }
 
@@ -12372,7 +12467,16 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi)
   store_record(m_table,record[1]);
 
   m_curr_row= m_curr_row_end;
+  message= "Update_rows_log_event::unpack_current_row()";
+#ifdef WSREP_PROC_INFO
+  my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+              "Update_rows_log_event::unpack_current_row(%lld)",
+              (long long) wsrep_thd_trx_seqno(thd));
+  message= thd->wsrep_info;
+#endif /* WSREP_PROC_INFO */
+
   /* this also updates m_curr_row_end */
+  thd_proc_info(thd, message);
   if ((error= unpack_current_row(rgi)))
     goto err;
 
@@ -12390,6 +12494,15 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi)
   DBUG_DUMP("new values", m_table->record[0], m_table->s->reclength);
 #endif
 
+  message= "Update_rows_log_event::ha_update_row()";
+#ifdef WSREP_PROC_INFO
+  my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+              "Update_rows_log_event::ha_update_row(%lld)",
+              (long long) wsrep_thd_trx_seqno(thd));
+  message= thd->wsrep_info;
+#endif /* WSREP_PROC_INFO */
+
+  thd_proc_info(thd, message);
   if (invoke_triggers &&
       process_triggers(TRG_EVENT_UPDATE, TRG_ACTION_BEFORE, TRUE))
   {
@@ -12405,6 +12518,8 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi)
       process_triggers(TRG_EVENT_UPDATE, TRG_ACTION_AFTER, TRUE))
     error= HA_ERR_GENERIC; // in case if error is not set yet
 
+  thd_proc_info(thd, tmp);
+
 err:
   m_table->file->ha_index_or_rnd_end();
   return error;
@@ -12494,6 +12609,49 @@ void Incident_log_event::pack_info(THD *thd, Protocol *protocol)
                        m_incident, description(), m_message.str);
   protocol->store(buf, bytes, &my_charset_bin);
 }
+#endif /* MYSQL_CLIENT */
+
+
+#if WITH_WSREP && !defined(MYSQL_CLIENT)
+Format_description_log_event *wsrep_format_desc; // TODO: free them at the end
+/*
+  read the first event from (*buf). The size of the (*buf) is (*buf_len).
+  At the end (*buf) is shitfed to point to the following event or NULL and
+  (*buf_len) will be changed to account just being read bytes of the 1st event.
+*/
+#define WSREP_MAX_ALLOWED_PACKET 1024*1024*1024 // current protocol max
+
+Log_event* wsrep_read_log_event(
+  char **arg_buf, size_t *arg_buf_len,
+  const Format_description_log_event *description_event)
+{
+  char *head= (*arg_buf);
+  uint data_len = uint4korr(head + EVENT_LEN_OFFSET);
+  char *buf= (*arg_buf);
+  const char *error= 0;
+  Log_event *res=  0;
+  DBUG_ENTER("wsrep_read_log_event");
+
+  if (data_len > WSREP_MAX_ALLOWED_PACKET)
+  {
+    error = "Event too big";
+    goto err;
+  }
+
+  res= Log_event::read_log_event(buf, data_len, &error, description_event, false);
+
+err:
+  if (!res)
+  {
+    DBUG_ASSERT(error != 0);
+    sql_print_error("Error in Log_event::read_log_event(): "
+                    "'%s', data_len: %d, event_type: %d",
+		    error,data_len,head[EVENT_TYPE_OFFSET]);
+  }
+  (*arg_buf)+= data_len;
+  (*arg_buf_len)-= data_len;
+  DBUG_RETURN(res);
+}
 #endif
 
 
diff --git a/sql/log_event.h b/sql/log_event.h
index 2091d968558..212215d97b6 100644
--- a/sql/log_event.h
+++ b/sql/log_event.h
@@ -1340,7 +1340,11 @@ public:
    */
   int apply_event(rpl_group_info *rgi)
   {
-    return do_apply_event(rgi);
+    int res;
+    THD_STAGE_INFO(thd, stage_apply_event);
+    res= do_apply_event(rgi);
+    THD_STAGE_INFO(thd, stage_after_apply_event);
+    return res;
   }
 
 
diff --git a/sql/mdl.cc b/sql/mdl.cc
index 2c2d64e96b2..07130b4d002 100644
--- a/sql/mdl.cc
+++ b/sql/mdl.cc
@@ -22,6 +22,8 @@
 #include <mysql/plugin.h>
 #include <mysql/service_thd_wait.h>
 #include <mysql/psi/mysql_stage.h>
+#include "wsrep_mysqld.h"
+#include "wsrep_thd.h"
 
 #ifdef HAVE_PSI_INTERFACE
 static PSI_mutex_key key_MDL_map_mutex;
@@ -1497,11 +1499,53 @@ void MDL_lock::Ticket_list::add_ticket(MDL_ticket *ticket)
     called by other threads.
   */
   DBUG_ASSERT(ticket->get_lock());
-  /*
-    Add ticket to the *back* of the queue to ensure fairness
-    among requests with the same priority.
-  */
-  m_list.push_back(ticket);
+#ifdef WITH_WSREP
+  if ((this == &(ticket->get_lock()->m_waiting)) &&
+      wsrep_thd_is_BF((void *)(ticket->get_ctx()->get_thd()), false))
+  {
+    Ticket_iterator itw(ticket->get_lock()->m_waiting);
+    Ticket_iterator itg(ticket->get_lock()->m_granted);
+
+    DBUG_ASSERT(WSREP_ON);
+    MDL_ticket *waiting, *granted;
+    MDL_ticket *prev=NULL;
+    bool added= false;
+
+    while ((waiting= itw++) && !added)
+    {
+      if (!wsrep_thd_is_BF((void *)(waiting->get_ctx()->get_thd()), true))
+      {
+        WSREP_DEBUG("MDL add_ticket inserted before: %lu %s",
+                    wsrep_thd_thread_id(waiting->get_ctx()->get_thd()),
+                    wsrep_thd_query(waiting->get_ctx()->get_thd()));
+        m_list.insert_after(prev, ticket);
+        added= true;
+      }
+      prev= waiting;
+    }
+    if (!added)   m_list.push_back(ticket);
+
+    while ((granted= itg++))
+    {
+      if (granted->get_ctx() != ticket->get_ctx() &&
+          granted->is_incompatible_when_granted(ticket->get_type()))
+      {
+        if (!wsrep_grant_mdl_exception(ticket->get_ctx(), granted))
+        {
+          WSREP_DEBUG("MDL victim killed at add_ticket");
+        }
+      }
+    }
+  }
+  else
+#endif /* WITH_WSREP */
+  {
+    /*
+      Add ticket to the *back* of the queue to ensure fairness
+      among requests with the same priority.
+    */
+    m_list.push_back(ticket);
+  }
   m_bitmap|= MDL_BIT(ticket->get_type());
 }
 
@@ -1842,6 +1886,7 @@ MDL_lock::can_grant_lock(enum_mdl_type type_arg,
   bool can_grant= FALSE;
   bitmap_t waiting_incompat_map= incompatible_waiting_types_bitmap()[type_arg];
   bitmap_t granted_incompat_map= incompatible_granted_types_bitmap()[type_arg];
+  bool  wsrep_can_grant= TRUE;
 
   /*
     New lock request can be satisfied iff:
@@ -1864,12 +1909,53 @@ MDL_lock::can_grant_lock(enum_mdl_type type_arg,
       {
         if (ticket->get_ctx() != requestor_ctx &&
             ticket->is_incompatible_when_granted(type_arg))
+        {
+#ifdef WITH_WSREP
+          if (wsrep_thd_is_BF((void *)(requestor_ctx->get_thd()),false) &&
+              key.mdl_namespace() == MDL_key::GLOBAL)
+          {
+            WSREP_DEBUG("global lock granted for BF: %lu %s",
+                        wsrep_thd_thread_id(requestor_ctx->get_thd()),
+                        wsrep_thd_query(requestor_ctx->get_thd()));
+            can_grant = true;
+          }
+          else if (!wsrep_grant_mdl_exception(requestor_ctx, ticket))
+          {
+            wsrep_can_grant= FALSE;
+            if (wsrep_log_conflicts)
+            {
+              MDL_lock * lock = ticket->get_lock();
+              WSREP_INFO(
+                         "MDL conflict db=%s table=%s ticket=%d solved by %s",
+                         lock->key.db_name(), lock->key.name(), ticket->get_type(),
+                         "abort" );
+            }
+          }
+          else
+            can_grant= TRUE;
+          /* Continue loop */
+#else
           break;
+#endif /* WITH_WSREP */
+        }
       }
-      if (ticket == NULL)             /* Incompatible locks are our own. */
-        can_grant= TRUE;
+      if ((ticket == NULL) && IF_WSREP(wsrep_can_grant, 1))
+        can_grant= TRUE; /* Incompatible locks are our own. */
     }
   }
+#ifdef WITH_WSREP
+  else
+  {
+    if (wsrep_thd_is_BF((void *)(requestor_ctx->get_thd()), false) &&
+	key.mdl_namespace() == MDL_key::GLOBAL)
+    {
+      WSREP_DEBUG("global lock granted for BF (waiting queue): %lu %s",
+		  wsrep_thd_thread_id(requestor_ctx->get_thd()), 
+		  wsrep_thd_query(requestor_ctx->get_thd()));
+      can_grant = true;
+    }
+  }
+#endif /* WITH_WSREP */
   return can_grant;
 }
 
@@ -3222,3 +3308,44 @@ void MDL_context::set_transaction_duration_for_all_locks()
     ticket->m_duration= MDL_TRANSACTION;
 #endif
 }
+
+
+#ifdef WITH_WSREP
+
+void MDL_context::release_explicit_locks()
+{
+  release_locks_stored_before(MDL_EXPLICIT, NULL);
+}
+
+
+void MDL_ticket::wsrep_report(bool debug)
+{
+  if (debug)
+  {
+      const PSI_stage_info *psi_stage = m_lock->key.get_wait_state_name();
+
+      WSREP_DEBUG("MDL ticket: type: %s space: %s db: %s name: %s (%s)",
+       	 (get_type()  == MDL_INTENTION_EXCLUSIVE)  ? "intention exclusive"  :
+       	 ((get_type() == MDL_SHARED)               ? "shared"               :
+       	 ((get_type() == MDL_SHARED_HIGH_PRIO      ? "shared high prio"     :
+       	 ((get_type() == MDL_SHARED_READ)          ? "shared read"          :
+       	 ((get_type() == MDL_SHARED_WRITE)         ? "shared write"         :
+       	 ((get_type() == MDL_SHARED_NO_WRITE)      ? "shared no write"      :
+         ((get_type() == MDL_SHARED_NO_READ_WRITE) ? "shared no read write" :
+       	 ((get_type() == MDL_EXCLUSIVE)            ? "exclusive"            :
+          "UNKNOWN")))))))),
+         (m_lock->key.mdl_namespace()  == MDL_key::GLOBAL) ? "GLOBAL"       :
+         ((m_lock->key.mdl_namespace() == MDL_key::SCHEMA) ? "SCHEMA"       :
+         ((m_lock->key.mdl_namespace() == MDL_key::TABLE)  ? "TABLE"        :
+         ((m_lock->key.mdl_namespace() == MDL_key::TABLE)  ? "FUNCTION"     :
+         ((m_lock->key.mdl_namespace() == MDL_key::TABLE)  ? "PROCEDURE"    :
+         ((m_lock->key.mdl_namespace() == MDL_key::TABLE)  ? "TRIGGER"      :
+         ((m_lock->key.mdl_namespace() == MDL_key::TABLE)  ? "EVENT"        :
+         ((m_lock->key.mdl_namespace() == MDL_key::COMMIT) ? "COMMIT"       :
+         (char *)"UNKNOWN"))))))),
+         m_lock->key.db_name(),
+       	 m_lock->key.name(),
+         psi_stage->m_name);
+    }
+}
+#endif /* WITH_WSREP */
diff --git a/sql/mdl.h b/sql/mdl.h
index 47c587eb3be..231f95d7419 100644
--- a/sql/mdl.h
+++ b/sql/mdl.h
@@ -586,6 +586,9 @@ public:
   MDL_ticket *next_in_lock;
   MDL_ticket **prev_in_lock;
 public:
+#ifdef WITH_WSREP
+  void wsrep_report(bool debug);
+#endif /* WITH_WSREP */
   bool has_pending_conflicting_lock() const;
 
   MDL_context *get_ctx() const { return m_ctx; }
@@ -773,6 +776,10 @@ public:
              m_tickets[MDL_TRANSACTION].is_empty() &&
              m_tickets[MDL_EXPLICIT].is_empty());
   }
+  inline bool has_transactional_locks() const
+  {
+    return !m_tickets[MDL_TRANSACTION].is_empty();
+  }
 
   MDL_savepoint mdl_savepoint()
   {
@@ -786,6 +793,9 @@ public:
 
   void release_statement_locks();
   void release_transactional_locks();
+#ifdef WITH_WSREP
+  void release_explicit_locks();
+#endif
   void rollback_to_savepoint(const MDL_savepoint &mdl_savepoint);
 
   MDL_context_owner *get_owner() { return m_owner; }
@@ -910,7 +920,6 @@ private:
    */
   MDL_wait_for_subgraph *m_waiting_for;
 private:
-  THD *get_thd() const { return m_owner->get_thd(); }
   MDL_ticket *find_ticket(MDL_request *mdl_req,
                           enum_mdl_duration *duration);
   void release_locks_stored_before(enum_mdl_duration duration, MDL_ticket *sentinel);
@@ -919,6 +928,7 @@ private:
                              MDL_ticket **out_ticket);
 
 public:
+  THD *get_thd() const { return m_owner->get_thd(); }
   void find_deadlock();
 
   ulong get_thread_id() const { return thd_get_thread_id(get_thd()); }
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 066dae224eb..97e9cff3b79 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -71,6 +71,11 @@
 #include "scheduler.h"
 #include <waiting_threads.h>
 #include "debug_sync.h"
+#include "wsrep_mysqld.h"
+#include "wsrep_var.h"
+#include "wsrep_thd.h"
+#include "wsrep_sst.h"
+
 #include "sql_callback.h"
 #include "threadpool.h"
 
@@ -352,7 +357,8 @@ static bool volatile select_thread_in_use, signal_thread_in_use;
 static volatile bool ready_to_exit;
 static my_bool opt_debugging= 0, opt_external_locking= 0, opt_console= 0;
 static my_bool opt_short_log_format= 0;
-static uint kill_cached_threads, wake_thread;
+uint kill_cached_threads;
+static uint wake_thread;
 ulong max_used_connections;
 static volatile ulong cached_thread_count= 0;
 static char *mysqld_user, *mysqld_chroot;
@@ -360,14 +366,15 @@ static char *default_character_set_name;
 static char *character_set_filesystem_name;
 static char *lc_messages;
 static char *lc_time_names_name;
-static char *my_bind_addr_str;
+char *my_bind_addr_str;
 static char *default_collation_name;
 char *default_storage_engine, *default_tmp_storage_engine;
 static char compiled_default_collation_name[]= MYSQL_DEFAULT_COLLATION_NAME;
 static I_List<THD> thread_cache;
 static bool binlog_format_used= false;
 LEX_STRING opt_init_connect, opt_init_slave;
-static mysql_cond_t COND_thread_cache, COND_flush_thread_cache;
+mysql_cond_t COND_thread_cache;
+static mysql_cond_t COND_flush_thread_cache;
 static DYNAMIC_ARRAY all_options;
 
 /* Global variables */
@@ -731,6 +738,27 @@ mysql_cond_t COND_server_started;
 int mysqld_server_started=0, mysqld_server_initialized= 0;
 File_parser_dummy_hook file_parser_dummy_hook;
 
+#ifdef WITH_WSREP
+mysql_mutex_t LOCK_wsrep_ready;
+mysql_cond_t  COND_wsrep_ready;
+mysql_mutex_t LOCK_wsrep_sst;
+mysql_cond_t  COND_wsrep_sst;
+mysql_mutex_t LOCK_wsrep_sst_init;
+mysql_cond_t  COND_wsrep_sst_init;
+mysql_mutex_t LOCK_wsrep_rollback;
+mysql_cond_t  COND_wsrep_rollback;
+wsrep_aborting_thd_t wsrep_aborting_thd= NULL;
+mysql_mutex_t LOCK_wsrep_replaying;
+mysql_cond_t  COND_wsrep_replaying;
+mysql_mutex_t LOCK_wsrep_slave_threads;
+mysql_mutex_t LOCK_wsrep_desync;
+int wsrep_replaying= 0;
+ulong  wsrep_running_threads = 0; // # of currently running wsrep threads
+ulong  my_bind_addr;
+const char *wsrep_binlog_format_names[]=
+                                   {"MIXED", "STATEMENT", "ROW", "NONE", NullS};
+#endif /* WITH_WSREP */
+
 /* replication parameters, if master_host is not NULL, we are a slave */
 uint report_port= 0;
 ulong master_retry_count=0;
@@ -868,6 +896,12 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list,
   key_LOCK_error_messages, key_LOG_INFO_lock,
   key_LOCK_thread_count, key_LOCK_thread_cache,
   key_PARTITION_LOCK_auto_inc;
+#ifdef WITH_WSREP
+PSI_mutex_key key_LOCK_wsrep_rollback, key_LOCK_wsrep_thd, 
+  key_LOCK_wsrep_replaying, key_LOCK_wsrep_ready, key_LOCK_wsrep_sst, 
+  key_LOCK_wsrep_sst_thread, key_LOCK_wsrep_sst_init,
+  key_LOCK_wsrep_slave_threads, key_LOCK_wsrep_desync;
+#endif
 PSI_mutex_key key_RELAYLOG_LOCK_index;
 PSI_mutex_key key_LOCK_slave_state, key_LOCK_binlog_state,
   key_LOCK_rpl_thread, key_LOCK_rpl_thread_pool, key_LOCK_parallel_entry;
@@ -942,6 +976,18 @@ static PSI_mutex_info all_server_mutexes[]=
   { &key_LOCK_prepare_ordered, "LOCK_prepare_ordered", PSI_FLAG_GLOBAL},
   { &key_LOCK_commit_ordered, "LOCK_commit_ordered", PSI_FLAG_GLOBAL},
   { &key_LOG_INFO_lock, "LOG_INFO::lock", 0},
+#ifdef WITH_WSREP
+  { &key_LOCK_wsrep_ready, "LOCK_wsrep_ready", PSI_FLAG_GLOBAL},
+  { &key_LOCK_wsrep_sst, "LOCK_wsrep_sst", PSI_FLAG_GLOBAL},
+  { &key_LOCK_wsrep_sst_thread, "wsrep_sst_thread", 0},
+  { &key_LOCK_wsrep_sst_init, "LOCK_wsrep_sst_init", PSI_FLAG_GLOBAL},
+  { &key_LOCK_wsrep_sst, "LOCK_wsrep_sst", PSI_FLAG_GLOBAL},
+  { &key_LOCK_wsrep_rollback, "LOCK_wsrep_rollback", PSI_FLAG_GLOBAL},
+  { &key_LOCK_wsrep_thd, "THD::LOCK_wsrep_thd", 0},
+  { &key_LOCK_wsrep_replaying, "LOCK_wsrep_replaying", PSI_FLAG_GLOBAL},
+  { &key_LOCK_wsrep_slave_threads, "LOCK_wsrep_slave_threads", PSI_FLAG_GLOBAL},
+  { &key_LOCK_wsrep_desync, "LOCK_wsrep_desync", PSI_FLAG_GLOBAL},
+#endif
   { &key_LOCK_thread_count, "LOCK_thread_count", PSI_FLAG_GLOBAL},
   { &key_LOCK_thread_cache, "LOCK_thread_cache", PSI_FLAG_GLOBAL},
   { &key_PARTITION_LOCK_auto_inc, "HA_DATA_PARTITION::LOCK_auto_inc", 0},
@@ -988,6 +1034,11 @@ PSI_cond_key key_BINLOG_COND_xid_list, key_BINLOG_update_cond,
   key_TABLE_SHARE_cond, key_user_level_lock_cond,
   key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache,
   key_BINLOG_COND_queue_busy;
+#ifdef WITH_WSREP
+PSI_cond_key key_COND_wsrep_rollback, key_COND_wsrep_thd, 
+  key_COND_wsrep_replaying, key_COND_wsrep_ready, key_COND_wsrep_sst,
+  key_COND_wsrep_sst_init, key_COND_wsrep_sst_thread;
+#endif /* WITH_WSREP */
 PSI_cond_key key_RELAYLOG_update_cond, key_COND_wakeup_ready,
   key_COND_wait_commit;
 PSI_cond_key key_RELAYLOG_COND_queue_busy;
@@ -1037,6 +1088,15 @@ static PSI_cond_info all_server_conds[]=
   { &key_user_level_lock_cond, "User_level_lock::cond", 0},
   { &key_COND_thread_count, "COND_thread_count", PSI_FLAG_GLOBAL},
   { &key_COND_thread_cache, "COND_thread_cache", PSI_FLAG_GLOBAL},
+#ifdef WITH_WSREP
+  { &key_COND_wsrep_ready, "COND_wsrep_ready", PSI_FLAG_GLOBAL},
+  { &key_COND_wsrep_sst, "COND_wsrep_sst", PSI_FLAG_GLOBAL},
+  { &key_COND_wsrep_sst_init, "COND_wsrep_sst_init", PSI_FLAG_GLOBAL},
+  { &key_COND_wsrep_sst_thread, "wsrep_sst_thread", 0},
+  { &key_COND_wsrep_rollback, "COND_wsrep_rollback", PSI_FLAG_GLOBAL},
+  { &key_COND_wsrep_thd, "THD::COND_wsrep_thd", 0},
+  { &key_COND_wsrep_replaying, "COND_wsrep_replaying", PSI_FLAG_GLOBAL},
+#endif
   { &key_COND_flush_thread_cache, "COND_flush_thread_cache", PSI_FLAG_GLOBAL},
   { &key_COND_rpl_thread, "COND_rpl_thread", 0},
   { &key_COND_rpl_thread_queue, "COND_rpl_thread_queue", 0},
@@ -1398,7 +1458,7 @@ bool mysqld_embedded=0;
 bool mysqld_embedded=1;
 #endif
 
-static my_bool plugins_are_initialized= FALSE;
+my_bool plugins_are_initialized= FALSE;
 
 #ifndef DBUG_OFF
 static const char* default_dbug_option;
@@ -1622,6 +1682,11 @@ static void close_connections(void)
     if (tmp->slave_thread)
       continue;
 
+#ifdef WITH_WSREP
+    /* skip wsrep system threads as well */
+    if (WSREP(tmp) && (tmp->wsrep_exec_mode==REPL_RECV || tmp->wsrep_applier))
+      continue;
+#endif
     tmp->killed= KILL_SERVER_HARD;
     MYSQL_CALLBACK(thread_scheduler, post_kill_notification, (tmp));
     mysql_mutex_lock(&tmp->LOCK_thd_data);
@@ -1698,6 +1763,34 @@ static void close_connections(void)
       close_connection(tmp,ER_SERVER_SHUTDOWN);
     }
 #endif
+#ifdef WITH_WSREP
+    /*
+     * WSREP_TODO:
+     *       this code block may turn out redundant. wsrep->disconnect()
+     *       should terminate slave threads gracefully, and we don't need
+     *       to signal them here. 
+     *       The code here makes sure mysqld will not hang during shutdown
+     *       even if wsrep provider has problems in shutting down.
+     */
+    if (WSREP(tmp) && tmp->wsrep_exec_mode==REPL_RECV)
+    {
+      sql_print_information("closing wsrep system thread");
+      tmp->killed= KILL_CONNECTION;
+      MYSQL_CALLBACK(thread_scheduler, post_kill_notification, (tmp));
+      if (tmp->mysys_var)
+      {
+        tmp->mysys_var->abort=1;
+        mysql_mutex_lock(&tmp->mysys_var->mutex);
+        if (tmp->mysys_var->current_cond)
+        {
+          mysql_mutex_lock(tmp->mysys_var->current_mutex);
+          mysql_cond_broadcast(tmp->mysys_var->current_cond);
+          mysql_mutex_unlock(tmp->mysys_var->current_mutex);
+        }
+        mysql_mutex_unlock(&tmp->mysys_var->mutex);
+      }
+    }
+#endif
     DBUG_PRINT("quit",("Unlocking LOCK_thread_count"));
     mysql_mutex_unlock(&LOCK_thread_count);
   }
@@ -1852,7 +1945,18 @@ static void __cdecl kill_server(int sig_ptr)
   }
 #endif
 
+#ifdef WITH_WSREP
+  if (WSREP_ON)
+    wsrep_stop_replication(NULL);
+#endif
+
   close_connections();
+
+#ifdef WITH_WSREP
+  if (wsrep_inited == 1)
+    wsrep_deinit(true);
+#endif
+
   if (sig != MYSQL_KILL_SIGNAL &&
       sig != 0)
     unireg_abort(1);				/* purecov: inspected */
@@ -1947,6 +2051,30 @@ extern "C" void unireg_abort(int exit_code)
     usage();
   if (exit_code)
     sql_print_error("Aborting\n");
+
+#ifdef WITH_WSREP
+  /* Check if wsrep class is used. If yes, then cleanup wsrep */
+  if (wsrep)
+  {
+    /*
+      This is an abort situation, we cannot expect to gracefully close all
+      wsrep threads here, we can only diconnect from service
+    */
+    wsrep_close_client_connections(FALSE);
+    shutdown_in_progress= 1;
+    THD *thd(0);
+    wsrep->disconnect(wsrep);
+    WSREP_INFO("Service disconnected.");
+    wsrep_close_threads(thd); /* this won't close all threads */
+    sleep(1); /* so give some time to exit for those which can */
+    WSREP_INFO("Some threads may fail to exit.");
+
+    /* In bootstrap mode we deinitialize wsrep here. */
+    if (opt_bootstrap && wsrep_inited)
+      wsrep_deinit(true);
+  }
+#endif // WITH_WSREP
+
   clean_up(!opt_abort && (exit_code || !opt_bootstrap)); /* purecov: inspected */
   DBUG_PRINT("quit",("done with cleanup in unireg_abort"));
   mysqld_exit(exit_code);
@@ -2166,6 +2294,20 @@ static void clean_up_mutexes()
   mysql_cond_destroy(&COND_thread_count);
   mysql_cond_destroy(&COND_thread_cache);
   mysql_cond_destroy(&COND_flush_thread_cache);
+#ifdef WITH_WSREP
+  (void) mysql_mutex_destroy(&LOCK_wsrep_ready);
+  (void) mysql_cond_destroy(&COND_wsrep_ready);
+  (void) mysql_mutex_destroy(&LOCK_wsrep_sst);
+  (void) mysql_cond_destroy(&COND_wsrep_sst);
+  (void) mysql_mutex_destroy(&LOCK_wsrep_sst_init);
+  (void) mysql_cond_destroy(&COND_wsrep_sst_init);
+  (void) mysql_mutex_destroy(&LOCK_wsrep_rollback);
+  (void) mysql_cond_destroy(&COND_wsrep_rollback);
+  (void) mysql_mutex_destroy(&LOCK_wsrep_replaying);
+  (void) mysql_cond_destroy(&COND_wsrep_replaying);
+  (void) mysql_mutex_destroy(&LOCK_wsrep_slave_threads);
+  (void) mysql_mutex_destroy(&LOCK_wsrep_desync);
+#endif
   mysql_mutex_destroy(&LOCK_server_started);
   mysql_cond_destroy(&COND_server_started);
   mysql_mutex_destroy(&LOCK_prepare_ordered);
@@ -2481,6 +2623,11 @@ static MYSQL_SOCKET activate_tcp_port(uint port)
                     socket_errno);
     unireg_abort(1);
   }
+#if defined(WITH_WSREP) && defined(HAVE_FCNTL) && defined(FD_CLOEXEC)
+  if (WSREP_ON)
+    (void) fcntl(mysql_socket_getfd(ip_sock), F_SETFD, FD_CLOEXEC);
+#endif /* WITH_WSREP */
+
   DBUG_RETURN(ip_sock);
 }
 
@@ -2607,6 +2754,10 @@ static void network_init(void)
     if (mysql_socket_listen(unix_sock,(int) back_log) < 0)
       sql_print_warning("listen() on Unix socket failed with error %d",
 		      socket_errno);
+#if defined(WITH_WSREP) && defined(HAVE_FCNTL) && defined(FD_CLOEXEC)
+    if (WSREP_ON)
+      (void) fcntl(mysql_socket_getfd(unix_sock), F_SETFD, FD_CLOEXEC);
+#endif /* WITH_WSREP */
   }
 #endif
   DBUG_PRINT("info",("server started"));
@@ -2681,6 +2832,16 @@ void thd_cleanup(THD *thd)
 
 void dec_connection_count(THD *thd)
 {
+#ifdef WITH_WSREP
+  /*
+    Do not decrement when its wsrep system thread. wsrep_applier is set for
+    applier as well as rollbacker threads.
+  */
+  if (thd->wsrep_applier)
+    return;
+#endif /* WITH_WSREP */
+
+  DBUG_ASSERT(*thd->scheduler->connection_count > 0);
   mysql_mutex_lock(&LOCK_connection_count);
   (*thd->scheduler->connection_count)--;
   mysql_mutex_unlock(&LOCK_connection_count);
@@ -2851,10 +3012,15 @@ static bool cache_thread()
 bool one_thread_per_connection_end(THD *thd, bool put_in_cache)
 {
   DBUG_ENTER("one_thread_per_connection_end");
+#ifdef WITH_WSREP
+  const bool wsrep_applier(thd->wsrep_applier);
+#endif
+
   unlink_thd(thd);
   /* Mark that current_thd is not valid anymore */
   set_current_thd(0);
-  if (put_in_cache && cache_thread())
+
+  if (put_in_cache && cache_thread() && IF_WSREP(!wsrep_applier, 1))
     DBUG_RETURN(0);                             // Thread is reused
 
   /*
@@ -3976,6 +4142,14 @@ static int init_common_variables()
   else
     opt_log_basename= glob_hostname;
 
+#ifdef WITH_WSREP
+  if (0 == wsrep_node_name || 0 == wsrep_node_name[0])
+  {
+    my_free((void *)wsrep_node_name);
+    wsrep_node_name= my_strdup(glob_hostname, MYF(MY_WME));
+  }
+#endif /* WITH_WSREP */
+
   if (!*pidfile_name)
   {
     strmake(pidfile_name, opt_log_basename, sizeof(pidfile_name)-5);
@@ -4036,6 +4210,15 @@ static int init_common_variables()
                      SQLCOM_END + 8);
 #endif
 
+#ifdef WITH_WSREP
+  /*
+    This is a protection against mutually incompatible option values.
+    Note WSREP_ON == global_system_variables.wsrep_on
+  */
+  if (WSREP_ON && wsrep_check_opts (remaining_argc, remaining_argv))
+    global_system_variables.wsrep_on= 0;
+#endif /* WITH_WSREP */
+
   if (get_options(&remaining_argc, &remaining_argv))
     return 1;
   set_server_version();
@@ -4430,6 +4613,28 @@ static int init_thread_environment()
   rpl_init_gtid_waiting();
 #endif
 
+#ifdef WITH_WSREP
+  mysql_mutex_init(key_LOCK_wsrep_ready,
+                   &LOCK_wsrep_ready, MY_MUTEX_INIT_FAST);
+  mysql_cond_init(key_COND_wsrep_ready, &COND_wsrep_ready, NULL);
+  mysql_mutex_init(key_LOCK_wsrep_sst,
+                   &LOCK_wsrep_sst, MY_MUTEX_INIT_FAST);
+  mysql_cond_init(key_COND_wsrep_sst, &COND_wsrep_sst, NULL);
+  mysql_mutex_init(key_LOCK_wsrep_sst_init,
+                   &LOCK_wsrep_sst_init, MY_MUTEX_INIT_FAST);
+  mysql_cond_init(key_COND_wsrep_sst_init, &COND_wsrep_sst_init, NULL);
+  mysql_mutex_init(key_LOCK_wsrep_rollback,
+                   &LOCK_wsrep_rollback, MY_MUTEX_INIT_FAST);
+  mysql_cond_init(key_COND_wsrep_rollback, &COND_wsrep_rollback, NULL);
+  mysql_mutex_init(key_LOCK_wsrep_replaying,
+                   &LOCK_wsrep_replaying, MY_MUTEX_INIT_FAST);
+  mysql_cond_init(key_COND_wsrep_replaying, &COND_wsrep_replaying, NULL);
+  mysql_mutex_init(key_LOCK_wsrep_slave_threads,
+                   &LOCK_wsrep_slave_threads, MY_MUTEX_INIT_FAST);
+  mysql_mutex_init(key_LOCK_wsrep_desync,
+                   &LOCK_wsrep_desync, MY_MUTEX_INIT_FAST);
+#endif
+
   DBUG_RETURN(0);
 }
 
@@ -4727,10 +4932,10 @@ static int init_server_components()
   /* need to configure logging before initializing storage engines */
   if (!opt_bin_log_used)
   {
-    if (opt_log_slave_updates)
+    if (IF_WSREP(!WSREP_ON,1) && opt_log_slave_updates)
       sql_print_warning("You need to use --log-bin to make "
                         "--log-slave-updates work.");
-    if (binlog_format_used)
+    if (IF_WSREP(!WSREP_ON, 1) && binlog_format_used)
       sql_print_warning("You need to use --log-bin to make "
                         "--binlog-format work.");
   }
@@ -4805,10 +5010,67 @@ a file name for --log-bin-index option", opt_binlog_index_name);
     {
       opt_bin_logname= my_once_strdup(buf, MYF(MY_WME));
     }
+#ifdef WITH_WSREP /* WSREP BEFORE SE */
+    /*
+      Wsrep initialization must happen at this point, because:
+      - opt_bin_logname must be known when starting replication
+        since SST may need it
+      - SST may modify binlog index file, so it must be opened
+        after SST has happened
+     */
+  }
+  if (WSREP_ON && !wsrep_recovery)
+  {
+    if (opt_bootstrap) // bootsrap option given - disable wsrep functionality
+    {
+      wsrep_provider_init(WSREP_NONE);
+      if (wsrep_init()) unireg_abort(1);
+    }
+    else // full wsrep initialization
+    {
+      // add basedir/bin to PATH to resolve wsrep script names
+      char* const tmp_path((char*)alloca(strlen(mysql_home) +
+                                           strlen("/bin") + 1));
+      if (tmp_path)
+      {
+        strcpy(tmp_path, mysql_home);
+        strcat(tmp_path, "/bin");
+        wsrep_prepend_PATH(tmp_path);
+      }
+      else
+      {
+        WSREP_ERROR("Could not append %s/bin to PATH", mysql_home);
+      }
+
+      if (wsrep_before_SE())
+      {
+        set_ports(); // this is also called in network_init() later but we need
+                     // to know mysqld_port now - lp:1071882
+        wsrep_init_startup(true);
+      }
+    }
+  }
+  if (opt_bin_log)
+  {
+    /*
+      Variable ln is not defined at this scope. We use opt_bin_logname instead.
+      It should be the same as ln since
+      - mysql_bin_log.generate_name() returns first argument if new log name
+        is not generated
+      - if new log name is generated, return value is assigned to ln and copied
+        to opt_bin_logname above
+     */
+    if (mysql_bin_log.open_index_file(opt_binlog_index_name, opt_bin_logname,
+                                      TRUE))
+    {
+      unireg_abort(1);
+    }
+#else
     if (mysql_bin_log.open_index_file(opt_binlog_index_name, ln, TRUE))
     {
       unireg_abort(1);
     }
+#endif /* WITH_WSREP */
   }
 
   /* call ha_init_key_cache() on all key caches to init them */
@@ -4930,10 +5192,40 @@ a file name for --log-bin-index option", opt_binlog_index_name);
   internal_tmp_table_max_key_segments= myisam_max_key_segments();
 #endif
 
+#ifdef WITH_WSREP
+  if (WSREP_ON && !opt_bin_log)
+  {
+    wsrep_emulate_bin_log= 1;
+  }
+#endif
+
+  /* if total_ha_2pc <= 1
+       tc_log = tc_log_dummy
+     else
+       if opt_bin_log == true
+         tc_log = mysql_bin_log
+       else
+         if WITH_WSREP
+           if WSREP_ON
+             tc_log = tc_log_dummy
+           else
+             tc_log = tc_log_mmap
+         else
+           tc_log=tc_log_mmap
+  */
   tc_log= (total_ha_2pc > 1 ? (opt_bin_log  ?
                                (TC_LOG *) &mysql_bin_log :
-                               (TC_LOG *) &tc_log_mmap) :
-           (TC_LOG *) &tc_log_dummy);
+		  IF_WSREP((WSREP_ON ?  (TC_LOG *) &tc_log_dummy :
+			  (TC_LOG *) &tc_log_mmap), (TC_LOG *) &tc_log_mmap)) :
+	  (TC_LOG *) &tc_log_dummy);
+
+#ifdef WITH_WSREP
+  WSREP_DEBUG("Initial TC log open: %s",
+              (tc_log == &mysql_bin_log) ? "binlog" :
+              (tc_log == &tc_log_mmap) ? "mmap" :
+              (tc_log == &tc_log_dummy) ? "dummy" : "unknown"
+              );
+#endif
 
   if (tc_log->open(opt_bin_log ? opt_bin_logname : opt_tc_log_file))
   {
@@ -5192,6 +5484,10 @@ int mysqld_main(int argc, char **argv)
     return 1;
   }
 #endif
+#ifdef WITH_WSREP
+  if (WSREP_ON)
+    wsrep_filter_new_cluster (&argc, argv);
+#endif /* WITH_WSREP */
 
   orig_argc= argc;
   orig_argv= argv;
@@ -5408,6 +5704,15 @@ int mysqld_main(int argc, char **argv)
   }
 #endif
 
+#ifdef WITH_WSREP /* WSREP AFTER SE */
+  if (WSREP_ON && wsrep_recovery)
+  {
+    select_thread_in_use= 0;
+    wsrep_recover();
+    unireg_abort(0);
+  }
+#endif /* WITH_WSREP */
+
   /*
     init signals & alarm
     After this we can't quit by a simple unireg_abort
@@ -5464,7 +5769,35 @@ int mysqld_main(int argc, char **argv)
   if (Events::init(opt_noacl || opt_bootstrap))
     unireg_abort(1);
 
-  if (opt_bootstrap)
+#ifdef WITH_WSREP /* WSREP AFTER SE */
+  if (WSREP_ON)
+  {
+    if (opt_bootstrap)
+    {
+      /*! bootstrap wsrep init was taken care of above */
+    }
+    else
+    {
+      wsrep_SE_initialized();
+
+      if (wsrep_before_SE())
+      {
+        /*! in case of no SST wsrep waits in view handler callback */
+        wsrep_SE_init_grab();
+        wsrep_SE_init_done();
+        /*! in case of SST wsrep waits for wsrep->sst_received */
+        wsrep_sst_continue();
+      }
+      else
+      {
+        wsrep_init_startup (false);
+      }
+
+      wsrep_create_appliers(wsrep_slave_threads - 1);
+    }
+  }
+#endif /* WITH_WSREP */
+ if (opt_bootstrap)
   {
     select_thread_in_use= 0;                    // Allow 'kill' to work
     bootstrap(mysql_stdin);
@@ -5532,6 +5865,7 @@ int mysqld_main(int argc, char **argv)
 #ifdef EXTRA_DEBUG2
   sql_print_error("Before Lock_thread_count");
 #endif
+  WSREP_DEBUG("Before Lock_thread_count");
   mysql_mutex_lock(&LOCK_thread_count);
   DBUG_PRINT("quit", ("Got thread_count mutex"));
   select_thread_in_use=0;			// For close_connections
@@ -5797,6 +6131,9 @@ static void bootstrap(MYSQL_FILE *file)
   DBUG_ENTER("bootstrap");
 
   THD *thd= new THD;
+#ifdef WITH_WSREP
+  thd->variables.wsrep_on= 0;
+#endif
   thd->bootstrap=1;
   my_net_init(&thd->net,(st_vio*) 0, MYF(0));
   thd->max_client_packet_length= thd->net.max_packet;
@@ -6198,6 +6535,9 @@ void handle_connections_sockets()
 	sleep(1);				// Give other threads some time
       continue;
     }
+#if defined(WITH_WSREP) && defined(HAVE_FCNTL) && defined(FD_CLOEXEC)
+    (void) fcntl(mysql_socket_getfd(new_sock), F_SETFD, FD_CLOEXEC);
+#endif /* WITH_WSREP */
 
 #ifdef HAVE_LIBWRAP
     {
@@ -7971,6 +8311,21 @@ SHOW_VAR status_vars[]= {
 #ifdef ENABLED_PROFILING
   {"Uptime_since_flush_status",(char*) &show_flushstatustime,   SHOW_SIMPLE_FUNC},
 #endif
+#ifdef WITH_WSREP
+  {"wsrep_connected",          (char*) &wsrep_connected,         SHOW_BOOL},
+  {"wsrep_ready",              (char*) &wsrep_ready,             SHOW_BOOL},
+  {"wsrep_cluster_state_uuid", (char*) &wsrep_cluster_state_uuid,SHOW_CHAR_PTR},
+  {"wsrep_cluster_conf_id",    (char*) &wsrep_cluster_conf_id,   SHOW_LONGLONG},
+  {"wsrep_cluster_status",     (char*) &wsrep_cluster_status,    SHOW_CHAR_PTR},
+  {"wsrep_cluster_size",       (char*) &wsrep_cluster_size,      SHOW_LONG_NOFLUSH},
+  {"wsrep_local_index",        (char*) &wsrep_local_index,       SHOW_LONG_NOFLUSH},
+  {"wsrep_local_bf_aborts",    (char*) &wsrep_show_bf_aborts,    SHOW_SIMPLE_FUNC},
+  {"wsrep_provider_name",      (char*) &wsrep_provider_name,     SHOW_CHAR_PTR},
+  {"wsrep_provider_version",   (char*) &wsrep_provider_version,  SHOW_CHAR_PTR},
+  {"wsrep_provider_vendor",    (char*) &wsrep_provider_vendor,   SHOW_CHAR_PTR},
+  {"wsrep_thread_count",       (char*) &wsrep_running_threads,   SHOW_LONG_NOFLUSH},
+  {"wsrep",                    (char*) &wsrep_show_status,       SHOW_FUNC},
+#endif
   {NullS, NullS, SHOW_LONG}
 };
 
@@ -8314,6 +8669,10 @@ static int mysql_init_variables(void)
     tmpenv = DEFAULT_MYSQL_HOME;
   strmake_buf(mysql_home, tmpenv);
 #endif
+#ifdef WITH_WSREP
+  if (WSREP_ON && wsrep_init_vars())
+    return 1;
+#endif
   return 0;
 }
 
@@ -8561,6 +8920,14 @@ mysqld_get_one_option(int optid,
   case OPT_LOWER_CASE_TABLE_NAMES:
     lower_case_table_names_used= 1;
     break;
+#ifdef WITH_WSREP
+  case OPT_WSREP_START_POSITION:
+    wsrep_start_position_init (argument);
+    break;
+  case OPT_WSREP_SST_AUTH:
+    wsrep_sst_auth_init (argument);
+    break;
+#endif
 #if defined(ENABLED_DEBUG_SYNC)
   case OPT_DEBUG_SYNC_TIMEOUT:
     /*
@@ -9029,6 +9396,9 @@ void set_server_version(void)
 #ifdef EMBEDDED_LIBRARY
   end= strmov(end, "-embedded");
 #endif
+#ifdef WITH_WSREP
+  end= strmov(end, "-wsrep");
+#endif
 #ifndef DBUG_OFF
   if (!strstr(MYSQL_SERVER_SUFFIX_STR, "-debug"))
     end= strmov(end, "-debug");
@@ -9321,6 +9691,10 @@ void refresh_status(THD *thd)
 
   /* Reset some global variables */
   reset_status_vars();
+#ifdef WITH_WSREP
+  if (WSREP_ON)
+    wsrep->stats_reset(wsrep);
+#endif /* WITH_WSREP */
 
   /* Reset the counters of all key caches (default and named). */
   process_key_caches(reset_key_cache_counters, 0);
@@ -9372,6 +9746,7 @@ static PSI_file_info all_server_files[]=
 };
 #endif /* HAVE_PSI_INTERFACE */
 
+PSI_stage_info stage_after_apply_event= { 0, "after apply log event", 0};
 PSI_stage_info stage_after_create= { 0, "After create", 0};
 PSI_stage_info stage_after_opening_tables= { 0, "After opening tables", 0};
 PSI_stage_info stage_after_table_lock= { 0, "After table lock", 0};
@@ -9379,6 +9754,7 @@ PSI_stage_info stage_allocating_local_table= { 0, "allocating local table", 0};
 PSI_stage_info stage_alter_inplace_prepare= { 0, "preparing for alter table", 0};
 PSI_stage_info stage_alter_inplace= { 0, "altering table", 0};
 PSI_stage_info stage_alter_inplace_commit= { 0, "committing alter table to storage engine", 0};
+PSI_stage_info stage_apply_event= { 0, "apply log event", 0};
 PSI_stage_info stage_changing_master= { 0, "Changing master", 0};
 PSI_stage_info stage_checking_master_version= { 0, "Checking master version", 0};
 PSI_stage_info stage_checking_permissions= { 0, "checking permissions", 0};
@@ -9452,6 +9828,7 @@ PSI_stage_info stage_sql_thd_waiting_until_delay= { 0, "Waiting until MASTER_DEL
 PSI_stage_info stage_storing_result_in_query_cache= { 0, "storing result in query cache", 0};
 PSI_stage_info stage_storing_row_into_queue= { 0, "storing row into queue", 0};
 PSI_stage_info stage_system_lock= { 0, "System lock", 0};
+PSI_stage_info stage_unlocking_tables= { 0, "Unlocking tables", 0};
 PSI_stage_info stage_update= { 0, "update", 0};
 PSI_stage_info stage_updating= { 0, "updating", 0};
 PSI_stage_info stage_updating_main_table= { 0, "updating main table", 0};
@@ -9496,6 +9873,7 @@ PSI_stage_info stage_gtid_wait_other_connection= { 0, "Waiting for other master
 
 PSI_stage_info *all_server_stages[]=
 {
+  & stage_after_apply_event,
   & stage_after_create,
   & stage_after_opening_tables,
   & stage_after_table_lock,
@@ -9503,6 +9881,7 @@ PSI_stage_info *all_server_stages[]=
   & stage_alter_inplace,
   & stage_alter_inplace_commit,
   & stage_alter_inplace_prepare,
+  & stage_apply_event,
   & stage_binlog_processing_checkpoint_notify,
   & stage_binlog_stopping_background_thread,
   & stage_binlog_waiting_background_tasks,
@@ -9584,6 +9963,7 @@ PSI_stage_info *all_server_stages[]=
   & stage_storing_result_in_query_cache,
   & stage_storing_row_into_queue,
   & stage_system_lock,
+  & stage_unlocking_tables,
   & stage_update,
   & stage_updating,
   & stage_updating_main_table,
diff --git a/sql/mysqld.h b/sql/mysqld.h
index a1656a49047..8a6794cc30c 100644
--- a/sql/mysqld.h
+++ b/sql/mysqld.h
@@ -25,6 +25,7 @@
 #include "sql_list.h"                      /* I_List */
 #include "sql_cmd.h"
 #include <my_rnd.h>
+#include "my_pthread.h"
 
 class THD;
 struct handlerton;
@@ -241,6 +242,11 @@ extern PSI_mutex_key key_PAGE_lock, key_LOCK_sync, key_LOCK_active,
        key_LOCK_pool, key_LOCK_pending_checkpoint;
 #endif /* HAVE_MMAP */
 
+#ifdef WITH_WSREP
+extern PSI_mutex_key key_LOCK_wsrep_thd;
+extern PSI_cond_key  key_COND_wsrep_thd;
+#endif /* WITH_WSREP */
+
 #ifdef HAVE_OPENSSL
 extern PSI_mutex_key key_LOCK_des_key_file;
 #endif
@@ -328,6 +334,7 @@ void init_server_psi_keys();
   MAINTAINER: Please keep this list in order, to limit merge collisions.
   Hint: grep PSI_stage_info | sort -u
 */
+extern PSI_stage_info stage_apply_event;
 extern PSI_stage_info stage_after_create;
 extern PSI_stage_info stage_after_opening_tables;
 extern PSI_stage_info stage_after_table_lock;
@@ -335,6 +342,7 @@ extern PSI_stage_info stage_allocating_local_table;
 extern PSI_stage_info stage_alter_inplace_prepare;
 extern PSI_stage_info stage_alter_inplace;
 extern PSI_stage_info stage_alter_inplace_commit;
+extern PSI_stage_info stage_after_apply_event;
 extern PSI_stage_info stage_changing_master;
 extern PSI_stage_info stage_checking_master_version;
 extern PSI_stage_info stage_checking_permissions;
@@ -408,6 +416,7 @@ extern PSI_stage_info stage_statistics;
 extern PSI_stage_info stage_storing_result_in_query_cache;
 extern PSI_stage_info stage_storing_row_into_queue;
 extern PSI_stage_info stage_system_lock;
+extern PSI_stage_info stage_unlocking_tables;
 extern PSI_stage_info stage_update;
 extern PSI_stage_info stage_updating;
 extern PSI_stage_info stage_updating_main_table;
@@ -595,6 +604,15 @@ enum options_mysqld
   OPT_WANT_CORE,
   OPT_MYSQL_COMPATIBILITY,
   OPT_MYSQL_TO_BE_IMPLEMENTED,
+#ifdef WITH_WSREP
+  OPT_WSREP_PROVIDER,
+  OPT_WSREP_PROVIDER_OPTIONS,
+  OPT_WSREP_CLUSTER_ADDRESS,
+  OPT_WSREP_START_POSITION,
+  OPT_WSREP_SST_AUTH,
+  OPT_WSREP_RECOVER,
+#endif /* WITH_WSREP */
+
   OPT_which_is_always_the_last
 };
 #endif
diff --git a/sql/protocol.cc b/sql/protocol.cc
index 2400dadfadc..caa00dd80e3 100644
--- a/sql/protocol.cc
+++ b/sql/protocol.cc
@@ -488,6 +488,8 @@ static uchar *net_store_length_fast(uchar *packet, uint length)
 
 void Protocol::end_statement()
 {
+  /* sanity check*/
+  DBUG_ASSERT_IF_WSREP(!(WSREP_ON && WSREP(thd) && thd->wsrep_conflict_state == REPLAYING));
   DBUG_ENTER("Protocol::end_statement");
   DBUG_ASSERT(! thd->get_stmt_da()->is_sent());
   bool error= FALSE;
diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc
index a6d93d10f11..29a1b9728e5 100644
--- a/sql/rpl_record.cc
+++ b/sql/rpl_record.cc
@@ -305,9 +305,8 @@ unpack_row(rpl_group_info *rgi,
           normal unpack operation.
         */
         uint16 const metadata= tabledef->field_metadata(i);
-#ifndef DBUG_OFF
         uchar const *const old_pack_ptr= pack_ptr;
-#endif
+
         pack_ptr= f->unpack(f->ptr, pack_ptr, row_end, metadata);
 	DBUG_PRINT("debug", ("field: %s; metadata: 0x%x;"
                              " pack_ptr: 0x%lx; pack_ptr': 0x%lx; bytes: %d",
@@ -316,6 +315,24 @@ unpack_row(rpl_group_info *rgi,
                              (int) (pack_ptr - old_pack_ptr)));
         if (!pack_ptr)
         {
+#ifdef WITH_WSREP
+	  if (WSREP_ON)
+          {
+            /*
+              Debug message to troubleshoot bug:
+              https://mariadb.atlassian.net/browse/MDEV-4404
+              Galera Node throws "Could not read field" error and drops out of cluster
+            */
+            WSREP_WARN("ROW event unpack field: %s  metadata: 0x%x;"
+                       " pack_ptr: 0x%lx; conv_table %p conv_field %p table %s"
+                       " row_end: 0x%lx",
+                       f->field_name, metadata,
+                       (ulong) old_pack_ptr, conv_table, conv_field,
+                       (table_found) ? "found" : "not found", (ulong)row_end
+            );
+	  }
+#endif /* WITH_WSREP */
+
           rgi->rli->report(ERROR_LEVEL, ER_SLAVE_CORRUPT_EVENT,
                       "Could not read field '%s' of table '%s.%s'",
                       f->field_name, table->s->db.str,
diff --git a/sql/set_var.cc b/sql/set_var.cc
index 5c1e00af33e..7ad5dc31f6e 100644
--- a/sql/set_var.cc
+++ b/sql/set_var.cc
@@ -556,7 +556,11 @@ int mysql_del_sys_var_chain(sys_var *first)
 
 static int show_cmp(SHOW_VAR *a, SHOW_VAR *b)
 {
+#ifdef WITH_WSREP
+  return my_strcasecmp(system_charset_info, a->name, b->name);
+#else
   return strcmp(a->name, b->name);
+#endif /* WITH_WSREP */
 }
 
 
diff --git a/sql/set_var.h b/sql/set_var.h
index bb92e555aa7..e72a8af6210 100644
--- a/sql/set_var.h
+++ b/sql/set_var.h
@@ -249,6 +249,9 @@ public:
   int check(THD *thd);
   int update(THD *thd);
   int light_check(THD *thd);
+#ifdef WITH_WSREP
+  int wsrep_store_variable(THD *thd);
+#endif
 };
 
 
@@ -355,6 +358,9 @@ extern sys_var *Sys_autocommit_ptr;
 
 CHARSET_INFO *get_old_charset_by_name(const char *old_name);
 
+#ifdef WITH_WSREP
+int sql_set_wsrep_variables(THD *thd, List<set_var_base> *var_list);
+#endif
 int sys_var_init();
 int sys_var_add_options(DYNAMIC_ARRAY *long_options, int parse_flags);
 void sys_var_end(void);
diff --git a/sql/slave.cc b/sql/slave.cc
index f7d019a6c39..746b1d48458 100644
--- a/sql/slave.cc
+++ b/sql/slave.cc
@@ -52,6 +52,7 @@
 #include "log_event.h"                          // Rotate_log_event,
                                                 // Create_file_log_event,
                                                 // Format_description_log_event
+#include "wsrep_mysqld.h"
 
 #ifdef HAVE_REPLICATION
 
@@ -4367,6 +4368,7 @@ pthread_handler_t handle_slave_sql(void *arg)
   my_off_t saved_skip= 0;
   Master_info *mi= ((Master_info*)arg);
   Relay_log_info* rli = &mi->rli;
+  my_bool wsrep_node_dropped= FALSE;
   const char *errmsg;
   rpl_group_info *serial_rgi;
   rpl_sql_thread_info sql_info(mi->rpl_filter);
@@ -4375,6 +4377,8 @@ pthread_handler_t handle_slave_sql(void *arg)
   my_thread_init();
   DBUG_ENTER("handle_slave_sql");
 
+ wsrep_restart_point:
+
   LINT_INIT(saved_master_log_pos);
   LINT_INIT(saved_log_pos);
 
@@ -4503,6 +4507,12 @@ pthread_handler_t handle_slave_sql(void *arg)
   }
 #endif
 
+#ifdef WITH_WSREP
+  thd->wsrep_exec_mode= LOCAL_STATE;
+  /* synchronize with wsrep replication */
+  if (WSREP_ON)
+    wsrep_ready_wait();
+#endif
   DBUG_PRINT("master_info",("log_file_name: %s  position: %s",
                             rli->group_master_log_name,
                             llstr(rli->group_master_log_pos,llbuff)));
@@ -4603,13 +4613,21 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME,
         rli->group_master_log_name, (ulong) rli->group_master_log_pos);
       saved_skip= 0;
     }
-    
+
     if (exec_relay_log_event(thd, rli, serial_rgi))
     {
       DBUG_PRINT("info", ("exec_relay_log_event() failed"));
       // do not scare the user if SQL thread was simply killed or stopped
       if (!sql_slave_killed(serial_rgi))
+      {
         slave_output_error_info(rli, thd);
+#ifdef WITH_WSREP
+        if (WSREP_ON && rli->last_error().number == ER_UNKNOWN_COM_ERROR)
+        {
+	  wsrep_node_dropped= TRUE;
+	}
+#endif /* WITH_WSREP */
+      }
       goto err;
     }
   }
@@ -4694,6 +4712,27 @@ err_during_init:
   delete serial_rgi;
   delete thd;
   mysql_mutex_unlock(&LOCK_thread_count);
+#ifdef WITH_WSREP
+  /* if slave stopped due to node going non primary, we set global flag to
+     trigger automatic restart of slave when node joins back to cluster
+  */
+   if (WSREP_ON && wsrep_node_dropped && wsrep_restart_slave)
+   {
+     if (wsrep_ready)
+     {
+       WSREP_INFO("Slave error due to node temporarily non-primary"
+		  "SQL slave will continue");
+       wsrep_node_dropped= FALSE;
+       mysql_mutex_unlock(&rli->run_lock);
+       goto wsrep_restart_point;
+     } else {
+       WSREP_INFO("Slave error due to node going non-primary");
+       WSREP_INFO("wsrep_restart_slave was set and therefore slave will be "
+		  "automatically restarted when node joins back to cluster");
+       wsrep_restart_slave_activated= TRUE;
+     }
+   }
+#endif /* WITH_WSREP */
  /*
   Note: the order of the broadcast and unlock calls below (first broadcast, then unlock)
   is important. Otherwise a killer_thread can execute between the calls and
diff --git a/sql/sp.cc b/sql/sp.cc
index 188b311ae86..261299464c5 100644
--- a/sql/sp.cc
+++ b/sql/sp.cc
@@ -32,7 +32,7 @@
 
 #include <my_user.h>
 
-static bool
+bool
 create_string(THD *thd, String *buf,
 	      stored_procedure_type sp_type,
 	      const char *db, ulong dblen,
@@ -924,7 +924,7 @@ end:
 }
 
 
-static void
+void
 sp_returns_type(THD *thd, String &result, sp_head *sp)
 {
   TABLE table;
@@ -2126,7 +2126,7 @@ int sp_cache_routine(THD *thd, enum stored_procedure_type type, sp_name *name,
   @return
     Returns TRUE on success, FALSE on (alloc) failure.
 */
-static bool
+bool
 create_string(THD *thd, String *buf,
               stored_procedure_type type,
               const char *db, ulong dblen,
@@ -2271,3 +2271,4 @@ sp_load_for_information_schema(THD *thd, TABLE *proc_table, String *db,
   thd->lex= old_lex;
   return sp;
 }
+
diff --git a/sql/sp.h b/sql/sp.h
index 3353132346b..de32b5454f8 100644
--- a/sql/sp.h
+++ b/sql/sp.h
@@ -214,4 +214,19 @@ bool load_collation(MEM_ROOT *mem_root,
                     CHARSET_INFO *dflt_cl,
                     CHARSET_INFO **cl);
 
+void sp_returns_type(THD *thd,
+                     String &result,
+                     sp_head *sp);
+
+bool create_string(THD *thd, String *buf,
+              stored_procedure_type type,
+              const char *db, ulong dblen,
+              const char *name, ulong namelen,
+              const char *params, ulong paramslen,
+              const char *returns, ulong returnslen,
+              const char *body, ulong bodylen,
+              st_sp_chistics *chistics,
+              const LEX_STRING *definer_user,
+              const LEX_STRING *definer_host,
+	      ulonglong sql_mode);
 #endif /* _SP_H_ */
diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc
index 106036e1e83..e67a019d1f7 100644
--- a/sql/sql_acl.cc
+++ b/sql/sql_acl.cc
@@ -2559,7 +2559,9 @@ int check_alter_user(THD *thd, const char *host, const char *user)
     my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), "--skip-grant-tables");
     goto end;
   }
-  if (!thd->slave_thread && !thd->security_ctx->priv_user[0])
+
+  if (IF_WSREP((!WSREP(thd) || !thd->wsrep_applier), 1) &&
+      !thd->slave_thread && !thd->security_ctx->priv_user[0])
   {
     my_message(ER_PASSWORD_ANONYMOUS_USER, ER(ER_PASSWORD_ANONYMOUS_USER),
                MYF(0));
@@ -2570,7 +2572,9 @@ int check_alter_user(THD *thd, const char *host, const char *user)
     my_error(ER_PASSWORD_NO_MATCH, MYF(0));
     goto end;
   }
+
   if (!thd->slave_thread &&
+      IF_WSREP((!WSREP(thd) || !thd->wsrep_applier),1) &&
       (strcmp(thd->security_ctx->priv_user, user) ||
        my_strcasecmp(system_charset_info, host,
                      thd->security_ctx->priv_host)))
@@ -2635,10 +2639,12 @@ bool change_password(THD *thd, const char *host, const char *user,
   TABLE_LIST tables[TABLES_MAX];
   /* Buffer should be extended when password length is extended. */
   char buff[512];
-  ulong query_length;
+  ulong query_length=0;
   enum_binlog_format save_binlog_format;
   uint new_password_len= (uint) strlen(new_password);
-  int result;
+  int result=0;
+  const CSET_STRING query_save = thd->query_string;
+
   DBUG_ENTER("change_password");
   DBUG_PRINT("enter",("host: '%s'  user: '%s'  new_password: '%s'",
 		      host,user,new_password));
@@ -2647,6 +2653,19 @@ bool change_password(THD *thd, const char *host, const char *user,
   if (check_change_password(thd, host, user, new_password, new_password_len))
     DBUG_RETURN(1);
 
+#ifdef WITH_WSREP
+  if (WSREP(thd) && !thd->wsrep_applier)
+  {
+    query_length= sprintf(buff, "SET PASSWORD FOR '%-.120s'@'%-.120s'='%-.120s'",
+			    user ? user : "",
+			    host ? host : "",
+			    new_password);
+    thd->set_query_inner(buff, query_length, system_charset_info);
+
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, (char*)"user", NULL);
+  }
+#endif /* WITH_WSREP */
+
   if ((result= open_grant_tables(thd, tables, TL_WRITE, Table_user)))
     DBUG_RETURN(result != 1);
 
@@ -2708,9 +2727,23 @@ bool change_password(THD *thd, const char *host, const char *user,
   }
 end:
   close_mysql_tables(thd);
+#ifdef WITH_WSREP
+  if (WSREP(thd) && !thd->wsrep_applier)
+  {
+    WSREP_TO_ISOLATION_END;
+
+    thd->query_string     = query_save;
+    thd->wsrep_exec_mode  = LOCAL_STATE;
+  }
+#endif /* WITH_WSREP */
   thd->restore_stmt_binlog_format(save_binlog_format);
 
   DBUG_RETURN(result);
+
+error:
+  WSREP_ERROR("Repliation of SET PASSWORD failed: %s", buff);
+  DBUG_RETURN(result);
+
 }
 
 int acl_check_set_default_role(THD *thd, const char *host, const char *user)
@@ -2727,8 +2760,9 @@ int acl_set_default_role(THD *thd, const char *host, const char *user,
   int result= 1;
   int error;
   bool clear_role= FALSE;
+  char buff[512];
   enum_binlog_format save_binlog_format;
-
+  const CSET_STRING query_save = thd->query_string;
 
   DBUG_ENTER("acl_set_default_role");
   DBUG_PRINT("enter",("host: '%s'  user: '%s'  rolename: '%s'",
@@ -2823,7 +2857,6 @@ int acl_set_default_role(THD *thd, const char *host, const char *user,
   result= 0;
   if (mysql_bin_log.is_open())
   {
-    char buff[512];
     int query_length=
       sprintf(buff,"SET DEFAULT ROLE '%-.120s' FOR '%-.120s'@'%-.120s'",
               safe_str(acl_user->default_rolename.str),
@@ -2835,6 +2868,17 @@ int acl_set_default_role(THD *thd, const char *host, const char *user,
   }
 end:
   close_mysql_tables(thd);
+
+#ifdef WITH_WSREP
+  if (WSREP(thd) && !thd->wsrep_applier)
+  {
+    WSREP_TO_ISOLATION_END;
+
+    thd->query_string     = query_save;
+    thd->wsrep_exec_mode  = LOCAL_STATE;
+  }
+#endif /* WITH_WSREP */
+
   thd->restore_stmt_binlog_format(save_binlog_format);
 
   DBUG_RETURN(result);
diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc
index 34a076cc327..0b610718cd0 100644
--- a/sql/sql_admin.cc
+++ b/sql/sql_admin.cc
@@ -1143,6 +1143,8 @@ bool Sql_cmd_analyze_table::execute(THD *thd)
                          FALSE, UINT_MAX, FALSE))
     goto error;
   thd->enable_slow_log= opt_log_slow_admin_statements;
+  WSREP_TO_ISOLATION_BEGIN(first_table->db, first_table->table_name, NULL);
+
   res= mysql_admin_table(thd, first_table, &m_lex->check_opt,
                          "analyze", lock_type, 1, 0, 0, 0,
                          &handler::ha_analyze, 0);
@@ -1197,6 +1199,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd)
   if (check_table_access(thd, SELECT_ACL | INSERT_ACL, first_table,
                          FALSE, UINT_MAX, FALSE))
     goto error; /* purecov: inspected */
+  WSREP_TO_ISOLATION_BEGIN(first_table->db, first_table->table_name, NULL)
   thd->enable_slow_log= opt_log_slow_admin_statements;
   res= (specialflag & SPECIAL_NO_NEW_FUNC) ?
     mysql_recreate_table(thd, first_table, true) :
@@ -1230,6 +1233,7 @@ bool Sql_cmd_repair_table::execute(THD *thd)
                          FALSE, UINT_MAX, FALSE))
     goto error; /* purecov: inspected */
   thd->enable_slow_log= opt_log_slow_admin_statements;
+  WSREP_TO_ISOLATION_BEGIN(first_table->db, first_table->table_name, NULL)
   res= mysql_admin_table(thd, first_table, &m_lex->check_opt, "repair",
                          TL_WRITE, 1,
                          MY_TEST(m_lex->check_opt.sql_flags & TT_USEFRM),
diff --git a/sql/sql_alter.cc b/sql/sql_alter.cc
index 97b9c127c22..cfe360217c2 100644
--- a/sql/sql_alter.cc
+++ b/sql/sql_alter.cc
@@ -18,6 +18,7 @@
                                              // mysql_exchange_partition
 #include "sql_base.h"                        // open_temporary_tables
 #include "sql_alter.h"
+#include "wsrep_mysqld.h"
 
 Alter_info::Alter_info(const Alter_info &rhs, MEM_ROOT *mem_root)
   :drop_list(rhs.drop_list, mem_root),
@@ -303,6 +304,22 @@ bool Sql_cmd_alter_table::execute(THD *thd)
 
   thd->enable_slow_log= opt_log_slow_admin_statements;
 
+#ifdef WITH_WSREP
+  TABLE *find_temporary_table(THD *thd, const TABLE_LIST *tl);
+
+  if (WSREP(thd) &&
+      (!thd->is_current_stmt_binlog_format_row() ||
+       !find_temporary_table(thd, first_table))  &&
+      wsrep_to_isolation_begin(thd,
+                               lex->name.str ? select_lex->db : NULL,
+                               lex->name.str ? lex->name.str : NULL,
+                               first_table))
+  {
+    WSREP_WARN("ALTER TABLE isolation failure");
+    DBUG_RETURN(TRUE);
+  }
+#endif /* WITH_WSREP */
+
   result= mysql_alter_table(thd, select_lex->db, lex->name.str,
                             &create_info,
                             first_table,
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index d60506dcad7..68f325e20ab 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -61,7 +61,8 @@
 #ifdef  __WIN__
 #include <io.h>
 #endif
-
+#include "wsrep_mysqld.h"
+#include "wsrep_thd.h"
 
 bool
 No_such_table_error_handler::handle_condition(THD *,
@@ -3557,7 +3558,7 @@ thr_lock_type read_lock_type_for_table(THD *thd,
   */
   bool log_on= mysql_bin_log.is_open() && thd->variables.sql_log_bin;
   ulong binlog_format= thd->variables.binlog_format;
-  if ((log_on == FALSE) || (binlog_format == BINLOG_FORMAT_ROW) ||
+  if ((log_on == FALSE) || (WSREP_FORMAT(binlog_format) == BINLOG_FORMAT_ROW) ||
       (table_list->table->s->table_category == TABLE_CATEGORY_LOG) ||
       (table_list->table->s->table_category == TABLE_CATEGORY_PERFORMANCE) ||
       !(is_update_query(prelocking_ctx->sql_command) ||
@@ -4585,8 +4586,29 @@ restart:
     }
   }
 
+#ifdef WITH_WSREP
+  if (WSREP_ON &&
+      (thd->lex->sql_command== SQLCOM_INSERT         ||
+       thd->lex->sql_command== SQLCOM_INSERT_SELECT  ||
+       thd->lex->sql_command== SQLCOM_REPLACE        ||
+       thd->lex->sql_command== SQLCOM_REPLACE_SELECT ||
+       thd->lex->sql_command== SQLCOM_UPDATE         ||
+       thd->lex->sql_command== SQLCOM_UPDATE_MULTI   ||
+       thd->lex->sql_command== SQLCOM_LOAD           ||
+       thd->lex->sql_command== SQLCOM_DELETE)        &&
+      wsrep_replicate_myisam                         &&
+      (*start)                                       &&
+      (*start)->table && (*start)->table->file->ht->db_type == DB_TYPE_MYISAM)
+    {
+      WSREP_TO_ISOLATION_BEGIN(NULL, NULL, (*start));
+    }
+ error:
+#endif
+
 err:
   THD_STAGE_INFO(thd, stage_after_opening_tables);
+  thd_proc_info(thd, 0);
+
   free_root(&new_frm_mem, MYF(0));              // Free pre-alloced block
 
   if (error && *table_to_open)
@@ -5040,6 +5062,8 @@ end:
     close_thread_tables(thd);
   }
   THD_STAGE_INFO(thd, stage_after_opening_tables);
+
+  thd_proc_info(thd, 0);
   DBUG_RETURN(table);
 }
 
@@ -5303,7 +5327,7 @@ bool lock_tables(THD *thd, TABLE_LIST *tables, uint count,
         We can solve these problems in mixed mode by switching to binlogging 
         if at least one updated table is used by sub-statement
       */
-      if (thd->variables.binlog_format != BINLOG_FORMAT_ROW && tables && 
+      if (WSREP_FORMAT(thd->variables.binlog_format) != BINLOG_FORMAT_ROW && tables && 
           has_write_table_with_auto_increment(thd->lex->first_not_own_table()))
         thd->lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_AUTOINC_COLUMNS);
     }
@@ -8993,7 +9017,17 @@ bool mysql_notify_thread_having_shared_lock(THD *thd, THD *in_use,
         (e.g. see partitioning code).
       */
       if (!thd_table->needs_reopen())
-        signalled|= mysql_lock_abort_for_thread(thd, thd_table);
+      {
+	signalled|= mysql_lock_abort_for_thread(thd, thd_table);
+#ifdef WITH_WSREP
+	if (thd && WSREP(thd) && wsrep_thd_is_BF((void *)thd, true))
+	{
+	  WSREP_DEBUG("remove_table_from_cache: %llu",
+		      (unsigned long long) thd->real_id);
+	  wsrep_abort_thd((void *)thd, (void *)in_use, FALSE);
+	}
+#endif
+      }
     }
     mysql_mutex_unlock(&in_use->LOCK_thd_data);
   }
diff --git a/sql/sql_builtin.cc.in b/sql/sql_builtin.cc.in
index 63850650ac9..2de475b0a76 100644
--- a/sql/sql_builtin.cc.in
+++ b/sql/sql_builtin.cc.in
@@ -25,7 +25,11 @@ extern
 #endif
 builtin_maria_plugin 
   @mysql_mandatory_plugins@ @mysql_optional_plugins@
-  builtin_maria_binlog_plugin, builtin_maria_mysql_password_plugin;
+  builtin_maria_binlog_plugin,
+#ifdef WITH_WSREP
+	builtin_wsrep_plugin@mysql_plugin_defs@,
+#endif /* WITH_WSREP */
+ builtin_maria_mysql_password_plugin;
 
 struct st_maria_plugin *mysql_optional_plugins[]=
 {
@@ -35,5 +39,8 @@ struct st_maria_plugin *mysql_optional_plugins[]=
 struct st_maria_plugin *mysql_mandatory_plugins[]=
 {
   builtin_maria_binlog_plugin, builtin_maria_mysql_password_plugin,
+#ifdef WITH_WSREP
+  builtin_wsrep_plugin@mysql_plugin_defs@,
+#endif /* WITH_WSREP */
   @mysql_mandatory_plugins@ 0
 };
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index 8d6ddc0bb08..443da8557ad 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -64,6 +64,8 @@
 #include "sql_parse.h"                          // is_update_query
 #include "sql_callback.h"
 #include "lock.h"
+#include "wsrep_mysqld.h"
+#include "wsrep_thd.h"
 #include "sql_connect.h"
 
 /*
@@ -859,7 +861,7 @@ bool Drop_table_error_handler::handle_condition(THD *thd,
 }
 
 
-THD::THD()
+THD::THD(bool is_applier)
    :Statement(&main_lex, &main_mem_root, STMT_CONVENTIONAL_EXECUTION,
               /* statement id */ 0),
    rli_fake(0), rgi_fake(0), rgi_slave(NULL),
@@ -889,6 +891,16 @@ THD::THD()
    bootstrap(0),
    derived_tables_processing(FALSE),
    spcont(NULL),
+#ifdef WITH_WSREP
+   wsrep_applier(is_applier),
+   wsrep_applier_closing(false),
+   wsrep_client_thread(false),
+   wsrep_po_handle(WSREP_PO_INITIALIZER),
+   wsrep_po_cnt(0),
+//   wsrep_po_in_trans(false),
+   wsrep_apply_format(0),
+   wsrep_apply_toi(false),
+#endif
    m_parser_state(NULL),
 #if defined(ENABLED_DEBUG_SYNC)
    debug_sync_control(0),
@@ -1004,6 +1016,22 @@ THD::THD()
   m_command=COM_CONNECT;
   *scramble= '\0';
 
+#ifdef WITH_WSREP
+  mysql_mutex_init(key_LOCK_wsrep_thd, &LOCK_wsrep_thd, MY_MUTEX_INIT_FAST);
+  mysql_cond_init(key_COND_wsrep_thd, &COND_wsrep_thd, NULL);
+  wsrep_ws_handle.trx_id = WSREP_UNDEFINED_TRX_ID;
+  wsrep_ws_handle.opaque = NULL;
+  wsrep_retry_counter     = 0;
+  wsrep_PA_safe           = true;
+  wsrep_retry_query       = NULL;
+  wsrep_retry_query_len   = 0;
+  wsrep_retry_command     = COM_CONNECT;
+  wsrep_consistency_check = NO_CONSISTENCY_CHECK;
+  wsrep_status_vars       = 0;
+  wsrep_mysql_replicated  = 0;
+  wsrep_TOI_pre_query     = NULL;
+  wsrep_TOI_pre_query_len = 0;
+#endif
   /* Call to init() below requires fully initialized Open_tables_state. */
   reset_open_tables_state(this);
 
@@ -1043,6 +1071,9 @@ THD::THD()
   my_rnd_init(&rand, tmp + (ulong) &rand, tmp + (ulong) ::global_query_id);
   substitute_null_with_insert_id = FALSE;
   thr_lock_info_init(&lock_info); /* safety: will be reset after start */
+  lock_info.mysql_thd= (void *)this;
+
+  wsrep_info[sizeof(wsrep_info) - 1] = '\0'; /* make sure it is 0-terminated */
 
   m_internal_handler= NULL;
   m_binlog_invoker= INVOKER_NONE;
@@ -1387,7 +1418,24 @@ void THD::init(void)
   bzero((char *) &org_status_var, sizeof(org_status_var));
   start_bytes_received= 0;
   last_commit_gtid.seq_no= 0;
-
+#ifdef WITH_WSREP
+  wsrep_exec_mode= wsrep_applier ? REPL_RECV :  LOCAL_STATE;
+  wsrep_conflict_state= NO_CONFLICT;
+  wsrep_query_state= QUERY_IDLE;
+  wsrep_last_query_id= 0;
+  wsrep_trx_meta.gtid= WSREP_GTID_UNDEFINED;
+  wsrep_trx_meta.depends_on= WSREP_SEQNO_UNDEFINED;
+  wsrep_converted_lock_session= false;
+  wsrep_retry_counter= 0;
+  wsrep_rli= NULL;
+  wsrep_rgi= NULL;
+  wsrep_PA_safe= true;
+  wsrep_consistency_check = NO_CONSISTENCY_CHECK;
+  wsrep_mysql_replicated  = 0;
+
+  wsrep_TOI_pre_query     = NULL;
+  wsrep_TOI_pre_query_len = 0;
+#endif
   if (variables.sql_log_bin)
     variables.option_bits|= OPTION_BIN_LOG;
   else
@@ -1582,6 +1630,14 @@ THD::~THD()
   mysql_mutex_lock(&LOCK_thd_data);
   mysql_mutex_unlock(&LOCK_thd_data);
 
+#ifdef WITH_WSREP
+  mysql_mutex_lock(&LOCK_wsrep_thd);
+  mysql_mutex_unlock(&LOCK_wsrep_thd);
+  mysql_mutex_destroy(&LOCK_wsrep_thd);
+  if (wsrep_rli) delete wsrep_rli;
+  if (wsrep_rgi) delete wsrep_rgi;
+  if (wsrep_status_vars) wsrep->stats_free(wsrep, wsrep_status_vars);
+#endif
   /* Close connection */
 #ifndef EMBEDDED_LIBRARY
   if (net.vio)
@@ -1892,7 +1948,17 @@ bool THD::notify_shared_lock(MDL_context_owner *ctx_in_use,
         (e.g. see partitioning code).
       */
       if (!thd_table->needs_reopen())
+      {
         signalled|= mysql_lock_abort_for_thread(this, thd_table);
+#if WITH_WSREP
+        if (this && WSREP(this) && wsrep_thd_is_BF((void *)this, FALSE))
+        {
+          WSREP_DEBUG("remove_table_from_cache: %llu",
+                      (unsigned long long) this->real_id);
+          wsrep_abort_thd((void *)this, (void *)in_use, FALSE);
+        }
+#endif /* WITH_WSREP */
+      }
     }
     mysql_mutex_unlock(&in_use->LOCK_thd_data);
   }
@@ -2074,6 +2140,12 @@ void THD::cleanup_after_query()
   /* reset table map for multi-table update */
   table_map_for_update= 0;
   m_binlog_invoker= INVOKER_NONE;
+#ifdef WITH_WSREP
+  if (TOTAL_ORDER == wsrep_exec_mode)
+  {
+    wsrep_exec_mode = LOCAL_STATE;
+  }
+#endif  /* WITH_WSREP */
 
 #ifndef EMBEDDED_LIBRARY
   if (rgi_slave)
@@ -2496,6 +2568,13 @@ bool sql_exchange::escaped_given(void)
 bool select_send::send_result_set_metadata(List<Item> &list, uint flags)
 {
   bool res;
+#ifdef WITH_WSREP
+  if (WSREP(thd) && thd->wsrep_retry_query)
+  {
+    WSREP_DEBUG("skipping select metadata");
+    return FALSE;
+  }
+#endif /* WITH_WSREP */
   if (!(res= thd->protocol->send_result_set_metadata(&list, flags)))
     is_result_set_started= 1;
   return res;
@@ -4241,8 +4320,10 @@ extern "C" int thd_non_transactional_update(const MYSQL_THD thd)
 
 extern "C" int thd_binlog_format(const MYSQL_THD thd)
 {
-  if (mysql_bin_log.is_open() && (thd->variables.option_bits & OPTION_BIN_LOG))
-    return (int) thd->variables.binlog_format;
+  if (IF_WSREP(((WSREP(thd) &&  wsrep_emulate_bin_log) || mysql_bin_log.is_open()),
+      mysql_bin_log.is_open()) &&
+      thd->variables.option_bits & OPTION_BIN_LOG)
+    return (int) WSREP_FORMAT(thd->variables.binlog_format);
   else
     return BINLOG_FORMAT_UNSPEC;
 }
@@ -4972,7 +5053,7 @@ int THD::decide_logging_format(TABLE_LIST *tables)
     binlog by filtering rules.
   */
   if (mysql_bin_log.is_open() && (variables.option_bits & OPTION_BIN_LOG) &&
-      !(variables.binlog_format == BINLOG_FORMAT_STMT &&
+      !(WSREP_FORMAT(variables.binlog_format) == BINLOG_FORMAT_STMT &&
         !binlog_filter->db_ok(db)))
   {
     /*
@@ -5182,7 +5263,7 @@ int THD::decide_logging_format(TABLE_LIST *tables)
         */
         my_error((error= ER_BINLOG_ROW_INJECTION_AND_STMT_ENGINE), MYF(0));
       }
-      else if (variables.binlog_format == BINLOG_FORMAT_ROW &&
+      else if (WSREP_FORMAT(variables.binlog_format) == BINLOG_FORMAT_ROW &&
                sqlcom_can_generate_row_events(this))
       {
         /*
@@ -5211,7 +5292,7 @@ int THD::decide_logging_format(TABLE_LIST *tables)
     else
     {
       /* binlog_format = STATEMENT */
-      if (variables.binlog_format == BINLOG_FORMAT_STMT)
+      if (WSREP_FORMAT(variables.binlog_format) == BINLOG_FORMAT_STMT)
       {
         if (lex->is_stmt_row_injection())
         {
@@ -5228,7 +5309,10 @@ int THD::decide_logging_format(TABLE_LIST *tables)
             5. Error: Cannot modify table that uses a storage engine
                limited to row-logging when binlog_format = STATEMENT
           */
-          my_error((error= ER_BINLOG_STMT_MODE_AND_ROW_ENGINE), MYF(0), "");
+	  if (IF_WSREP((!WSREP(this) || wsrep_exec_mode == LOCAL_STATE),1))
+	  {
+            my_error((error= ER_BINLOG_STMT_MODE_AND_ROW_ENGINE), MYF(0), "");
+	  }
         }
         else if (is_write && (unsafe_flags= lex->get_stmt_unsafe_flags()) != 0)
         {
@@ -5340,7 +5424,7 @@ int THD::decide_logging_format(TABLE_LIST *tables)
                         "and binlog_filter->db_ok(db) = %d",
                         mysql_bin_log.is_open(),
                         (variables.option_bits & OPTION_BIN_LOG),
-                        variables.binlog_format,
+                        WSREP_FORMAT(variables.binlog_format),
                         binlog_filter->db_ok(db)));
 #endif
 
@@ -5576,9 +5660,11 @@ CPP_UNNAMED_NS_END
 int THD::binlog_write_row(TABLE* table, bool is_trans, 
                           MY_BITMAP const* cols, size_t colcnt, 
                           uchar const *record) 
-{ 
-  DBUG_ASSERT(is_current_stmt_binlog_format_row() && mysql_bin_log.is_open());
+{
 
+  DBUG_ASSERT(is_current_stmt_binlog_format_row() &&
+    IF_WSREP(((WSREP(this) && wsrep_emulate_bin_log) || mysql_bin_log.is_open()),
+              mysql_bin_log.is_open()));
   /*
     Pack records into format for transfer. We are allocating more
     memory than needed, but that doesn't matter.
@@ -5610,8 +5696,10 @@ int THD::binlog_update_row(TABLE* table, bool is_trans,
                            MY_BITMAP const* cols, size_t colcnt,
                            const uchar *before_record,
                            const uchar *after_record)
-{ 
-  DBUG_ASSERT(is_current_stmt_binlog_format_row() && mysql_bin_log.is_open());
+{
+  DBUG_ASSERT(is_current_stmt_binlog_format_row() &&
+    IF_WSREP(((WSREP(this) && wsrep_emulate_bin_log) || mysql_bin_log.is_open()),
+              mysql_bin_log.is_open()));
 
   size_t const before_maxlen = max_row_length(table, before_record);
   size_t const after_maxlen  = max_row_length(table, after_record);
@@ -5659,8 +5747,10 @@ int THD::binlog_update_row(TABLE* table, bool is_trans,
 int THD::binlog_delete_row(TABLE* table, bool is_trans, 
                            MY_BITMAP const* cols, size_t colcnt,
                            uchar const *record)
-{ 
-  DBUG_ASSERT(is_current_stmt_binlog_format_row() && mysql_bin_log.is_open());
+{
+  DBUG_ASSERT(is_current_stmt_binlog_format_row() &&
+    IF_WSREP(((WSREP(this) && wsrep_emulate_bin_log) || mysql_bin_log.is_open()),
+              mysql_bin_log.is_open()));
 
   /* 
      Pack records into format for transfer. We are allocating more
@@ -5695,7 +5785,8 @@ int THD::binlog_remove_pending_rows_event(bool clear_maps,
 {
   DBUG_ENTER("THD::binlog_remove_pending_rows_event");
 
-  if (!mysql_bin_log.is_open())
+  IF_WSREP(!(WSREP_EMULATE_BINLOG(this) || mysql_bin_log.is_open()),
+    !mysql_bin_log.is_open());
     DBUG_RETURN(0);
 
   /* Ensure that all events in a GTID group are in the same cache */
@@ -5718,7 +5809,8 @@ int THD::binlog_flush_pending_rows_event(bool stmt_end, bool is_transactional)
     mode: it might be the case that we left row-based mode before
     flushing anything (e.g., if we have explicitly locked tables).
    */
-  if (!mysql_bin_log.is_open())
+  if(IF_WSREP(!(WSREP_EMULATE_BINLOG(this) || mysql_bin_log.is_open()),
+              !mysql_bin_log.is_open()))
     DBUG_RETURN(0);
 
   /* Ensure that all events in a GTID group are in the same cache */
@@ -5970,7 +6062,10 @@ int THD::binlog_query(THD::enum_binlog_query_type qtype, char const *query_arg,
   DBUG_ENTER("THD::binlog_query");
   DBUG_PRINT("enter", ("qtype: %s  query: '%-.*s'",
                        show_query_type(qtype), (int) query_len, query_arg));
-  DBUG_ASSERT(query_arg && mysql_bin_log.is_open());
+
+  DBUG_ASSERT(query_arg &&
+    IF_WSREP((WSREP_EMULATE_BINLOG(this) || mysql_bin_log.is_open()),
+             mysql_bin_log.is_open()));
 
   /* If this is withing a BEGIN ... COMMIT group, don't log it */
   if (variables.option_bits & OPTION_GTID_BEGIN)
diff --git a/sql/sql_class.h b/sql/sql_class.h
index f3537086132..d3b305d34f4 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -50,12 +50,13 @@ void set_thd_stage_info(void *thd,
                         const char *calling_func,
                         const char *calling_file,
                         const unsigned int calling_line);
-                        
+
 #define THD_STAGE_INFO(thd, stage) \
   (thd)->enter_stage(& stage, NULL, __func__, __FILE__, __LINE__)
 
 #include "my_apc.h"
 #include "rpl_gtid.h"
+#include "wsrep_mysqld.h"
 
 class Reprepare_observer;
 class Relay_log_info;
@@ -638,6 +639,12 @@ typedef struct system_variables
   ulong wt_timeout_short, wt_deadlock_search_depth_short;
   ulong wt_timeout_long, wt_deadlock_search_depth_long;
 
+#ifdef WITH_WSREP
+  my_bool wsrep_on;
+  my_bool wsrep_causal_reads;
+  uint wsrep_sync_wait;
+  ulong wsrep_retry_autocommit;
+#endif
   double long_query_time_double;
 
   my_bool pseudo_slave_mode;
@@ -2072,7 +2079,7 @@ public:
   int is_current_stmt_binlog_format_row() const {
     DBUG_ASSERT(current_stmt_binlog_format == BINLOG_FORMAT_STMT ||
                 current_stmt_binlog_format == BINLOG_FORMAT_ROW);
-    return current_stmt_binlog_format == BINLOG_FORMAT_ROW;
+    return (WSREP_FORMAT((ulong)current_stmt_binlog_format) == BINLOG_FORMAT_ROW);
   }
 
   enum binlog_filter_state
@@ -2725,6 +2732,45 @@ public:
     query_id_t first_query_id;
   } binlog_evt_union;
 
+#ifdef WITH_WSREP
+  const bool                wsrep_applier; /* dedicated slave applier thread */
+  bool                      wsrep_applier_closing; /* applier marked to close */
+  bool                      wsrep_client_thread; /* to identify client threads*/
+  bool                      wsrep_PA_safe;
+  bool                      wsrep_converted_lock_session;
+  bool                      wsrep_apply_toi; /* applier processing in TOI */
+  enum wsrep_exec_mode      wsrep_exec_mode;
+  query_id_t                wsrep_last_query_id;
+  enum wsrep_query_state    wsrep_query_state;
+  enum wsrep_conflict_state wsrep_conflict_state;
+  mysql_mutex_t             LOCK_wsrep_thd;
+  mysql_cond_t              COND_wsrep_thd;
+  // changed from wsrep_seqno_t to wsrep_trx_meta_t in wsrep API rev 75
+  // wsrep_seqno_t             wsrep_trx_seqno;
+  wsrep_trx_meta_t          wsrep_trx_meta;
+  uint32                    wsrep_rand;
+  Relay_log_info*           wsrep_rli;
+  rpl_group_info*           wsrep_rgi;
+  wsrep_ws_handle_t         wsrep_ws_handle;
+  ulong                     wsrep_retry_counter; // of autocommit
+  char*                     wsrep_retry_query;
+  size_t                    wsrep_retry_query_len;
+  enum enum_server_command  wsrep_retry_command;
+  enum wsrep_consistency_check_mode
+                            wsrep_consistency_check;
+  wsrep_stats_var*          wsrep_status_vars;
+  int                       wsrep_mysql_replicated;
+  const char*               wsrep_TOI_pre_query; /* a query to apply before
+                                                    the actual TOI query */
+  size_t                    wsrep_TOI_pre_query_len;
+  wsrep_po_handle_t         wsrep_po_handle;
+  size_t                    wsrep_po_cnt;
+#ifdef GTID_SUPPORT
+  rpl_sid                   wsrep_po_sid;
+#endif /*  GTID_SUPPORT */
+  void*                     wsrep_apply_format;
+#endif /* WITH_WSREP */
+  char                      wsrep_info[128]; /* string for dynamic proc info */
   /**
     Internal parser state.
     Note that since the parser is not re-entrant, we keep only one parser
@@ -2756,7 +2802,8 @@ public:
   /* Debug Sync facility. See debug_sync.cc. */
   struct st_debug_sync_control *debug_sync_control;
 #endif /* defined(ENABLED_DEBUG_SYNC) */
-  THD();
+  THD(bool is_applier= false);
+
   ~THD();
 
   void init(void);
@@ -3270,7 +3317,7 @@ public:
       tests fail and so force them to propagate the
       lex->binlog_row_based_if_mixed upwards to the caller.
     */
-    if ((variables.binlog_format == BINLOG_FORMAT_MIXED) &&
+    if ((WSREP_FORMAT(variables.binlog_format) == BINLOG_FORMAT_MIXED) &&
         (in_sub_stmt == 0))
       set_current_stmt_binlog_format_row();
 
@@ -3322,7 +3369,7 @@ public:
                 show_system_thread(system_thread)));
     if (in_sub_stmt == 0)
     {
-      if (variables.binlog_format == BINLOG_FORMAT_ROW)
+      if (WSREP_FORMAT(variables.binlog_format) == BINLOG_FORMAT_ROW)
         set_current_stmt_binlog_format_row();
       else if (temporary_tables == NULL)
         set_current_stmt_binlog_format_stmt();
diff --git a/sql/sql_connect.cc b/sql/sql_connect.cc
index 433f3303ad7..0065edcc14d 100644
--- a/sql/sql_connect.cc
+++ b/sql/sql_connect.cc
@@ -37,6 +37,7 @@
                       // reset_host_errors
 #include "sql_acl.h"  // acl_getroot, NO_ACCESS, SUPER_ACL
 #include "sql_callback.h"
+#include "wsrep_mysqld.h"
 
 HASH global_user_stats, global_client_stats, global_table_stats;
 HASH global_index_stats;
@@ -1172,6 +1173,17 @@ bool login_connection(THD *thd)
 void end_connection(THD *thd)
 {
   NET *net= &thd->net;
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    wsrep_status_t rcode= wsrep->free_connection(wsrep, thd->thread_id);
+    if (rcode) {
+      WSREP_WARN("wsrep failed to free connection context: %lu, code: %d",
+                 thd->thread_id, rcode);
+    }
+  }
+  thd->wsrep_client_thread= 0;
+#endif
   plugin_thdvar_cleanup(thd);
 
   if (thd->user_connect)
@@ -1307,6 +1319,9 @@ bool thd_prepare_connection(THD *thd)
                          (char *) thd->security_ctx->host_or_ip);
 
   prepare_new_connection_state(thd);
+#ifdef WITH_WSREP
+  thd->wsrep_client_thread= 1;
+#endif /* WITH_WSREP */
   return FALSE;
 }
 
@@ -1380,7 +1395,15 @@ void do_handle_one_connection(THD *thd_arg)
 	break;
     }
     end_connection(thd);
-   
+
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+    thd->wsrep_query_state= QUERY_EXITING;
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  }
+#endif
 end_thread:
     close_connection(thd);
 
diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc
index 38fb897c4f8..5eb4c405c4b 100644
--- a/sql/sql_delete.cc
+++ b/sql/sql_delete.cc
@@ -642,18 +642,19 @@ cleanup:
   if (!transactional_table && deleted > 0)
     thd->transaction.stmt.modified_non_trans_table=
       thd->transaction.all.modified_non_trans_table= TRUE;
-  
+
   /* See similar binlogging code in sql_update.cc, for comments */
   if ((error < 0) || thd->transaction.stmt.modified_non_trans_table)
   {
-    if (mysql_bin_log.is_open())
+    if(IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+		mysql_bin_log.is_open()))
     {
       int errcode= 0;
       if (error < 0)
         thd->clear_error();
       else
         errcode= query_error_code(thd, killed_status == NOT_KILLED);
-      
+
       /*
         [binlog]: If 'handler::delete_all_rows()' was called and the
         storage engine does not inject the rows itself, we replicate
@@ -1107,13 +1108,14 @@ void multi_delete::abort_result_set()
     DBUG_ASSERT(error_handled);
     DBUG_VOID_RETURN;
   }
-  
+
   if (thd->transaction.stmt.modified_non_trans_table)
   {
-    /* 
+    /*
        there is only side effects; to binlog with the error
     */
-    if (mysql_bin_log.is_open())
+    if (IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+		 mysql_bin_log.is_open()))
     {
       int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
       /* possible error of writing binary log is ignored deliberately */
@@ -1289,7 +1291,8 @@ bool multi_delete::send_eof()
   }
   if ((local_error == 0) || thd->transaction.stmt.modified_non_trans_table)
   {
-    if (mysql_bin_log.is_open())
+    if(IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+		mysql_bin_log.is_open()))
     {
       int errcode= 0;
       if (local_error == 0)
diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc
index d61af758ced..f90e9cd2439 100644
--- a/sql/sql_insert.cc
+++ b/sql/sql_insert.cc
@@ -1014,7 +1014,8 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list,
         thd->transaction.stmt.modified_non_trans_table ||
 	was_insert_delayed)
     {
-      if (mysql_bin_log.is_open())
+      if(IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+                  mysql_bin_log.is_open()))
       {
         int errcode= 0;
 	if (error <= 0)
@@ -3195,6 +3196,13 @@ bool Delayed_insert::handle_inserts(void)
         mysql_cond_broadcast(&cond_client);     // If waiting clients
     }
   }
+
+#ifdef WITH_WSREP
+  if (WSREP((&thd)))
+    thd_proc_info(&thd, "insert done");
+  else
+#endif /* WITH_WSREP */
+  thd_proc_info(&thd, 0);
   mysql_mutex_unlock(&mutex);
 
   /*
@@ -3647,8 +3655,11 @@ bool select_insert::send_eof()
   DBUG_PRINT("enter", ("trans_table=%d, table_type='%s'",
                        trans_table, table->file->table_type()));
 
-  error= (thd->locked_tables_mode <= LTM_LOCK_TABLES ?
-          table->file->ha_end_bulk_insert() : 0);
+  error = IF_WSREP((thd->wsrep_conflict_state == MUST_ABORT ||
+		  thd->wsrep_conflict_state == CERT_FAILURE) ? -1 :, )
+	  (thd->locked_tables_mode <= LTM_LOCK_TABLES ?
+           table->file->ha_end_bulk_insert() : 0);
+
   if (!error && thd->is_error())
     error= thd->get_stmt_da()->sql_errno();
 
@@ -3676,8 +3687,9 @@ bool select_insert::send_eof()
     events are in the transaction cache and will be written when
     ha_autocommit_or_rollback() is issued below.
   */
-  if (mysql_bin_log.is_open() &&
-      (!error || thd->transaction.stmt.modified_non_trans_table))
+  if(IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+              mysql_bin_log.is_open()) &&
+     (!error || thd->transaction.stmt.modified_non_trans_table))
   {
     int errcode= 0;
     if (!error)
@@ -3761,7 +3773,8 @@ void select_insert::abort_result_set() {
         if (!can_rollback_data())
           thd->transaction.all.modified_non_trans_table= TRUE;
 
-        if (mysql_bin_log.is_open())
+        if(IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+		    mysql_bin_log.is_open()))
         {
           int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
           /* error of writing binary log is ignored */
@@ -4169,7 +4182,8 @@ select_create::binlog_show_create_table(TABLE **tables, uint count)
                             create_info->table_was_deleted);
   DBUG_ASSERT(result == 0); /* store_create_info() always return 0 */
 
-  if (mysql_bin_log.is_open())
+  if(IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+	      mysql_bin_log.is_open()))
   {
     int errcode= query_error_code(thd, thd->killed == NOT_KILLED);
     result= thd->binlog_query(THD::STMT_QUERY_TYPE,
@@ -4179,6 +4193,9 @@ select_create::binlog_show_create_table(TABLE **tables, uint count)
                               /* suppress_use */ FALSE,
                               errcode);
   }
+
+  IF_WSREP(ha_fake_trx_id(thd), );
+
   return result;
 }
 
@@ -4208,6 +4225,21 @@ bool select_create::send_eof()
     trans_commit_stmt(thd);
     if (!(thd->variables.option_bits & OPTION_GTID_BEGIN))
       trans_commit_implicit(thd);
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+    {
+      mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+      if (thd->wsrep_conflict_state != NO_CONFLICT)
+      {
+        WSREP_DEBUG("select_create commit failed, thd: %lu err: %d %s",
+                    thd->thread_id, thd->wsrep_conflict_state, thd->query());
+        mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+        abort_result_set();
+	return TRUE;
+      }
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    }
+#endif /* WITH_WSREP */
   }
   else if (!thd->is_current_stmt_binlog_format_row())
     table->s->table_creation_was_logged= 1;
diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc
index cd9f9238f71..926aa7d5e5a 100644
--- a/sql/sql_lex.cc
+++ b/sql/sql_lex.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2013, Monty Program Ab.
+   Copyright (c) 2009, 2014, Monty Program Ab.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1579,6 +1579,17 @@ int lex_one_token(void *arg, THD *thd)
           }
           else
           {
+#ifdef WITH_WSREP
+	    if (WSREP(thd) && version == 99997 && thd->wsrep_exec_mode == LOCAL_STATE)
+	    {
+	      WSREP_DEBUG("consistency check: %s", thd->query());
+	      thd->wsrep_consistency_check= CONSISTENCY_CHECK_DECLARED;
+	      lip->yySkipn(5);
+	      lip->set_echo(TRUE);
+	      state=MY_LEX_START;
+	      break;  /* Do not treat contents as a comment.  */
+	    }
+#endif /* WITH_WSREP */
             /*
               Patch and skip the conditional comment to avoid it
               being propagated infinitely (eg. to a slave).
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index 019fd55e3d8..066acfb015a 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2008, 2013, Monty Program Ab
+   Copyright (c) 2008, 2014, Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -103,6 +103,14 @@
 #include "../storage/maria/ha_maria.h"
 #endif
 
+#include "wsrep_mysqld.h"
+#include "wsrep_thd.h"
+
+#ifdef WITH_WSREP
+static void wsrep_mysql_parse(THD *thd, char *rawbuf, uint length,
+                              Parser_state *parser_state);
+#endif /* WITH_WSREP */
+
 /**
   @defgroup Runtime_Environment Runtime Environment
   @{
@@ -890,6 +898,19 @@ bool do_command(THD *thd)
   enum enum_server_command command;
   DBUG_ENTER("do_command");
 
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+    thd->wsrep_query_state= QUERY_IDLE;
+    if (thd->wsrep_conflict_state==MUST_ABORT)
+    {
+      wsrep_client_rollback(thd);
+    }
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  }
+#endif /* WITH_WSREP */
+
   /*
     indicator of uninitialized lex => normal flow of errors handling
     (see my_message_sql)
@@ -937,6 +958,29 @@ bool do_command(THD *thd)
   thd->m_server_idle= TRUE;
   packet_length= my_net_read(net);
   thd->m_server_idle= FALSE;
+#ifdef WITH_WSREP
+  if (WSREP(thd)) {
+    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+
+    /* these THD's are aborted or are aborting during being idle */
+    if (thd->wsrep_conflict_state == ABORTING)
+    {
+      while (thd->wsrep_conflict_state == ABORTING) {
+        mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+        my_sleep(1000);
+        mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+      }
+      thd->store_globals();
+    }
+    else if (thd->wsrep_conflict_state == ABORTED)
+    {
+      thd->store_globals();
+    }
+
+    thd->wsrep_query_state= QUERY_EXEC;
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  }
+#endif /* WITH_WSREP */
 
   if (packet_length == packet_error)
   {
@@ -944,6 +988,19 @@ bool do_command(THD *thd)
 		       net->error,
 		       vio_description(net->vio)));
 
+#ifdef WITH_WSREP
+    if (WSREP(thd))
+    {
+      mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+      if (thd->wsrep_conflict_state == MUST_ABORT)
+      {
+        DBUG_PRINT("wsrep",("aborted for wsrep rollback: %lu", thd->real_id));
+        wsrep_client_rollback(thd);
+      }
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    }
+#endif /* WITH_WSREP */
+
     /* Instrument this broken statement as "statement/com/error" */
     thd->m_statement_psi= MYSQL_REFINE_STATEMENT(thd->m_statement_psi,
                                                  com_statement_info[COM_END].
@@ -998,12 +1055,71 @@ bool do_command(THD *thd)
                      vio_description(net->vio), command,
                      command_name[command].str));
 
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    /*
+     * bail out if DB snapshot has not been installed. We however,
+     * allow queries "SET" and "SHOW", they are trapped later in execute_command
+     */
+    if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready &&
+        command != COM_QUERY        &&
+        command != COM_PING         &&
+        command != COM_QUIT         &&
+        command != COM_PROCESS_INFO &&
+        command != COM_PROCESS_KILL &&
+        command != COM_SET_OPTION   &&
+        command != COM_SHUTDOWN     &&
+        command != COM_SLEEP        &&
+        command != COM_STATISTICS   &&
+        command != COM_TIME         &&
+        command != COM_END
+    ) {
+      my_message(ER_UNKNOWN_COM_ERROR,
+                 "WSREP has not yet prepared node for application use",
+                 MYF(0));
+      thd->protocol->end_statement();
+      return_value= FALSE;
+      goto out;
+    }
+  }
+#endif /* WITH_WSREP */
   /* Restore read timeout value */
   my_net_set_read_timeout(net, thd->variables.net_read_timeout);
 
   DBUG_ASSERT(packet_length);
   DBUG_ASSERT(!thd->apc_target.is_enabled());
   return_value= dispatch_command(command, thd, packet+1, (uint) (packet_length-1));
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    while (thd->wsrep_conflict_state== RETRY_AUTOCOMMIT)
+    {
+      WSREP_DEBUG("Retry autocommit for: %s\n", thd->wsrep_retry_query);
+      CHARSET_INFO *current_charset = thd->variables.character_set_client;
+      if (!is_supported_parser_charset(current_charset))
+      {
+        /* Do not use non-supported parser character sets */
+        WSREP_WARN("Current client character set is non-supported parser "
+                   "character set: %s", current_charset->csname);
+        thd->variables.character_set_client = &my_charset_latin1;
+        WSREP_WARN("For retry temporally setting character set to : %s",
+                   my_charset_latin1.csname);
+      }
+      return_value= dispatch_command(command, thd, thd->wsrep_retry_query,
+                                     thd->wsrep_retry_query_len);
+      thd->variables.character_set_client = current_charset;
+    }
+
+    if (thd->wsrep_retry_query && thd->wsrep_conflict_state != REPLAYING)
+    {
+      my_free(thd->wsrep_retry_query);
+      thd->wsrep_retry_query      = NULL;
+      thd->wsrep_retry_query_len  = 0;
+      thd->wsrep_retry_command    = COM_CONNECT;
+    }
+  }
+#endif /* WITH_WSREP */
   DBUG_ASSERT(!thd->apc_target.is_enabled());
 
 out:
@@ -1027,7 +1143,7 @@ out:
     @retval FALSE The statement isn't updating any relevant tables.
 */
 
-static my_bool deny_updates_if_read_only_option(THD *thd,
+my_bool deny_updates_if_read_only_option(THD *thd,
                                                 TABLE_LIST *all_tables)
 {
   DBUG_ENTER("deny_updates_if_read_only_option");
@@ -1081,6 +1197,7 @@ static my_bool deny_updates_if_read_only_option(THD *thd,
   DBUG_RETURN(FALSE);
 }
 
+
 /**
   Perform one connection-level (COM_XXXX) command.
 
@@ -1110,6 +1227,43 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
   DBUG_ENTER("dispatch_command");
   DBUG_PRINT("info", ("command: %d", command));
 
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    if (!thd->in_multi_stmt_transaction_mode())
+    {
+      thd->wsrep_PA_safe= true;
+    }
+
+    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+    thd->wsrep_query_state= QUERY_EXEC;
+    if (thd->wsrep_conflict_state== RETRY_AUTOCOMMIT)
+    {
+      thd->wsrep_conflict_state= NO_CONFLICT;
+    }
+    if (thd->wsrep_conflict_state== MUST_ABORT)
+    {
+      wsrep_client_rollback(thd);
+    }
+    if (thd->wsrep_conflict_state== ABORTED)
+    {
+      my_error(ER_LOCK_DEADLOCK, MYF(0), "wsrep aborted transaction");
+      WSREP_DEBUG("Deadlock error for: %s", thd->query());
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+      thd->killed               = NOT_KILLED;
+      thd->mysys_var->abort     = 0;
+      thd->wsrep_conflict_state = NO_CONFLICT;
+      thd->wsrep_retry_counter  = 0;
+      /*
+        Increment threads running to compensate dec_thread_running() called
+        after dispatch_end label.
+      */
+      inc_thread_running();
+      goto dispatch_end;
+    }
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  }
+#endif /* WITH_WSREP */
 #if defined(ENABLED_PROFILING)
   thd->profiling.start_new_query();
 #endif
@@ -1306,7 +1460,12 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
     if (parser_state.init(thd, thd->query(), thd->query_length()))
       break;
 
-    mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+      wsrep_mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
+    else
+#endif /* WITH_WSREP */
+      mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
 
     while (!thd->killed && (parser_state.m_lip.found_semicolon != NULL) &&
            ! thd->is_error())
@@ -1380,10 +1539,20 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
         Count each statement from the client.
       */
       statistic_increment(thd->status_var.questions, &LOCK_status);
-      thd->set_time(); /* Reset the query start time. */
+
+      if(IF_WSREP(!WSREP(thd), 1))
+	thd->set_time(); /* Reset the query start time. */
+
       parser_state.reset(beginning_of_next_stmt, length);
       /* TODO: set thd->lex->sql_command to SQLCOM_END here */
-      mysql_parse(thd, beginning_of_next_stmt, length, &parser_state);
+
+#ifdef WITH_WSREP
+      if (WSREP_ON)
+        wsrep_mysql_parse(thd, beginning_of_next_stmt, length, &parser_state);
+      else
+#endif /* WITH_WSREP */
+        mysql_parse(thd, beginning_of_next_stmt, length, &parser_state);
+
     }
 
     DBUG_PRINT("info",("query ready"));
@@ -1720,15 +1889,39 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
     my_message(ER_UNKNOWN_COM_ERROR, ER(ER_UNKNOWN_COM_ERROR), MYF(0));
     break;
   }
-  DBUG_ASSERT(thd->derived_tables == NULL &&
-              (thd->open_tables == NULL ||
+#ifdef WITH_WSREP
+ dispatch_end:
+
+  if (WSREP(thd))
+  {
+    /* wsrep BF abort in query exec phase */
+    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+    if ((thd->wsrep_conflict_state != REPLAYING) &&
+        (thd->wsrep_conflict_state != RETRY_AUTOCOMMIT))
+    {
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+      thd->update_server_status();
+      thd->protocol->end_statement();
+      query_cache_end_of_result(thd);
+    }
+    else
+    {
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    }
+  }
+  else
+#endif /* WITH_WSREP */
+  {
+    DBUG_ASSERT(thd->derived_tables == NULL &&
+               (thd->open_tables == NULL ||
                (thd->locked_tables_mode == LTM_LOCK_TABLES)));
 
-  thd_proc_info(thd, "updating status");
-  /* Finalize server status flags after executing a command. */
-  thd->update_server_status();
-  thd->protocol->end_statement();
-  query_cache_end_of_result(thd);
+    thd_proc_info(thd, "updating status");
+    /* Finalize server status flags after executing a command. */
+    thd->update_server_status();
+    thd->protocol->end_statement();
+    query_cache_end_of_result(thd);
+  }
 
   if (!thd->is_error() && !thd->killed_errno())
     mysql_audit_general(thd, MYSQL_AUDIT_GENERAL_RESULT, 0, 0);
@@ -2381,7 +2574,47 @@ mysql_execute_command(THD *thd)
 #ifdef HAVE_REPLICATION
   } /* endif unlikely slave */
 #endif
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    /*
+      change LOCK TABLE WRITE to transaction
+    */
+    if (lex->sql_command== SQLCOM_LOCK_TABLES && wsrep_convert_LOCK_to_trx)
+    {
+      for (TABLE_LIST *table= all_tables; table; table= table->next_global)
+      {
+	if (table->lock_type >= TL_WRITE_ALLOW_WRITE)
+        {
+	  lex->sql_command= SQLCOM_BEGIN;
+	  thd->wsrep_converted_lock_session= true;
+	  break;
+	}
+      }
+    }
+    if (lex->sql_command== SQLCOM_UNLOCK_TABLES &&
+	thd->wsrep_converted_lock_session)
+    {
+      thd->wsrep_converted_lock_session= false;
+      lex->sql_command= SQLCOM_COMMIT;
+      lex->tx_release= TVL_NO;
+    }
 
+    /*
+     * bail out if DB snapshot has not been installed. We however,
+     * allow SET and SHOW queries
+     */
+    if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready &&
+        lex->sql_command != SQLCOM_SET_OPTION &&
+        !wsrep_is_show_query(lex->sql_command))
+    {
+      my_message(ER_UNKNOWN_COM_ERROR,
+                 "WSREP has not yet prepared node for application use",
+                 MYF(0));
+      goto error;
+    }
+  }
+#endif /* WITH_WSREP */
   status_var_increment(thd->status_var.com_stat[lex->sql_command]);
   thd->progress.report_to_client= MY_TEST(sql_command_flags[lex->sql_command] &
                                           CF_REPORT_PROGRESS);
@@ -2423,12 +2656,16 @@ mysql_execute_command(THD *thd)
     {
       /* Commit the normal transaction if one is active. */
       if (trans_commit_implicit(thd))
+      {
+        thd->mdl_context.release_transactional_locks();
+        WSREP_DEBUG("implicit commit failed, MDL released: %lu", thd->thread_id);
         goto error;
+      }
       /* Release metadata locks acquired in this transaction. */
       thd->mdl_context.release_transactional_locks();
     }
   }
-  
+
 #ifndef DBUG_OFF
   if (lex->sql_command != SQLCOM_SET_OPTION)
     DEBUG_SYNC(thd,"before_execute_sql_command");
@@ -2481,6 +2718,10 @@ mysql_execute_command(THD *thd)
 #endif
   case SQLCOM_SHOW_STATUS:
   {
+#ifdef WITH_WSREP
+    if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd))
+      goto error;
+#endif /* WITH_WSREP */
     execute_show_status(thd, all_tables);
     break;
   }
@@ -2513,6 +2754,11 @@ mysql_execute_command(THD *thd)
   }
   case SQLCOM_SHOW_STATUS_PROC:
   case SQLCOM_SHOW_STATUS_FUNC:
+#ifdef WITH_WSREP
+    if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd))
+      goto error;
+#endif /* WITH_WSREP */
+
   case SQLCOM_SHOW_DATABASES:
   case SQLCOM_SHOW_TABLES:
   case SQLCOM_SHOW_TRIGGERS:
@@ -2521,16 +2767,27 @@ mysql_execute_command(THD *thd)
   case SQLCOM_SHOW_PLUGINS:
   case SQLCOM_SHOW_FIELDS:
   case SQLCOM_SHOW_KEYS:
+#ifndef WITH_WSREP
   case SQLCOM_SHOW_VARIABLES:
   case SQLCOM_SHOW_CHARSETS:
   case SQLCOM_SHOW_COLLATIONS:
   case SQLCOM_SHOW_STORAGE_ENGINES:
   case SQLCOM_SHOW_PROFILE:
+#endif /* WITH_WSREP */
   case SQLCOM_SHOW_CLIENT_STATS:
   case SQLCOM_SHOW_USER_STATS:
   case SQLCOM_SHOW_TABLE_STATS:
   case SQLCOM_SHOW_INDEX_STATS:
   case SQLCOM_SELECT:
+#ifdef WITH_WSREP
+    if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd))
+      goto error;
+  case SQLCOM_SHOW_VARIABLES:
+  case SQLCOM_SHOW_CHARSETS:
+  case SQLCOM_SHOW_COLLATIONS:
+  case SQLCOM_SHOW_STORAGE_ENGINES:
+  case SQLCOM_SHOW_PROFILE:
+#endif /* WITH_WSREP */
   {
     thd->status_var.last_query_cost= 0.0;
 
@@ -2554,7 +2811,7 @@ mysql_execute_command(THD *thd)
     res= execute_sqlcom_select(thd, all_tables);
     break;
   }
-case SQLCOM_PREPARE:
+  case SQLCOM_PREPARE:
   {
     mysql_sql_stmt_prepare(thd);
     break;
@@ -2888,7 +3145,7 @@ case SQLCOM_PREPARE:
       */
       if(lex->ignore)
         lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_CREATE_IGNORE_SELECT);
-      
+
       if(lex->duplicates == DUP_REPLACE)
         lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_CREATE_REPLACE_SELECT);
 
@@ -2900,9 +3157,9 @@ case SQLCOM_PREPARE:
         raise a warning, as it may cause problems
         (see 'NAME_CONST issues' in 'Binary Logging of Stored Programs')
        */
-      if (thd->query_name_consts && 
+      if (thd->query_name_consts &&
           mysql_bin_log.is_open() &&
-          thd->variables.binlog_format == BINLOG_FORMAT_STMT &&
+          WSREP_FORMAT(thd->variables.binlog_format) == BINLOG_FORMAT_STMT &&
           !mysql_bin_log.is_query_in_union(thd, thd->query_id))
       {
         List_iterator_fast<Item> it(select_lex->item_list);
@@ -3017,6 +3274,15 @@ case SQLCOM_PREPARE:
       }
       else
       {
+#ifdef WITH_WSREP
+        /* in STATEMENT format, we probably have to replicate also temporary
+           tables, like mysql replication does
+        */
+	  if (WSREP_ON && (!thd->is_current_stmt_binlog_format_row() ||
+	      !(create_info.options & HA_LEX_CREATE_TMP_TABLE)))
+          WSREP_TO_ISOLATION_BEGIN(create_table->db, create_table->table_name,
+                                   NULL)
+#endif /* WITH_WSREP */
         /* Regular CREATE TABLE */
         res= mysql_create_table(thd, create_table,
                                 &create_info, &alter_info);
@@ -3055,6 +3321,7 @@ end_with_restore_list:
     DBUG_ASSERT(first_table == all_tables && first_table != 0);
     if (check_one_table_access(thd, INDEX_ACL, all_tables))
       goto error; /* purecov: inspected */
+    WSREP_TO_ISOLATION_BEGIN(first_table->db, first_table->table_name, NULL)
     /*
       Currently CREATE INDEX or DROP INDEX cause a full table rebuild
       and thus classify as slow administrative statements just like
@@ -3172,6 +3439,7 @@ end_with_restore_list:
 #endif /* HAVE_REPLICATION */
   case SQLCOM_RENAME_TABLE:
   {
+    WSREP_TO_ISOLATION_BEGIN(0, 0, first_table)
     if (execute_rename_table(thd, first_table, all_tables))
       goto error;
     break;
@@ -3192,13 +3460,19 @@ end_with_restore_list:
 #endif
 #endif /* EMBEDDED_LIBRARY */
   case SQLCOM_SHOW_CREATE:
+  {
     DBUG_ASSERT(first_table == all_tables && first_table != 0);
 #ifdef DONT_ALLOW_SHOW_COMMANDS
     my_message(ER_NOT_ALLOWED_COMMAND, ER(ER_NOT_ALLOWED_COMMAND),
                MYF(0)); /* purecov: inspected */
     goto error;
 #else
-    {
+
+#ifdef WITH_WSREP
+      if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd))
+        goto error;
+#endif /* WITH_WSREP */
+
      /*
         Access check:
         SHOW CREATE TABLE require any privileges on the table level (ie
@@ -3256,11 +3530,16 @@ end_with_restore_list:
       /* Access is granted. Execute the command.  */
       res= mysqld_show_create(thd, first_table);
       break;
-    }
 #endif
+  }
   case SQLCOM_CHECKSUM:
   {
     DBUG_ASSERT(first_table == all_tables && first_table != 0);
+#ifdef WITH_WSREP
+    if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd))
+      goto error;
+#endif /* WITH_WSREP */
+
     if (check_table_access(thd, SELECT_ACL, all_tables,
                            FALSE, UINT_MAX, FALSE))
       goto error; /* purecov: inspected */
@@ -3272,6 +3551,12 @@ end_with_restore_list:
   {
     ha_rows found= 0, updated= 0;
     DBUG_ASSERT(first_table == all_tables && first_table != 0);
+#ifdef WITH_WSREP
+      if (WSREP_CLIENT(thd) &&
+          wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE))
+        goto error;
+#endif /* WITH_WSREP */
+
     if (update_precheck(thd, all_tables))
       break;
 
@@ -3308,6 +3593,11 @@ end_with_restore_list:
     /* if we switched from normal update, rights are checked */
     if (up_result != 2)
     {
+#ifdef WITH_WSREP
+      if (WSREP_CLIENT(thd) &&
+          wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE))
+        goto error;
+#endif /* WITH_WSREP */
       if ((res= multi_update_precheck(thd, all_tables)))
         break;
     }
@@ -3377,6 +3667,12 @@ end_with_restore_list:
     break;
   }
   case SQLCOM_REPLACE:
+  {
+#ifdef WITH_WSREP
+    if (WSREP_CLIENT(thd) &&
+        wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE))
+      goto error;
+#endif /* WITH_WSREP */
 #ifndef DBUG_OFF
     if (mysql_bin_log.is_open())
     {
@@ -3411,10 +3707,17 @@ end_with_restore_list:
       DBUG_PRINT("debug", ("Just after generate_incident()"));
     }
 #endif
+  }
   case SQLCOM_INSERT:
   {
     DBUG_ASSERT(first_table == all_tables && first_table != 0);
 
+#ifdef WITH_WSREP
+    if (WSREP_CLIENT(thd) &&
+        wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE))
+      goto error;
+#endif /* WITH_WSREP */
+
     /*
       Since INSERT DELAYED doesn't support temporary tables, we could
       not pre-open temporary tables for SQLCOM_INSERT / SQLCOM_REPLACE.
@@ -3469,8 +3772,22 @@ end_with_restore_list:
     select_result *sel_result;
     bool explain= MY_TEST(lex->describe);
     DBUG_ASSERT(first_table == all_tables && first_table != 0);
+#ifdef WITH_WSREP
+    if (WSREP_CLIENT(thd) &&
+        wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE))
+      goto error;
+#endif /* WITH_WSREP */
+
     if ((res= insert_precheck(thd, all_tables)))
       break;
+#ifdef WITH_WSREP
+    if (WSREP(thd) && thd->wsrep_consistency_check == CONSISTENCY_CHECK_DECLARED)
+    {
+      thd->wsrep_consistency_check = CONSISTENCY_CHECK_RUNNING;
+      WSREP_TO_ISOLATION_BEGIN(first_table->db, first_table->table_name, NULL);
+    }
+#endif /* WITH_WSREP */
+
     /*
       INSERT...SELECT...ON DUPLICATE KEY UPDATE/REPLACE SELECT/
       INSERT...IGNORE...SELECT can be unsafe, unless ORDER BY PRIMARY KEY
@@ -3561,6 +3878,12 @@ end_with_restore_list:
   {
     select_result *sel_result=lex->result;
     DBUG_ASSERT(first_table == all_tables && first_table != 0);
+#ifdef WITH_WSREP
+    if (WSREP_CLIENT(thd) &&
+        wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE))
+      goto error;
+#endif /* WITH_WSREP */
+
     if ((res= delete_precheck(thd, all_tables)))
       break;
     DBUG_ASSERT(select_lex->offset_limit == 0);
@@ -3617,6 +3940,11 @@ end_with_restore_list:
     DBUG_ASSERT(first_table == all_tables && first_table != 0);
     TABLE_LIST *aux_tables= thd->lex->auxiliary_table_list.first;
     multi_delete *result;
+#ifdef WITH_WSREP
+    if (WSREP_CLIENT(thd) &&
+        wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE))
+      goto error;
+#endif /* WITH_WSREP */
 
     if ((res= multi_delete_precheck(thd, all_tables)))
       break;
@@ -3687,6 +4015,21 @@ end_with_restore_list:
       /* So that DROP TEMPORARY TABLE gets to binlog at commit/rollback */
       thd->variables.option_bits|= OPTION_KEEP_LOG;
     }
+#ifdef WITH_WSREP
+   if (WSREP(thd))
+   {
+     for (TABLE_LIST *table= all_tables; table; table= table->next_global)
+     {
+       if (!lex->drop_temporary                       &&
+          (!thd->is_current_stmt_binlog_format_row() ||
+	   !find_temporary_table(thd, table)))
+       {
+         WSREP_TO_ISOLATION_BEGIN(NULL, NULL, all_tables);
+         break;
+       }
+     }
+   }
+#endif /* WITH_WSREP */
     /*
       If we are a slave, we should add IF EXISTS if the query executed
       on the master without an error. This will help a slave to
@@ -3700,8 +4043,8 @@ end_with_restore_list:
     /* DDL and binlog write order are protected by metadata locks. */
     res= mysql_rm_table(thd, first_table, lex->check_exists,
 			lex->drop_temporary);
+    break;
   }
-  break;
   case SQLCOM_SHOW_PROCESSLIST:
     if (!thd->security_ctx->priv_user[0] &&
         check_global_access(thd,PROCESS_ACL))
@@ -3891,6 +4234,7 @@ end_with_restore_list:
 #endif
     if (check_access(thd, CREATE_ACL, lex->name.str, NULL, NULL, 1, 0))
       break;
+    WSREP_TO_ISOLATION_BEGIN(lex->name.str, NULL, NULL)
     res= mysql_create_db(thd, lex->name.str, &create_info, 0);
     break;
   }
@@ -3922,6 +4266,7 @@ end_with_restore_list:
 #endif
     if (check_access(thd, DROP_ACL, lex->name.str, NULL, NULL, 1, 0))
       break;
+    WSREP_TO_ISOLATION_BEGIN(lex->name.str, NULL, NULL)
     res= mysql_rm_db(thd, lex->name.str, lex->check_exists, 0);
     break;
   }
@@ -3953,6 +4298,7 @@ end_with_restore_list:
       res= 1;
       break;
     }
+    WSREP_TO_ISOLATION_BEGIN(db->str, NULL, NULL)
     res= mysql_upgrade_db(thd, db);
     if (!res)
       my_ok(thd);
@@ -3988,6 +4334,7 @@ end_with_restore_list:
 #endif
     if (check_access(thd, ALTER_ACL, db->str, NULL, NULL, 1, 0))
       break;
+    WSREP_TO_ISOLATION_BEGIN(db->str, NULL, NULL)
     res= mysql_alter_db(thd, db->str, &create_info);
     break;
   }
@@ -4026,6 +4373,7 @@ end_with_restore_list:
     if (res)
       break;
 
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     switch (lex->sql_command) {
     case SQLCOM_CREATE_EVENT:
     {
@@ -4060,6 +4408,7 @@ end_with_restore_list:
                                    lex->spname->m_name);
     break;
   case SQLCOM_DROP_EVENT:
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     if (!(res= Events::drop_event(thd,
                                   lex->spname->m_db, lex->spname->m_name,
                                   lex->check_exists)))
@@ -4074,6 +4423,7 @@ end_with_restore_list:
     if (check_access(thd, INSERT_ACL, "mysql", NULL, NULL, 1, 0))
       break;
 #ifdef HAVE_DLOPEN
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     if (!(res = mysql_create_function(thd, &lex->udf)))
       my_ok(thd);
 #else
@@ -4089,6 +4439,7 @@ end_with_restore_list:
     if (check_access(thd, INSERT_ACL, "mysql", NULL, NULL, 1, 1) &&
         check_global_access(thd,CREATE_USER_ACL))
       break;
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     /* Conditionally writes to binlog */
     if (!(res= mysql_create_user(thd, lex->users_list,
                                  lex->sql_command == SQLCOM_CREATE_ROLE)))
@@ -4102,6 +4453,7 @@ end_with_restore_list:
         check_global_access(thd,CREATE_USER_ACL))
       break;
     /* Conditionally writes to binlog */
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     if (!(res= mysql_drop_user(thd, lex->users_list,
                                lex->sql_command == SQLCOM_DROP_ROLE)))
       my_ok(thd);
@@ -4113,6 +4465,7 @@ end_with_restore_list:
         check_global_access(thd,CREATE_USER_ACL))
       break;
     /* Conditionally writes to binlog */
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     if (!(res= mysql_rename_user(thd, lex->users_list)))
       my_ok(thd);
     break;
@@ -4124,6 +4477,7 @@ end_with_restore_list:
       break;
 
     /* Conditionally writes to binlog */
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     if (!(res = mysql_revoke_all(thd, lex->users_list)))
       my_ok(thd);
     break;
@@ -4206,6 +4560,7 @@ end_with_restore_list:
                                 lex->type == TYPE_ENUM_PROCEDURE, 0))
 	  goto error;
         /* Conditionally writes to binlog */
+        WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
         res= mysql_routine_grant(thd, all_tables,
                                  lex->type == TYPE_ENUM_PROCEDURE, 
                                  lex->users_list, grants,
@@ -4219,6 +4574,7 @@ end_with_restore_list:
                         all_tables, FALSE, UINT_MAX, FALSE))
 	  goto error;
         /* Conditionally writes to binlog */
+        WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
         res= mysql_table_grant(thd, all_tables, lex->users_list,
 			       lex->columns, lex->grant,
 			       lex->sql_command == SQLCOM_REVOKE);
@@ -4234,6 +4590,7 @@ end_with_restore_list:
       }
       else
       {
+          WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
         /* Conditionally writes to binlog */
         res= mysql_grant(thd, select_lex->db, lex->users_list, lex->grant,
                          lex->sql_command == SQLCOM_REVOKE,
@@ -4402,6 +4759,7 @@ end_with_restore_list:
       able to open it (with SQLCOM_HA_OPEN) in the first place.
     */
     unit->set_limit(select_lex);
+
     res= mysql_ha_read(thd, first_table, lex->ha_read_mode, lex->ident.str,
                        lex->insert_list, lex->ha_rkey_mode, select_lex->where,
                        unit->select_limit_cnt, unit->offset_limit_cnt);
@@ -4410,7 +4768,11 @@ end_with_restore_list:
   case SQLCOM_BEGIN:
     DBUG_PRINT("info", ("Executing SQLCOM_BEGIN  thd: %p", thd));
     if (trans_begin(thd, lex->start_transaction_opt))
+    {
+      thd->mdl_context.release_transactional_locks();
+      WSREP_DEBUG("BEGIN failed, MDL released: %lu", thd->thread_id);
       goto error;
+    }
     my_ok(thd);
     break;
   case SQLCOM_COMMIT:
@@ -4424,7 +4786,11 @@ end_with_restore_list:
                       (thd->variables.completion_type == 2 &&
                        lex->tx_release != TVL_NO));
     if (trans_commit(thd))
+    {
+      thd->mdl_context.release_transactional_locks();
+      WSREP_DEBUG("COMMIT failed, MDL released: %lu", thd->thread_id);
       goto error;
+    }
     thd->mdl_context.release_transactional_locks();
     /* Begin transaction with the same isolation level. */
     if (tx_chain)
@@ -4444,7 +4810,17 @@ end_with_restore_list:
       thd->killed= KILL_CONNECTION;
       thd->print_aborted_warning(3, "RELEASE");
     }
-    my_ok(thd);
+#ifdef WITH_WSREP
+    if (WSREP(thd))
+    {
+      if (thd->wsrep_conflict_state == NO_CONFLICT ||
+          thd->wsrep_conflict_state == REPLAYING)
+      {
+        my_ok(thd);
+      }
+    } else
+#endif /* WITH_WSREP */
+      my_ok(thd);
     break;
   }
   case SQLCOM_ROLLBACK:
@@ -4459,7 +4835,11 @@ end_with_restore_list:
                        lex->tx_release != TVL_NO));
 
     if (trans_rollback(thd))
+    {
+      thd->mdl_context.release_transactional_locks();
+      WSREP_DEBUG("rollback failed, MDL released: %lu", thd->thread_id);
       goto error;
+    }
     thd->mdl_context.release_transactional_locks();
     /* Begin transaction with the same isolation level. */
     if (tx_chain)
@@ -4476,8 +4856,17 @@ end_with_restore_list:
     /* Disconnect the current client connection. */
     if (tx_release)
       thd->killed= KILL_CONNECTION;
-    my_ok(thd);
-    break;
+#ifdef WITH_WSREP
+    if (WSREP(thd))
+    {
+      if (thd->wsrep_conflict_state == NO_CONFLICT) {
+        my_ok(thd);
+      }
+    }
+    else
+#endif /* WITH_WSREP */
+      my_ok(thd);
+   break;
   }
   case SQLCOM_RELEASE_SAVEPOINT:
     if (trans_release_savepoint(thd, lex->ident))
@@ -4545,6 +4934,7 @@ end_with_restore_list:
     if (sp_process_definer(thd))
       goto create_sp_error;
 
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     res= (sp_result= sp_create_routine(thd, lex->sphead->m_type, lex->sphead));
     switch (sp_result) {
     case SP_OK: {
@@ -4826,6 +5216,7 @@ create_sp_error:
       if (check_routine_access(thd, ALTER_PROC_ACL, db, name,
                                lex->sql_command == SQLCOM_DROP_PROCEDURE, 0))
         goto error;
+      WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
 
       /* Conditionally writes to binlog */
       sp_result= sp_drop_routine(thd, type, lex->spname);
@@ -4943,6 +5334,7 @@ create_sp_error:
         Note: SQLCOM_CREATE_VIEW also handles 'ALTER VIEW' commands
         as specified through the thd->lex->create_view_mode flag.
       */
+      WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
       res= mysql_create_view(thd, first_table, thd->lex->create_view_mode);
       break;
     }
@@ -4951,12 +5343,14 @@ create_sp_error:
       if (check_table_access(thd, DROP_ACL, all_tables, FALSE, UINT_MAX, FALSE))
         goto error;
       /* Conditionally writes to binlog. */
+      WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
       res= mysql_drop_view(thd, first_table, thd->lex->drop_mode);
       break;
     }
   case SQLCOM_CREATE_TRIGGER:
   {
     /* Conditionally writes to binlog. */
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     res= mysql_create_or_drop_trigger(thd, all_tables, 1);
 
     break;
@@ -4964,6 +5358,7 @@ create_sp_error:
   case SQLCOM_DROP_TRIGGER:
   {
     /* Conditionally writes to binlog. */
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     res= mysql_create_or_drop_trigger(thd, all_tables, 0);
     break;
   }
@@ -4984,7 +5379,11 @@ create_sp_error:
     break;
   case SQLCOM_XA_COMMIT:
     if (trans_xa_commit(thd))
+    {
+      thd->mdl_context.release_transactional_locks();
+      WSREP_DEBUG("XA commit failed, MDL released: %lu", thd->thread_id);
       goto error;
+    }
     thd->mdl_context.release_transactional_locks();
     /*
       We've just done a commit, reset transaction
@@ -4996,7 +5395,11 @@ create_sp_error:
     break;
   case SQLCOM_XA_ROLLBACK:
     if (trans_xa_rollback(thd))
+    {
+      thd->mdl_context.release_transactional_locks();
+      WSREP_DEBUG("XA rollback failed, MDL released: %lu", thd->thread_id);
       goto error;
+    }
     thd->mdl_context.release_transactional_locks();
     /*
       We've just done a rollback, reset transaction
@@ -5016,11 +5419,13 @@ create_sp_error:
       my_ok(thd);
     break;
   case SQLCOM_INSTALL_PLUGIN:
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     if (! (res= mysql_install_plugin(thd, &thd->lex->comment,
                                      &thd->lex->ident)))
       my_ok(thd);
     break;
   case SQLCOM_UNINSTALL_PLUGIN:
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
     if (! (res= mysql_uninstall_plugin(thd, &thd->lex->comment,
                                        &thd->lex->ident)))
       my_ok(thd);
@@ -5169,6 +5574,9 @@ finish:
 
   /* Free tables */
   close_thread_tables(thd);
+#ifdef WITH_WSREP
+  thd->wsrep_consistency_check= NO_CONSISTENCY_CHECK;
+#endif /* WITH_WSREP */
 
 #ifndef DBUG_OFF
   if (lex->sql_command != SQLCOM_SET_OPTION && ! thd->in_sub_stmt)
@@ -5222,6 +5630,22 @@ finish:
   {
     thd->mdl_context.release_statement_locks();
   }
+  WSREP_TO_ISOLATION_END;
+
+#ifdef WITH_WSREP
+  /*
+    Force release of transactional locks if not in active MST and wsrep is on.
+  */
+  if (WSREP(thd) &&
+      ! thd->in_sub_stmt &&
+      ! thd->in_active_multi_stmt_transaction() &&
+      thd->mdl_context.has_transactional_locks())
+  {
+    WSREP_DEBUG("Forcing release of transactional locks for thd %lu",
+               thd->thread_id);
+    thd->mdl_context.release_transactional_locks();
+  }
+#endif /* WITH_WSREP */
 
   DBUG_RETURN(res || thd->is_error());
 }
@@ -5322,6 +5746,10 @@ static bool execute_sqlcom_select(THD *thd, TABLE_LIST *all_tables)
     status_var_increment(thd->status_var.empty_queries);
   else
     status_var_add(thd->status_var.rows_sent, thd->get_sent_row_count());
+#ifdef WITH_WSREP
+    if (WSREP_ON && lex->sql_command == SQLCOM_SHOW_STATUS)
+      wsrep_free_status(thd);
+#endif /* WITH_WSREP */
   return res;
 }
 
@@ -6160,6 +6588,27 @@ void THD::reset_for_next_command()
   thd->auto_inc_intervals_in_cur_stmt_for_binlog.empty();
   thd->stmt_depends_on_first_successful_insert_id_in_prev_stmt= 0;
 
+#ifdef WITH_WSREP
+  /*
+    Autoinc variables should be adjusted only for locally executed
+    transactions. Appliers and replayers are either processing ROW
+    events or get autoinc variable values from Query_log_event.
+  */
+  if (WSREP(thd) && thd->wsrep_exec_mode == LOCAL_STATE)
+  {
+    if (wsrep_auto_increment_control)
+    {
+      if (thd->variables.auto_increment_offset !=
+	  global_system_variables.auto_increment_offset)
+	thd->variables.auto_increment_offset=
+	  global_system_variables.auto_increment_offset;
+      if (thd->variables.auto_increment_increment !=
+	  global_system_variables.auto_increment_increment)
+	thd->variables.auto_increment_increment=
+	  global_system_variables.auto_increment_increment;
+    }
+  }
+#endif /* WITH_WSREP */
   thd->query_start_used= 0;
   thd->query_start_sec_part_used= 0;
   thd->is_fatal_error= thd->time_zone_used= 0;
@@ -6363,6 +6812,109 @@ void mysql_init_multi_delete(LEX *lex)
   lex->query_tables_last= &lex->query_tables;
 }
 
+#ifdef WITH_WSREP
+static void wsrep_mysql_parse(THD *thd, char *rawbuf, uint length,
+                 Parser_state *parser_state)
+{
+  bool is_autocommit=
+    !thd->in_multi_stmt_transaction_mode()                  &&
+    thd->wsrep_conflict_state == NO_CONFLICT                &&
+    !thd->wsrep_applier                                     &&
+    wsrep_read_only_option(thd, thd->lex->query_tables);
+
+  do
+  {
+    if (thd->wsrep_conflict_state== RETRY_AUTOCOMMIT)
+    {
+      thd->wsrep_conflict_state= NO_CONFLICT;
+      /* Performance Schema Interface instrumentation, begin */
+      thd->m_statement_psi= MYSQL_REFINE_STATEMENT(thd->m_statement_psi,
+	      com_statement_info[thd->get_command()].m_key);
+      MYSQL_SET_STATEMENT_TEXT(thd->m_statement_psi, thd->query(),
+	                       thd->query_length());
+    }
+    mysql_parse(thd, rawbuf, length, parser_state);
+
+    if (WSREP(thd)) {
+      /* wsrep BF abort in query exec phase */
+      mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+      if (thd->wsrep_conflict_state == MUST_ABORT) {
+        wsrep_client_rollback(thd);
+
+        WSREP_DEBUG("abort in exec query state, avoiding autocommit");
+      }
+
+      if (thd->wsrep_conflict_state== MUST_REPLAY) 
+      {
+        wsrep_replay_transaction(thd);
+      }
+
+      /* setting error code for BF aborted trxs */
+      if (thd->wsrep_conflict_state == ABORTED ||
+          thd->wsrep_conflict_state == CERT_FAILURE)
+      {
+        mysql_reset_thd_for_next_command(thd);
+        thd->killed= NOT_KILLED;
+        if (is_autocommit                           &&
+            thd->lex->sql_command != SQLCOM_SELECT  &&
+            (thd->wsrep_retry_counter < thd->variables.wsrep_retry_autocommit))
+        {
+          WSREP_DEBUG("wsrep retrying AC query: %s", 
+                      (thd->query()) ? thd->query() : "void");
+
+	  /* Performance Schema Interface instrumentation, end */
+	  MYSQL_END_STATEMENT(thd->m_statement_psi, thd->get_stmt_da());
+	  thd->m_statement_psi= NULL;
+          close_thread_tables(thd);
+
+          thd->wsrep_conflict_state= RETRY_AUTOCOMMIT;
+          thd->wsrep_retry_counter++;            // grow
+          wsrep_copy_query(thd);
+          thd->set_time();
+          parser_state->reset(rawbuf, length);
+        }
+        else
+        {
+          WSREP_DEBUG("%s, thd: %lu is_AC: %d, retry: %lu - %lu SQL: %s", 
+                      (thd->wsrep_conflict_state == ABORTED) ? 
+                      "BF Aborted" : "cert failure",
+                      thd->thread_id, is_autocommit, thd->wsrep_retry_counter, 
+                      thd->variables.wsrep_retry_autocommit, thd->query());
+          my_error(ER_LOCK_DEADLOCK, MYF(0), "wsrep aborted transaction");
+          thd->killed= NOT_KILLED;
+          thd->wsrep_conflict_state= NO_CONFLICT;
+          if (thd->wsrep_conflict_state != REPLAYING)
+            thd->wsrep_retry_counter= 0;             //  reset
+        }
+      }
+      else
+      {
+        set_if_smaller(thd->wsrep_retry_counter, 0); // reset; eventually ok
+      }
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    }
+
+    /* If retry is requested clean up explain structure */
+    if (thd->wsrep_conflict_state == RETRY_AUTOCOMMIT && thd->lex->explain)
+        delete_explain_query(thd->lex);
+
+  }  while (thd->wsrep_conflict_state== RETRY_AUTOCOMMIT);
+
+  if (thd->wsrep_retry_query)
+  {
+    WSREP_DEBUG("releasing retry_query: conf %d sent %d kill %d  errno %d SQL %s",
+                thd->wsrep_conflict_state,
+	    thd->get_stmt_da()->is_sent(),
+                thd->killed,
+	    thd->get_stmt_da()->is_error() ? thd->get_stmt_da()->sql_errno() : 0,
+                thd->wsrep_retry_query);
+    my_free(thd->wsrep_retry_query);
+    thd->wsrep_retry_query      = NULL;
+    thd->wsrep_retry_query_len  = 0;
+    thd->wsrep_retry_command    = COM_CONNECT;
+  }
+}
+#endif /* WITH_WSREP */
 
 /*
   When you modify mysql_parse(), you may need to mofify
@@ -7392,8 +7944,9 @@ kill_one_thread(THD *thd, longlong id, killed_state kill_signal, killed_type typ
       faster and do a harder kill than KILL_SYSTEM_THREAD;
     */
 
-    if ((thd->security_ctx->master_access & SUPER_ACL) ||
-        thd->security_ctx->user_matches(tmp->security_ctx))
+    if (((thd->security_ctx->master_access & SUPER_ACL) ||
+        thd->security_ctx->user_matches(tmp->security_ctx)) &&
+	IF_WSREP(!wsrep_thd_is_BF((void *)tmp, true), 1))
     {
       tmp->awake(kill_signal);
       error=0;
diff --git a/sql/sql_parse.h b/sql/sql_parse.h
index 926a4d800ad..87f8854e050 100644
--- a/sql/sql_parse.h
+++ b/sql/sql_parse.h
@@ -203,5 +203,4 @@ inline bool is_supported_parser_charset(CHARSET_INFO *cs)
   return MY_TEST(cs->mbminlen == 1);
 }
 
-
 #endif /* SQL_PARSE_INCLUDED */
diff --git a/sql/sql_partition_admin.cc b/sql/sql_partition_admin.cc
index 8c59febeb77..9db8b1c136a 100644
--- a/sql/sql_partition_admin.cc
+++ b/sql/sql_partition_admin.cc
@@ -1,4 +1,5 @@
 /* Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2014, SkySQL Ab.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -763,6 +764,23 @@ bool Sql_cmd_alter_table_truncate_partition::execute(THD *thd)
   if (check_one_table_access(thd, DROP_ACL, first_table))
     DBUG_RETURN(TRUE);
 
+#ifdef WITH_WSREP
+  if (WSREP_ON)
+  {
+    TABLE *find_temporary_table(THD *thd, const TABLE_LIST *tl);
+
+    if ((!thd->is_current_stmt_binlog_format_row() ||
+         !find_temporary_table(thd, first_table))  &&
+        wsrep_to_isolation_begin(
+          thd, first_table->db, first_table->table_name, NULL)
+       )
+    {
+      WSREP_WARN("ALTER TABLE isolation failure");
+      DBUG_RETURN(TRUE);
+    }
+  }
+#endif /* WITH_WSREP */
+
   if (open_tables(thd, &first_table, &table_counter, 0))
     DBUG_RETURN(true);
 
diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc
index b2dd38ad720..ea9748c8cb2 100644
--- a/sql/sql_plugin.cc
+++ b/sql/sql_plugin.cc
@@ -3087,15 +3087,19 @@ void plugin_thdvar_init(THD *thd)
   thd->variables.dynamic_variables_size= 0;
   thd->variables.dynamic_variables_ptr= 0;
 
-  mysql_mutex_lock(&LOCK_plugin);
-  thd->variables.table_plugin=
+  if (IF_WSREP((!WSREP(thd) || !thd->wsrep_applier),1))
+  {
+    mysql_mutex_lock(&LOCK_plugin);
+    thd->variables.table_plugin=
         intern_plugin_lock(NULL, global_system_variables.table_plugin);
-  if (global_system_variables.tmp_table_plugin)
-    thd->variables.tmp_table_plugin=
+    if (global_system_variables.tmp_table_plugin)
+      thd->variables.tmp_table_plugin=
             intern_plugin_lock(NULL, global_system_variables.tmp_table_plugin);
-  intern_plugin_unlock(NULL, old_table_plugin);
-  intern_plugin_unlock(NULL, old_tmp_table_plugin);
-  mysql_mutex_unlock(&LOCK_plugin);
+    intern_plugin_unlock(NULL, old_table_plugin);
+    intern_plugin_unlock(NULL, old_tmp_table_plugin);
+    mysql_mutex_unlock(&LOCK_plugin);
+  }
+
   DBUG_VOID_RETURN;
 }
 
diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc
index ebceae70ee5..d9365e14494 100644
--- a/sql/sql_prepare.cc
+++ b/sql/sql_prepare.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2002, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2008, 2013, Monty Program Ab
+   Copyright (c) 2008, 2014, Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -117,6 +117,7 @@ When one supplies long data for a placeholder:
 #include "lock.h"                               // MYSQL_OPEN_FORCE_SHARED_MDL
 #include "sql_handler.h"
 #include "transaction.h"                        // trans_rollback_implicit
+#include "wsrep_mysqld.h"
 
 /**
   A result class used to send cursor rows using the binary protocol.
@@ -3626,6 +3627,27 @@ reexecute:
   error= execute(expanded_query, open_cursor) || thd->is_error();
 
   thd->m_reprepare_observer= NULL;
+#ifdef WITH_WSREP
+
+  if (WSREP_ON)
+  {
+    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+    switch (thd->wsrep_conflict_state)
+    {
+      case CERT_FAILURE:
+        WSREP_DEBUG("PS execute fail for CERT_FAILURE: thd: %ld err: %d",
+	            thd->thread_id, thd->get_stmt_da()->sql_errno() );
+        thd->wsrep_conflict_state = NO_CONFLICT;
+        break;
+
+      case MUST_REPLAY:
+        (void)wsrep_replay_transaction(thd);
+      default:
+        break;
+    }
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  }
+#endif /* WITH_WSREP */
 
   if ((sql_command_flags[lex->sql_command] & CF_REEXECUTION_FRAGILE) &&
       error && !thd->is_fatal_error && !thd->killed &&
diff --git a/sql/sql_reload.cc b/sql/sql_reload.cc
index bb3d5bb899a..f1f9ba9452f 100644
--- a/sql/sql_reload.cc
+++ b/sql/sql_reload.cc
@@ -253,7 +253,18 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options,
       }
       if (options & REFRESH_CHECKPOINT)
         disable_checkpoints(thd);
-    }
+#ifdef WITH_WSREP
+      /*
+        We need to do it second time after wsrep appliers were blocked in
+        make_global_read_lock_block_commit(thd) above since they could have
+        modified the tables too.
+      */
+      if (WSREP(thd) &&
+          close_cached_tables(thd, tables, (options & REFRESH_FAST) ?
+                              FALSE : TRUE, TRUE))
+          result= 1;
+#endif /* WITH_WSREP */
+     }
     else
     {
       if (thd && thd->locked_tables_mode)
diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc
index c7bd28259ae..d8db5c55c3b 100644
--- a/sql/sql_repl.cc
+++ b/sql/sql_repl.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2008, 2013, Monty Program Ab
+   Copyright (c) 2008, 2014, Monty Program Ab
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -2946,6 +2946,7 @@ int start_slave(THD* thd , Master_info* mi,  bool net_report)
 
 err:
   unlock_slave_threads(mi);
+  thd_proc_info(thd, 0);
 
   if (slave_errno)
   {
diff --git a/sql/sql_show.cc b/sql/sql_show.cc
index 79c444ea442..bb8ae16189a 100644
--- a/sql/sql_show.cc
+++ b/sql/sql_show.cc
@@ -2931,11 +2931,39 @@ static bool show_status_array(THD *thd, const char *wild,
     *prefix_end++= '_';
   len=name_buffer + sizeof(name_buffer) - prefix_end;
 
+#ifdef WITH_WSREP
+  bool is_wsrep_var= FALSE;
+  /*
+    This is a workaround for lp:1306875 (PBX) to skip switching of wsrep
+    status variable name's first letter to uppercase. This is an optimization
+    for status variables defined under wsrep plugin.
+    TODO: remove once lp:1306875 has been addressed.
+  */
+  if (*prefix && !my_strcasecmp(system_charset_info, prefix, "wsrep"))
+  {
+    is_wsrep_var= TRUE;
+  }
+#endif /* WITH_WSREP */
+
   for (; variables->name; variables++)
   {
     bool wild_checked;
     strnmov(prefix_end, variables->name, len);
     name_buffer[sizeof(name_buffer)-1]=0;       /* Safety */
+
+#ifdef WITH_WSREP
+    /*
+      If the prefix is NULL, that means we are looking into the status variables
+      defined directly under mysqld.cc. Do not capitalize wsrep status variable
+      names until lp:1306875 has been fixed.
+      TODO: remove once lp:1306875 has been addressed.
+     */
+    if (!(*prefix) && !strncasecmp(name_buffer, "wsrep", strlen("wsrep")))
+    {
+      is_wsrep_var= TRUE;
+    }
+#endif /* WITH_WSREP */
+
     if (ucase_names)
       my_caseup_str(system_charset_info, name_buffer);
     else
@@ -2944,8 +2972,9 @@ static bool show_status_array(THD *thd, const char *wild,
       DBUG_ASSERT(name_buffer[0] >= 'a');
       DBUG_ASSERT(name_buffer[0] <= 'z');
 
-      /* traditionally status variables have a first letter uppercased */
-      if (status_var)
+      // WSREP_TODO: remove once lp:1306875 has been addressed.
+      if (IF_WSREP(is_wsrep_var == FALSE, 1) &&
+          status_var)
         name_buffer[0]-= 'a' - 'A';
     }
 
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 80ac0978834..b86de2f6cc1 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -5264,6 +5264,12 @@ bool mysql_create_like_table(THD* thd, TABLE_LIST* table,
   int create_res;
   DBUG_ENTER("mysql_create_like_table");
 
+#ifdef WITH_WSREP
+  if (WSREP_ON && !thd->wsrep_applier &&
+      wsrep_create_like_table(thd, table, src_table, create_info))
+    goto end;
+#endif
+
   /*
     We the open source table to get its description in HA_CREATE_INFO
     and Alter_info objects. This also acquires a shared metadata lock
@@ -5525,6 +5531,8 @@ err:
                            thd->query_length(), is_trans))
       res= 1;
   }
+
+end:
   DBUG_RETURN(res);
 }
 
@@ -8117,6 +8125,7 @@ simple_rename_or_index_change(THD *thd, TABLE_LIST *table_list,
   if (!error)
   {
     error= write_bin_log(thd, TRUE, thd->query(), thd->query_length());
+
     if (!error)
       my_ok(thd);
   }
diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc
index 70ac6265046..268fad5d90e 100644
--- a/sql/sql_trigger.cc
+++ b/sql/sql_trigger.cc
@@ -434,7 +434,8 @@ bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create)
     binlogged, so they share the same danger, so trust_function_creators
     applies to them too.
   */
-  if (!trust_function_creators && mysql_bin_log.is_open() &&
+  if (!trust_function_creators                                &&
+      IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()), mysql_bin_log.is_open()) &&
       !(thd->security_ctx->master_access & SUPER_ACL))
   {
     my_error(ER_BINLOG_CREATE_ROUTINE_NEED_SUPER, MYF(0));
@@ -2437,3 +2438,4 @@ bool load_table_name_for_trigger(THD *thd,
 
   DBUG_RETURN(FALSE);
 }
+
diff --git a/sql/sql_truncate.cc b/sql/sql_truncate.cc
index e98679b1d51..5e23c6ee32b 100644
--- a/sql/sql_truncate.cc
+++ b/sql/sql_truncate.cc
@@ -24,6 +24,7 @@
 #include "sql_acl.h"     // DROP_ACL
 #include "sql_parse.h"   // check_one_table_access()
 #include "sql_truncate.h"
+#include "wsrep_mysqld.h"
 #include "sql_show.h"    //append_identifier()
 
 
@@ -411,6 +412,12 @@ bool Sql_cmd_truncate_table::truncate_table(THD *thd, TABLE_LIST *table_ref)
   {
     bool hton_can_recreate;
 
+#ifdef WITH_WSREP
+    if (WSREP(thd) && wsrep_to_isolation_begin(thd, 
+                                                table_ref->db, 
+                                                table_ref->table_name, NULL))
+        DBUG_RETURN(TRUE);
+#endif /* WITH_WSREP */
     if (lock_table(thd, table_ref, &hton_can_recreate))
       DBUG_RETURN(TRUE);
 
diff --git a/sql/sql_update.cc b/sql/sql_update.cc
index faf8e4c61d2..c458f01303d 100644
--- a/sql/sql_update.cc
+++ b/sql/sql_update.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2011, 2013, Monty Program Ab.
+   Copyright (c) 2011, 2014, Monty Program Ab.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -976,7 +976,8 @@ int mysql_update(THD *thd,
   */
   if ((error < 0) || thd->transaction.stmt.modified_non_trans_table)
   {
-    if (mysql_bin_log.is_open())
+    if (IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+	         mysql_bin_log.is_open()))
     {
       int errcode= 0;
       if (error < 0)
@@ -2219,7 +2220,8 @@ void multi_update::abort_result_set()
       The query has to binlog because there's a modified non-transactional table
       either from the query's list or via a stored routine: bug#13270,23333
     */
-    if (mysql_bin_log.is_open())
+    if (IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+	         mysql_bin_log.is_open()))
     {
       /*
         THD::killed status might not have been set ON at time of an error
@@ -2488,7 +2490,8 @@ bool multi_update::send_eof()
 
   if (local_error == 0 || thd->transaction.stmt.modified_non_trans_table)
   {
-    if (mysql_bin_log.is_open())
+    if (IF_WSREP((WSREP_EMULATE_BINLOG(thd) || mysql_bin_log.is_open()),
+	         mysql_bin_log.is_open()))
     {
       int errcode= 0;
       if (local_error == 0)
diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy
index a261d611aa6..a06d702f620 100644
--- a/sql/sql_yacc.yy
+++ b/sql/sql_yacc.yy
@@ -7156,7 +7156,7 @@ alter:
           }
           view_tail
           {}
-        | ALTER definer_opt EVENT_SYM sp_name
+        | ALTER definer_opt remember_name EVENT_SYM sp_name
           {
             /* 
               It is safe to use Lex->spname because
@@ -7168,9 +7168,12 @@ alter:
 
             if (!(Lex->event_parse_data= Event_parse_data::new_instance(thd)))
               MYSQL_YYABORT;
-            Lex->event_parse_data->identifier= $4;
+            Lex->event_parse_data->identifier= $5;
 
             Lex->sql_command= SQLCOM_ALTER_EVENT;
+#ifdef WITH_WSREP
+            Lex->stmt_definition_begin= $3;
+#endif
           }
           ev_alter_on_schedule_completion
           opt_ev_rename_to
@@ -7178,7 +7181,7 @@ alter:
           opt_ev_comment
           opt_ev_sql_stmt
           {
-            if (!($6 || $7 || $8 || $9 || $10))
+            if (!($7 || $8 || $9 || $10 || $11))
             {
               my_parse_error(ER(ER_SYNTAX_ERROR));
               MYSQL_YYABORT;
@@ -7188,6 +7191,9 @@ alter:
               can overwrite it
             */
             Lex->sql_command= SQLCOM_ALTER_EVENT;
+#ifdef WITH_WSREP
+            Lex->stmt_definition_end= (char*)YYLIP->get_cpp_ptr();
+#endif
           }
         | ALTER TABLESPACE alter_tablespace_info
           {
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index 9495d16247d..c76ee5e6358 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -465,6 +465,26 @@ static bool binlog_format_check(sys_var *self, THD *thd, set_var *var)
          ER_INSIDE_TRANSACTION_PREVENTS_SWITCH_BINLOG_FORMAT))
     return true;
 
+#ifdef WITH_WSREP
+  /* MariaDB Galera does not support STATEMENT or MIXED binlog
+  format currently */
+  if (WSREP(thd) &&
+     (var->save_result.ulonglong_value == BINLOG_FORMAT_STMT ||
+      var->save_result.ulonglong_value == BINLOG_FORMAT_MIXED))
+  {
+    WSREP_DEBUG("MariaDB Galera does not support binlog format : %s",
+                var->save_result.ulonglong_value == BINLOG_FORMAT_STMT ?
+                "STATEMENT" : "MIXED");
+    /* Push also warning, because error message is general */
+    push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
+                        ER_UNKNOWN_ERROR,
+                        "MariaDB Galera does not support binlog format: %s",
+                        var->save_result.ulonglong_value == BINLOG_FORMAT_STMT ?
+                        "STATEMENT" : "MIXED");
+    return true;
+  }
+#endif
+
   return false;
 }
 
@@ -3317,6 +3337,8 @@ static bool fix_autocommit(sys_var *self, THD *thd, enum_var_type type)
     if (trans_commit_stmt(thd) || trans_commit(thd))
     {
       thd->variables.option_bits&= ~OPTION_AUTOCOMMIT;
+      thd->mdl_context.release_transactional_locks();
+      WSREP_DEBUG("autocommit, MDL TRX lock released: %lu", thd->thread_id);
       return true;
     }
     /*
@@ -4412,6 +4434,270 @@ static Sys_var_tz Sys_time_zone(
        SESSION_VAR(time_zone), NO_CMD_LINE,
        DEFAULT(&default_tz), NO_MUTEX_GUARD, IN_BINLOG);
 
+#ifdef WITH_WSREP
+#include "wsrep_var.h"
+#include "wsrep_sst.h"
+#include "wsrep_binlog.h"
+
+static Sys_var_charptr Sys_wsrep_provider(
+       "wsrep_provider", "Path to replication provider library",
+       PREALLOCATED GLOBAL_VAR(wsrep_provider), CMD_LINE(REQUIRED_ARG, OPT_WSREP_PROVIDER),
+       IN_FS_CHARSET, DEFAULT(WSREP_NONE),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_provider_check), ON_UPDATE(wsrep_provider_update));
+
+static Sys_var_charptr Sys_wsrep_provider_options(
+       "wsrep_provider_options", "provider specific options",
+       PREALLOCATED GLOBAL_VAR(wsrep_provider_options), 
+       CMD_LINE(REQUIRED_ARG, OPT_WSREP_PROVIDER_OPTIONS),
+       IN_FS_CHARSET, DEFAULT(""), NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_provider_options_check), 
+       ON_UPDATE(wsrep_provider_options_update));
+
+static Sys_var_charptr Sys_wsrep_data_home_dir(
+       "wsrep_data_home_dir", "home directory for wsrep provider",
+       READ_ONLY GLOBAL_VAR(wsrep_data_home_dir), CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(""), 
+       NO_MUTEX_GUARD, NOT_IN_BINLOG);
+
+static Sys_var_charptr Sys_wsrep_cluster_name(
+       "wsrep_cluster_name", "Name for the cluster",
+       PREALLOCATED GLOBAL_VAR(wsrep_cluster_name), CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(WSREP_CLUSTER_NAME),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_cluster_name_check),
+       ON_UPDATE(wsrep_cluster_name_update));
+
+static PolyLock_mutex PLock_wsrep_slave_threads(&LOCK_wsrep_slave_threads);
+static Sys_var_charptr Sys_wsrep_cluster_address (
+       "wsrep_cluster_address", "Address to initially connect to cluster",
+       PREALLOCATED GLOBAL_VAR(wsrep_cluster_address), 
+       CMD_LINE(REQUIRED_ARG, OPT_WSREP_CLUSTER_ADDRESS),
+       IN_FS_CHARSET, DEFAULT(""),
+       &PLock_wsrep_slave_threads, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_cluster_address_check), 
+       ON_UPDATE(wsrep_cluster_address_update));
+
+static Sys_var_charptr Sys_wsrep_node_name (
+       "wsrep_node_name", "Node name",
+       PREALLOCATED GLOBAL_VAR(wsrep_node_name), CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(""), NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       wsrep_node_name_check, wsrep_node_name_update);
+
+static Sys_var_charptr Sys_wsrep_node_address (
+       "wsrep_node_address", "Node address",
+       PREALLOCATED GLOBAL_VAR(wsrep_node_address), CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(""),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_node_address_check),
+       ON_UPDATE(wsrep_node_address_update));
+
+static Sys_var_charptr Sys_wsrep_node_incoming_address(
+       "wsrep_node_incoming_address", "Client connection address",
+       PREALLOCATED GLOBAL_VAR(wsrep_node_incoming_address),CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(WSREP_NODE_INCOMING_AUTO),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG);
+
+static Sys_var_ulong Sys_wsrep_slave_threads(
+       "wsrep_slave_threads", "Number of slave appliers to launch",
+       GLOBAL_VAR(wsrep_slave_threads), CMD_LINE(REQUIRED_ARG),
+       VALID_RANGE(1, 512), DEFAULT(1), BLOCK_SIZE(1),
+       &PLock_wsrep_slave_threads, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_slave_threads_check), 
+       ON_UPDATE(wsrep_slave_threads_update));
+
+static Sys_var_charptr Sys_wsrep_dbug_option(
+       "wsrep_dbug_option", "DBUG options to provider library",
+       GLOBAL_VAR(wsrep_dbug_option),CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(""),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG);
+
+static Sys_var_mybool Sys_wsrep_debug(
+       "wsrep_debug", "To enable debug level logging",
+       GLOBAL_VAR(wsrep_debug), CMD_LINE(OPT_ARG), DEFAULT(FALSE));
+
+static Sys_var_mybool Sys_wsrep_convert_LOCK_to_trx(
+       "wsrep_convert_LOCK_to_trx", "To convert locking sessions "
+       "into transactions",
+       GLOBAL_VAR(wsrep_convert_LOCK_to_trx), 
+       CMD_LINE(OPT_ARG), DEFAULT(FALSE));
+
+static Sys_var_ulong Sys_wsrep_retry_autocommit(
+      "wsrep_retry_autocommit", "Max number of times to retry "
+      "a failed autocommit statement",
+       SESSION_VAR(wsrep_retry_autocommit), CMD_LINE(REQUIRED_ARG),
+       VALID_RANGE(0, 10000), DEFAULT(1), BLOCK_SIZE(1));
+
+static Sys_var_mybool Sys_wsrep_auto_increment_control(
+       "wsrep_auto_increment_control", "To automatically control the "
+       "assignment of autoincrement variables",
+       GLOBAL_VAR(wsrep_auto_increment_control), 
+       CMD_LINE(OPT_ARG), DEFAULT(TRUE));
+
+static Sys_var_mybool Sys_wsrep_drupal_282555_workaround(
+       "wsrep_drupal_282555_workaround", "To use a workaround for"
+       "bad autoincrement value", 
+       GLOBAL_VAR(wsrep_drupal_282555_workaround), 
+       CMD_LINE(OPT_ARG), DEFAULT(FALSE));
+
+static Sys_var_charptr sys_wsrep_sst_method(
+       "wsrep_sst_method", "State snapshot transfer method",
+       GLOBAL_VAR(wsrep_sst_method),CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(WSREP_SST_DEFAULT), NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_sst_method_check),
+       ON_UPDATE(wsrep_sst_method_update)); 
+
+static Sys_var_charptr Sys_wsrep_sst_receive_address( 
+       "wsrep_sst_receive_address", "Address where node is waiting for "
+       "SST contact", 
+       GLOBAL_VAR(wsrep_sst_receive_address),CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(WSREP_SST_ADDRESS_AUTO), NO_MUTEX_GUARD,
+       NOT_IN_BINLOG,
+       ON_CHECK(wsrep_sst_receive_address_check),
+       ON_UPDATE(wsrep_sst_receive_address_update)); 
+
+static Sys_var_charptr Sys_wsrep_sst_auth(
+       "wsrep_sst_auth", "Authentication for SST connection",
+       PREALLOCATED GLOBAL_VAR(wsrep_sst_auth), CMD_LINE(REQUIRED_ARG, OPT_WSREP_SST_AUTH),
+       IN_FS_CHARSET, DEFAULT(NULL), NO_MUTEX_GUARD,
+       NOT_IN_BINLOG,
+       ON_CHECK(wsrep_sst_auth_check),
+       ON_UPDATE(wsrep_sst_auth_update)); 
+
+static Sys_var_charptr Sys_wsrep_sst_donor(
+       "wsrep_sst_donor", "preferred donor node for the SST",
+       GLOBAL_VAR(wsrep_sst_donor),CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(""), NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_sst_donor_check),
+       ON_UPDATE(wsrep_sst_donor_update)); 
+
+static Sys_var_mybool Sys_wsrep_sst_donor_rejects_queries(
+       "wsrep_sst_donor_rejects_queries", "Reject client queries "
+       "when donating state snapshot transfer", 
+       GLOBAL_VAR(wsrep_sst_donor_rejects_queries), 
+       CMD_LINE(OPT_ARG), DEFAULT(FALSE));
+
+static Sys_var_mybool Sys_wsrep_on (
+       "wsrep_on", "To enable wsrep replication ",
+       SESSION_VAR(wsrep_on), 
+       CMD_LINE(OPT_ARG), DEFAULT(TRUE), 
+       NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0),
+       ON_UPDATE(wsrep_on_update));
+
+static Sys_var_charptr Sys_wsrep_start_position (
+       "wsrep_start_position", "global transaction position to start from ",
+       PREALLOCATED GLOBAL_VAR(wsrep_start_position), 
+       CMD_LINE(REQUIRED_ARG, OPT_WSREP_START_POSITION),
+       IN_FS_CHARSET, DEFAULT(WSREP_START_POSITION_ZERO),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_start_position_check), 
+       ON_UPDATE(wsrep_start_position_update));
+
+static Sys_var_ulong Sys_wsrep_max_ws_size (
+       "wsrep_max_ws_size", "Max write set size (bytes)",
+       GLOBAL_VAR(wsrep_max_ws_size), CMD_LINE(REQUIRED_ARG),
+       /* Upper limit is 65K short of 4G to avoid overlows on 32-bit systems */
+       VALID_RANGE(1024, WSREP_MAX_WS_SIZE), DEFAULT(1073741824UL), BLOCK_SIZE(1));
+
+static Sys_var_ulong Sys_wsrep_max_ws_rows (
+       "wsrep_max_ws_rows", "Max number of rows in write set",
+       GLOBAL_VAR(wsrep_max_ws_rows), CMD_LINE(REQUIRED_ARG),
+       VALID_RANGE(1, 1048576), DEFAULT(131072), BLOCK_SIZE(1));
+
+static Sys_var_charptr Sys_wsrep_notify_cmd(
+       "wsrep_notify_cmd", "",
+       GLOBAL_VAR(wsrep_notify_cmd),CMD_LINE(REQUIRED_ARG),
+       IN_FS_CHARSET, DEFAULT(""), NO_MUTEX_GUARD, NOT_IN_BINLOG);
+
+static Sys_var_mybool Sys_wsrep_certify_nonPK(
+       "wsrep_certify_nonPK", "Certify tables with no primary key",
+       GLOBAL_VAR(wsrep_certify_nonPK), 
+       CMD_LINE(OPT_ARG), DEFAULT(TRUE));
+
+static Sys_var_mybool Sys_wsrep_causal_reads(
+       "wsrep_causal_reads", "(DEPRECATED) Setting this variable is equivalent "
+       "to setting wsrep_sync_wait READ flag",
+       SESSION_VAR(wsrep_causal_reads), CMD_LINE(OPT_ARG), DEFAULT(FALSE),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0),
+       ON_UPDATE(wsrep_causal_reads_update));
+
+static Sys_var_uint Sys_wsrep_sync_wait(
+       "wsrep_sync_wait", "Ensure \"synchronous\" read view before executing "
+       "an operation of the type specified by bitmask: 1 - READ(includes "
+       "SELECT, SHOW and BEGIN/START TRANSACTION); 2 - UPDATE and DELETE; 4 - "
+       "INSERT and REPLACE",
+       SESSION_VAR(wsrep_sync_wait), CMD_LINE(OPT_ARG),
+       VALID_RANGE(WSREP_SYNC_WAIT_NONE, WSREP_SYNC_WAIT_MAX),
+       DEFAULT(WSREP_SYNC_WAIT_NONE), BLOCK_SIZE(1),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0),
+       ON_UPDATE(wsrep_sync_wait_update));
+
+static const char *wsrep_OSU_method_names[]= { "TOI", "RSU", NullS };
+static Sys_var_enum Sys_wsrep_OSU_method(
+       "wsrep_OSU_method", "Method for Online Schema Upgrade",
+       GLOBAL_VAR(wsrep_OSU_method_options), CMD_LINE(OPT_ARG),
+       wsrep_OSU_method_names, DEFAULT(WSREP_OSU_TOI),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0),
+       ON_UPDATE(0));
+
+static PolyLock_mutex PLock_wsrep_desync(&LOCK_wsrep_desync);
+static Sys_var_mybool Sys_wsrep_desync (
+       "wsrep_desync", "To desynchronize the node from the cluster",
+       GLOBAL_VAR(wsrep_desync), 
+       CMD_LINE(OPT_ARG), DEFAULT(FALSE),
+       &PLock_wsrep_desync, NOT_IN_BINLOG,
+       ON_CHECK(wsrep_desync_check),
+       ON_UPDATE(wsrep_desync_update));
+
+static Sys_var_enum Sys_wsrep_forced_binlog_format(
+       "wsrep_forced_binlog_format", "binlog format to take effect over user's choice",
+       GLOBAL_VAR(wsrep_forced_binlog_format), 
+       CMD_LINE(REQUIRED_ARG, OPT_BINLOG_FORMAT),
+       wsrep_binlog_format_names, DEFAULT(BINLOG_FORMAT_UNSPEC),
+       NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0),
+       ON_UPDATE(0));
+
+static Sys_var_mybool Sys_wsrep_recover_datadir(
+       "wsrep_recover", "Recover database state after crash and exit",
+       READ_ONLY GLOBAL_VAR(wsrep_recovery),
+       CMD_LINE(OPT_ARG, OPT_WSREP_RECOVER), DEFAULT(FALSE));
+
+static Sys_var_mybool Sys_wsrep_replicate_myisam(
+       "wsrep_replicate_myisam", "To enable myisam replication",
+       GLOBAL_VAR(wsrep_replicate_myisam), CMD_LINE(OPT_ARG), DEFAULT(FALSE));
+
+static Sys_var_mybool Sys_wsrep_log_conflicts(
+       "wsrep_log_conflicts", "To log multi-master conflicts",
+       GLOBAL_VAR(wsrep_log_conflicts), CMD_LINE(OPT_ARG), DEFAULT(FALSE));
+
+static Sys_var_ulong Sys_wsrep_mysql_replication_bundle(
+      "wsrep_mysql_replication_bundle", "mysql replication group commit ",
+       GLOBAL_VAR(wsrep_mysql_replication_bundle), CMD_LINE(REQUIRED_ARG),
+       VALID_RANGE(0, 1000), DEFAULT(0), BLOCK_SIZE(1));
+
+static Sys_var_mybool Sys_wsrep_load_data_splitting(
+       "wsrep_load_data_splitting", "To commit LOAD DATA "
+       "transaction after every 10K rows inserted",
+       GLOBAL_VAR(wsrep_load_data_splitting), 
+       CMD_LINE(OPT_ARG), DEFAULT(TRUE));
+
+static Sys_var_mybool Sys_wsrep_slave_FK_checks(
+       "wsrep_slave_FK_checks", "Should slave thread do "
+       "foreign key constraint checks",
+       GLOBAL_VAR(wsrep_slave_FK_checks), 
+       CMD_LINE(OPT_ARG), DEFAULT(TRUE));
+
+static Sys_var_mybool Sys_wsrep_slave_UK_checks(
+       "wsrep_slave_UK_checks", "Should slave thread do "
+       "secondary index uniqueness checks",
+       GLOBAL_VAR(wsrep_slave_UK_checks), 
+       CMD_LINE(OPT_ARG), DEFAULT(FALSE));
+
+static Sys_var_mybool Sys_wsrep_restart_slave(
+       "wsrep_restart_slave", "Should MySQL slave be restarted automatically, when node joins back to cluster",
+       GLOBAL_VAR(wsrep_restart_slave), CMD_LINE(OPT_ARG), DEFAULT(FALSE));
+#endif /* WITH_WSREP */
+
 static bool fix_host_cache_size(sys_var *, THD *, enum_var_type)
 {
   hostname_cache_resize((uint) host_cache_size);
diff --git a/sql/table.cc b/sql/table.cc
index d720a16f6e5..735dbf6eedd 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -40,6 +40,7 @@
 #include "sql_statistics.h"
 #include "discover.h"
 #include "mdl.h"                 // MDL_wait_for_graph_visitor
+#include "ha_partition.h"
 
 /* INFORMATION_SCHEMA name */
 LEX_STRING INFORMATION_SCHEMA_NAME= {C_STRING_WITH_LEN("information_schema")};
diff --git a/sql/transaction.cc b/sql/transaction.cc
index 933e39ae357..9b3507de2eb 100644
--- a/sql/transaction.cc
+++ b/sql/transaction.cc
@@ -97,6 +97,10 @@ static bool xa_trans_force_rollback(THD *thd)
     by ha_rollback()/THD::transaction::cleanup().
   */
   thd->transaction.xid_state.rm_error= 0;
+#ifdef WITH_WSREP
+  if (WSREP_ON)
+    wsrep_register_hton(thd, TRUE);
+#endif /* WITH_WSREP */
   if (ha_rollback_trans(thd, true))
   {
     my_error(ER_XAER_RMERR, MYF(0));
@@ -135,10 +139,18 @@ bool trans_begin(THD *thd, uint flags)
       (thd->variables.option_bits & OPTION_TABLE_LOCK))
   {
     thd->variables.option_bits&= ~OPTION_TABLE_LOCK;
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+      wsrep_register_hton(thd, TRUE);
+#endif /* WITH_WSREP */
     thd->server_status&=
       ~(SERVER_STATUS_IN_TRANS | SERVER_STATUS_IN_TRANS_READONLY);
     DBUG_PRINT("info", ("clearing SERVER_STATUS_IN_TRANS"));
     res= MY_TEST(ha_commit_trans(thd, TRUE));
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+      wsrep_post_commit(thd, TRUE);
+#endif /* WITH_WSREP */
   }
 
   thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_KEEP_LOG);
@@ -181,6 +193,12 @@ bool trans_begin(THD *thd, uint flags)
     thd->tx_read_only= false;
   }
 
+#ifdef WITH_WSREP
+  thd->wsrep_PA_safe= true;
+  if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd))
+    DBUG_RETURN(TRUE);
+#endif /* WITH_WSREP */
+
   thd->variables.option_bits|= OPTION_BEGIN;
   thd->server_status|= SERVER_STATUS_IN_TRANS;
   if (thd->tx_read_only)
@@ -212,10 +230,18 @@ bool trans_commit(THD *thd)
   if (trans_check(thd))
     DBUG_RETURN(TRUE);
 
+#ifdef WITH_WSREP
+  if (WSREP_ON)
+    wsrep_register_hton(thd, TRUE);
+#endif /* WITH_WSREP */
   thd->server_status&=
     ~(SERVER_STATUS_IN_TRANS | SERVER_STATUS_IN_TRANS_READONLY);
   DBUG_PRINT("info", ("clearing SERVER_STATUS_IN_TRANS"));
   res= ha_commit_trans(thd, TRUE);
+#ifdef WITH_WSREP
+  if (WSREP_ON)
+    wsrep_post_commit(thd, TRUE);
+#endif /* WITH_WSREP */
     /*
       if res is non-zero, then ha_commit_trans has rolled back the
       transaction, so the hooks for rollback will be called.
@@ -261,10 +287,18 @@ bool trans_commit_implicit(THD *thd)
     /* Safety if one did "drop table" on locked tables */
     if (!thd->locked_tables_mode)
       thd->variables.option_bits&= ~OPTION_TABLE_LOCK;
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+      wsrep_register_hton(thd, TRUE);
+#endif /* WITH_WSREP */
     thd->server_status&=
       ~(SERVER_STATUS_IN_TRANS | SERVER_STATUS_IN_TRANS_READONLY);
     DBUG_PRINT("info", ("clearing SERVER_STATUS_IN_TRANS"));
     res= MY_TEST(ha_commit_trans(thd, TRUE));
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+      wsrep_post_commit(thd, TRUE);
+#endif /* WITH_WSREP */
   }
 
   thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_KEEP_LOG);
@@ -297,9 +331,16 @@ bool trans_rollback(THD *thd)
   int res;
   DBUG_ENTER("trans_rollback");
 
+#ifdef WITH_WSREP
+  thd->wsrep_PA_safe= true;
+#endif /* WITH_WSREP */
   if (trans_check(thd))
     DBUG_RETURN(TRUE);
 
+#ifdef WITH_WSREP
+  if (WSREP_ON)
+    wsrep_register_hton(thd, TRUE);
+#endif /* WITH_WSREP */
   thd->server_status&=
     ~(SERVER_STATUS_IN_TRANS | SERVER_STATUS_IN_TRANS_READONLY);
   DBUG_PRINT("info", ("clearing SERVER_STATUS_IN_TRANS"));
@@ -390,11 +431,19 @@ bool trans_commit_stmt(THD *thd)
 
   if (thd->transaction.stmt.ha_list)
   {
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+      wsrep_register_hton(thd, FALSE);
+#endif /* WITH_WSREP */
     res= ha_commit_trans(thd, FALSE);
     if (! thd->in_active_multi_stmt_transaction())
     {
       thd->tx_isolation= (enum_tx_isolation) thd->variables.tx_isolation;
       thd->tx_read_only= thd->variables.tx_read_only;
+#ifdef WITH_WSREP
+      if (WSREP_ON)
+        wsrep_post_commit(thd, FALSE);
+#endif /* WITH_WSREP */
     }
   }
 
@@ -435,6 +484,10 @@ bool trans_rollback_stmt(THD *thd)
 
   if (thd->transaction.stmt.ha_list)
   {
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+      wsrep_register_hton(thd, FALSE);
+#endif /* WITH_WSREP */
     ha_rollback_trans(thd, FALSE);
     if (! thd->in_active_multi_stmt_transaction())
     {
@@ -812,9 +865,17 @@ bool trans_xa_commit(THD *thd)
   }
   else if (xa_state == XA_IDLE && thd->lex->xa_opt == XA_ONE_PHASE)
   {
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+      wsrep_register_hton(thd, TRUE);
+#endif /* WITH_WSREP */
     int r= ha_commit_trans(thd, TRUE);
     if ((res= MY_TEST(r)))
       my_error(r == 1 ? ER_XA_RBROLLBACK : ER_XAER_RMERR, MYF(0));
+#ifdef WITH_WSREP
+    if (WSREP_ON)
+      wsrep_post_commit(thd, TRUE);
+#endif /* WITH_WSREP */
   }
   else if (xa_state == XA_PREPARED && thd->lex->xa_opt == XA_NONE)
   {
@@ -833,6 +894,10 @@ bool trans_xa_commit(THD *thd)
     if (thd->mdl_context.acquire_lock(&mdl_request,
                                       thd->variables.lock_wait_timeout))
     {
+#ifdef WITH_WSREP
+      if (WSREP_ON)
+        wsrep_register_hton(thd, TRUE);
+#endif /* WITH_WSREP */
       ha_rollback_trans(thd, TRUE);
       my_error(ER_XAER_RMERR, MYF(0));
     }
diff --git a/sql/tztime.cc b/sql/tztime.cc
index d3b4fec6335..577474cd78f 100644
--- a/sql/tztime.cc
+++ b/sql/tztime.cc
@@ -2708,6 +2708,13 @@ main(int argc, char **argv)
     free_defaults(default_argv);
     return 1;
   }
+
+#ifdef WITH_WSREP
+  // Replicate MyISAM DDL for this session, cf. lp:1161432
+  // timezone info unfixable in XtraDB Cluster
+    printf("SET GLOBAL wsrep_replicate_myisam= ON;\n");
+#endif /* WITH_WSREP */
+
   if (argc == 1 && !opt_leap)
   {
     /* Argument is timezonedir */
@@ -2755,6 +2762,11 @@ main(int argc, char **argv)
     free_root(&tz_storage, MYF(0));
   }
 
+#ifdef WITH_WSREP
+  // Reset wsrep_replicate_myisam. lp:1161432
+  printf("SET GLOBAL wsrep_replicate_myisam= OFF;\n");
+#endif /* WITH_WSREP */
+
   free_defaults(default_argv);
   my_end(0);
   return 0;
diff --git a/sql/wsrep_applier.cc b/sql/wsrep_applier.cc
new file mode 100644
index 00000000000..23687e98c32
--- /dev/null
+++ b/sql/wsrep_applier.cc
@@ -0,0 +1,387 @@
+/* Copyright (C) 2013 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with this program; if not, write to the Free Software Foundation, Inc.,
+   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+
+#include "wsrep_priv.h"
+#include "wsrep_binlog.h" // wsrep_dump_rbr_buf()
+
+#include "log_event.h" // EVENT_LEN_OFFSET, etc.
+#include "wsrep_applier.h"
+
+/*
+  read the first event from (*buf). The size of the (*buf) is (*buf_len).
+  At the end (*buf) is shitfed to point to the following event or NULL and
+  (*buf_len) will be changed to account just being read bytes of the 1st event.
+*/
+
+static Log_event* wsrep_read_log_event(
+    char **arg_buf, size_t *arg_buf_len,
+    const Format_description_log_event *description_event)
+{
+  DBUG_ENTER("wsrep_read_log_event");
+  char *head= (*arg_buf);
+
+  uint data_len = uint4korr(head + EVENT_LEN_OFFSET);
+  char *buf= (*arg_buf);
+  const char *error= 0;
+  Log_event *res=  0;
+
+  if (data_len > wsrep_max_ws_size)
+  {
+    error = "Event too big";
+    goto err;
+  }
+
+  res= Log_event::read_log_event(buf, data_len, &error, description_event, true);
+
+err:
+  if (!res)
+  {
+    DBUG_ASSERT(error != 0);
+    sql_print_error("Error in Log_event::read_log_event(): "
+                    "'%s', data_len: %d, event_type: %d",
+                    error,data_len,head[EVENT_TYPE_OFFSET]);
+  }
+  (*arg_buf)+= data_len;
+  (*arg_buf_len)-= data_len;
+  DBUG_RETURN(res);
+}
+
+#include "transaction.h" // trans_commit(), trans_rollback()
+#include "rpl_rli.h"     // class Relay_log_info;
+#include "sql_base.h"    // close_temporary_table()
+
+static inline void
+wsrep_set_apply_format(THD* thd, Format_description_log_event* ev)
+{
+  if (thd->wsrep_apply_format)
+  {
+      delete (Format_description_log_event*)thd->wsrep_apply_format;
+  }
+  thd->wsrep_apply_format= ev;
+}
+
+static inline Format_description_log_event*
+wsrep_get_apply_format(THD* thd)
+{
+  if (thd->wsrep_apply_format)
+      return (Format_description_log_event*) thd->wsrep_apply_format;
+  /* TODO: mariadb does not support rli->get_rli_description_event()
+   * => look for alternative way to remember last FDE in replication
+   */
+  //return thd->wsrep_rli->get_rli_description_event();
+  thd->wsrep_apply_format = new Format_description_log_event(4);
+  return (Format_description_log_event*) thd->wsrep_apply_format;
+}
+
+static wsrep_cb_status_t wsrep_apply_events(THD*        thd,
+                                            const void* events_buf,
+                                            size_t      buf_len)
+{
+  char *buf= (char *)events_buf;
+  int rcode= 0;
+  int event= 1;
+
+  DBUG_ENTER("wsrep_apply_events");
+
+  if (thd->killed == KILL_CONNECTION)
+  {
+    WSREP_INFO("applier has been aborted, skipping apply_rbr: %lld",
+               (long long) wsrep_thd_trx_seqno(thd));
+    DBUG_RETURN(WSREP_CB_FAILURE);
+  }
+
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  thd->wsrep_query_state= QUERY_EXEC;
+  if (thd->wsrep_conflict_state!= REPLAYING)
+    thd->wsrep_conflict_state= NO_CONFLICT;
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+  if (!buf_len) WSREP_DEBUG("empty rbr buffer to apply: %lld",
+                            (long long) wsrep_thd_trx_seqno(thd));
+
+  while(buf_len)
+  {
+    int exec_res;
+    Log_event* ev= wsrep_read_log_event(&buf, &buf_len,
+                                        wsrep_get_apply_format(thd));
+
+    if (!ev)
+    {
+      WSREP_ERROR("applier could not read binlog event, seqno: %lld, len: %zu",
+                  (long long)wsrep_thd_trx_seqno(thd), buf_len);
+      rcode= 1;
+      goto error;
+    }
+
+    switch (ev->get_type_code()) {
+    case FORMAT_DESCRIPTION_EVENT:
+      wsrep_set_apply_format(thd, (Format_description_log_event*)ev);
+      continue;
+#ifdef GTID_SUPPORT
+    case GTID_LOG_EVENT:
+    {
+      Gtid_log_event* gev= (Gtid_log_event*)ev;
+      if (gev->get_gno() == 0)
+      {
+        /* Skip GTID log event to make binlog to generate LTID on commit */
+        delete ev;
+        continue;
+      }
+    }
+#endif /* GTID_SUPPORT */
+    default:
+      break;
+    }
+
+    thd->set_server_id(ev->server_id); // use the original server id for logging
+    thd->set_time();                // time the query
+    wsrep_xid_init(&thd->transaction.xid_state.xid,
+                   &thd->wsrep_trx_meta.gtid.uuid,
+                   thd->wsrep_trx_meta.gtid.seqno);
+    thd->lex->current_select= 0;
+    if (!ev->when)
+    {
+      my_hrtime_t hrtime= my_hrtime();
+      ev->when= hrtime_to_my_time(hrtime);
+      ev->when_sec_part= hrtime_sec_part(hrtime);
+    }
+
+    ev->thd = thd;
+    //exec_res = ev->apply_event(thd->wsrep_rli);
+    exec_res = ev->apply_event(thd->wsrep_rgi);
+    DBUG_PRINT("info", ("exec_event result: %d", exec_res));
+
+    if (exec_res)
+    {
+      WSREP_WARN("RBR event %d %s apply warning: %d, %lld",
+                 event, ev->get_type_str(), exec_res,
+                 (long long) wsrep_thd_trx_seqno(thd));
+      rcode= exec_res;
+      /* stop processing for the first error */
+      delete ev;
+      goto error;
+    }
+    event++;
+
+    if (thd->wsrep_conflict_state!= NO_CONFLICT &&
+        thd->wsrep_conflict_state!= REPLAYING)
+      WSREP_WARN("conflict state after RBR event applying: %d, %lld",
+                 thd->wsrep_query_state, (long long)wsrep_thd_trx_seqno(thd));
+
+    if (thd->wsrep_conflict_state == MUST_ABORT) {
+      WSREP_WARN("RBR event apply failed, rolling back: %lld",
+                 (long long) wsrep_thd_trx_seqno(thd));
+      trans_rollback(thd);
+      thd->locked_tables_list.unlock_locked_tables(thd);
+      /* Release transactional metadata locks. */
+      thd->mdl_context.release_transactional_locks();
+      thd->wsrep_conflict_state= NO_CONFLICT;
+      DBUG_RETURN(WSREP_CB_FAILURE);
+    }
+
+    delete ev;
+  }
+
+ error:
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  thd->wsrep_query_state= QUERY_IDLE;
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+  assert(thd->wsrep_exec_mode== REPL_RECV);
+
+  if (thd->killed == KILL_CONNECTION)
+    WSREP_INFO("applier aborted: %lld", (long long)wsrep_thd_trx_seqno(thd));
+
+  if (rcode) DBUG_RETURN(WSREP_CB_FAILURE);
+  DBUG_RETURN(WSREP_CB_SUCCESS);
+}
+
+wsrep_cb_status_t wsrep_apply_cb(void* const             ctx,
+                                 const void* const       buf,
+                                 size_t const            buf_len,
+                                 uint32_t const          flags,
+                                 const wsrep_trx_meta_t* meta)
+{
+  THD* const thd((THD*)ctx);
+
+  thd->wsrep_trx_meta = *meta;
+
+#ifdef WSREP_PROC_INFO
+  snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+           "applying write set %lld: %p, %zu",
+           (long long)wsrep_thd_trx_seqno(thd), buf, buf_len);
+  thd_proc_info(thd, thd->wsrep_info);
+#else
+  thd_proc_info(thd, "applying write set");
+#endif /* WSREP_PROC_INFO */
+
+  /* tune FK and UK checking policy */
+  if (wsrep_slave_UK_checks == FALSE) 
+    thd->variables.option_bits|= OPTION_RELAXED_UNIQUE_CHECKS;
+  else
+    thd->variables.option_bits&= ~OPTION_RELAXED_UNIQUE_CHECKS;
+
+  if (wsrep_slave_FK_checks == FALSE) 
+    thd->variables.option_bits|= OPTION_NO_FOREIGN_KEY_CHECKS;
+  else
+    thd->variables.option_bits&= ~OPTION_NO_FOREIGN_KEY_CHECKS;
+
+  if (flags & WSREP_FLAG_ISOLATION)
+  {
+    thd->wsrep_apply_toi= true;
+    /*
+      Don't run in transaction mode with TOI actions.
+     */
+    thd->variables.option_bits&= ~OPTION_BEGIN;
+    thd->server_status&= ~SERVER_STATUS_IN_TRANS;
+  }
+  wsrep_cb_status_t rcode(wsrep_apply_events(thd, buf, buf_len));
+
+#ifdef WSREP_PROC_INFO
+  snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+           "applied write set %lld", (long long)wsrep_thd_trx_seqno(thd));
+  thd_proc_info(thd, thd->wsrep_info);
+#else
+  thd_proc_info(thd, "applied write set");
+#endif /* WSREP_PROC_INFO */
+
+  if (WSREP_CB_SUCCESS != rcode)
+  {
+    wsrep_dump_rbr_buf(thd, buf, buf_len);
+  }
+
+  TABLE *tmp;
+  while ((tmp = thd->temporary_tables))
+  {
+    WSREP_DEBUG("Applier %lu, has temporary tables: %s.%s",
+		thd->thread_id,
+		(tmp->s) ? tmp->s->db.str : "void",
+		(tmp->s) ? tmp->s->table_name.str : "void");
+    close_temporary_table(thd, tmp, 1, 1);
+  }
+
+  return rcode;
+}
+
+static wsrep_cb_status_t wsrep_commit(THD* const thd,
+                                      wsrep_seqno_t const global_seqno)
+{
+#ifdef WSREP_PROC_INFO
+  snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+           "committing %lld", (long long)wsrep_thd_trx_seqno(thd));
+  thd_proc_info(thd, thd->wsrep_info);
+#else
+  thd_proc_info(thd, "committing");
+#endif /* WSREP_PROC_INFO */
+
+  wsrep_cb_status_t const rcode(trans_commit(thd) ?
+                                WSREP_CB_FAILURE : WSREP_CB_SUCCESS);
+
+#ifdef WSREP_PROC_INFO
+  snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+           "committed %lld", (long long)wsrep_thd_trx_seqno(thd));
+  thd_proc_info(thd, thd->wsrep_info);
+#else
+  thd_proc_info(thd, "committed");
+#endif /* WSREP_PROC_INFO */
+
+  if (WSREP_CB_SUCCESS == rcode)
+  {
+    thd->wsrep_rgi->cleanup_context(thd, false);
+#ifdef GTID_SUPPORT
+    thd->variables.gtid_next.set_automatic();
+#endif /* GTID_SUPPORT */
+    // TODO: mark snapshot with global_seqno.
+  }
+
+  return rcode;
+}
+
+static wsrep_cb_status_t wsrep_rollback(THD* const thd,
+                                        wsrep_seqno_t const global_seqno)
+{
+#ifdef WSREP_PROC_INFO
+  snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+           "rolling back %lld", (long long)wsrep_thd_trx_seqno(thd));
+  thd_proc_info(thd, thd->wsrep_info);
+#else
+  thd_proc_info(thd, "rolling back");
+#endif /* WSREP_PROC_INFO */
+
+  wsrep_cb_status_t const rcode(trans_rollback(thd) ?
+                                WSREP_CB_FAILURE : WSREP_CB_SUCCESS);
+
+#ifdef WSREP_PROC_INFO
+  snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1,
+           "rolled back %lld", (long long)wsrep_thd_trx_seqno(thd));
+  thd_proc_info(thd, thd->wsrep_info);
+#else
+  thd_proc_info(thd, "rolled back");
+#endif /* WSREP_PROC_INFO */
+
+  return rcode;
+}
+
+wsrep_cb_status_t wsrep_commit_cb(void*         const     ctx,
+                                  uint32_t      const     flags,
+                                  const wsrep_trx_meta_t* meta,
+                                  wsrep_bool_t* const     exit,
+                                  bool          const     commit)
+{
+  THD* const thd((THD*)ctx);
+
+  assert(meta->gtid.seqno == wsrep_thd_trx_seqno(thd));
+
+  wsrep_cb_status_t rcode;
+
+  if (commit)
+    rcode = wsrep_commit(thd, meta->gtid.seqno);
+  else
+    rcode = wsrep_rollback(thd, meta->gtid.seqno);
+
+  wsrep_set_apply_format(thd, NULL);
+  thd->mdl_context.release_transactional_locks();
+  free_root(thd->mem_root,MYF(MY_KEEP_PREALLOC));
+  thd->tx_isolation= (enum_tx_isolation) thd->variables.tx_isolation;
+
+  if (wsrep_slave_count_change < 0 && commit && WSREP_CB_SUCCESS == rcode)
+  {
+    mysql_mutex_lock(&LOCK_wsrep_slave_threads);
+    if (wsrep_slave_count_change < 0)
+    {
+      wsrep_slave_count_change++;
+      *exit = true;
+    }
+    mysql_mutex_unlock(&LOCK_wsrep_slave_threads);
+  }
+
+  if (*exit == false && thd->wsrep_applier)
+  {
+    /* From trans_begin() */
+    thd->variables.option_bits|= OPTION_BEGIN;
+    thd->server_status|= SERVER_STATUS_IN_TRANS;
+    thd->wsrep_apply_toi= false;
+  }
+
+  return rcode;
+}
+
+
+wsrep_cb_status_t wsrep_unordered_cb(void*       const ctx,
+                                     const void* const data,
+                                     size_t      const size)
+{
+    return WSREP_CB_SUCCESS;
+}
diff --git a/sql/wsrep_applier.h b/sql/wsrep_applier.h
new file mode 100644
index 00000000000..816970db67c
--- /dev/null
+++ b/sql/wsrep_applier.h
@@ -0,0 +1,38 @@
+/* Copyright 2013 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef WSREP_APPLIER_H
+#define WSREP_APPLIER_H
+
+#include <sys/types.h>
+
+/* wsrep callback prototypes */
+
+wsrep_cb_status_t wsrep_apply_cb(void *ctx,
+                                 const void* buf, size_t buf_len,
+                                 uint32_t flags,
+                                 const wsrep_trx_meta_t* meta);
+
+wsrep_cb_status_t wsrep_commit_cb(void *ctx,
+                                  uint32_t flags,
+                                  const wsrep_trx_meta_t* meta,
+                                  wsrep_bool_t* exit,
+                                  bool commit);
+
+wsrep_cb_status_t wsrep_unordered_cb(void*       ctx,
+                                     const void* data,
+                                     size_t      size);
+
+#endif /* WSREP_APPLIER_H */
diff --git a/sql/wsrep_binlog.cc b/sql/wsrep_binlog.cc
new file mode 100644
index 00000000000..ee8a9cb130b
--- /dev/null
+++ b/sql/wsrep_binlog.cc
@@ -0,0 +1,414 @@
+/* Copyright (C) 2013 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with this program; if not, write to the Free Software Foundation, Inc.,
+   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+
+#include "wsrep_binlog.h"
+#include "wsrep_priv.h"
+#include "log.h"
+
+extern handlerton *binlog_hton;
+/*
+  Write the contents of a cache to a memory buffer.
+
+  This function quite the same as MYSQL_BIN_LOG::write_cache(),
+  with the exception that here we write in buffer instead of log file.
+ */
+int wsrep_write_cache_buf(IO_CACHE *cache, uchar **buf, size_t *buf_len)
+{
+  *buf= NULL;
+  *buf_len= 0;
+
+  my_off_t const saved_pos(my_b_tell(cache));
+
+  if (reinit_io_cache(cache, READ_CACHE, 0, 0, 0))
+  {
+    WSREP_ERROR("failed to initialize io-cache");
+    return ER_ERROR_ON_WRITE;
+  }
+
+  uint length = my_b_bytes_in_cache(cache);
+  if (unlikely(0 == length)) length = my_b_fill(cache);
+
+  size_t total_length = 0;
+
+  if (likely(length > 0)) do
+  {
+      total_length += length;
+      /*
+        Bail out if buffer grows too large.
+        A temporary fix to avoid allocating indefinitely large buffer,
+        not a real limit on a writeset size which includes other things
+        like header and keys.
+      */
+      if (total_length > wsrep_max_ws_size)
+      {
+          WSREP_WARN("transaction size limit (%lu) exceeded: %zu",
+                     wsrep_max_ws_size, total_length);
+          goto error;
+      }
+      uchar* tmp = (uchar *)my_realloc(*buf, total_length,
+                                       MYF(MY_ALLOW_ZERO_PTR));
+      if (!tmp)
+      {
+          WSREP_ERROR("could not (re)allocate buffer: %zu + %u",
+                      *buf_len, length);
+          goto error;
+      }
+      *buf = tmp;
+
+      memcpy(*buf + *buf_len, cache->read_pos, length);
+      *buf_len = total_length;
+      cache->read_pos = cache->read_end;
+  } while ((cache->file >= 0) && (length = my_b_fill(cache)));
+
+  if (reinit_io_cache(cache, WRITE_CACHE, saved_pos, 0, 0))
+  {
+    WSREP_WARN("failed to initialize io-cache");
+    goto cleanup;
+  }
+
+  return 0;
+
+error:
+  if (reinit_io_cache(cache, WRITE_CACHE, saved_pos, 0, 0))
+  {
+    WSREP_WARN("failed to initialize io-cache");
+  }
+cleanup:
+  my_free(*buf);
+  *buf= NULL;
+  *buf_len= 0;
+  return ER_ERROR_ON_WRITE;
+}
+
+#define STACK_SIZE 4096 /* 4K - for buffer preallocated on the stack:
+                         * many transactions would fit in there
+                         * so there is no need to reach for the heap */
+
+/* Returns minimum multiple of HEAP_PAGE_SIZE that is >= length */
+static inline size_t
+heap_size(size_t length)
+{
+    return (length + HEAP_PAGE_SIZE - 1)/HEAP_PAGE_SIZE*HEAP_PAGE_SIZE;
+}
+
+/* append data to writeset */
+static inline wsrep_status_t
+wsrep_append_data(wsrep_t*           const wsrep,
+                  wsrep_ws_handle_t* const ws,
+                  const void*        const data,
+                  size_t             const len)
+{
+    struct wsrep_buf const buff = { data, len };
+    wsrep_status_t const rc(wsrep->append_data(wsrep, ws, &buff, 1,
+                                               WSREP_DATA_ORDERED, true));
+    if (rc != WSREP_OK)
+    {
+        WSREP_WARN("append_data() returned %d", rc);
+    }
+
+    return rc;
+}
+
+/*
+  Write the contents of a cache to wsrep provider.
+
+  This function quite the same as MYSQL_BIN_LOG::write_cache(),
+  with the exception that here we write in buffer instead of log file.
+
+  This version reads all of cache into single buffer and then appends to a
+  writeset at once.
+ */
+static int wsrep_write_cache_once(wsrep_t*  const wsrep,
+                                  THD*      const thd,
+                                  IO_CACHE* const cache,
+                                  size_t*   const len)
+{
+    my_off_t const saved_pos(my_b_tell(cache));
+
+    if (reinit_io_cache(cache, READ_CACHE, 0, 0, 0))
+    {
+        WSREP_ERROR("failed to initialize io-cache");
+        return ER_ERROR_ON_WRITE;
+    }
+
+    int err(WSREP_OK);
+
+    size_t total_length(0);
+    uchar  stack_buf[STACK_SIZE]; /* to avoid dynamic allocations for few data*/
+    uchar* heap_buf(NULL);
+    uchar* buf(stack_buf);
+    size_t allocated(sizeof(stack_buf));
+    size_t used(0);
+
+    uint length(my_b_bytes_in_cache(cache));
+    if (unlikely(0 == length)) length = my_b_fill(cache);
+
+    if (likely(length > 0)) do
+    {
+        total_length += length;
+        /*
+          Bail out if buffer grows too large.
+          A temporary fix to avoid allocating indefinitely large buffer,
+          not a real limit on a writeset size which includes other things
+          like header and keys.
+        */
+        if (unlikely(total_length > wsrep_max_ws_size))
+        {
+            WSREP_WARN("transaction size limit (%lu) exceeded: %zu",
+                       wsrep_max_ws_size, total_length);
+	    err = WSREP_TRX_SIZE_EXCEEDED;
+            goto cleanup;
+        }
+
+        if (total_length > allocated)
+        {
+            size_t const new_size(heap_size(total_length));
+            uchar* tmp = (uchar *)my_realloc(heap_buf, new_size,
+                                             MYF(MY_ALLOW_ZERO_PTR));
+            if (!tmp)
+            {
+                WSREP_ERROR("could not (re)allocate buffer: %zu + %u",
+                            allocated, length);
+                err = WSREP_TRX_SIZE_EXCEEDED;
+                goto cleanup;
+            }
+
+            heap_buf = tmp;
+            buf = heap_buf;
+            allocated = new_size;
+
+            if (used <= STACK_SIZE && used > 0) // there's data in stack_buf
+            {
+                DBUG_ASSERT(buf == stack_buf);
+                memcpy(heap_buf, stack_buf, used);
+            }
+        }
+
+        memcpy(buf + used, cache->read_pos, length);
+        used = total_length;
+        cache->read_pos = cache->read_end;
+    } while ((cache->file >= 0) && (length = my_b_fill(cache)));
+
+    if (used > 0)
+        err = wsrep_append_data(wsrep, &thd->wsrep_ws_handle, buf, used);
+
+    if (WSREP_OK == err) *len = total_length;
+
+cleanup:
+    if (reinit_io_cache(cache, WRITE_CACHE, saved_pos, 0, 0))
+    {
+        WSREP_ERROR("failed to reinitialize io-cache");
+    }
+
+    if (unlikely(WSREP_OK != err)) wsrep_dump_rbr_buf(thd, buf, used);
+
+    my_free(heap_buf);
+    return err;
+}
+
+/*
+  Write the contents of a cache to wsrep provider.
+
+  This function quite the same as MYSQL_BIN_LOG::write_cache(),
+  with the exception that here we write in buffer instead of log file.
+
+  This version uses incremental data appending as it reads it from cache.
+ */
+static int wsrep_write_cache_inc(wsrep_t*  const wsrep,
+                                 THD*      const thd,
+                                 IO_CACHE* const cache,
+                                 size_t*   const len)
+{
+    my_off_t const saved_pos(my_b_tell(cache));
+
+    if (reinit_io_cache(cache, READ_CACHE, 0, 0, 0))
+    {
+      WSREP_ERROR("failed to initialize io-cache");
+      return WSREP_TRX_ERROR;
+    }
+
+    int err(WSREP_OK);
+
+    size_t total_length(0);
+
+    uint length(my_b_bytes_in_cache(cache));
+    if (unlikely(0 == length)) length = my_b_fill(cache);
+
+    if (likely(length > 0)) do
+    {
+        total_length += length;
+        /* bail out if buffer grows too large
+           not a real limit on a writeset size which includes other things
+           like header and keys.
+        */
+        if (unlikely(total_length > wsrep_max_ws_size))
+        {
+            WSREP_WARN("transaction size limit (%lu) exceeded: %zu",
+                       wsrep_max_ws_size, total_length);
+            err = WSREP_TRX_SIZE_EXCEEDED;
+            goto cleanup;
+        }
+
+        if(WSREP_OK != (err=wsrep_append_data(wsrep, &thd->wsrep_ws_handle,
+                                              cache->read_pos, length)))
+                goto cleanup;
+
+        cache->read_pos = cache->read_end;
+    } while ((cache->file >= 0) && (length = my_b_fill(cache)));
+
+    if (WSREP_OK == err) *len = total_length;
+
+cleanup:
+    if (reinit_io_cache(cache, WRITE_CACHE, saved_pos, 0, 0))
+    {
+        WSREP_ERROR("failed to reinitialize io-cache");
+    }
+
+    return err;
+}
+
+/*
+  Write the contents of a cache to wsrep provider.
+
+  This function quite the same as MYSQL_BIN_LOG::write_cache(),
+  with the exception that here we write in buffer instead of log file.
+ */
+int wsrep_write_cache(wsrep_t*  const wsrep,
+                      THD*      const thd,
+                      IO_CACHE* const cache,
+                      size_t*   const len)
+{
+    if (wsrep_incremental_data_collection) {
+        return wsrep_write_cache_inc(wsrep, thd, cache, len);
+    }
+    else {
+        return wsrep_write_cache_once(wsrep, thd, cache, len);
+    }
+}
+
+void wsrep_dump_rbr_buf(THD *thd, const void* rbr_buf, size_t buf_len)
+{
+  char filename[PATH_MAX]= {0};
+  int len= snprintf(filename, PATH_MAX, "%s/GRA_%ld_%lld.log",
+                    wsrep_data_home_dir, thd->thread_id,
+                    (long long)wsrep_thd_trx_seqno(thd));
+  if (len >= PATH_MAX)
+  {
+    WSREP_ERROR("RBR dump path too long: %d, skipping dump.", len);
+    return;
+  }
+
+  FILE *of= fopen(filename, "wb");
+  if (of)
+  {
+    fwrite (rbr_buf, buf_len, 1, of);
+    fclose(of);
+  }
+  else
+  {
+    WSREP_ERROR("Failed to open file '%s': %d (%s)",
+                filename, errno, strerror(errno));
+  }
+}
+
+/*
+  wsrep exploits binlog's caches even if binlogging itself is not
+  activated. In such case connection close needs calling
+  actual binlog's method.
+  Todo: split binlog hton from its caches to use ones by wsrep
+  without referring to binlog's stuff.
+*/
+int wsrep_binlog_close_connection(THD* thd)
+{
+  DBUG_ENTER("wsrep_binlog_close_connection");
+  if (thd_get_ha_data(thd, binlog_hton) != NULL)
+    binlog_hton->close_connection (binlog_hton, thd);
+  DBUG_RETURN(0);
+}
+
+int wsrep_binlog_savepoint_set(THD *thd,  void *sv)
+{
+  if (!wsrep_emulate_bin_log) return 0;
+  int rcode = binlog_hton->savepoint_set(binlog_hton, thd, sv);
+  return rcode;
+}
+
+int wsrep_binlog_savepoint_rollback(THD *thd, void *sv)
+{
+  if (!wsrep_emulate_bin_log) return 0;
+  int rcode = binlog_hton->savepoint_rollback(binlog_hton, thd, sv);
+  return rcode;
+}
+
+void wsrep_dump_rbr_direct(THD* thd, IO_CACHE* cache)
+{
+  char filename[PATH_MAX]= {0};
+  int len= snprintf(filename, PATH_MAX, "%s/GRA_%ld_%lld.log",
+                    wsrep_data_home_dir, thd->thread_id,
+                    (long long)wsrep_thd_trx_seqno(thd));
+  size_t bytes_in_cache = 0;
+  // check path
+  if (len >= PATH_MAX)
+  {
+    WSREP_ERROR("RBR dump path too long: %d, skipping dump.", len);
+    return ;
+  }
+  // init cache
+  my_off_t const saved_pos(my_b_tell(cache));
+  if (reinit_io_cache(cache, READ_CACHE, 0, 0, 0))
+  {
+    WSREP_ERROR("failed to initialize io-cache");
+    return ;
+  }
+  // open file
+  FILE* of = fopen(filename, "wb");
+  if (!of)
+  {
+    WSREP_ERROR("Failed to open file '%s': %d (%s)",
+                filename, errno, strerror(errno));
+    goto cleanup;
+  }
+  // ready to write
+  bytes_in_cache= my_b_bytes_in_cache(cache);
+  if (unlikely(bytes_in_cache == 0)) bytes_in_cache = my_b_fill(cache);
+  if (likely(bytes_in_cache > 0)) do
+  {
+    if (my_fwrite(of, cache->read_pos, bytes_in_cache,
+                  MYF(MY_WME | MY_NABP)) == (size_t) -1)
+    {
+      WSREP_ERROR("Failed to write file '%s'", filename);
+      goto cleanup;
+    }
+    cache->read_pos= cache->read_end;
+  } while ((cache->file >= 0) && (bytes_in_cache= my_b_fill(cache)));
+  if(cache->error == -1)
+  {
+    WSREP_ERROR("RBR inconsistent");
+    goto cleanup;
+  }
+cleanup:
+  // init back
+  if (reinit_io_cache(cache, WRITE_CACHE, saved_pos, 0, 0))
+  {
+    WSREP_ERROR("failed to reinitialize io-cache");
+  }
+  // close file
+  if (of) fclose(of);
+}
+
+void thd_binlog_flush_pending_rows_event(THD *thd, bool stmt_end)
+{
+  thd->binlog_flush_pending_rows_event(stmt_end);
+}
diff --git a/sql/wsrep_binlog.h b/sql/wsrep_binlog.h
new file mode 100644
index 00000000000..a3d8ec6ec2c
--- /dev/null
+++ b/sql/wsrep_binlog.h
@@ -0,0 +1,59 @@
+/* Copyright (C) 2013 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with this program; if not, write to the Free Software Foundation, Inc.,
+   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+
+#ifndef WSREP_BINLOG_H
+#define WSREP_BINLOG_H
+
+#include "sql_class.h" // THD, IO_CACHE
+
+#define HEAP_PAGE_SIZE 65536 /* 64K */
+#define WSREP_MAX_WS_SIZE (0xFFFFFFFFUL - HEAP_PAGE_SIZE)
+
+/*
+  Write the contents of a cache to a memory buffer.
+
+  This function quite the same as MYSQL_BIN_LOG::write_cache(),
+  with the exception that here we write in buffer instead of log file.
+ */
+int wsrep_write_cache_buf(IO_CACHE *cache, uchar **buf, size_t *buf_len);
+
+/*
+  Write the contents of a cache to wsrep provider.
+
+  This function quite the same as MYSQL_BIN_LOG::write_cache(),
+  with the exception that here we write in buffer instead of log file.
+
+  @param len  total amount of data written
+  @return     wsrep error status
+ */
+int wsrep_write_cache (wsrep_t*  wsrep,
+                       THD*      thd,
+                       IO_CACHE* cache,
+                       size_t*   len);
+
+/* Dump replication buffer to disk */
+void wsrep_dump_rbr_buf(THD *thd, const void* rbr_buf, size_t buf_len);
+
+/* Dump replication buffer to disk without intermediate buffer */
+void wsrep_dump_rbr_direct(THD* thd, IO_CACHE* cache);
+
+int wsrep_binlog_close_connection(THD* thd);
+int wsrep_binlog_savepoint_set(THD *thd,  void *sv);
+int wsrep_binlog_savepoint_rollback(THD *thd, void *sv);
+
+/* Dump replication buffer to disk without intermediate buffer */
+void wsrep_dump_rbr_direct(THD* thd, IO_CACHE* cache);
+
+#endif /* WSREP_BINLOG_H */
diff --git a/sql/wsrep_check_opts.cc b/sql/wsrep_check_opts.cc
new file mode 100644
index 00000000000..5ec18c79978
--- /dev/null
+++ b/sql/wsrep_check_opts.cc
@@ -0,0 +1,379 @@
+/* Copyright 2011 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+//#include <mysqld.h>
+#include <sql_class.h>
+//#include <sql_plugin.h>
+//#include <set_var.h>
+
+#include "wsrep_mysqld.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <ctype.h>
+
+/* This file is about checking for correctness of mysql configuration options */
+
+struct opt
+{
+    const char* const name;
+    const char*       value;
+};
+
+/* A list of options to check.
+ * At first we assume default values and then see if they are changed on CLI or
+ * in my.cnf */
+static struct opt opts[] =
+{
+    { "wsrep_slave_threads",     "1" }, // mysqld.cc
+    { "bind_address",      "0.0.0.0" }, // mysqld.cc
+    { "wsrep_sst_method",    "rsync" }, // mysqld.cc
+    { "wsrep_sst_receive_address","AUTO"}, // mysqld.cc
+    { "binlog_format",         "ROW" }, // mysqld.cc
+    { "wsrep_provider",       "none" }, // mysqld.cc
+    { "query_cache_type",        "0" }, // mysqld.cc
+    { "query_cache_size",        "0" }, // mysqld.cc
+    { "locked_in_memory",        "0" }, // mysqld.cc
+    { "wsrep_cluster_address",   "0" }, // mysqld.cc
+    { "locks_unsafe_for_binlog", "0" }, // ha_innodb.cc
+    { "autoinc_lock_mode",       "1" }, // ha_innodb.cc
+    { 0, 0 }
+};
+
+enum
+{
+    WSREP_SLAVE_THREADS,
+    BIND_ADDRESS,
+    WSREP_SST_METHOD,
+    WSREP_SST_RECEIVE_ADDRESS,
+    BINLOG_FORMAT,
+    WSREP_PROVIDER,
+    QUERY_CACHE_TYPE,
+    QUERY_CACHE_SIZE,
+    LOCKED_IN_MEMORY,
+    WSREP_CLUSTER_ADDRESS,
+    LOCKS_UNSAFE_FOR_BINLOG,
+    AUTOINC_LOCK_MODE
+};
+
+
+/* A class to make a copy of argv[] vector */
+struct argv_copy
+{
+    int    const argc_;
+    char**       argv_;
+
+    argv_copy (int const argc, const char* const argv[]) :
+        argc_ (argc),
+        argv_ (reinterpret_cast<char**>(calloc(argc_, sizeof(char*))))
+    {
+        if (argv_)
+        {
+            for (int i = 0; i < argc_; ++i)
+            {
+                argv_[i] = strdup(argv[i]);
+
+                if (!argv_[i])
+                {
+                    argv_free (); // free whatever bee allocated
+                    return;
+                }
+            }
+        }
+    }
+
+    ~argv_copy () { argv_free (); }
+
+private:
+    argv_copy (const argv_copy&);
+    argv_copy& operator= (const argv_copy&);
+
+    void argv_free()
+    {
+        if (argv_)
+        {
+            for (int i = 0; (i < argc_) && argv_[i] ; ++i) free (argv_[i]);
+            free (argv_);
+            argv_ = 0;
+        }
+    }
+};
+
+/* a short corresponding to '--' byte sequence */
+static short const long_opt_prefix ('-' + ('-' << 8));
+
+/* Normalizes long options to have '_' instead of '-' */
+static int
+normalize_opts (argv_copy& a)
+{
+    if (a.argv_)
+    {
+        for (int i = 0; i < a.argc_; ++i)
+        {
+            char* ptr = a.argv_[i];
+            if (long_opt_prefix == *(short*)ptr) // long option
+            {
+                ptr += 2;
+                const char* end = strchr(ptr, '=');
+
+                if (!end) end = ptr + strlen(ptr);
+
+                for (; ptr != end; ++ptr) if ('-' == *ptr) *ptr = '_';
+            }
+        }
+
+        return 0;
+    }
+
+    return EINVAL;
+}
+
+/* Find required options in the argument list and change their values */
+static int
+find_opts (argv_copy& a, struct opt* const opts)
+{
+    for (int i = 0; i < a.argc_; ++i)
+    {
+        char* ptr = a.argv_[i] + 2; // we're interested only in long options
+
+        struct opt* opt = opts;
+        for (; 0 != opt->name; ++opt)
+        {
+            if (!strstr(ptr, opt->name)) continue; // try next option
+
+            /* 1. try to find value after the '=' */
+            opt->value = strchr(ptr, '=') + 1;
+
+            /* 2. if no '=', try next element in the argument vector */
+            if (reinterpret_cast<void*>(1) == opt->value)
+            {
+                /* also check that the next element is not an option itself */
+                if (i + 1 < a.argc_ && *(a.argv_[i + 1]) != '-')
+                {
+                    ++i;
+                    opt->value = a.argv_[i];
+                }
+                else opt->value = ""; // no value supplied (like boolean opt)
+            }
+
+            break; // option found, break inner loop
+        }
+    }
+
+    return 0;
+}
+
+/* Parses string for an integer. Returns 0 on success. */
+int get_long_long (const struct opt& opt, long long* const val, int const base)
+{
+    const char* const str = opt.value;
+
+    if ('\0' != *str)
+    {
+        char* endptr;
+
+        *val = strtoll (str, &endptr, base);
+
+        if ('k' == *endptr || 'K' == *endptr) 
+        { 
+            *val *= 1024L;
+            endptr++;
+        } 
+        else if ('m' == *endptr || 'M' == *endptr) 
+        {
+            *val *= 1024L * 1024L;
+            endptr++;
+        }
+        else if ('g' == *endptr || 'G' == *endptr) 
+        {
+            *val *= 1024L * 1024L * 1024L;
+            endptr++;
+        }
+
+        if ('\0' == *endptr) return 0; // the whole string was a valid integer
+    }
+
+    WSREP_ERROR ("Bad value for *%s: '%s'. Should be integer.",
+                 opt.name, opt.value);
+
+    return EINVAL;
+}
+
+/* This is flimzy coz hell knows how mysql interprets boolean strings...
+ * and, no, I'm not going to become versed in how mysql handles options -
+ * I'd rather sing.
+
+ Aha, http://dev.mysql.com/doc/refman/5.1/en/dynamic-system-variables.html:
+ Variables that have a type of “boolean” can be set to 0, 1, ON or OFF. (If you
+ set them on the command line or in an option file, use the numeric values.)
+
+ So it is '0' for FALSE, '1' or empty string for TRUE
+
+ */
+int get_bool (const struct opt& opt, bool* const val)
+{
+    const char* str = opt.value;
+
+    while (isspace(*str)) ++str; // skip initial whitespaces
+
+    ssize_t str_len = strlen(str);
+    switch (str_len)
+    {
+    case 0:
+        *val = true;
+        return 0;
+    case 1:
+        if ('0' == *str || '1' == *str)
+        {
+            *val = ('1' == *str);
+            return 0;
+        }
+    }
+
+    WSREP_ERROR ("Bad value for *%s: '%s'. Should be '0', '1' or empty string.",
+                 opt.name, opt.value);
+
+    return EINVAL;
+}
+
+static int
+check_opts (int const argc, const char* const argv[], struct opt opts[])
+{
+    /* First, make a copy of argv to be able to manipulate it */
+    argv_copy a(argc, argv);
+
+    if (!a.argv_)
+    {
+        WSREP_ERROR ("Could not copy argv vector: not enough memory.");
+        return ENOMEM;
+    }
+
+    int err = normalize_opts (a);
+    if (err)
+    {
+        WSREP_ERROR ("Failed to normalize options.");
+        return err;
+    }
+
+    err = find_opts (a, opts);
+    if (err)
+    {
+        WSREP_ERROR ("Failed to parse options.");
+        return err;
+    }
+
+    /* At this point we have updated default values in our option list to
+       what has been specified on the command line / my.cnf */
+
+    long long slave_threads;
+    err = get_long_long (opts[WSREP_SLAVE_THREADS], &slave_threads, 10);
+    if (err) return err;
+
+    int rcode = 0;
+
+    if (slave_threads > 1)
+        /* Need to check AUTOINC_LOCK_MODE and LOCKS_UNSAFE_FOR_BINLOG */
+    {
+        long long autoinc_lock_mode;
+        err = get_long_long (opts[AUTOINC_LOCK_MODE], &autoinc_lock_mode, 10);
+        if (err) return err;
+
+        bool locks_unsafe_for_binlog;
+        err = get_bool (opts[LOCKS_UNSAFE_FOR_BINLOG],&locks_unsafe_for_binlog);
+        if (err) return err;
+
+        if (autoinc_lock_mode != 2)
+        {
+            WSREP_ERROR ("Parallel applying (wsrep_slave_threads > 1) requires"
+                         " innodb_autoinc_lock_mode = 2.");
+            rcode = EINVAL;
+        }
+    }
+
+    bool locked_in_memory;
+    err = get_bool (opts[LOCKED_IN_MEMORY], &locked_in_memory);
+    if (err) { WSREP_ERROR("get_bool error: %s", strerror(err)); return err; }
+    if (locked_in_memory)
+    {
+        WSREP_ERROR ("Memory locking is not supported (locked_in_memory=%s)",
+                     locked_in_memory ? "ON" : "OFF");
+        rcode = EINVAL;
+    }
+
+    if (!strcasecmp(opts[WSREP_SST_METHOD].value,"mysqldump"))
+    {
+        if (!strcasecmp(opts[BIND_ADDRESS].value, "127.0.0.1") ||
+            !strcasecmp(opts[BIND_ADDRESS].value, "localhost"))
+        {
+            WSREP_ERROR ("wsrep_sst_method is set to 'mysqldump' yet "
+                         "mysqld bind_address is set to '%s', which makes it "
+                         "impossible to receive state transfer from another "
+                         "node, since mysqld won't accept such connections. "
+                         "If you wish to use mysqldump state transfer method, "
+                         "set bind_address to allow mysql client connections "
+                         "from other cluster members (e.g. 0.0.0.0).",
+                         opts[BIND_ADDRESS].value);
+            rcode = EINVAL;
+        }
+    }
+    else
+    {
+        // non-mysqldump SST requires wsrep_cluster_address on startup
+        if (strlen(opts[WSREP_CLUSTER_ADDRESS].value) == 0)
+        {
+            WSREP_ERROR ("%s SST method requires wsrep_cluster_address to be "
+                         "configured on startup.",opts[WSREP_SST_METHOD].value);
+            rcode = EINVAL;
+        }
+    }
+
+    if (strcasecmp(opts[WSREP_SST_RECEIVE_ADDRESS].value, "AUTO"))
+    {
+        if (!strncasecmp(opts[WSREP_SST_RECEIVE_ADDRESS].value,
+                         "127.0.0.1", strlen("127.0.0.1"))       ||
+            !strncasecmp(opts[WSREP_SST_RECEIVE_ADDRESS].value,
+                         "localhost", strlen("localhost")))
+        {
+            WSREP_WARN  ("wsrep_sst_receive_address is set to '%s' which "
+                         "makes it impossible for another host to reach this "
+                         "one. Please set it to the address which this node "
+                         "can be connected at by other cluster members.",
+                         opts[WSREP_SST_RECEIVE_ADDRESS].value);
+//            rcode = EINVAL;
+        }
+    }
+
+    if (strcasecmp(opts[WSREP_PROVIDER].value, "none"))
+    {
+        if (strcasecmp(opts[BINLOG_FORMAT].value, "ROW"))
+        {
+            WSREP_ERROR ("Only binlog_format = 'ROW' is currently supported. "
+                         "Configured value: '%s'. Please adjust your "
+                         "configuration.", opts[BINLOG_FORMAT].value);
+
+            rcode = EINVAL;
+        }
+    }
+
+    return rcode;
+}
+
+int
+wsrep_check_opts (int const argc, char* const* const argv)
+{
+    return check_opts (argc, argv, opts);
+}
+
diff --git a/sql/wsrep_hton.cc b/sql/wsrep_hton.cc
new file mode 100644
index 00000000000..96f1431a82f
--- /dev/null
+++ b/sql/wsrep_hton.cc
@@ -0,0 +1,577 @@
+/* Copyright 2008 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include <mysqld.h>
+#include "sql_base.h"
+#include "rpl_filter.h"
+#include <sql_class.h>
+#include "wsrep_mysqld.h"
+#include "wsrep_binlog.h"
+#include <cstdio>
+#include <cstdlib>
+
+extern ulonglong thd_to_trx_id(THD *thd);
+
+extern "C" int thd_binlog_format(const MYSQL_THD thd);
+// todo: share interface with ha_innodb.c
+
+enum wsrep_trx_status wsrep_run_wsrep_commit(THD *thd, handlerton *hton,
+                                             bool all);
+
+/*
+  Cleanup after local transaction commit/rollback, replay or TOI.
+*/
+void wsrep_cleanup_transaction(THD *thd)
+{
+  if (wsrep_emulate_bin_log) thd_binlog_trx_reset(thd);
+  thd->wsrep_ws_handle.trx_id= WSREP_UNDEFINED_TRX_ID;
+  thd->wsrep_trx_meta.gtid= WSREP_GTID_UNDEFINED;
+  thd->wsrep_trx_meta.depends_on= WSREP_SEQNO_UNDEFINED;
+  thd->wsrep_exec_mode= LOCAL_STATE;
+  return;
+}
+
+/*
+  wsrep hton
+*/
+handlerton *wsrep_hton;
+
+
+/*
+  Registers wsrep hton at commit time if transaction has registered htons
+  for supported engine types.
+
+  Hton should not be registered for TOTAL_ORDER operations.
+
+  Registration is needed for both LOCAL_MODE and REPL_RECV transactions to run
+  commit in 2pc so that wsrep position gets properly recorded in storage
+  engines.
+
+  Note that all hton calls should immediately return for threads that are
+  in REPL_RECV mode as their states are controlled by wsrep appliers or
+  replaying code. Only threads in LOCAL_MODE should run wsrep callbacks
+  from hton methods.
+*/
+void wsrep_register_hton(THD* thd, bool all)
+{
+  if (thd->wsrep_exec_mode != TOTAL_ORDER && !thd->wsrep_apply_toi)
+  {
+    THD_TRANS *trans=all ? &thd->transaction.all : &thd->transaction.stmt;
+    for (Ha_trx_info *i= trans->ha_list; WSREP(thd) && i; i = i->next())
+    {
+      if ((i->ht()->db_type == DB_TYPE_INNODB) ||
+	  (i->ht()->db_type == DB_TYPE_TOKUDB))
+      {
+        trans_register_ha(thd, all, wsrep_hton);
+
+        /* follow innodb read/write settting
+         * but, as an exception: CTAS with empty result set will not be
+         * replicated unless we declare wsrep hton as read/write here
+	 */
+        if (i->is_trx_read_write() ||
+            (thd->lex->sql_command == SQLCOM_CREATE_TABLE &&
+             thd->wsrep_exec_mode == LOCAL_STATE))
+        {
+          thd->ha_data[wsrep_hton->slot].ha_info[all].set_trx_read_write();
+        }
+        break;
+      }
+    }
+  }
+}
+
+/*
+  Calls wsrep->post_commit() for locally executed transactions that have
+  got seqno from provider (must commit) and don't require replaying.
+ */
+void wsrep_post_commit(THD* thd, bool all)
+{
+  if (thd->wsrep_exec_mode == LOCAL_COMMIT)
+  {
+    DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED);
+    if (wsrep->post_commit(wsrep, &thd->wsrep_ws_handle))
+    {
+        DBUG_PRINT("wsrep", ("set committed fail"));
+        WSREP_WARN("set committed fail: %llu %d",
+                   (long long)thd->real_id, thd->get_stmt_da()->status());
+    }
+    wsrep_cleanup_transaction(thd);
+  }
+}
+
+/*
+  wsrep exploits binlog's caches even if binlogging itself is not
+  activated. In such case connection close needs calling
+  actual binlog's method.
+  Todo: split binlog hton from its caches to use ones by wsrep
+  without referring to binlog's stuff.
+*/
+static int
+wsrep_close_connection(handlerton*  hton, THD* thd)
+{
+  DBUG_ENTER("wsrep_close_connection");
+
+  if (thd->wsrep_exec_mode == REPL_RECV)
+  {
+    DBUG_RETURN(0);
+  }
+  DBUG_RETURN(wsrep_binlog_close_connection (thd));
+}
+
+/*
+  prepare/wsrep_run_wsrep_commit can fail in two ways
+  - certification test or an equivalent. As a result,
+    the current transaction just rolls back
+    Error codes:
+           WSREP_TRX_CERT_FAIL, WSREP_TRX_SIZE_EXCEEDED, WSREP_TRX_ERROR
+  - a post-certification failure makes this server unable to
+    commit its own WS and therefore the server must abort
+*/
+static int wsrep_prepare(handlerton *hton, THD *thd, bool all)
+{
+  DBUG_ENTER("wsrep_prepare");
+
+  if (thd->wsrep_exec_mode == REPL_RECV)
+  {
+    DBUG_RETURN(0);
+  }
+
+  DBUG_ASSERT(thd->ha_data[wsrep_hton->slot].ha_info[all].is_trx_read_write());
+  DBUG_ASSERT(thd->wsrep_exec_mode == LOCAL_STATE);
+  DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno == WSREP_SEQNO_UNDEFINED);
+
+  if ((all ||
+      !thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) &&
+      (thd->variables.wsrep_on && !wsrep_trans_cache_is_empty(thd)))
+  {
+    DBUG_RETURN (wsrep_run_wsrep_commit(thd, hton, all));
+  }
+  DBUG_RETURN(0);
+}
+
+static int wsrep_savepoint_set(handlerton *hton, THD *thd,  void *sv)
+{
+  DBUG_ENTER("wsrep_savepoint_set");
+
+  if (thd->wsrep_exec_mode == REPL_RECV)
+  {
+    DBUG_RETURN(0);
+  }
+
+  if (!wsrep_emulate_bin_log) DBUG_RETURN(0);
+  int rcode = wsrep_binlog_savepoint_set(thd, sv);
+  DBUG_RETURN(rcode);
+}
+
+static int wsrep_savepoint_rollback(handlerton *hton, THD *thd, void *sv)
+{
+  DBUG_ENTER("wsrep_savepoint_rollback");
+
+  if (thd->wsrep_exec_mode == REPL_RECV)
+  {
+    DBUG_RETURN(0);
+  }
+
+  if (!wsrep_emulate_bin_log) DBUG_RETURN(0);
+  int rcode = wsrep_binlog_savepoint_rollback(thd, sv);
+  DBUG_RETURN(rcode);
+}
+
+static int wsrep_rollback(handlerton *hton, THD *thd, bool all)
+{
+  DBUG_ENTER("wsrep_rollback");
+
+  if (thd->wsrep_exec_mode == REPL_RECV)
+  {
+    DBUG_RETURN(0);
+  }
+
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  switch (thd->wsrep_exec_mode)
+  {
+  case TOTAL_ORDER:
+  case REPL_RECV:
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+      WSREP_DEBUG("Avoiding wsrep rollback for failed DDL: %s", thd->query());
+      DBUG_RETURN(0);
+  default: break;
+  }
+
+  if ((all || !thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) &&
+      (thd->variables.wsrep_on && thd->wsrep_conflict_state != MUST_REPLAY))
+  {
+    if (wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle))
+    {
+      DBUG_PRINT("wsrep", ("setting rollback fail"));
+      WSREP_ERROR("settting rollback fail: thd: %llu SQL: %s",
+                  (long long)thd->real_id, thd->query());
+    }
+    wsrep_cleanup_transaction(thd);
+  }
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  DBUG_RETURN(0);
+}
+
+int wsrep_commit(handlerton *hton, THD *thd, bool all)
+{
+  DBUG_ENTER("wsrep_commit");
+
+  if (thd->wsrep_exec_mode == REPL_RECV)
+  {
+    DBUG_RETURN(0);
+  }
+
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  if ((all || !thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) &&
+      (thd->variables.wsrep_on && thd->wsrep_conflict_state != MUST_REPLAY))
+  {
+    if (thd->wsrep_exec_mode == LOCAL_COMMIT)
+    {
+      DBUG_ASSERT(thd->ha_data[wsrep_hton->slot].ha_info[all].is_trx_read_write());
+      /*
+        Call to wsrep->post_commit() (moved to wsrep_post_commit()) must
+        be done only after commit has done for all involved htons.
+      */
+      DBUG_PRINT("wsrep", ("commit"));
+    }
+    else
+    {
+      /*
+        Transaction didn't go through wsrep->pre_commit() so just roll back
+        possible changes to clean state.
+      */
+      if (WSREP_PROVIDER_EXISTS) {
+        if (wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle))
+        {
+          DBUG_PRINT("wsrep", ("setting rollback fail"));
+          WSREP_ERROR("settting rollback fail: thd: %llu SQL: %s",
+                      (long long)thd->real_id, thd->query());
+        }
+      }
+      wsrep_cleanup_transaction(thd);
+    }
+  }
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  DBUG_RETURN(0);
+}
+
+
+extern Rpl_filter* binlog_filter;
+extern my_bool opt_log_slave_updates;
+
+enum wsrep_trx_status
+wsrep_run_wsrep_commit(THD *thd, handlerton *hton, bool all)
+{
+  int rcode= -1;
+  size_t data_len= 0;
+  IO_CACHE *cache;
+  int replay_round= 0;
+
+  if (thd->get_stmt_da()->is_error()) {
+    WSREP_ERROR("commit issue, error: %d %s",
+                thd->get_stmt_da()->sql_errno(), thd->get_stmt_da()->message());
+  }
+
+  DBUG_ENTER("wsrep_run_wsrep_commit");
+
+  if (thd->slave_thread && !opt_log_slave_updates) DBUG_RETURN(WSREP_TRX_OK);
+
+  if (thd->wsrep_exec_mode == REPL_RECV) {
+
+    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+    if (thd->wsrep_conflict_state == MUST_ABORT) {
+      if (wsrep_debug)
+        WSREP_INFO("WSREP: must abort for BF");
+      DBUG_PRINT("wsrep", ("BF apply commit fail"));
+      thd->wsrep_conflict_state = NO_CONFLICT;
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+      //
+      // TODO: test all calls of the rollback.
+      // rollback must happen automagically innobase_rollback(hton, thd, 1);
+      //
+      DBUG_RETURN(WSREP_TRX_ERROR);
+    }
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  }
+
+  if (thd->wsrep_exec_mode != LOCAL_STATE) DBUG_RETURN(WSREP_TRX_OK);
+
+  if (thd->wsrep_consistency_check == CONSISTENCY_CHECK_RUNNING) {
+    WSREP_DEBUG("commit for consistency check: %s", thd->query());
+    DBUG_RETURN(WSREP_TRX_OK);
+  }
+
+  DBUG_PRINT("wsrep", ("replicating commit"));
+
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  if (thd->wsrep_conflict_state == MUST_ABORT) {
+    DBUG_PRINT("wsrep", ("replicate commit fail"));
+    thd->wsrep_conflict_state = ABORTED;
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    if (wsrep_debug) {
+      WSREP_INFO("innobase_commit, abort %s",
+                 (thd->query()) ? thd->query() : "void");
+    }
+    DBUG_RETURN(WSREP_TRX_CERT_FAIL);
+  }
+
+  mysql_mutex_lock(&LOCK_wsrep_replaying);
+
+  while (wsrep_replaying > 0                       &&
+         thd->wsrep_conflict_state == NO_CONFLICT  &&
+         thd->killed == NOT_KILLED            &&
+         !shutdown_in_progress)
+  {
+
+    mysql_mutex_unlock(&LOCK_wsrep_replaying);
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+    mysql_mutex_lock(&thd->mysys_var->mutex);
+    thd_proc_info(thd, "wsrep waiting on replaying");
+    thd->mysys_var->current_mutex= &LOCK_wsrep_replaying;
+    thd->mysys_var->current_cond=  &COND_wsrep_replaying;
+    mysql_mutex_unlock(&thd->mysys_var->mutex);
+
+    mysql_mutex_lock(&LOCK_wsrep_replaying);
+    // Using timedwait is a hack to avoid deadlock in case if BF victim
+    // misses the signal.
+    struct timespec wtime = {0, 1000000};
+    mysql_cond_timedwait(&COND_wsrep_replaying, &LOCK_wsrep_replaying,
+			 &wtime);
+
+    if (replay_round++ % 100000 == 0)
+      WSREP_DEBUG("commit waiting for replaying: replayers %d, thd: (%lu) "
+                  "conflict: %d (round: %d)",
+		  wsrep_replaying, thd->thread_id,
+                  thd->wsrep_conflict_state, replay_round);
+
+    mysql_mutex_unlock(&LOCK_wsrep_replaying);
+
+    mysql_mutex_lock(&thd->mysys_var->mutex);
+    thd->mysys_var->current_mutex= 0;
+    thd->mysys_var->current_cond=  0;
+    mysql_mutex_unlock(&thd->mysys_var->mutex);
+
+    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+    mysql_mutex_lock(&LOCK_wsrep_replaying);
+  }
+  mysql_mutex_unlock(&LOCK_wsrep_replaying);
+
+  if (thd->wsrep_conflict_state == MUST_ABORT) {
+    DBUG_PRINT("wsrep", ("replicate commit fail"));
+    thd->wsrep_conflict_state = ABORTED;
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    WSREP_DEBUG("innobase_commit abort after replaying wait %s",
+                (thd->query()) ? thd->query() : "void");
+    DBUG_RETURN(WSREP_TRX_CERT_FAIL);
+  }
+
+  thd->wsrep_query_state = QUERY_COMMITTING;
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+  cache = get_trans_log(thd);
+  rcode = 0;
+  if (cache) {
+    thd->binlog_flush_pending_rows_event(true);
+    rcode = wsrep_write_cache(wsrep, thd, cache, &data_len);
+    if (WSREP_OK != rcode) {
+      WSREP_ERROR("rbr write fail, data_len: %zu, %d", data_len, rcode);
+      DBUG_RETURN(WSREP_TRX_SIZE_EXCEEDED);
+    }
+  }
+
+  if (data_len == 0)
+  {
+    if (thd->get_stmt_da()->is_ok()              &&
+        thd->get_stmt_da()->affected_rows() > 0  &&
+        !binlog_filter->is_on())
+    {
+      WSREP_DEBUG("empty rbr buffer, query: %s, "
+                 "affected rows: %llu, "
+                 "changed tables: %d, "
+                 "sql_log_bin: %d, "
+                 "wsrep status (%d %d %d)",
+                 thd->query(), thd->get_stmt_da()->affected_rows(),
+                 stmt_has_updated_trans_table(thd), thd->variables.sql_log_bin,
+                 thd->wsrep_exec_mode, thd->wsrep_query_state,
+                 thd->wsrep_conflict_state);
+    }
+    else
+    {
+      WSREP_DEBUG("empty rbr buffer, query: %s", thd->query());
+    }
+    thd->wsrep_query_state= QUERY_EXEC;
+    DBUG_RETURN(WSREP_TRX_OK);
+  }
+
+  if (WSREP_UNDEFINED_TRX_ID == thd->wsrep_ws_handle.trx_id)
+  {
+    WSREP_WARN("SQL statement was ineffective, THD: %lu, buf: %zu\n"
+	       "QUERY: %s\n"
+	       " => Skipping replication",
+	       thd->thread_id, data_len, thd->query());
+    rcode = WSREP_TRX_FAIL;
+  }
+  else if (!rcode)
+  {
+    if (WSREP_OK == rcode)
+      rcode = wsrep->pre_commit(wsrep,
+                                (wsrep_conn_id_t)thd->thread_id,
+                                &thd->wsrep_ws_handle,
+                                WSREP_FLAG_COMMIT |
+                                ((thd->wsrep_PA_safe) ?
+                                 0ULL : WSREP_FLAG_PA_UNSAFE),
+                                &thd->wsrep_trx_meta);
+
+    if (rcode == WSREP_TRX_MISSING) {
+      WSREP_WARN("Transaction missing in provider, thd: %ld, SQL: %s",
+                 thd->thread_id, thd->query());
+      rcode = WSREP_TRX_FAIL;
+    } else if (rcode == WSREP_BF_ABORT) {
+      WSREP_DEBUG("thd %lu seqno %lld BF aborted by provider, will replay",
+                  thd->thread_id, (long long)thd->wsrep_trx_meta.gtid.seqno);
+      mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+      thd->wsrep_conflict_state = MUST_REPLAY;
+      DBUG_ASSERT(wsrep_thd_trx_seqno(thd) > 0);
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+      mysql_mutex_lock(&LOCK_wsrep_replaying);
+      wsrep_replaying++;
+      WSREP_DEBUG("replaying increased: %d, thd: %lu",
+                  wsrep_replaying, thd->thread_id);
+      mysql_mutex_unlock(&LOCK_wsrep_replaying);
+    }
+  } else {
+    WSREP_ERROR("I/O error reading from thd's binlog iocache: "
+                "errno=%d, io cache code=%d", my_errno, cache->error);
+    DBUG_ASSERT(0); // failure like this can not normally happen
+    DBUG_RETURN(WSREP_TRX_ERROR);
+  }
+
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  switch(rcode) {
+  case 0:
+    /*
+      About MUST_ABORT: We assume that even if thd conflict state was set
+      to MUST_ABORT, underlying transaction was not rolled back or marked
+      as deadlock victim in QUERY_COMMITTING state. Conflict state is
+      set to NO_CONFLICT and commit proceeds as usual.
+    */
+    if (thd->wsrep_conflict_state == MUST_ABORT)
+        thd->wsrep_conflict_state= NO_CONFLICT;
+
+    if (thd->wsrep_conflict_state != NO_CONFLICT)
+    {
+      WSREP_WARN("thd %lu seqno %lld: conflict state %d after post commit",
+                 thd->thread_id,
+                 (long long)thd->wsrep_trx_meta.gtid.seqno,
+                 thd->wsrep_conflict_state);
+    }
+    thd->wsrep_exec_mode= LOCAL_COMMIT;
+    DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED);
+    /* Override XID iff it was generated by mysql */
+    if (thd->transaction.xid_state.xid.get_my_xid())
+    {
+      wsrep_xid_init(&thd->transaction.xid_state.xid,
+                     &thd->wsrep_trx_meta.gtid.uuid,
+                     thd->wsrep_trx_meta.gtid.seqno);
+    }
+    DBUG_PRINT("wsrep", ("replicating commit success"));
+    break;
+  case WSREP_BF_ABORT:
+    DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED);
+  case WSREP_TRX_FAIL:
+    WSREP_DEBUG("commit failed for reason: %d", rcode);
+    DBUG_PRINT("wsrep", ("replicating commit fail"));
+
+    thd->wsrep_query_state= QUERY_EXEC;
+
+    if (thd->wsrep_conflict_state == MUST_ABORT) {
+      thd->wsrep_conflict_state= ABORTED;
+    }
+    else
+    {
+      WSREP_DEBUG("conflict state: %d", thd->wsrep_conflict_state);
+      if (thd->wsrep_conflict_state == NO_CONFLICT)
+      {
+        thd->wsrep_conflict_state = CERT_FAILURE;
+        WSREP_LOG_CONFLICT(NULL, thd, FALSE);
+      }
+    }
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+    DBUG_RETURN(WSREP_TRX_CERT_FAIL);
+
+  case WSREP_SIZE_EXCEEDED:
+    WSREP_ERROR("transaction size exceeded");
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    DBUG_RETURN(WSREP_TRX_SIZE_EXCEEDED);
+  case WSREP_CONN_FAIL:
+    WSREP_ERROR("connection failure");
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    DBUG_RETURN(WSREP_TRX_ERROR);
+  default:
+    WSREP_ERROR("unknown connection failure");
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    DBUG_RETURN(WSREP_TRX_ERROR);
+  }
+
+  thd->wsrep_query_state= QUERY_EXEC;
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+  DBUG_RETURN(WSREP_TRX_OK);
+}
+
+
+static int wsrep_hton_init(void *p)
+{
+  wsrep_hton= (handlerton *)p;
+  //wsrep_hton->state=opt_bin_log ? SHOW_OPTION_YES : SHOW_OPTION_NO;
+  wsrep_hton->state= SHOW_OPTION_YES;
+  wsrep_hton->db_type=(legacy_db_type)0;
+  wsrep_hton->savepoint_offset= sizeof(my_off_t);
+  wsrep_hton->close_connection= wsrep_close_connection;
+  wsrep_hton->savepoint_set= wsrep_savepoint_set;
+  wsrep_hton->savepoint_rollback= wsrep_savepoint_rollback;
+  wsrep_hton->commit= wsrep_commit;
+  wsrep_hton->rollback= wsrep_rollback;
+  wsrep_hton->prepare= wsrep_prepare;
+  wsrep_hton->flags= HTON_NOT_USER_SELECTABLE | HTON_HIDDEN; // todo: fix flags
+  wsrep_hton->slot= 0;
+  return 0;
+}
+
+
+struct st_mysql_storage_engine wsrep_storage_engine=
+{ MYSQL_HANDLERTON_INTERFACE_VERSION };
+
+
+mysql_declare_plugin(wsrep)
+{
+  MYSQL_STORAGE_ENGINE_PLUGIN,
+  &wsrep_storage_engine,
+  "wsrep",
+  "Codership Oy",
+  "A pseudo storage engine to represent transactions in multi-master "
+  "synchornous replication",
+  PLUGIN_LICENSE_GPL,
+  wsrep_hton_init, /* Plugin Init */
+  NULL, /* Plugin Deinit */
+  0x0100 /* 1.0 */,
+  NULL,                       /* status variables                */
+  NULL,                       /* system variables                */
+  NULL,                       /* config options                  */
+  0,                          /* flags                           */
+}
+mysql_declare_plugin_end;
diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
new file mode 100644
index 00000000000..c1b15bd02c2
--- /dev/null
+++ b/sql/wsrep_mysqld.cc
@@ -0,0 +1,2433 @@
+/* Copyright 2008-2014 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include <mysqld.h>
+#include <sql_class.h>
+#include <sql_parse.h>
+#include "slave.h"
+#include "rpl_mi.h"
+#include "sql_repl.h"
+#include "rpl_filter.h"
+#include "sql_callback.h"
+#include "sp_head.h"
+#include "sp.h"
+#include "wsrep_priv.h"
+#include "wsrep_thd.h"
+#include "wsrep_sst.h"
+#include "wsrep_utils.h"
+#include "wsrep_var.h"
+#include "wsrep_binlog.h"
+#include "wsrep_applier.h"
+#include <cstdio>
+#include <cstdlib>
+#include "log_event.h"
+#include <slave.h>
+
+wsrep_t *wsrep                  = NULL;
+my_bool wsrep_emulate_bin_log   = FALSE; // activating parts of binlog interface
+#ifdef GTID_SUPPORT
+/* Sidno in global_sid_map corresponding to group uuid */
+rpl_sidno wsrep_sidno= -1;
+#endif /* GTID_SUPPORT */
+my_bool wsrep_preordered_opt= FALSE;
+
+/*
+ * Begin configuration options and their default values
+ */
+
+extern int wsrep_replaying;
+extern ulong wsrep_running_threads;
+extern ulong my_bind_addr;
+extern my_bool plugins_are_initialized;
+extern uint kill_cached_threads;
+extern mysql_cond_t COND_thread_cache;
+
+const char* wsrep_data_home_dir = NULL;
+const char* wsrep_dbug_option   = "";
+
+long    wsrep_slave_threads            = 1; // # of slave action appliers wanted
+int     wsrep_slave_count_change       = 0; // # of appliers to stop or start
+my_bool wsrep_debug                    = 0; // enable debug level logging
+my_bool wsrep_convert_LOCK_to_trx      = 1; // convert locking sessions to trx
+ulong   wsrep_retry_autocommit         = 5; // retry aborted autocommit trx
+my_bool wsrep_auto_increment_control   = 1; // control auto increment variables
+my_bool wsrep_drupal_282555_workaround = 1; // retry autoinc insert after dupkey
+my_bool wsrep_incremental_data_collection = 0; // incremental data collection
+ulong   wsrep_max_ws_size              = 1073741824UL;//max ws (RBR buffer) size
+ulong   wsrep_max_ws_rows              = 65536; // max number of rows in ws
+int     wsrep_to_isolation             = 0; // # of active TO isolation threads
+my_bool wsrep_certify_nonPK            = 1; // certify, even when no primary key
+long    wsrep_max_protocol_version     = 3; // maximum protocol version to use
+ulong   wsrep_forced_binlog_format     = BINLOG_FORMAT_UNSPEC;
+my_bool wsrep_recovery                 = 0; // recovery
+my_bool wsrep_replicate_myisam         = 0; // enable myisam replication
+my_bool wsrep_log_conflicts            = 0;
+ulong   wsrep_mysql_replication_bundle = 0;
+my_bool wsrep_desync                   = 0; // desynchronize the node from the
+                                            // cluster
+my_bool wsrep_load_data_splitting      = 1; // commit load data every 10K intervals
+my_bool wsrep_restart_slave            = 0; // should mysql slave thread be
+                                            // restarted, if node joins back
+my_bool wsrep_restart_slave_activated  = 0; // node has dropped, and slave
+                                            // restart will be needed
+my_bool wsrep_slave_UK_checks          = 0; // slave thread does UK checks
+my_bool wsrep_slave_FK_checks          = 0; // slave thread does FK checks
+/*
+ * End configuration options
+ */
+
+/*
+ * Other wsrep global variables.
+ */
+my_bool wsrep_inited                   = 0; // initialized ?
+
+static const wsrep_uuid_t cluster_uuid = WSREP_UUID_UNDEFINED;
+static char         cluster_uuid_str[40]= { 0, };
+static const char*  cluster_status_str[WSREP_VIEW_MAX] =
+{
+    "Primary",
+    "non-Primary",
+    "Disconnected"
+};
+
+static char provider_name[256]= { 0, };
+static char provider_version[256]= { 0, };
+static char provider_vendor[256]= { 0, };
+
+/*
+ * wsrep status variables
+ */
+my_bool     wsrep_connected          = FALSE;
+my_bool     wsrep_ready              = FALSE; // node can accept queries
+const char* wsrep_cluster_state_uuid = cluster_uuid_str;
+long long   wsrep_cluster_conf_id    = WSREP_SEQNO_UNDEFINED;
+const char* wsrep_cluster_status = cluster_status_str[WSREP_VIEW_DISCONNECTED];
+long        wsrep_cluster_size       = 0;
+long        wsrep_local_index        = -1;
+long long   wsrep_local_bf_aborts    = 0;
+const char* wsrep_provider_name      = provider_name;
+const char* wsrep_provider_version   = provider_version;
+const char* wsrep_provider_vendor    = provider_vendor;
+/* End wsrep status variables */
+
+wsrep_uuid_t     local_uuid   = WSREP_UUID_UNDEFINED;
+wsrep_seqno_t    local_seqno  = WSREP_SEQNO_UNDEFINED;
+wsp::node_status local_status;
+long             wsrep_protocol_version = 3;
+
+// Boolean denoting if server is in initial startup phase. This is needed
+// to make sure that main thread waiting in wsrep_sst_wait() is signaled
+// if there was no state gap on receiving first view event.
+static my_bool   wsrep_startup = TRUE;
+
+
+static void wsrep_log_cb(wsrep_log_level_t level, const char *msg) {
+  switch (level) {
+  case WSREP_LOG_INFO:
+    sql_print_information("WSREP: %s", msg);
+    break;
+  case WSREP_LOG_WARN:
+    sql_print_warning("WSREP: %s", msg);
+    break;
+  case WSREP_LOG_ERROR:
+  case WSREP_LOG_FATAL:
+    sql_print_error("WSREP: %s", msg);
+    break;
+  case WSREP_LOG_DEBUG:
+    if (wsrep_debug) sql_print_information ("[Debug] WSREP: %s", msg);
+  default:
+    break;
+  }
+}
+
+static void wsrep_log_states (wsrep_log_level_t   const level,
+                              const wsrep_uuid_t* const group_uuid,
+                              wsrep_seqno_t       const group_seqno,
+                              const wsrep_uuid_t* const node_uuid,
+                              wsrep_seqno_t       const node_seqno)
+{
+  char uuid_str[37];
+  char msg[256];
+
+  wsrep_uuid_print (group_uuid, uuid_str, sizeof(uuid_str));
+  snprintf (msg, 255, "WSREP: Group state: %s:%lld",
+            uuid_str, (long long)group_seqno);
+  wsrep_log_cb (level, msg);
+
+  wsrep_uuid_print (node_uuid, uuid_str, sizeof(uuid_str));
+  snprintf (msg, 255, "WSREP: Local state: %s:%lld",
+            uuid_str, (long long)node_seqno);
+  wsrep_log_cb (level, msg);
+}
+
+static my_bool set_SE_checkpoint(THD* unused, plugin_ref plugin, void* arg)
+{
+  XID* xid= reinterpret_cast<XID*>(arg);
+  handlerton* hton= plugin_data(plugin, handlerton *);
+  if (hton->set_checkpoint)
+  {
+    const wsrep_uuid_t* uuid(wsrep_xid_uuid(xid));
+    char uuid_str[40] = {0, };
+    wsrep_uuid_print(uuid, uuid_str, sizeof(uuid_str));
+    WSREP_DEBUG("Set WSREPXid for InnoDB:  %s:%lld",
+                uuid_str, (long long)wsrep_xid_seqno(xid));
+    hton->set_checkpoint(hton, xid);
+  }
+  return FALSE;
+}
+
+void wsrep_set_SE_checkpoint(XID* xid)
+{
+  plugin_foreach(NULL, set_SE_checkpoint, MYSQL_STORAGE_ENGINE_PLUGIN, xid);
+}
+
+static my_bool get_SE_checkpoint(THD* unused, plugin_ref plugin, void* arg)
+{
+  XID* xid= reinterpret_cast<XID*>(arg);
+  handlerton* hton= plugin_data(plugin, handlerton *);
+  if (hton->get_checkpoint)
+  {
+    hton->get_checkpoint(hton, xid);
+    const wsrep_uuid_t* uuid(wsrep_xid_uuid(xid));
+    char uuid_str[40] = {0, };
+    wsrep_uuid_print(uuid, uuid_str, sizeof(uuid_str));
+    WSREP_DEBUG("Read WSREPXid from InnoDB:  %s:%lld",
+                uuid_str, (long long)wsrep_xid_seqno(xid));
+  }
+  return FALSE;
+}
+
+void wsrep_get_SE_checkpoint(XID* xid)
+{
+  plugin_foreach(NULL, get_SE_checkpoint, MYSQL_STORAGE_ENGINE_PLUGIN, xid);
+}
+
+#ifdef GTID_SUPPORT
+void wsrep_init_sidno(const wsrep_uuid_t& uuid)
+{
+  /* generate new Sid map entry from inverted uuid */
+  rpl_sid sid;
+  wsrep_uuid_t ltid_uuid;
+  for (size_t i= 0; i < sizeof(ltid_uuid.data); ++i)
+  {
+      ltid_uuid.data[i] = ~local_uuid.data[i];
+  }
+  sid.copy_from(ltid_uuid.data);
+  global_sid_lock->wrlock();
+  wsrep_sidno= global_sid_map->add_sid(sid);
+  WSREP_INFO("inited wsrep sidno %d", wsrep_sidno);
+  global_sid_lock->unlock();
+}
+#endif /* GTID_SUPPORT */
+
+static wsrep_cb_status_t
+wsrep_view_handler_cb (void*                    app_ctx,
+                       void*                    recv_ctx,
+                       const wsrep_view_info_t* view,
+                       const char*              state,
+                       size_t                   state_len,
+                       void**                   sst_req,
+                       size_t*                  sst_req_len)
+{
+  *sst_req     = NULL;
+  *sst_req_len = 0;
+
+  wsrep_member_status_t new_status= local_status.get();
+
+  if (memcmp(&cluster_uuid, &view->state_id.uuid, sizeof(wsrep_uuid_t)))
+  {
+    memcpy((wsrep_uuid_t*)&cluster_uuid, &view->state_id.uuid,
+           sizeof(cluster_uuid));
+
+    wsrep_uuid_print (&cluster_uuid, cluster_uuid_str,
+                      sizeof(cluster_uuid_str));
+  }
+
+  wsrep_cluster_conf_id= view->view;
+  wsrep_cluster_status= cluster_status_str[view->status];
+  wsrep_cluster_size= view->memb_num;
+  wsrep_local_index= view->my_idx;
+
+  WSREP_INFO("New cluster view: global state: %s:%lld, view# %lld: %s, "
+             "number of nodes: %ld, my index: %ld, protocol version %d",
+             wsrep_cluster_state_uuid, (long long)view->state_id.seqno,
+             (long long)wsrep_cluster_conf_id, wsrep_cluster_status,
+             wsrep_cluster_size, wsrep_local_index, view->proto_ver);
+
+  /* Proceed further only if view is PRIMARY */
+  if (WSREP_VIEW_PRIMARY != view->status) {
+    wsrep_ready_set(FALSE);
+    new_status= WSREP_MEMBER_UNDEFINED;
+    /* Always record local_uuid and local_seqno in non-prim since this
+     * may lead to re-initializing provider and start position is
+     * determined according to these variables */
+    // WRONG! local_uuid should be the last primary configuration uuid we were
+    // a member of. local_seqno should be updated in commit calls.
+    // local_uuid= cluster_uuid;
+    // local_seqno= view->first - 1;
+    goto out;
+  }
+
+  switch (view->proto_ver)
+  {
+  case 0:
+  case 1:
+  case 2:
+  case 3:
+      // version change
+      if (view->proto_ver != wsrep_protocol_version)
+      {
+          my_bool wsrep_ready_saved= wsrep_ready;
+          wsrep_ready_set(FALSE);
+          WSREP_INFO("closing client connections for "
+                     "protocol change %ld -> %d",
+                     wsrep_protocol_version, view->proto_ver);
+          wsrep_close_client_connections(TRUE);
+          wsrep_protocol_version= view->proto_ver;
+          wsrep_ready_set(wsrep_ready_saved);
+      }
+      break;
+  default:
+      WSREP_ERROR("Unsupported application protocol version: %d",
+                  view->proto_ver);
+      unireg_abort(1);
+  }
+
+  if (view->state_gap)
+  {
+    WSREP_WARN("Gap in state sequence. Need state transfer.");
+
+    /* After that wsrep will call wsrep_sst_prepare. */
+    /* keep ready flag 0 until we receive the snapshot */
+    wsrep_ready_set(FALSE);
+
+    /* Close client connections to ensure that they don't interfere
+     * with SST */
+    WSREP_DEBUG("[debug]: closing client connections for PRIM");
+    wsrep_close_client_connections(TRUE);
+
+    ssize_t const req_len= wsrep_sst_prepare (sst_req);
+
+    if (req_len < 0)
+    {
+      WSREP_ERROR("SST preparation failed: %zd (%s)", -req_len,
+                  strerror(-req_len));
+      new_status= WSREP_MEMBER_UNDEFINED;
+    }
+    else
+    {
+      assert(sst_req != NULL);
+      *sst_req_len= req_len;
+      new_status= WSREP_MEMBER_JOINER;
+    }
+  }
+  else
+  {
+    /*
+     *  NOTE: Initialize wsrep_group_uuid here only if it wasn't initialized
+     *  before - OR - it was reinitilized on startup (lp:992840)
+     */
+    if (wsrep_startup)
+    {
+      if (wsrep_before_SE())
+      {
+        wsrep_SE_init_grab();
+        // Signal mysqld init thread to continue
+        wsrep_sst_complete (&cluster_uuid, view->state_id.seqno, false);
+        // and wait for SE initialization
+        wsrep_SE_init_wait();
+      }
+      else
+      {
+        local_uuid=  cluster_uuid;
+        local_seqno= view->state_id.seqno;
+      }
+      /* Init storage engine XIDs from first view */
+      XID xid;
+      wsrep_xid_init(&xid, &local_uuid, local_seqno);
+      wsrep_set_SE_checkpoint(&xid);
+      new_status= WSREP_MEMBER_JOINED;
+#ifdef GTID_SUPPORT
+      wsrep_init_sidno(local_uuid);
+#endif /* GTID_SUPPORT */
+    }
+
+    // just some sanity check
+    if (memcmp (&local_uuid, &cluster_uuid, sizeof (wsrep_uuid_t)))
+    {
+      WSREP_ERROR("Undetected state gap. Can't continue.");
+      wsrep_log_states(WSREP_LOG_FATAL, &cluster_uuid, view->state_id.seqno,
+                       &local_uuid, -1);
+      unireg_abort(1);
+    }
+  }
+
+  if (wsrep_auto_increment_control)
+  {
+    global_system_variables.auto_increment_offset= view->my_idx + 1;
+    global_system_variables.auto_increment_increment= view->memb_num;
+  }
+
+  { /* capabilities may be updated on new configuration */
+    uint64_t const caps(wsrep->capabilities (wsrep));
+
+    my_bool const idc((caps & WSREP_CAP_INCREMENTAL_WRITESET) != 0);
+    if (TRUE == wsrep_incremental_data_collection && FALSE == idc)
+    {
+      WSREP_WARN("Unsupported protocol downgrade: "
+                 "incremental data collection disabled. Expect abort.");
+    }
+    wsrep_incremental_data_collection = idc;
+  }
+
+out:
+  if (view->status == WSREP_VIEW_PRIMARY) wsrep_startup= FALSE;
+  local_status.set(new_status, view);
+
+  return WSREP_CB_SUCCESS;
+}
+
+void wsrep_ready_set (my_bool x)
+{
+  WSREP_DEBUG("Setting wsrep_ready to %d", x);
+  if (mysql_mutex_lock (&LOCK_wsrep_ready)) abort();
+  if (wsrep_ready != x)
+  {
+    wsrep_ready= x;
+    mysql_cond_signal (&COND_wsrep_ready);
+  }
+  mysql_mutex_unlock (&LOCK_wsrep_ready);
+}
+
+// Wait until wsrep has reached ready state
+void wsrep_ready_wait ()
+{
+  if (mysql_mutex_lock (&LOCK_wsrep_ready)) abort();
+  while (!wsrep_ready)
+  {
+    WSREP_INFO("Waiting to reach ready state");
+    mysql_cond_wait (&COND_wsrep_ready, &LOCK_wsrep_ready);
+  }
+  WSREP_INFO("ready state reached");
+  mysql_mutex_unlock (&LOCK_wsrep_ready);
+}
+
+static void wsrep_synced_cb(void* app_ctx)
+{
+  WSREP_INFO("Synchronized with group, ready for connections");
+  bool signal_main= false;
+  if (mysql_mutex_lock (&LOCK_wsrep_ready)) abort();
+  if (!wsrep_ready)
+  {
+    wsrep_ready= TRUE;
+    mysql_cond_signal (&COND_wsrep_ready);
+    signal_main= true;
+
+  }
+  local_status.set(WSREP_MEMBER_SYNCED);
+  mysql_mutex_unlock (&LOCK_wsrep_ready);
+
+  if (signal_main)
+  {
+      wsrep_SE_init_grab();
+      // Signal mysqld init thread to continue
+      wsrep_sst_complete (&local_uuid, local_seqno, false);
+      // and wait for SE initialization
+      wsrep_SE_init_wait();
+  }
+  if (wsrep_restart_slave_activated)
+  {
+    int rcode;
+    WSREP_INFO("MySQL slave restart");
+    wsrep_restart_slave_activated= FALSE;
+
+    mysql_mutex_lock(&LOCK_active_mi);
+    if ((rcode = start_slave_threads(1 /* need mutex */,
+                            0 /* no wait for start*/,
+                            active_mi,
+                            master_info_file,
+                            relay_log_info_file,
+                            SLAVE_SQL)))
+    {
+      WSREP_WARN("Failed to create slave threads: %d", rcode);
+    }
+    mysql_mutex_unlock(&LOCK_active_mi);
+
+  }
+}
+
+static void wsrep_init_position()
+{
+  /* read XIDs from storage engines */
+  XID xid;
+  memset(&xid, 0, sizeof(xid));
+  xid.formatID= -1;
+  wsrep_get_SE_checkpoint(&xid);
+
+  if (xid.formatID == -1)
+  {
+    WSREP_INFO("Read nil XID from storage engines, skipping position init");
+    return;
+  }
+  else if (!wsrep_is_wsrep_xid(&xid))
+  {
+    WSREP_WARN("Read non-wsrep XID from storage engines, skipping position init");
+    return;
+  }
+
+  const wsrep_uuid_t* uuid= wsrep_xid_uuid(&xid);
+  const wsrep_seqno_t seqno= wsrep_xid_seqno(&xid);
+
+  char uuid_str[40] = {0, };
+  wsrep_uuid_print(uuid, uuid_str, sizeof(uuid_str));
+  WSREP_INFO("Initial position: %s:%lld", uuid_str, (long long)seqno);
+
+
+  if (!memcmp(&local_uuid, &WSREP_UUID_UNDEFINED, sizeof(local_uuid)) &&
+      local_seqno == WSREP_SEQNO_UNDEFINED)
+  {
+    // Initial state
+    local_uuid= *uuid;
+    local_seqno= seqno;
+  }
+  else if (memcmp(&local_uuid, uuid, sizeof(local_uuid)) ||
+           local_seqno != seqno)
+  {
+    WSREP_WARN("Initial position was provided by configuration or SST, "
+               "avoiding override");
+  }
+}
+
+extern char* my_bind_addr_str;
+
+int wsrep_init()
+{
+  int rcode= -1;
+  DBUG_ASSERT(wsrep_inited == 0);
+
+  wsrep_ready_set(FALSE);
+  assert(wsrep_provider);
+
+  wsrep_init_position();
+
+  if ((rcode= wsrep_load(wsrep_provider, &wsrep, wsrep_log_cb)) != WSREP_OK)
+  {
+    if (strcasecmp(wsrep_provider, WSREP_NONE))
+    {
+      WSREP_ERROR("wsrep_load(%s) failed: %s (%d). Reverting to no provider.",
+                  wsrep_provider, strerror(rcode), rcode);
+      strcpy((char*)wsrep_provider, WSREP_NONE); // damn it's a dirty hack
+      (void) wsrep_init();
+      return rcode;
+    }
+    else /* this is for recursive call above */
+    {
+      WSREP_ERROR("Could not revert to no provider: %s (%d). Need to abort.",
+                  strerror(rcode), rcode);
+      unireg_abort(1);
+    }
+  }
+
+  if (!WSREP_PROVIDER_EXISTS)
+  {
+    // enable normal operation in case no provider is specified
+    wsrep_ready_set(TRUE);
+    wsrep_inited= 1;
+    global_system_variables.wsrep_on = 0;
+    wsrep_init_args args;
+    args.logger_cb = wsrep_log_cb;
+    args.options = (wsrep_provider_options) ?
+            wsrep_provider_options : "";
+    rcode = wsrep->init(wsrep, &args);
+    if (rcode)
+    {
+      DBUG_PRINT("wsrep",("wsrep::init() failed: %d", rcode));
+      WSREP_ERROR("wsrep::init() failed: %d, must shutdown", rcode);
+      wsrep->free(wsrep);
+      free(wsrep);
+      wsrep = NULL;
+    }
+    return rcode;
+  }
+  else
+  {
+    global_system_variables.wsrep_on = 1;
+    strncpy(provider_name,
+            wsrep->provider_name,    sizeof(provider_name) - 1);
+    strncpy(provider_version,
+            wsrep->provider_version, sizeof(provider_version) - 1);
+    strncpy(provider_vendor,
+            wsrep->provider_vendor,  sizeof(provider_vendor) - 1);
+  }
+
+  if (!wsrep_data_home_dir || strlen(wsrep_data_home_dir) == 0)
+    wsrep_data_home_dir = mysql_real_data_home;
+
+  char node_addr[512]= { 0, };
+  size_t const node_addr_max= sizeof(node_addr) - 1;
+  if (!wsrep_node_address || !strcmp(wsrep_node_address, ""))
+  {
+    size_t const ret= wsrep_guess_ip(node_addr, node_addr_max);
+    if (!(ret > 0 && ret < node_addr_max))
+    {
+      WSREP_WARN("Failed to guess base node address. Set it explicitly via "
+                 "wsrep_node_address.");
+      node_addr[0]= '\0';
+    }
+  }
+  else
+  {
+    strncpy(node_addr, wsrep_node_address, node_addr_max);
+  }
+
+  char inc_addr[512]= { 0, };
+  size_t const inc_addr_max= sizeof (inc_addr);
+  if ((!wsrep_node_incoming_address ||
+       !strcmp (wsrep_node_incoming_address, WSREP_NODE_INCOMING_AUTO)))
+  {
+    unsigned int my_bind_ip= INADDR_ANY; // default if not set
+    if (my_bind_addr_str && strlen(my_bind_addr_str))
+    {
+      my_bind_ip= wsrep_check_ip(my_bind_addr_str);
+    }
+
+    if (INADDR_ANY != my_bind_ip)
+    {
+      if (INADDR_NONE != my_bind_ip && INADDR_LOOPBACK != my_bind_ip)
+      {
+        snprintf(inc_addr, inc_addr_max, "%s:%u",
+                 my_bind_addr_str, (int)mysqld_port);
+      } // else leave inc_addr an empty string - mysqld is not listening for
+        // client connections on network interfaces.
+    }
+    else // mysqld binds to 0.0.0.0, take IP from wsrep_node_address if possible
+    {
+      size_t const node_addr_len= strlen(node_addr);
+      if (node_addr_len > 0)
+      {
+        const char* const colon= strrchr(node_addr, ':');
+        if (strchr(node_addr, ':') == colon) // 1 or 0 ':'
+        {
+          size_t const ip_len= colon ? colon - node_addr : node_addr_len;
+          if (ip_len + 7 /* :55555\0 */ < inc_addr_max)
+          {
+            memcpy (inc_addr, node_addr, ip_len);
+            snprintf(inc_addr + ip_len, inc_addr_max - ip_len, ":%u",
+                     (int)mysqld_port);
+          }
+          else
+          {
+            WSREP_WARN("Guessing address for incoming client connections: "
+                       "address too long.");
+            inc_addr[0]= '\0';
+          }
+        }
+        else
+        {
+          WSREP_WARN("Guessing address for incoming client connections: "
+                     "too many colons :) .");
+          inc_addr[0]= '\0';
+        }
+      }
+
+      if (!strlen(inc_addr))
+      {
+          WSREP_WARN("Guessing address for incoming client connections failed. "
+                     "Try setting wsrep_node_incoming_address explicitly.");
+      }
+    }
+  }
+  else if (!strchr(wsrep_node_incoming_address, ':')) // no port included
+  {
+    if ((int)inc_addr_max <=
+        snprintf(inc_addr, inc_addr_max, "%s:%u",
+                 wsrep_node_incoming_address,(int)mysqld_port))
+    {
+      WSREP_WARN("Guessing address for incoming client connections: "
+                 "address too long.");
+      inc_addr[0]= '\0';
+    }
+  }
+  else
+  {
+    size_t const need = strlen (wsrep_node_incoming_address);
+    if (need >= inc_addr_max) {
+      WSREP_WARN("wsrep_node_incoming_address too long: %zu", need);
+      inc_addr[0]= '\0';
+    }
+    else {
+      memcpy (inc_addr, wsrep_node_incoming_address, need);
+    }
+  }
+
+  struct wsrep_init_args wsrep_args;
+
+  struct wsrep_gtid const state_id = { local_uuid, local_seqno };
+
+  wsrep_args.data_dir        = wsrep_data_home_dir;
+  wsrep_args.node_name       = (wsrep_node_name) ? wsrep_node_name : "";
+  wsrep_args.node_address    = node_addr;
+  wsrep_args.node_incoming   = inc_addr;
+  wsrep_args.options         = (wsrep_provider_options) ?
+                                wsrep_provider_options : "";
+  wsrep_args.proto_ver       = wsrep_max_protocol_version;
+
+  wsrep_args.state_id        = &state_id;
+
+  wsrep_args.logger_cb       = wsrep_log_cb;
+  wsrep_args.view_handler_cb = wsrep_view_handler_cb;
+  wsrep_args.apply_cb        = wsrep_apply_cb;
+  wsrep_args.commit_cb       = wsrep_commit_cb;
+  wsrep_args.unordered_cb    = wsrep_unordered_cb;
+  wsrep_args.sst_donate_cb   = wsrep_sst_donate_cb;
+  wsrep_args.synced_cb       = wsrep_synced_cb;
+
+  rcode = wsrep->init(wsrep, &wsrep_args);
+
+  if (rcode)
+  {
+    DBUG_PRINT("wsrep",("wsrep::init() failed: %d", rcode));
+    WSREP_ERROR("wsrep::init() failed: %d, must shutdown", rcode);
+    wsrep->free(wsrep);
+    free(wsrep);
+    wsrep = NULL;
+  } else {
+    wsrep_inited= 1;
+  }
+
+  return rcode;
+}
+
+extern int wsrep_on(void *);
+
+void wsrep_init_startup (bool first)
+{
+  if (wsrep_init()) unireg_abort(1);
+
+  wsrep_thr_lock_init(wsrep_thd_is_BF, wsrep_abort_thd,
+                      wsrep_debug, wsrep_convert_LOCK_to_trx, wsrep_on);
+
+  /* Skip replication start if no cluster address */
+  if (!wsrep_cluster_address || strlen(wsrep_cluster_address) == 0) return;
+
+  if (first) wsrep_sst_grab(); // do it so we can wait for SST below
+
+  if (!wsrep_start_replication()) unireg_abort(1);
+
+  wsrep_create_rollbacker();
+  wsrep_create_appliers(1);
+
+  if (first && !wsrep_sst_wait()) unireg_abort(1);// wait until SST is completed
+}
+
+
+void wsrep_deinit(bool free_options)
+{
+  DBUG_ASSERT(wsrep_inited == 1);
+  wsrep_unload(wsrep);
+  wsrep= 0;
+  provider_name[0]=    '\0';
+  provider_version[0]= '\0';
+  provider_vendor[0]=  '\0';
+  wsrep_inited= 0;
+
+  if (free_options)
+  {
+    wsrep_sst_auth_free();
+  }
+}
+
+void wsrep_recover()
+{
+  if (!memcmp(&local_uuid, &WSREP_UUID_UNDEFINED, sizeof(wsrep_uuid_t)) &&
+      local_seqno == -2)
+  {
+    char uuid_str[40];
+    wsrep_uuid_print(&local_uuid, uuid_str, sizeof(uuid_str));
+    WSREP_INFO("Position %s:%lld given at startup, skipping position recovery",
+               uuid_str, (long long)local_seqno);
+    return;
+  }
+  XID xid;
+  memset(&xid, 0, sizeof(xid));
+  xid.formatID= -1;
+  wsrep_get_SE_checkpoint(&xid);
+  char uuid_str[40];
+  wsrep_uuid_print(wsrep_xid_uuid(&xid), uuid_str, sizeof(uuid_str));
+  WSREP_INFO("Recovered position: %s:%lld", uuid_str,
+             (long long)wsrep_xid_seqno(&xid));
+}
+
+
+void wsrep_stop_replication(THD *thd)
+{
+  WSREP_INFO("Stop replication");
+  if (!wsrep)
+  {
+    WSREP_INFO("Provider was not loaded, in stop replication");
+    return;
+  }
+
+  /* disconnect from group first to get wsrep_ready == FALSE */
+  WSREP_DEBUG("Provider disconnect");
+  wsrep->disconnect(wsrep);
+
+  wsrep_connected= FALSE;
+
+  wsrep_close_client_connections(TRUE);
+
+  /* wait until appliers have stopped */
+  wsrep_wait_appliers_close(thd);
+
+  return;
+}
+
+/* This one is set to true when --wsrep-new-cluster is found in the command
+ * line arguments */
+static my_bool wsrep_new_cluster= FALSE;
+#define WSREP_NEW_CLUSTER "--wsrep-new-cluster"
+/* Finds and hides --wsrep-new-cluster from the arguments list
+ * by moving it to the end of the list and decrementing argument count */
+void wsrep_filter_new_cluster (int* argc, char* argv[])
+{
+  int i;
+  for (i= *argc - 1; i > 0; i--)
+  {
+    /* make a copy of the argument to convert possible underscores to hyphens.
+     * the copy need not to be longer than WSREP_NEW_CLUSTER option */
+    char arg[sizeof(WSREP_NEW_CLUSTER) + 1]= { 0, };
+    strncpy(arg, argv[i], sizeof(arg) - 1);
+    char* underscore(arg);
+    while (NULL != (underscore= strchr(underscore, '_'))) *underscore= '-';
+
+    if (!strcmp(arg, WSREP_NEW_CLUSTER))
+    {
+      wsrep_new_cluster= TRUE;
+      *argc -= 1;
+      /* preserve the order of remaining arguments AND
+       * preserve the original argument pointers - just in case */
+      char* wnc= argv[i];
+      memmove(&argv[i], &argv[i + 1], (*argc - i)*sizeof(argv[i]));
+      argv[*argc]= wnc; /* this will be invisible to the rest of the program */
+    }
+  }
+}
+
+bool wsrep_start_replication()
+{
+  wsrep_status_t rcode;
+
+  /*
+    if provider is trivial, don't even try to connect,
+    but resume local node operation
+  */
+  if (!WSREP_PROVIDER_EXISTS)
+  {
+    // enable normal operation in case no provider is specified
+    wsrep_ready_set(TRUE);
+    return true;
+  }
+
+  if (!wsrep_cluster_address || strlen(wsrep_cluster_address)== 0)
+  {
+    // if provider is non-trivial, but no address is specified, wait for address
+    wsrep_ready_set(FALSE);
+    return true;
+  }
+
+  bool const bootstrap(TRUE == wsrep_new_cluster);
+  wsrep_new_cluster= FALSE;
+
+  WSREP_INFO("Start replication");
+
+  if ((rcode = wsrep->connect(wsrep,
+                              wsrep_cluster_name,
+                              wsrep_cluster_address,
+                              wsrep_sst_donor,
+                              bootstrap)))
+  {
+    if (-ESOCKTNOSUPPORT == rcode)
+    {
+      DBUG_PRINT("wsrep",("unrecognized cluster address: '%s', rcode: %d",
+                          wsrep_cluster_address, rcode));
+      WSREP_ERROR("unrecognized cluster address: '%s', rcode: %d",
+                  wsrep_cluster_address, rcode);
+    }
+    else
+    {
+      DBUG_PRINT("wsrep",("wsrep->connect() failed: %d", rcode));
+      WSREP_ERROR("wsrep::connect() failed: %d", rcode);
+    }
+
+    return false;
+  }
+  else
+  {
+    wsrep_connected= TRUE;
+
+    char* opts= wsrep->options_get(wsrep);
+    if (opts)
+    {
+      wsrep_provider_options_init(opts);
+      free(opts);
+    }
+    else
+    {
+      WSREP_WARN("Failed to get wsrep options");
+    }
+  }
+
+  return true;
+}
+
+bool wsrep_sync_wait (THD* thd, uint mask)
+{
+  if ((thd->variables.wsrep_sync_wait & mask) &&
+      thd->variables.wsrep_on &&
+      !thd->in_active_multi_stmt_transaction() &&
+      thd->wsrep_conflict_state != REPLAYING)
+  {
+    WSREP_DEBUG("wsrep_sync_wait: thd->variables.wsrep_sync_wait = %u, mask = %u",
+                thd->variables.wsrep_sync_wait, mask);
+    // This allows autocommit SELECTs and a first SELECT after SET AUTOCOMMIT=0
+    // TODO: modify to check if thd has locked any rows.
+    wsrep_gtid_t  gtid;
+    wsrep_status_t ret= wsrep->causal_read (wsrep, &gtid);
+
+    if (unlikely(WSREP_OK != ret))
+    {
+      const char* msg;
+      int err;
+
+      // Possibly relevant error codes:
+      // ER_CHECKREAD, ER_ERROR_ON_READ, ER_INVALID_DEFAULT, ER_EMPTY_QUERY,
+      // ER_FUNCTION_NOT_DEFINED, ER_NOT_ALLOWED_COMMAND, ER_NOT_SUPPORTED_YET,
+      // ER_FEATURE_DISABLED, ER_QUERY_INTERRUPTED
+
+      switch (ret)
+      {
+      case WSREP_NOT_IMPLEMENTED:
+        msg= "synchronous reads by wsrep backend. "
+             "Please unset wsrep_causal_reads variable.";
+        err= ER_NOT_SUPPORTED_YET;
+        break;
+      default:
+        msg= "Synchronous wait failed.";
+        err= ER_LOCK_WAIT_TIMEOUT; // NOTE: the above msg won't be displayed
+                                   //       with ER_LOCK_WAIT_TIMEOUT
+      }
+
+      my_error(err, MYF(0), msg);
+
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/*
+ * Helpers to deal with TOI key arrays
+ */
+typedef struct wsrep_key_arr
+{
+    wsrep_key_t* keys;
+    size_t       keys_len;
+} wsrep_key_arr_t;
+
+
+static void wsrep_keys_free(wsrep_key_arr_t* key_arr)
+{
+    for (size_t i= 0; i < key_arr->keys_len; ++i)
+    {
+        my_free((void*)key_arr->keys[i].key_parts);
+    }
+    my_free(key_arr->keys);
+    key_arr->keys= 0;
+    key_arr->keys_len= 0;
+}
+
+
+/*!
+ * @param db      Database string
+ * @param table   Table string
+ * @param key     Array of wsrep_key_t
+ * @param key_len In: number of elements in key array, Out: number of
+ *                elements populated
+ *
+ * @return true if preparation was successful, otherwise false.
+ */
+
+static bool wsrep_prepare_key_for_isolation(const char* db,
+                                            const char* table,
+                                            wsrep_buf_t* key,
+                                            size_t* key_len)
+{
+    if (*key_len < 2) return false;
+
+    switch (wsrep_protocol_version)
+    {
+    case 0:
+        *key_len= 0;
+        break;
+    case 1:
+    case 2:
+    case 3:
+    {
+        *key_len= 0;
+        if (db)
+        {
+            // sql_print_information("%s.%s", db, table);
+            if (db)
+            {
+                key[*key_len].ptr= db;
+                key[*key_len].len= strlen(db);
+                ++(*key_len);
+                if (table)
+                {
+                    key[*key_len].ptr= table;
+                    key[*key_len].len= strlen(table);
+                    ++(*key_len);
+                }
+            }
+        }
+        break;
+    }
+    default:
+        return false;
+    }
+
+    return true;
+}
+
+/* Prepare key list from db/table and table_list */
+static bool wsrep_prepare_keys_for_isolation(THD*              thd,
+                                             const char*       db,
+                                             const char*       table,
+                                             const TABLE_LIST* table_list,
+                                             wsrep_key_arr_t*  ka)
+{
+    ka->keys= 0;
+    ka->keys_len= 0;
+
+    extern TABLE* find_temporary_table(THD*, const TABLE_LIST*);
+
+    if (db || table)
+    {
+        TABLE_LIST tmp_table;
+	MDL_request mdl_request;
+
+        memset(&tmp_table, 0, sizeof(tmp_table));
+        tmp_table.table_name= (char*)table;
+        tmp_table.db= (char*)db;
+	tmp_table.mdl_request.init(MDL_key::GLOBAL, (db) ? db :  "",
+				   (table) ? table : "",
+				   MDL_INTENTION_EXCLUSIVE, MDL_STATEMENT);
+
+        if (!table || !find_temporary_table(thd, &tmp_table))
+        {
+            if (!(ka->keys= (wsrep_key_t*)my_malloc(sizeof(wsrep_key_t), MYF(0))))
+            {
+                WSREP_ERROR("Can't allocate memory for key_array");
+                goto err;
+            }
+            ka->keys_len= 1;
+            if (!(ka->keys[0].key_parts= (wsrep_buf_t*)
+                  my_malloc(sizeof(wsrep_buf_t)*2, MYF(0))))
+            {
+                WSREP_ERROR("Can't allocate memory for key_parts");
+                goto err;
+            }
+            ka->keys[0].key_parts_num= 2;
+            if (!wsrep_prepare_key_for_isolation(
+                    db, table,
+                    (wsrep_buf_t*)ka->keys[0].key_parts,
+                    &ka->keys[0].key_parts_num))
+            {
+                WSREP_ERROR("Preparing keys for isolation failed");
+                goto err;
+            }
+        }
+    }
+
+    for (const TABLE_LIST* table= table_list; table; table= table->next_global)
+    {
+        if (!find_temporary_table(thd, table))
+        {
+            wsrep_key_t* tmp;
+            tmp= (wsrep_key_t*)my_realloc(
+                ka->keys, (ka->keys_len + 1) * sizeof(wsrep_key_t), 
+                 MYF(MY_ALLOW_ZERO_PTR));
+
+            if (!tmp)
+            {
+                WSREP_ERROR("Can't allocate memory for key_array");
+                goto err;
+            }
+            ka->keys= tmp;
+            if (!(ka->keys[ka->keys_len].key_parts= (wsrep_buf_t*)
+                  my_malloc(sizeof(wsrep_buf_t)*2, MYF(0))))
+            {
+                WSREP_ERROR("Can't allocate memory for key_parts");
+                goto err;
+            }
+            ka->keys[ka->keys_len].key_parts_num= 2;
+            ++ka->keys_len;
+            if (!wsrep_prepare_key_for_isolation(
+                    table->db, table->table_name,
+                    (wsrep_buf_t*)ka->keys[ka->keys_len - 1].key_parts,
+                    &ka->keys[ka->keys_len - 1].key_parts_num))
+            {
+                WSREP_ERROR("Preparing keys for isolation failed");
+                goto err;
+            }
+        }
+    }
+    return true;
+err:
+    wsrep_keys_free(ka);
+    return false;
+}
+
+
+bool wsrep_prepare_key_for_innodb(const uchar* cache_key,
+                                  size_t cache_key_len,
+                                  const uchar* row_id,
+                                  size_t row_id_len,
+                                  wsrep_buf_t* key,
+                                  size_t* key_len)
+{
+    if (*key_len < 3) return false;
+
+    *key_len= 0;
+    switch (wsrep_protocol_version)
+    {
+    case 0:
+    {
+        key[0].ptr = cache_key;
+        key[0].len = cache_key_len;
+
+        *key_len = 1;
+        break;
+    }
+    case 1:
+    case 2:
+    case 3:
+    {
+        key[0].ptr = cache_key;
+        key[0].len = strlen( (char*)cache_key );
+
+        key[1].ptr = cache_key + strlen( (char*)cache_key ) + 1;
+        key[1].len = strlen( (char*)(key[1].ptr) );
+
+        *key_len = 2;
+        break;
+    }
+    default:
+        return false;
+    }
+
+    key[*key_len].ptr = row_id;
+    key[*key_len].len = row_id_len;
+    ++(*key_len);
+
+    return true;
+}
+
+
+/*
+ * Construct Query_log_Event from thd query and serialize it
+ * into buffer.
+ *
+ * Return 0 in case of success, 1 in case of error.
+ */
+int wsrep_to_buf_helper(
+    THD* thd, const char *query, uint query_len, uchar** buf, size_t* buf_len)
+{
+  IO_CACHE tmp_io_cache;
+  if (open_cached_file(&tmp_io_cache, mysql_tmpdir, TEMP_PREFIX,
+                       65536, MYF(MY_WME)))
+    return 1;
+  int ret(0);
+
+#ifdef GTID_SUPPORT
+  if (thd->variables.gtid_next.type == GTID_GROUP)
+  {
+      Gtid_log_event gtid_ev(thd, FALSE, &thd->variables.gtid_next);
+      if (!gtid_ev.is_valid()) ret= 0;
+      if (!ret && gtid_ev.write(&tmp_io_cache)) ret= 1;
+  }
+#endif /* GTID_SUPPORT */
+
+  /* if there is prepare query, add event for it */
+  if (!ret && thd->wsrep_TOI_pre_query)
+  {
+    Query_log_event ev(thd, thd->wsrep_TOI_pre_query,
+		       thd->wsrep_TOI_pre_query_len,
+		       FALSE, FALSE, FALSE, 0);
+    if (ev.write(&tmp_io_cache)) ret= 1;
+  }
+
+  /* continue to append the actual query */
+  Query_log_event ev(thd, query, query_len, FALSE, FALSE, FALSE, 0);
+  if (!ret && ev.write(&tmp_io_cache)) ret= 1;
+  if (!ret && wsrep_write_cache_buf(&tmp_io_cache, buf, buf_len)) ret= 1;
+  close_cached_file(&tmp_io_cache);
+  return ret;
+}
+
+#include "sql_show.h"
+static int
+create_view_query(THD *thd, uchar** buf, size_t* buf_len)
+{
+    LEX *lex= thd->lex;
+    SELECT_LEX *select_lex= &lex->select_lex;
+    TABLE_LIST *first_table= select_lex->table_list.first;
+    TABLE_LIST *views = first_table;
+
+    String buff;
+    const LEX_STRING command[3]=
+      {{ C_STRING_WITH_LEN("CREATE ") },
+       { C_STRING_WITH_LEN("ALTER ") },
+       { C_STRING_WITH_LEN("CREATE OR REPLACE ") }};
+
+    buff.append(command[thd->lex->create_view_mode].str,
+                command[thd->lex->create_view_mode].length);
+
+    if (!lex->definer)
+    {
+      /*
+        DEFINER-clause is missing; we have to create default definer in
+        persistent arena to be PS/SP friendly.
+        If this is an ALTER VIEW then the current user should be set as
+        the definer.
+      */
+
+      if (!(lex->definer= create_default_definer(thd, false)))
+      {
+        WSREP_WARN("view default definer issue");
+      }
+    }
+
+    views->algorithm    = lex->create_view_algorithm;
+    views->definer.user = lex->definer->user;
+    views->definer.host = lex->definer->host;
+    views->view_suid    = lex->create_view_suid;
+    views->with_check   = lex->create_view_check;
+
+    view_store_options(thd, views, &buff);
+    buff.append(STRING_WITH_LEN("VIEW "));
+    /* Test if user supplied a db (ie: we did not use thd->db) */
+    if (views->db && views->db[0] &&
+        (thd->db == NULL || strcmp(views->db, thd->db)))
+    {
+      append_identifier(thd, &buff, views->db,
+                        views->db_length);
+      buff.append('.');
+    }
+    append_identifier(thd, &buff, views->table_name,
+                      views->table_name_length);
+    if (lex->view_list.elements)
+    {
+      List_iterator_fast<LEX_STRING> names(lex->view_list);
+      LEX_STRING *name;
+      int i;
+
+      for (i= 0; (name= names++); i++)
+      {
+        buff.append(i ? ", " : "(");
+        append_identifier(thd, &buff, name->str, name->length);
+      }
+      buff.append(')');
+    }
+    buff.append(STRING_WITH_LEN(" AS "));
+    //buff.append(views->source.str, views->source.length);
+    buff.append(thd->lex->create_view_select.str,
+                thd->lex->create_view_select.length);
+    //int errcode= query_error_code(thd, TRUE);
+    //if (thd->binlog_query(THD::STMT_QUERY_TYPE,
+    //                      buff.ptr(), buff.length(), FALSE, FALSE, FALSE, errcod
+    return wsrep_to_buf_helper(thd, buff.ptr(), buff.length(), buf, buf_len);
+}
+
+static int wsrep_TOI_begin(THD *thd, char *db_, char *table_,
+                           const TABLE_LIST* table_list)
+{
+  wsrep_status_t ret(WSREP_WARNING);
+  uchar* buf(0);
+  size_t buf_len(0);
+  int buf_err;
+
+  WSREP_DEBUG("TO BEGIN: %lld, %d : %s", (long long)wsrep_thd_trx_seqno(thd),
+              thd->wsrep_exec_mode, thd->query() );
+  switch (thd->lex->sql_command)
+  {
+  case SQLCOM_CREATE_VIEW:
+    buf_err= create_view_query(thd, &buf, &buf_len);
+    break;
+  case SQLCOM_CREATE_PROCEDURE:
+  case SQLCOM_CREATE_SPFUNCTION:
+    buf_err= wsrep_create_sp(thd, &buf, &buf_len);
+    break;
+  case SQLCOM_CREATE_TRIGGER:
+    buf_err= wsrep_create_trigger_query(thd, &buf, &buf_len);
+    break;
+  case SQLCOM_CREATE_EVENT:
+    buf_err= wsrep_create_event_query(thd, &buf, &buf_len);
+    break;
+  case SQLCOM_ALTER_EVENT:
+    buf_err= wsrep_alter_event_query(thd, &buf, &buf_len);
+    break;
+  default:
+    buf_err= wsrep_to_buf_helper(thd, thd->query(), thd->query_length(), &buf,
+                                 &buf_len);
+    break;
+  }
+
+  wsrep_key_arr_t key_arr= {0, 0};
+  struct wsrep_buf buff = { buf, buf_len };
+  if (!buf_err                                                    &&
+      wsrep_prepare_keys_for_isolation(thd, db_, table_, table_list, &key_arr)&&
+      WSREP_OK == (ret = wsrep->to_execute_start(wsrep, thd->thread_id,
+                                                 key_arr.keys, key_arr.keys_len,
+                                                 &buff, 1,
+                                                 &thd->wsrep_trx_meta)))
+  {
+    thd->wsrep_exec_mode= TOTAL_ORDER;
+    wsrep_to_isolation++;
+    my_free(buf);
+    wsrep_keys_free(&key_arr);
+    WSREP_DEBUG("TO BEGIN: %lld, %d",(long long)wsrep_thd_trx_seqno(thd),
+                thd->wsrep_exec_mode);
+  }
+  else {
+    /* jump to error handler in mysql_execute_command() */
+    WSREP_WARN("TO isolation failed for: %d, sql: %s. Check wsrep "
+               "connection state and retry the query.",
+               ret, (thd->query()) ? thd->query() : "void");
+    my_error(ER_LOCK_DEADLOCK, MYF(0), "WSREP replication failed. Check "
+            "your wsrep connection state and retry the query.");
+    if (buf) my_free(buf);
+    wsrep_keys_free(&key_arr);
+    return -1;
+  }
+  return 0;
+}
+
+static void wsrep_TOI_end(THD *thd) {
+  wsrep_status_t ret;
+  wsrep_to_isolation--;
+
+  WSREP_DEBUG("TO END: %lld, %d : %s", (long long)wsrep_thd_trx_seqno(thd),
+              thd->wsrep_exec_mode, (thd->query()) ? thd->query() : "void");
+  
+  XID xid;
+  wsrep_xid_init(&xid, &thd->wsrep_trx_meta.gtid.uuid,
+                 thd->wsrep_trx_meta.gtid.seqno);
+  wsrep_set_SE_checkpoint(&xid);
+  WSREP_DEBUG("TO END: %lld, update seqno",
+              (long long)wsrep_thd_trx_seqno(thd));
+  
+  if (WSREP_OK == (ret = wsrep->to_execute_end(wsrep, thd->thread_id))) {
+    WSREP_DEBUG("TO END: %lld", (long long)wsrep_thd_trx_seqno(thd));
+  }
+  else {
+    WSREP_WARN("TO isolation end failed for: %d, sql: %s",
+               ret, (thd->query()) ? thd->query() : "void");
+  }
+}
+
+static int wsrep_RSU_begin(THD *thd, char *db_, char *table_)
+{
+  wsrep_status_t ret(WSREP_WARNING);
+  WSREP_DEBUG("RSU BEGIN: %lld, %d : %s", (long long)wsrep_thd_trx_seqno(thd),
+               thd->wsrep_exec_mode, thd->query() );
+
+  ret = wsrep->desync(wsrep);
+  if (ret != WSREP_OK)
+  {
+    WSREP_WARN("RSU desync failed %d for %s", ret, thd->query());
+    my_error(ER_LOCK_DEADLOCK, MYF(0));
+    return(ret);
+  }
+  mysql_mutex_lock(&LOCK_wsrep_replaying);
+  wsrep_replaying++;
+  mysql_mutex_unlock(&LOCK_wsrep_replaying);
+
+  if (wsrep_wait_committing_connections_close(5000))
+  {
+    /* no can do, bail out from DDL */
+    WSREP_WARN("RSU failed due to pending transactions, %s", thd->query());
+    mysql_mutex_lock(&LOCK_wsrep_replaying);
+    wsrep_replaying--;
+    mysql_mutex_unlock(&LOCK_wsrep_replaying);
+
+    ret = wsrep->resync(wsrep);
+    if (ret != WSREP_OK)
+    {
+      WSREP_WARN("resync failed %d for %s", ret, thd->query());
+    }
+    my_error(ER_LOCK_DEADLOCK, MYF(0));
+    return(1);
+  }
+
+  wsrep_seqno_t seqno = wsrep->pause(wsrep);
+  if (seqno == WSREP_SEQNO_UNDEFINED)
+  {
+    WSREP_WARN("pause failed %lld for %s", (long long)seqno, thd->query());
+    return(1);
+  }
+  WSREP_DEBUG("paused at %lld", (long long)seqno);
+  thd->variables.wsrep_on = 0;
+  return 0;
+}
+
+static void wsrep_RSU_end(THD *thd)
+{
+  wsrep_status_t ret(WSREP_WARNING);
+  WSREP_DEBUG("RSU END: %lld, %d : %s", (long long)wsrep_thd_trx_seqno(thd),
+               thd->wsrep_exec_mode, thd->query() );
+
+
+  mysql_mutex_lock(&LOCK_wsrep_replaying);
+  wsrep_replaying--;
+  mysql_mutex_unlock(&LOCK_wsrep_replaying);
+
+  ret = wsrep->resume(wsrep);
+  if (ret != WSREP_OK)
+  {
+    WSREP_WARN("resume failed %d for %s", ret, thd->query());
+  }
+  ret = wsrep->resync(wsrep);
+  if (ret != WSREP_OK)
+  {
+    WSREP_WARN("resync failed %d for %s", ret, thd->query());
+    return;
+  }
+  thd->variables.wsrep_on = 1;
+}
+
+int wsrep_to_isolation_begin(THD *thd, char *db_, char *table_,
+                             const TABLE_LIST* table_list)
+{
+  int ret= 0;
+
+  /*
+    No isolation for applier or replaying threads.
+   */
+  if (thd->wsrep_exec_mode == REPL_RECV)
+    return 0;
+
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+
+  if (thd->wsrep_conflict_state == MUST_ABORT)
+  {
+    WSREP_INFO("thread: %lu, %s has been aborted due to multi-master conflict",
+               thd->thread_id, thd->query());
+    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    return WSREP_TRX_FAIL;
+  }
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+  DBUG_ASSERT(thd->wsrep_exec_mode == LOCAL_STATE);
+  DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno == WSREP_SEQNO_UNDEFINED);
+
+  if (thd->global_read_lock.can_acquire_protection())
+  {
+    WSREP_DEBUG("Aborting TOI: Global Read-Lock (FTWRL) in place: %s %lu",
+                thd->query(), thd->thread_id);
+    return -1;
+  }
+
+  if (wsrep_debug && thd->mdl_context.has_locks())
+  {
+    WSREP_DEBUG("thread holds MDL locks at TI begin: %s %lu",
+                thd->query(), thd->thread_id);
+  }
+
+  /*
+    It makes sense to set auto_increment_* to defaults in TOI operations.
+    Must be done before wsrep_TOI_begin() since Query_log_event encapsulating
+    TOI statement and auto inc variables for wsrep replication is constructed
+    there. Variables are reset back in THD::reset_for_next_command() before
+    processing of next command.
+   */
+  if (wsrep_auto_increment_control)
+  {
+    thd->variables.auto_increment_offset = 1;
+    thd->variables.auto_increment_increment = 1;
+  }
+
+  if (thd->variables.wsrep_on && thd->wsrep_exec_mode==LOCAL_STATE)
+  {
+    switch (wsrep_OSU_method_options) {
+    case WSREP_OSU_TOI: ret =  wsrep_TOI_begin(thd, db_, table_,
+                                               table_list); break;
+    case WSREP_OSU_RSU: ret =  wsrep_RSU_begin(thd, db_, table_); break;
+    }
+    if (!ret)
+    {
+      thd->wsrep_exec_mode= TOTAL_ORDER;
+    }
+  }
+  return ret;
+}
+
+void wsrep_to_isolation_end(THD *thd)
+{
+  if (thd->wsrep_exec_mode == TOTAL_ORDER)
+  {
+    switch(wsrep_OSU_method_options)
+    {
+    case WSREP_OSU_TOI: wsrep_TOI_end(thd); break;
+    case WSREP_OSU_RSU: wsrep_RSU_end(thd); break;
+    }
+    wsrep_cleanup_transaction(thd);
+  }
+}
+
+#define WSREP_MDL_LOG(severity, msg, req, gra)	                               \
+    WSREP_##severity(                                                          \
+      "%s\n"                                                                   \
+      "request: (%lu \tseqno %lld \twsrep (%d, %d, %d) cmd %d %d \t%s)\n"      \
+      "granted: (%lu \tseqno %lld \twsrep (%d, %d, %d) cmd %d %d \t%s)",       \
+      msg,                                                                     \
+      req->thread_id, (long long)wsrep_thd_trx_seqno(req),                     \
+      req->wsrep_exec_mode, req->wsrep_query_state, req->wsrep_conflict_state, \
+      req->get_command(), req->lex->sql_command, req->query(),		       \
+      gra->thread_id, (long long)wsrep_thd_trx_seqno(gra),                     \
+      gra->wsrep_exec_mode, gra->wsrep_query_state, gra->wsrep_conflict_state, \
+      gra->get_command(), gra->lex->sql_command, gra->query());
+
+bool
+wsrep_grant_mdl_exception(MDL_context *requestor_ctx,
+                          MDL_ticket *ticket
+) {
+  if (!WSREP_ON) return FALSE;
+
+  THD *request_thd  = requestor_ctx->get_thd();
+  THD *granted_thd  = ticket->get_ctx()->get_thd();
+  bool ret          = FALSE;
+
+  mysql_mutex_lock(&request_thd->LOCK_wsrep_thd);
+  if (request_thd->wsrep_exec_mode == TOTAL_ORDER ||
+      request_thd->wsrep_exec_mode == REPL_RECV)
+  {
+    mysql_mutex_unlock(&request_thd->LOCK_wsrep_thd);
+    WSREP_MDL_LOG(DEBUG, "MDL conflict ", request_thd, granted_thd);
+    ticket->wsrep_report(wsrep_debug);
+
+    mysql_mutex_lock(&granted_thd->LOCK_wsrep_thd);
+    if (granted_thd->wsrep_exec_mode == TOTAL_ORDER ||
+        granted_thd->wsrep_exec_mode == REPL_RECV)
+    {
+      WSREP_MDL_LOG(INFO, "MDL BF-BF conflict", request_thd, granted_thd);
+      ticket->wsrep_report(true);
+      mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd);
+      ret = TRUE;
+    }
+    else if (granted_thd->lex->sql_command == SQLCOM_FLUSH)
+    {
+      WSREP_DEBUG("mdl granted over FLUSH BF");
+      ticket->wsrep_report(wsrep_debug);
+      mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd);
+      ret = TRUE;
+    }
+    else if (request_thd->lex->sql_command == SQLCOM_DROP_TABLE)
+    {
+      WSREP_DEBUG("DROP caused BF abort");
+      ticket->wsrep_report(wsrep_debug);
+      mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd);
+      wsrep_abort_thd((void*)request_thd, (void*)granted_thd, 1);
+      ret = FALSE;
+    }
+    else if (granted_thd->wsrep_query_state == QUERY_COMMITTING)
+    {
+      WSREP_DEBUG("mdl granted, but commiting thd abort scheduled");
+      ticket->wsrep_report(wsrep_debug);
+      mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd);
+      wsrep_abort_thd((void*)request_thd, (void*)granted_thd, 1);
+      ret = FALSE;
+    }
+    else
+    {
+      WSREP_MDL_LOG(DEBUG, "MDL conflict-> BF abort", request_thd, granted_thd);
+      ticket->wsrep_report(wsrep_debug);
+      mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd);
+      wsrep_abort_thd((void*)request_thd, (void*)granted_thd, 1);
+      ret = FALSE;
+    }
+  }
+  else
+  {
+    mysql_mutex_unlock(&request_thd->LOCK_wsrep_thd);
+  }
+  return ret;
+}
+
+
+pthread_handler_t start_wsrep_THD(void *arg)
+{
+  THD *thd;
+  rpl_sql_thread_info sql_info(NULL);
+  wsrep_thd_processor_fun processor= (wsrep_thd_processor_fun)arg;
+
+  if (my_thread_init())
+  {
+    WSREP_ERROR("Could not initialize thread");
+    return(NULL);
+  }
+
+  if (!(thd= new THD(true)))
+  {
+    return(NULL);
+  }
+  mysql_mutex_lock(&LOCK_thread_count);
+  thd->thread_id=thread_id++;
+
+  thd->real_id=pthread_self(); // Keep purify happy
+  thread_count++;
+  thread_created++;
+  threads.append(thd);
+
+  my_net_init(&thd->net,(st_vio*) 0, MYF(0));
+
+  DBUG_PRINT("wsrep",(("creating thread %lld"), (long long)thd->thread_id));
+  thd->prior_thr_create_utime= thd->start_utime= microsecond_interval_timer();
+  (void) mysql_mutex_unlock(&LOCK_thread_count);
+
+  /* from bootstrap()... */
+  thd->bootstrap=1;
+  thd->max_client_packet_length= thd->net.max_packet;
+  thd->security_ctx->master_access= ~(ulong)0;
+  thd->system_thread_info.rpl_sql_info= &sql_info;
+
+  /* from handle_one_connection... */
+  pthread_detach_this_thread();
+
+  mysql_thread_set_psi_id(thd->thread_id);
+  thd->thr_create_utime=  microsecond_interval_timer();
+  if (MYSQL_CALLBACK_ELSE(thread_scheduler, init_new_connection_thread, (), 0))
+  {
+    close_connection(thd, ER_OUT_OF_RESOURCES);
+    statistic_increment(aborted_connects,&LOCK_status);
+    MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
+
+    return(NULL);
+  }
+
+// </5.1.17>
+  /*
+    handle_one_connection() is normally the only way a thread would
+    start and would always be on the very high end of the stack ,
+    therefore, the thread stack always starts at the address of the
+    first local variable of handle_one_connection, which is thd. We
+    need to know the start of the stack so that we could check for
+    stack overruns.
+  */
+  DBUG_PRINT("wsrep", ("handle_one_connection called by thread %lld\n",
+                       (long long)thd->thread_id));
+  /* now that we've called my_thread_init(), it is safe to call DBUG_* */
+
+  thd->thread_stack= (char*) &thd;
+  if (thd->store_globals())
+  {
+    close_connection(thd, ER_OUT_OF_RESOURCES);
+    statistic_increment(aborted_connects,&LOCK_status);
+    MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
+    delete thd;
+
+    return(NULL);
+  }
+
+  thd->system_thread= SYSTEM_THREAD_SLAVE_SQL;
+  thd->security_ctx->skip_grants();
+
+  /* handle_one_connection() again... */
+  //thd->version= refresh_version;
+  thd->proc_info= 0;
+  thd->set_command(COM_SLEEP);
+  thd->set_time();
+  thd->init_for_queries();
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  wsrep_running_threads++;
+  mysql_cond_broadcast(&COND_thread_count);
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  processor(thd);
+
+  close_connection(thd, 0);
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  wsrep_running_threads--;
+  WSREP_DEBUG("wsrep running threads now: %lu", wsrep_running_threads);
+  mysql_cond_broadcast(&COND_thread_count);
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  // Note: We can't call THD destructor without crashing
+  // if plugins have not been initialized. However, in most of the
+  // cases this means that pre SE initialization SST failed and
+  // we are going to exit anyway.
+  if (plugins_are_initialized)
+  {
+    net_end(&thd->net);
+    MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 1));
+  }
+  else
+  {
+    // TODO: lightweight cleanup to get rid of:
+    // 'Error in my_thread_global_end(): 2 threads didn't exit'
+    // at server shutdown
+  }
+
+  my_thread_end();
+  if (thread_handling > SCHEDULER_ONE_THREAD_PER_CONNECTION)
+  {
+    mysql_mutex_lock(&LOCK_thread_count);
+    delete thd;
+    thread_count--;
+    mysql_mutex_unlock(&LOCK_thread_count);
+  }
+  return(NULL);
+}
+
+
+/**/
+static bool abort_replicated(THD *thd)
+{
+  bool ret_code= false;
+  if (thd->wsrep_query_state== QUERY_COMMITTING)
+  {
+    if (wsrep_debug) WSREP_INFO("aborting replicated trx: %lu", thd->real_id);
+
+    (void)wsrep_abort_thd(thd, thd, TRUE);
+    ret_code= true;
+  }
+  return ret_code;
+}
+
+
+/**/
+static inline bool is_client_connection(THD *thd)
+{
+  return (thd->wsrep_client_thread && thd->variables.wsrep_on);
+}
+
+
+static inline bool is_replaying_connection(THD *thd)
+{
+  bool ret;
+
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  ret=  (thd->wsrep_conflict_state == REPLAYING) ? true : false;
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+  return ret;
+}
+
+
+static inline bool is_committing_connection(THD *thd)
+{
+  bool ret;
+
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  ret=  (thd->wsrep_query_state == QUERY_COMMITTING) ? true : false;
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+  return ret;
+}
+
+
+static bool have_client_connections()
+{
+  THD *tmp;
+
+  I_List_iterator<THD> it(threads);
+  while ((tmp=it++))
+  {
+    DBUG_PRINT("quit",("Informing thread %ld that it's time to die",
+                        tmp->thread_id));
+    if (is_client_connection(tmp) && tmp->killed == KILL_CONNECTION)
+    {
+      (void)abort_replicated(tmp);
+      return true;
+    }
+  }
+  return false;
+}
+
+/*
+   returns the number of wsrep appliers running.
+   However, the caller (thd parameter) is not taken in account
+ */
+static int have_wsrep_appliers(THD *thd)
+{
+  int ret= 0;
+  THD *tmp;
+
+  I_List_iterator<THD> it(threads);
+  while ((tmp=it++))
+  {
+    ret+= (tmp != thd && tmp->wsrep_applier);
+  }
+  return ret;
+}
+
+
+static void wsrep_close_thread(THD *thd)
+{
+  thd->killed= KILL_CONNECTION;
+  MYSQL_CALLBACK(thread_scheduler, post_kill_notification, (thd));
+  if (thd->mysys_var)
+  {
+    thd->mysys_var->abort=1;
+    mysql_mutex_lock(&thd->mysys_var->mutex);
+    if (thd->mysys_var->current_cond)
+    {
+      mysql_mutex_lock(thd->mysys_var->current_mutex);
+      mysql_cond_broadcast(thd->mysys_var->current_cond);
+      mysql_mutex_unlock(thd->mysys_var->current_mutex);
+    }
+    mysql_mutex_unlock(&thd->mysys_var->mutex);
+  }
+}
+
+
+static my_bool have_committing_connections()
+{
+  THD *tmp;
+  mysql_mutex_lock(&LOCK_thread_count); // For unlink from list
+
+  I_List_iterator<THD> it(threads);
+  while ((tmp=it++))
+  {
+    if (!is_client_connection(tmp))
+      continue;
+
+    if (is_committing_connection(tmp))
+    {
+      return TRUE;
+    }
+  }
+  mysql_mutex_unlock(&LOCK_thread_count);
+  return FALSE;
+}
+
+
+int wsrep_wait_committing_connections_close(int wait_time)
+{
+  int sleep_time= 100;
+
+  while (have_committing_connections() && wait_time > 0)
+  {
+    WSREP_DEBUG("wait for committing transaction to close: %d", wait_time);
+    my_sleep(sleep_time);
+    wait_time -= sleep_time;
+  }
+  if (have_committing_connections())
+  {
+    return 1;
+  }
+  return 0;
+}
+
+
+void wsrep_close_client_connections(my_bool wait_to_end)
+{
+  /*
+    First signal all threads that it's time to die
+  */
+
+  THD *tmp;
+  mysql_mutex_lock(&LOCK_thread_count); // For unlink from list
+
+  bool kill_cached_threads_saved= kill_cached_threads;
+  kill_cached_threads= true; // prevent future threads caching
+  mysql_cond_broadcast(&COND_thread_cache); // tell cached threads to die
+
+  I_List_iterator<THD> it(threads);
+  while ((tmp=it++))
+  {
+    DBUG_PRINT("quit",("Informing thread %ld that it's time to die",
+                        tmp->thread_id));
+    /* We skip slave threads & scheduler on this first loop through. */
+    if (!is_client_connection(tmp))
+      continue;
+
+    if (is_replaying_connection(tmp))
+    {
+      tmp->killed= KILL_CONNECTION;
+      continue;
+    }
+
+    /* replicated transactions must be skipped */
+    if (abort_replicated(tmp))
+      continue;
+
+    WSREP_DEBUG("closing connection %ld", tmp->thread_id);
+    wsrep_close_thread(tmp);
+  }
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  if (thread_count)
+    sleep(2);                               // Give threads time to die
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  /*
+    Force remaining threads to die by closing the connection to the client
+  */
+
+  I_List_iterator<THD> it2(threads);
+  while ((tmp=it2++))
+  {
+#ifndef __bsdi__				// Bug in BSDI kernel
+    if (is_client_connection(tmp) && 
+        !abort_replicated(tmp)    &&
+	!is_replaying_connection(tmp))
+    {
+      WSREP_INFO("killing local connection: %ld",tmp->thread_id);
+      close_connection(tmp,0);
+    }
+#endif
+  }
+
+  DBUG_PRINT("quit",("Waiting for threads to die (count=%u)",thread_count));
+  if (wsrep_debug)
+    WSREP_INFO("waiting for client connections to close: %u", thread_count);
+
+  while (wait_to_end && have_client_connections())
+  {
+    mysql_cond_wait(&COND_thread_count, &LOCK_thread_count);
+    DBUG_PRINT("quit",("One thread died (count=%u)", thread_count));
+  }
+
+  kill_cached_threads= kill_cached_threads_saved;
+
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  /* All client connection threads have now been aborted */
+}
+
+
+void wsrep_close_applier(THD *thd)
+{
+  WSREP_DEBUG("closing applier %ld", thd->thread_id);
+  wsrep_close_thread(thd);
+}
+
+
+void wsrep_close_threads(THD *thd)
+{
+  THD *tmp;
+  mysql_mutex_lock(&LOCK_thread_count); // For unlink from list
+
+  I_List_iterator<THD> it(threads);
+  while ((tmp=it++))
+  {
+    DBUG_PRINT("quit",("Informing thread %ld that it's time to die",
+                       tmp->thread_id));
+    /* We skip slave threads & scheduler on this first loop through. */
+    if (tmp->wsrep_applier && tmp != thd)
+    {
+      WSREP_DEBUG("closing wsrep thread %ld", tmp->thread_id);
+      wsrep_close_thread (tmp);
+    }
+  }
+
+  mysql_mutex_unlock(&LOCK_thread_count);
+}
+
+
+void wsrep_close_applier_threads(int count)
+{
+  THD *tmp;
+  mysql_mutex_lock(&LOCK_thread_count); // For unlink from list
+
+  I_List_iterator<THD> it(threads);
+  while ((tmp=it++) && count)
+  {
+    DBUG_PRINT("quit",("Informing thread %ld that it's time to die",
+                       tmp->thread_id));
+    /* We skip slave threads & scheduler on this first loop through. */
+    if (tmp->wsrep_applier)
+    {
+      WSREP_DEBUG("closing wsrep applier thread %ld", tmp->thread_id);
+      tmp->wsrep_applier_closing= TRUE;
+      count--;
+    }
+  }
+
+  mysql_mutex_unlock(&LOCK_thread_count);
+}
+
+
+void wsrep_wait_appliers_close(THD *thd)
+{
+  /* Wait for wsrep appliers to gracefully exit */
+  mysql_mutex_lock(&LOCK_thread_count);
+  while (have_wsrep_appliers(thd) > 1)
+  // 1 is for rollbacker thread which needs to be killed explicitly.
+  // This gotta be fixed in a more elegant manner if we gonna have arbitrary
+  // number of non-applier wsrep threads.
+  {
+    if (thread_handling > SCHEDULER_ONE_THREAD_PER_CONNECTION)
+    {
+      mysql_mutex_unlock(&LOCK_thread_count);
+      my_sleep(100);
+      mysql_mutex_lock(&LOCK_thread_count);
+    }
+    else
+      mysql_cond_wait(&COND_thread_count,&LOCK_thread_count);
+    DBUG_PRINT("quit",("One applier died (count=%u)",thread_count));
+  }
+  mysql_mutex_unlock(&LOCK_thread_count);
+  /* Now kill remaining wsrep threads: rollbacker */
+  wsrep_close_threads (thd);
+  /* and wait for them to die */
+  mysql_mutex_lock(&LOCK_thread_count);
+  while (have_wsrep_appliers(thd) > 0)
+  {
+   if (thread_handling > SCHEDULER_ONE_THREAD_PER_CONNECTION)
+    {
+      mysql_mutex_unlock(&LOCK_thread_count);
+      my_sleep(100);
+      mysql_mutex_lock(&LOCK_thread_count);
+    }
+    else
+      mysql_cond_wait(&COND_thread_count,&LOCK_thread_count);
+    DBUG_PRINT("quit",("One thread died (count=%u)",thread_count));
+  }
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  /* All wsrep applier threads have now been aborted. However, if this thread
+     is also applier, we are still running...
+  */
+}
+
+
+void wsrep_kill_mysql(THD *thd)
+{
+  if (mysqld_server_started)
+  {
+    if (!shutdown_in_progress)
+    {
+      WSREP_INFO("starting shutdown");
+      kill_mysql();
+    }
+  }
+  else
+  {
+    unireg_abort(1);
+  }
+}
+
+
+int wsrep_create_sp(THD *thd, uchar** buf, size_t* buf_len)
+{
+  String log_query;
+  sp_head *sp = thd->lex->sphead;
+  ulong saved_mode= thd->variables.sql_mode;
+  String retstr(64);
+  retstr.set_charset(system_charset_info);
+
+  log_query.set_charset(system_charset_info);
+
+  if (sp->m_type == TYPE_ENUM_FUNCTION)
+  {
+    sp_returns_type(thd, retstr, sp);
+  }
+
+  if (!create_string(thd, &log_query,
+                     sp->m_type,
+                     (sp->m_explicit_name ? sp->m_db.str : NULL),
+                     (sp->m_explicit_name ? sp->m_db.length : 0),
+                     sp->m_name.str, sp->m_name.length,
+                     sp->m_params.str, sp->m_params.length,
+                     retstr.c_ptr(), retstr.length(),
+                     sp->m_body.str, sp->m_body.length,
+                     sp->m_chistics, &(thd->lex->definer->user),
+                     &(thd->lex->definer->host),
+                     saved_mode))
+  {
+      WSREP_WARN("SP create string failed: %s", thd->query());
+      return 1;
+  }
+
+  return wsrep_to_buf_helper(thd, log_query.ptr(), log_query.length(), buf, buf_len);
+}
+
+
+extern int wsrep_on(void *thd)
+{
+  return (int)(WSREP(((THD*)thd)));
+}
+
+
+extern "C" bool wsrep_thd_is_wsrep_on(THD *thd)
+{
+  return thd->variables.wsrep_on;
+}
+
+
+extern "C" bool wsrep_consistency_check(void *thd)
+{
+  return ((THD*)thd)->wsrep_consistency_check == CONSISTENCY_CHECK_RUNNING;
+}
+
+
+extern "C" void wsrep_thd_set_exec_mode(THD *thd, enum wsrep_exec_mode mode)
+{
+  thd->wsrep_exec_mode= mode;
+}
+
+
+extern "C" void wsrep_thd_set_query_state(
+	THD *thd, enum wsrep_query_state state)
+{
+  thd->wsrep_query_state= state;
+}
+
+
+extern "C" void wsrep_thd_set_conflict_state(
+	THD *thd, enum wsrep_conflict_state state)
+{
+  thd->wsrep_conflict_state= state;
+}
+
+
+extern "C" enum wsrep_exec_mode wsrep_thd_exec_mode(THD *thd)
+{
+  return thd->wsrep_exec_mode;
+}
+
+
+extern "C" const char *wsrep_thd_exec_mode_str(THD *thd)
+{
+  return 
+    (!thd) ? "void" :
+    (thd->wsrep_exec_mode == LOCAL_STATE)  ? "local"         :
+    (thd->wsrep_exec_mode == REPL_RECV)    ? "applier"       :
+    (thd->wsrep_exec_mode == TOTAL_ORDER)  ? "total order"   : 
+    (thd->wsrep_exec_mode == LOCAL_COMMIT) ? "local commit"  : "void";
+}
+
+
+extern "C" enum wsrep_query_state wsrep_thd_query_state(THD *thd)
+{
+  return thd->wsrep_query_state;
+}
+
+
+extern "C" const char *wsrep_thd_query_state_str(THD *thd)
+{
+  return 
+    (!thd) ? "void" : 
+    (thd->wsrep_query_state == QUERY_IDLE)        ? "idle"          :
+    (thd->wsrep_query_state == QUERY_EXEC)        ? "executing"     :
+    (thd->wsrep_query_state == QUERY_COMMITTING)  ? "committing"    :
+    (thd->wsrep_query_state == QUERY_EXITING)     ? "exiting"       : 
+    (thd->wsrep_query_state == QUERY_ROLLINGBACK) ? "rolling back"  : "void";
+}
+
+
+extern "C" enum wsrep_conflict_state wsrep_thd_conflict_state(THD *thd)
+{
+  return thd->wsrep_conflict_state;
+}
+
+
+extern "C" const char *wsrep_thd_conflict_state_str(THD *thd)
+{
+  return 
+    (!thd) ? "void" :
+    (thd->wsrep_conflict_state == NO_CONFLICT)      ? "no conflict"  :
+    (thd->wsrep_conflict_state == MUST_ABORT)       ? "must abort"   :
+    (thd->wsrep_conflict_state == ABORTING)         ? "aborting"     :
+    (thd->wsrep_conflict_state == MUST_REPLAY)      ? "must replay"  : 
+    (thd->wsrep_conflict_state == REPLAYING)        ? "replaying"    : 
+    (thd->wsrep_conflict_state == RETRY_AUTOCOMMIT) ? "retrying"     : 
+    (thd->wsrep_conflict_state == CERT_FAILURE)     ? "cert failure" : "void";
+}
+
+
+extern "C" wsrep_ws_handle_t* wsrep_thd_ws_handle(THD *thd)
+{
+  return &thd->wsrep_ws_handle;
+}
+
+
+extern "C" void wsrep_thd_LOCK(THD *thd)
+{
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+}
+
+
+extern "C" void wsrep_thd_UNLOCK(THD *thd)
+{
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+}
+
+
+extern "C" time_t wsrep_thd_query_start(THD *thd) 
+{
+  return thd->query_start();
+}
+
+
+extern "C" uint32 wsrep_thd_wsrep_rand(THD *thd) 
+{
+  return thd->wsrep_rand;
+}
+
+
+extern "C" my_thread_id wsrep_thd_thread_id(THD *thd) 
+{
+  return thd->thread_id;
+}
+
+
+extern "C" wsrep_seqno_t wsrep_thd_trx_seqno(THD *thd) 
+{
+  return (thd) ? thd->wsrep_trx_meta.gtid.seqno : WSREP_SEQNO_UNDEFINED;
+}
+
+
+extern "C" query_id_t wsrep_thd_query_id(THD *thd) 
+{
+  return thd->query_id;
+}
+
+
+extern "C" char *wsrep_thd_query(THD *thd) 
+{
+  return (thd) ? thd->query() : NULL;
+}
+
+
+extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd) 
+{
+  return thd->wsrep_last_query_id;
+}
+
+
+extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id) 
+{
+  thd->wsrep_last_query_id= id;
+}
+
+
+extern "C" void wsrep_thd_awake(THD *thd, my_bool signal)
+{
+  if (signal)
+  {
+    mysql_mutex_lock(&thd->LOCK_thd_data);
+    thd->awake(KILL_QUERY);
+    mysql_mutex_unlock(&thd->LOCK_thd_data);
+  }
+  else
+  {
+    mysql_mutex_lock(&LOCK_wsrep_replaying);
+    mysql_cond_broadcast(&COND_wsrep_replaying);
+    mysql_mutex_unlock(&LOCK_wsrep_replaying);
+  }
+}
+
+
+extern "C" int wsrep_thd_retry_counter(THD *thd) 
+{
+  return(thd->wsrep_retry_counter);
+}
+
+
+extern int
+wsrep_trx_order_before(void *thd1, void *thd2)
+{
+    if (wsrep_thd_trx_seqno((THD*)thd1) < wsrep_thd_trx_seqno((THD*)thd2)) {
+        WSREP_DEBUG("BF conflict, order: %lld %lld\n",
+                    (long long)wsrep_thd_trx_seqno((THD*)thd1),
+                    (long long)wsrep_thd_trx_seqno((THD*)thd2));
+        return 1;
+    }
+    WSREP_DEBUG("waiting for BF, trx order: %lld %lld\n",
+                (long long)wsrep_thd_trx_seqno((THD*)thd1),
+                (long long)wsrep_thd_trx_seqno((THD*)thd2));
+    return 0;
+}
+
+
+extern "C" int
+wsrep_trx_is_aborting(void *thd_ptr)
+{
+	if (thd_ptr) {
+		if ((((THD *)thd_ptr)->wsrep_conflict_state == MUST_ABORT) ||
+		    (((THD *)thd_ptr)->wsrep_conflict_state == ABORTING)) {
+		  return 1;
+		}
+	}
+	return 0;
+}
+
+
+my_bool wsrep_read_only_option(THD *thd, TABLE_LIST *all_tables)
+{
+  int opt_readonly_saved = opt_readonly;
+  ulong flag_saved = (ulong)(thd->security_ctx->master_access & SUPER_ACL);
+
+  opt_readonly = 0;
+  thd->security_ctx->master_access &= ~SUPER_ACL;
+
+  my_bool ret = !deny_updates_if_read_only_option(thd, all_tables);
+
+  opt_readonly = opt_readonly_saved;
+  thd->security_ctx->master_access |= flag_saved;
+
+  return ret;
+}
+
+
+void wsrep_copy_query(THD *thd)
+{
+  thd->wsrep_retry_command   = thd->get_command();
+  thd->wsrep_retry_query_len = thd->query_length();
+  if (thd->wsrep_retry_query) {
+	  my_free(thd->wsrep_retry_query);
+  }
+  thd->wsrep_retry_query     = (char *)my_malloc(
+                                 thd->wsrep_retry_query_len + 1, MYF(0));
+  strncpy(thd->wsrep_retry_query, thd->query(), thd->wsrep_retry_query_len);
+  thd->wsrep_retry_query[thd->wsrep_retry_query_len] = '\0';
+}
+
+
+bool wsrep_is_show_query(enum enum_sql_command command)
+{
+  DBUG_ASSERT(command >= 0 && command <= SQLCOM_END);
+  return (sql_command_flags[command] & CF_STATUS_COMMAND) != 0;
+}
+
+bool wsrep_create_like_table(THD* thd, TABLE_LIST* table,
+                             TABLE_LIST* src_table,
+                             HA_CREATE_INFO *create_info)
+{
+  TABLE *tmp_table;
+  bool is_tmp_table= FALSE;
+
+  for (tmp_table= thd->temporary_tables; tmp_table; tmp_table=tmp_table->next)
+  {
+      if (!strcmp(src_table->db, tmp_table->s->db.str)     &&
+          !strcmp(src_table->table_name, tmp_table->s->table_name.str))
+      {
+        is_tmp_table= TRUE;
+        break;
+      }
+  }
+  if (create_info->options & HA_LEX_CREATE_TMP_TABLE)
+  {
+
+    /* CREATE TEMPORARY TABLE LIKE must be skipped from replication */
+    WSREP_DEBUG("CREATE TEMPORARY TABLE LIKE... skipped replication\n %s", 
+                thd->query());
+  }
+  else if (!is_tmp_table)
+  {
+    /* this is straight CREATE TABLE LIKE... eith no tmp tables */
+    WSREP_TO_ISOLATION_BEGIN(table->db, table->table_name, NULL);
+  }
+  else
+  {
+    /* here we have CREATE TABLE LIKE <temporary table> 
+       the temporary table definition will be needed in slaves to
+       enable the create to succeed
+     */
+    TABLE_LIST tbl;
+    bzero((void*) &tbl, sizeof(tbl));
+    tbl.db= src_table->db;
+    tbl.table_name= tbl.alias= src_table->table_name;
+    tbl.table= tmp_table;
+    char buf[2048];
+    String query(buf, sizeof(buf), system_charset_info);
+    query.length(0);  // Have to zero it since constructor doesn't
+
+    (void)  store_create_info(thd, &tbl, &query, NULL, TRUE, FALSE);
+    WSREP_DEBUG("TMP TABLE: %s", query.ptr());
+
+    thd->wsrep_TOI_pre_query=     query.ptr();
+    thd->wsrep_TOI_pre_query_len= query.length();
+
+    WSREP_TO_ISOLATION_BEGIN(table->db, table->table_name, NULL);
+
+    thd->wsrep_TOI_pre_query=      NULL;
+    thd->wsrep_TOI_pre_query_len= 0;
+  }
+
+  return(false);
+
+error:
+  thd->wsrep_TOI_pre_query= NULL;
+  return (true);
+}
+
+
+int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len)
+{
+  LEX *lex= thd->lex;
+  String stmt_query;
+
+  LEX_STRING definer_user;
+  LEX_STRING definer_host;
+
+  if (!lex->definer)
+  {
+    if (!thd->slave_thread)
+    {
+      if (!(lex->definer= create_default_definer(thd, false)))
+        return 1;
+    }
+  }
+
+  if (lex->definer)
+  {
+    /* SUID trigger. */
+
+    definer_user= lex->definer->user;
+    definer_host= lex->definer->host;
+  }
+  else
+  {
+    /* non-SUID trigger. */
+
+    definer_user.str= 0;
+    definer_user.length= 0;
+
+    definer_host.str= 0;
+    definer_host.length= 0;
+  }
+
+  stmt_query.append(STRING_WITH_LEN("CREATE "));
+
+  append_definer(thd, &stmt_query, &definer_user, &definer_host);
+
+  LEX_STRING stmt_definition;
+  stmt_definition.str= (char*) thd->lex->stmt_definition_begin;
+  stmt_definition.length= thd->lex->stmt_definition_end
+    - thd->lex->stmt_definition_begin;
+  trim_whitespace(thd->charset(), & stmt_definition);
+
+  stmt_query.append(stmt_definition.str, stmt_definition.length);
+
+  return wsrep_to_buf_helper(thd, stmt_query.c_ptr(), stmt_query.length(),
+                             buf, buf_len);
+}
diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h
new file mode 100644
index 00000000000..2150ac85bf0
--- /dev/null
+++ b/sql/wsrep_mysqld.h
@@ -0,0 +1,370 @@
+/* Copyright 2008-2013 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include <wsrep.h>
+
+#if !defined(WSREP_MYSQLD_H) && defined(WITH_WSREP)
+#define WSREP_MYSQLD_H
+
+typedef struct st_mysql_show_var SHOW_VAR;
+#include <sql_priv.h>
+//#include "rpl_gtid.h"
+#include "../wsrep/wsrep_api.h"
+#include "mdl.h"
+#include "mysqld.h"
+#include "sql_table.h"
+
+#define WSREP_UNDEFINED_TRX_ID ULONGLONG_MAX
+
+class set_var;
+class THD;
+
+enum wsrep_exec_mode {
+    LOCAL_STATE,
+    REPL_RECV,
+    TOTAL_ORDER,
+    LOCAL_COMMIT
+};
+
+enum wsrep_query_state {
+    QUERY_IDLE,
+    QUERY_EXEC,
+    QUERY_COMMITTING,
+    QUERY_EXITING,
+    QUERY_ROLLINGBACK,
+};
+
+enum wsrep_conflict_state {
+    NO_CONFLICT,
+    MUST_ABORT,
+    ABORTING,
+    ABORTED,
+    MUST_REPLAY,
+    REPLAYING,
+    RETRY_AUTOCOMMIT,
+    CERT_FAILURE,
+};
+
+enum wsrep_consistency_check_mode {
+    NO_CONSISTENCY_CHECK,
+    CONSISTENCY_CHECK_DECLARED,
+    CONSISTENCY_CHECK_RUNNING,
+};
+
+struct wsrep_thd_shadow {
+  ulonglong            options;
+  uint                 server_status;
+  enum wsrep_exec_mode wsrep_exec_mode;
+  Vio                  *vio;
+  ulong                tx_isolation;
+  char                 *db;
+  size_t               db_length;
+};
+
+// Global wsrep parameters
+extern wsrep_t*    wsrep;
+
+// MySQL wsrep options
+extern const char* wsrep_provider;
+extern const char* wsrep_provider_options;
+extern const char* wsrep_cluster_name;
+extern const char* wsrep_cluster_address;
+extern const char* wsrep_node_name;
+extern const char* wsrep_node_address;
+extern const char* wsrep_node_incoming_address;
+extern const char* wsrep_data_home_dir;
+extern const char* wsrep_dbug_option;
+extern long        wsrep_slave_threads;
+extern int         wsrep_slave_count_change;
+extern MYSQL_PLUGIN_IMPORT my_bool wsrep_debug;
+extern my_bool     wsrep_convert_LOCK_to_trx;
+extern ulong       wsrep_retry_autocommit;
+extern my_bool     wsrep_auto_increment_control;
+extern my_bool     wsrep_drupal_282555_workaround;
+extern my_bool     wsrep_incremental_data_collection;
+extern const char* wsrep_start_position;
+extern ulong       wsrep_max_ws_size;
+extern ulong       wsrep_max_ws_rows;
+extern const char* wsrep_notify_cmd;
+extern my_bool     wsrep_certify_nonPK;
+extern long        wsrep_max_protocol_version;
+extern long        wsrep_protocol_version;
+extern ulong       wsrep_forced_binlog_format;
+extern ulong       wsrep_OSU_method_options;
+extern my_bool     wsrep_desync;
+extern my_bool     wsrep_recovery;
+extern my_bool     wsrep_replicate_myisam;
+extern my_bool     wsrep_log_conflicts;
+extern ulong       wsrep_mysql_replication_bundle;
+extern my_bool     wsrep_load_data_splitting;
+extern my_bool     wsrep_restart_slave;
+extern my_bool     wsrep_restart_slave_activated;
+extern my_bool     wsrep_slave_FK_checks;
+extern my_bool     wsrep_slave_UK_checks;
+
+enum enum_wsrep_OSU_method { WSREP_OSU_TOI, WSREP_OSU_RSU };
+enum enum_wsrep_sync_wait {
+    WSREP_SYNC_WAIT_NONE = 0x0,
+    // show, select, begin
+    WSREP_SYNC_WAIT_BEFORE_READ = 0x1,
+    WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE = 0x2,
+    WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE = 0x4,
+    WSREP_SYNC_WAIT_MAX = 0x7
+};
+
+// MySQL status variables
+extern my_bool     wsrep_connected;
+extern my_bool     wsrep_ready;
+extern const char* wsrep_cluster_state_uuid;
+extern long long   wsrep_cluster_conf_id;
+extern const char* wsrep_cluster_status;
+extern long        wsrep_cluster_size;
+extern long        wsrep_local_index;
+extern long long   wsrep_local_bf_aborts;
+extern const char* wsrep_provider_name;
+extern const char* wsrep_provider_version;
+extern const char* wsrep_provider_vendor;
+
+int  wsrep_show_status(THD *thd, SHOW_VAR *var, char *buff);
+void wsrep_free_status(THD *thd);
+
+/* Filters out --wsrep-new-cluster oprtion from argv[]
+ * should be called in the very beginning of main() */
+void wsrep_filter_new_cluster (int* argc, char* argv[]);
+
+int  wsrep_init();
+void wsrep_deinit(bool free_options);
+void wsrep_recover();
+bool wsrep_before_SE(); // initialize wsrep before storage
+                        // engines (true) or after (false)
+/* wsrep initialization sequence at startup
+ * @param before wsrep_before_SE() value */
+void wsrep_init_startup(bool before);
+
+// Other wsrep global variables
+extern my_bool     wsrep_inited; // whether wsrep is initialized ?
+
+extern "C" enum wsrep_exec_mode wsrep_thd_exec_mode(THD *thd);
+extern "C" enum wsrep_conflict_state wsrep_thd_conflict_state(THD *thd);
+extern "C" enum wsrep_query_state wsrep_thd_query_state(THD *thd);
+extern "C" const char * wsrep_thd_exec_mode_str(THD *thd);
+extern "C" const char * wsrep_thd_conflict_state_str(THD *thd);
+extern "C" const char * wsrep_thd_query_state_str(THD *thd);
+extern "C" wsrep_ws_handle_t* wsrep_thd_ws_handle(THD *thd);
+
+extern "C" void wsrep_thd_set_exec_mode(THD *thd, enum wsrep_exec_mode mode);
+extern "C" void wsrep_thd_set_query_state(
+        THD *thd, enum wsrep_query_state state);
+extern "C" void wsrep_thd_set_conflict_state(
+        THD *thd, enum wsrep_conflict_state state);
+
+extern "C" void wsrep_thd_set_trx_to_replay(THD *thd, uint64 trx_id);
+
+extern "C" void wsrep_thd_LOCK(THD *thd);
+extern "C" void wsrep_thd_UNLOCK(THD *thd);
+extern "C" uint32 wsrep_thd_wsrep_rand(THD *thd);
+extern "C" time_t wsrep_thd_query_start(THD *thd);
+extern "C" my_thread_id wsrep_thd_thread_id(THD *thd);
+extern "C" int64_t wsrep_thd_trx_seqno(THD *thd);
+extern "C" query_id_t wsrep_thd_query_id(THD *thd);
+extern "C" char * wsrep_thd_query(THD *thd);
+extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd);
+extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id);
+extern "C" void wsrep_thd_awake(THD *thd, my_bool signal);
+extern "C" int wsrep_thd_retry_counter(THD *thd);
+
+
+extern void wsrep_close_client_connections(my_bool wait_to_end);
+extern int  wsrep_wait_committing_connections_close(int wait_time);
+extern void wsrep_close_applier(THD *thd);
+extern void wsrep_wait_appliers_close(THD *thd);
+extern void wsrep_close_applier_threads(int count);
+extern void wsrep_kill_mysql(THD *thd);
+
+/* new defines */
+extern void wsrep_stop_replication(THD *thd);
+extern bool wsrep_start_replication();
+extern bool wsrep_sync_wait (THD* thd, uint mask = WSREP_SYNC_WAIT_BEFORE_READ);
+extern int  wsrep_check_opts (int argc, char* const* argv);
+extern void wsrep_prepend_PATH (const char* path);
+/* some inline functions are defined in wsrep_mysqld_inl.h */
+
+/* Other global variables */
+extern wsrep_seqno_t wsrep_locked_seqno;
+
+#define WSREP_ON \
+  (global_system_variables.wsrep_on)
+
+#define WSREP(thd) \
+  (WSREP_ON && (thd && thd->variables.wsrep_on))
+
+#define WSREP_CLIENT(thd) \
+    (WSREP(thd) && thd->wsrep_client_thread)
+
+#define WSREP_EMULATE_BINLOG(thd) \
+  (WSREP(thd) && wsrep_emulate_bin_log)
+
+// MySQL logging functions don't seem to understand long long length modifer.
+// This is a workaround. It also prefixes all messages with "WSREP"
+#define WSREP_LOG(fun, ...)                                       \
+    {                                                             \
+        char msg[1024] = {'\0'};                                  \
+        snprintf(msg, sizeof(msg) - 1, ## __VA_ARGS__);           \
+        fun("WSREP: %s", msg);                                    \
+    }
+
+#define WSREP_LOG_CONFLICT_THD(thd, role)                                      \
+    WSREP_LOG(sql_print_information, 	                                       \
+      "%s: \n "       	                                                       \
+      "  THD: %lu, mode: %s, state: %s, conflict: %s, seqno: %lld\n "          \
+      "  SQL: %s",							       \
+      role, wsrep_thd_thread_id(thd), wsrep_thd_exec_mode_str(thd),            \
+      wsrep_thd_query_state_str(thd),                                          \
+      wsrep_thd_conflict_state_str(thd), (long long)wsrep_thd_trx_seqno(thd),  \
+      wsrep_thd_query(thd)                                                     \
+    );
+
+#define WSREP_LOG_CONFLICT(bf_thd, victim_thd, bf_abort)		       \
+  if (wsrep_debug || wsrep_log_conflicts)				       \
+  {                                                                            \
+    WSREP_LOG(sql_print_information, "cluster conflict due to %s for threads:",\
+      (bf_abort) ? "high priority abort" : "certification failure"             \
+    );                                                                         \
+    if (bf_thd != NULL) WSREP_LOG_CONFLICT_THD(bf_thd, "Winning thread");      \
+    if (victim_thd) WSREP_LOG_CONFLICT_THD(victim_thd, "Victim thread");       \
+  }
+
+#define WSREP_PROVIDER_EXISTS                                                  \
+  (wsrep_provider && strncasecmp(wsrep_provider, WSREP_NONE, FN_REFLEN))
+
+extern void wsrep_ready_wait();
+
+enum wsrep_trx_status {
+    WSREP_TRX_OK,
+    WSREP_TRX_CERT_FAIL,      /* certification failure, must abort */
+    WSREP_TRX_SIZE_EXCEEDED,  /* trx size exceeded */
+    WSREP_TRX_ERROR,          /* native mysql error */
+};
+
+extern enum wsrep_trx_status
+wsrep_run_wsrep_commit(THD *thd, handlerton *hton, bool all);
+class Ha_trx_info;
+struct THD_TRANS;
+void wsrep_register_hton(THD* thd, bool all);
+void wsrep_post_commit(THD* thd, bool all);
+void wsrep_brute_force_killer(THD *thd);
+int  wsrep_hire_brute_force_killer(THD *thd, uint64_t trx_id);
+
+extern "C" bool wsrep_consistency_check(void *thd_ptr);
+
+/* this is visible for client build so that innodb plugin gets this */
+typedef struct wsrep_aborting_thd {
+  struct wsrep_aborting_thd *next;
+  THD *aborting_thd;
+} *wsrep_aborting_thd_t;
+
+extern mysql_mutex_t LOCK_wsrep_ready;
+extern mysql_cond_t  COND_wsrep_ready;
+extern mysql_mutex_t LOCK_wsrep_sst;
+extern mysql_cond_t  COND_wsrep_sst;
+extern mysql_mutex_t LOCK_wsrep_sst_init;
+extern mysql_cond_t  COND_wsrep_sst_init;
+extern mysql_mutex_t LOCK_wsrep_rollback;
+extern mysql_cond_t  COND_wsrep_rollback;
+extern int wsrep_replaying;
+extern mysql_mutex_t LOCK_wsrep_replaying;
+extern mysql_cond_t  COND_wsrep_replaying;
+extern mysql_mutex_t LOCK_wsrep_slave_threads;
+extern mysql_mutex_t LOCK_wsrep_desync;
+extern wsrep_aborting_thd_t wsrep_aborting_thd;
+extern my_bool       wsrep_emulate_bin_log;
+extern int           wsrep_to_isolation;
+#ifdef GTID_SUPPORT
+extern rpl_sidno     wsrep_sidno;
+#endif /* GTID_SUPPORT */
+extern my_bool       wsrep_preordered_opt;
+extern handlerton    *wsrep_hton;
+
+#ifdef HAVE_PSI_INTERFACE
+extern PSI_mutex_key key_LOCK_wsrep_ready;
+extern PSI_mutex_key key_COND_wsrep_ready;
+extern PSI_mutex_key key_LOCK_wsrep_sst;
+extern PSI_cond_key  key_COND_wsrep_sst;
+extern PSI_mutex_key key_LOCK_wsrep_sst_init;
+extern PSI_cond_key  key_COND_wsrep_sst_init;
+extern PSI_mutex_key key_LOCK_wsrep_sst_thread;
+extern PSI_cond_key  key_COND_wsrep_sst_thread;
+extern PSI_mutex_key key_LOCK_wsrep_rollback;
+extern PSI_cond_key  key_COND_wsrep_rollback;
+extern PSI_mutex_key key_LOCK_wsrep_replaying;
+extern PSI_cond_key  key_COND_wsrep_replaying;
+extern PSI_mutex_key key_LOCK_wsrep_slave_threads;
+extern PSI_mutex_key key_LOCK_wsrep_desync;
+#endif /* HAVE_PSI_INTERFACE */
+struct TABLE_LIST;
+int wsrep_to_isolation_begin(THD *thd, char *db_, char *table_,
+                             const TABLE_LIST* table_list);
+void wsrep_to_isolation_end(THD *thd);
+void wsrep_cleanup_transaction(THD *thd);
+int wsrep_to_buf_helper(
+  THD* thd, const char *query, uint query_len, uchar** buf, size_t* buf_len);
+int wsrep_create_sp(THD *thd, uchar** buf, size_t* buf_len);
+int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len);
+int wsrep_create_event_query(THD *thd, uchar** buf, size_t* buf_len);
+int wsrep_alter_event_query(THD *thd, uchar** buf, size_t* buf_len);
+
+struct xid_t;
+void wsrep_get_SE_checkpoint(xid_t*);
+void wsrep_set_SE_checkpoint(xid_t*);
+void wsrep_init_sidno(const wsrep_uuid_t&);
+void wsrep_xid_init(xid_t*, const wsrep_uuid_t*, wsrep_seqno_t);
+const wsrep_uuid_t* wsrep_xid_uuid(const xid_t*);
+wsrep_seqno_t wsrep_xid_seqno(const xid_t*);
+extern "C" int wsrep_is_wsrep_xid(const void* xid);
+
+extern "C" my_thread_id wsrep_thd_thread_id(THD *thd);
+extern "C" char *wsrep_thd_query(THD *thd);
+
+extern bool
+wsrep_grant_mdl_exception(MDL_context *requestor_ctx,
+                           MDL_ticket *ticket);
+IO_CACHE * get_trans_log(THD * thd);
+bool wsrep_trans_cache_is_empty(THD *thd);
+void thd_binlog_flush_pending_rows_event(THD *thd, bool stmt_end);
+void thd_binlog_rollback_stmt(THD * thd);
+void thd_binlog_trx_reset(THD * thd);
+
+typedef void (*wsrep_thd_processor_fun)(THD *);
+pthread_handler_t start_wsrep_THD(void *arg);
+int wsrep_wait_committing_connections_close(int wait_time);
+void wsrep_close_client_connections(my_bool wait_to_end);
+void wsrep_close_applier(THD *thd);
+void wsrep_close_applier_threads(int count);
+void wsrep_wait_appliers_close(THD *thd);
+void wsrep_kill_mysql(THD *thd);
+void wsrep_close_threads(THD *thd);
+int wsrep_create_sp(THD *thd, uchar** buf, size_t* buf_len);
+my_bool wsrep_read_only_option(THD *thd, TABLE_LIST *all_tables);
+void wsrep_copy_query(THD *thd);
+bool wsrep_is_show_query(enum enum_sql_command command);
+void wsrep_replay_transaction(THD *thd);
+bool wsrep_create_like_table(THD* thd, TABLE_LIST* table,
+                             TABLE_LIST* src_table,
+	                     HA_CREATE_INFO *create_info);
+int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len);
+
+extern my_bool deny_updates_if_read_only_option(THD *thd,
+	TABLE_LIST *all_tables);
+#endif /* WSREP_MYSQLD_H */
diff --git a/sql/wsrep_notify.cc b/sql/wsrep_notify.cc
new file mode 100644
index 00000000000..6eefb961b62
--- /dev/null
+++ b/sql/wsrep_notify.cc
@@ -0,0 +1,111 @@
+/* Copyright 2010 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include <mysqld.h>
+#include "wsrep_priv.h"
+#include "wsrep_utils.h"
+
+const char* wsrep_notify_cmd="";
+
+static const char* _status_str(wsrep_member_status_t status)
+{
+  switch (status)
+  {
+  case WSREP_MEMBER_UNDEFINED: return "Undefined";
+  case WSREP_MEMBER_JOINER:    return "Joiner";
+  case WSREP_MEMBER_DONOR:     return "Donor";
+  case WSREP_MEMBER_JOINED:    return "Joined";
+  case WSREP_MEMBER_SYNCED:    return "Synced";
+  default:                     return "Error(?)";
+  }
+}
+
+void wsrep_notify_status (wsrep_member_status_t    status,
+                          const wsrep_view_info_t* view)
+{
+  if (!wsrep_notify_cmd || 0 == strlen(wsrep_notify_cmd))
+  {
+    WSREP_INFO("wsrep_notify_cmd is not defined, skipping notification.");
+    return;
+  }
+
+  char  cmd_buf[1 << 16]; // this can be long
+  long  cmd_len = sizeof(cmd_buf) - 1;
+  char* cmd_ptr = cmd_buf;
+  long  cmd_off = 0;
+
+  cmd_off += snprintf (cmd_ptr + cmd_off, cmd_len - cmd_off, "%s",
+                       wsrep_notify_cmd);
+
+  if (status >= WSREP_MEMBER_UNDEFINED && status < WSREP_MEMBER_ERROR)
+  {
+    cmd_off += snprintf (cmd_ptr + cmd_off, cmd_len - cmd_off, " --status %s",
+                         _status_str(status));
+  }
+  else
+  {
+    /* here we preserve provider error codes */
+    cmd_off += snprintf (cmd_ptr + cmd_off, cmd_len - cmd_off,
+                         " --status 'Error(%d)'", status);
+  }
+
+  if (0 != view)
+  {
+    char uuid_str[40];
+
+    wsrep_uuid_print (&view->state_id.uuid, uuid_str, sizeof(uuid_str));
+    cmd_off += snprintf (cmd_ptr + cmd_off, cmd_len - cmd_off,
+                         " --uuid %s", uuid_str);
+
+    cmd_off += snprintf (cmd_ptr + cmd_off, cmd_len - cmd_off,
+                         " --primary %s", view->view >= 0 ? "yes" : "no");
+
+    cmd_off += snprintf (cmd_ptr + cmd_off, cmd_len - cmd_off,
+                         " --index %d", view->my_idx);
+
+    if (view->memb_num)
+    {
+        cmd_off += snprintf (cmd_ptr + cmd_off, cmd_len - cmd_off, " --members");
+
+        for (int i = 0; i < view->memb_num; i++)
+        {
+            wsrep_uuid_print (&view->members[i].id, uuid_str, sizeof(uuid_str));
+            cmd_off += snprintf (cmd_ptr + cmd_off, cmd_len - cmd_off,
+                                 "%c%s/%s/%s", i > 0 ? ',' : ' ',
+                                 uuid_str, view->members[i].name,
+                                 view->members[i].incoming);
+        }
+    }
+  }
+
+  if (cmd_off == cmd_len)
+  {
+    WSREP_ERROR("Notification buffer too short (%ld). Aborting notification.",
+               cmd_len);
+    return;
+  }
+
+  wsp::process p(cmd_ptr, "r");
+
+  p.wait();
+  int err = p.error();
+
+  if (err)
+  {
+    WSREP_ERROR("Notification command failed: %d (%s): \"%s\"",
+                err, strerror(err), cmd_ptr);
+  }
+}
+
diff --git a/sql/wsrep_priv.h b/sql/wsrep_priv.h
new file mode 100644
index 00000000000..5c66587d757
--- /dev/null
+++ b/sql/wsrep_priv.h
@@ -0,0 +1,53 @@
+/* Copyright 2010 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+//! @file declares symbols private to wsrep integration layer
+
+#ifndef WSREP_PRIV_H
+#define WSREP_PRIV_H
+
+#include "wsrep_mysqld.h"
+#include "../wsrep/wsrep_api.h"
+
+#include <log.h>
+#include <pthread.h>
+#include <cstdio>
+
+void    wsrep_ready_set (my_bool x);
+
+ssize_t wsrep_sst_prepare   (void** msg);
+wsrep_cb_status wsrep_sst_donate_cb (void* app_ctx,
+                                     void* recv_ctx,
+                                     const void* msg, size_t msg_len,
+                                     const wsrep_gtid_t* current_id,
+                                     const char* state, size_t state_len,
+                                     bool bypass);
+extern unsigned int wsrep_check_ip (const char* addr);
+extern size_t wsrep_guess_ip (char* buf, size_t buf_len);
+extern size_t wsrep_guess_address(char* buf, size_t buf_len);
+
+extern wsrep_uuid_t  local_uuid;
+extern wsrep_seqno_t local_seqno;
+
+// a helper function
+extern void wsrep_sst_received(wsrep_t*, const wsrep_uuid_t*, wsrep_seqno_t,
+                               const void*, size_t);
+/*! SST thread signals init thread about sst completion */
+extern void wsrep_sst_complete(const wsrep_uuid_t*, wsrep_seqno_t, bool);
+
+void wsrep_notify_status (wsrep_member_status_t new_status,
+                          const wsrep_view_info_t* view = 0);
+#endif /* WSREP_PRIV_H */
diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc
new file mode 100644
index 00000000000..c122d88f7cc
--- /dev/null
+++ b/sql/wsrep_sst.cc
@@ -0,0 +1,1127 @@
+/* Copyright 2008-2012 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include "wsrep_sst.h"
+
+#include <mysqld.h>
+#include <m_ctype.h>
+#include <my_sys.h>
+#include <strfunc.h>
+#include <sql_class.h>
+#include <set_var.h>
+#include <sql_acl.h>
+#include <sql_reload.h>
+#include <sql_parse.h>
+#include "wsrep_priv.h"
+#include "wsrep_utils.h"
+#include <cstdio>
+#include <cstdlib>
+
+extern const char wsrep_defaults_file[];
+
+const char* wsrep_sst_method          = WSREP_SST_DEFAULT;
+const char* wsrep_sst_receive_address = WSREP_SST_ADDRESS_AUTO;
+const char* wsrep_sst_donor           = "";
+      char* wsrep_sst_auth            = NULL;
+
+// container for real auth string
+static const char* sst_auth_real      = NULL;
+my_bool wsrep_sst_donor_rejects_queries = FALSE;
+
+bool wsrep_sst_method_check (sys_var *self, THD* thd, set_var* var)
+{
+  if ((! var->save_result.string_value.str) ||
+      (var->save_result.string_value.length == 0 ))
+  {
+    my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), var->var->name.str,
+             var->save_result.string_value.str ?
+             var->save_result.string_value.str : "NULL");
+    return 1;
+  }
+
+  return 0;
+}
+
+bool wsrep_sst_method_update (sys_var *self, THD* thd, enum_var_type type)
+{
+    return 0;
+}
+
+// TODO: Improve address verification.
+static bool sst_receive_address_check (const char* str)
+{
+    if (!strncasecmp(str, "127.0.0.1", strlen("127.0.0.1")) ||
+        !strncasecmp(str, "localhost", strlen("localhost")))
+    {
+        return 1;
+    }
+
+    return 0;
+}
+
+bool  wsrep_sst_receive_address_check (sys_var *self, THD* thd, set_var* var)
+{
+  char addr_buf[FN_REFLEN];
+
+  if ((! var->save_result.string_value.str) ||
+      (var->save_result.string_value.length > (FN_REFLEN - 1))) // safety
+  {
+    goto err;
+  }
+
+  memcpy(addr_buf, var->save_result.string_value.str,
+         var->save_result.string_value.length);
+  addr_buf[var->save_result.string_value.length]= 0;
+
+  if (sst_receive_address_check(addr_buf))
+  {
+    goto err;
+  }
+
+  return 0;
+
+err:
+  my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), var->var->name.str,
+           var->save_result.string_value.str ?
+           var->save_result.string_value.str : "NULL");
+  return 1;
+}
+
+bool wsrep_sst_receive_address_update (sys_var *self, THD* thd,
+                                       enum_var_type type)
+{
+    return 0;
+}
+
+bool wsrep_sst_auth_check (sys_var *self, THD* thd, set_var* var)
+{
+    return 0;
+}
+
+static bool sst_auth_real_set (const char* value)
+{
+  const char* v= NULL;
+
+  if (value)
+  {
+    v= my_strdup(value, MYF(0));
+  }
+  else                                          // its NULL
+  {
+    wsrep_sst_auth_free();
+    return 0;
+  }
+
+  if (v)
+  {
+    // set sst_auth_real
+    if (sst_auth_real) { my_free((void *) sst_auth_real); }
+    sst_auth_real = v;
+
+    // mask wsrep_sst_auth
+    if (strlen(sst_auth_real))
+    {
+      if (wsrep_sst_auth) { my_free((void*) wsrep_sst_auth); }
+      wsrep_sst_auth= my_strdup(WSREP_SST_AUTH_MASK, MYF(0));
+    }
+    return 0;
+  }
+  return 1;
+}
+
+void wsrep_sst_auth_free()
+{
+  if (wsrep_sst_auth) { my_free((void *) wsrep_sst_auth); }
+  if (sst_auth_real) { my_free((void *) sst_auth_real); }
+  wsrep_sst_auth= NULL;
+  sst_auth_real= NULL;
+}
+
+bool wsrep_sst_auth_update (sys_var *self, THD* thd, enum_var_type type)
+{
+    return sst_auth_real_set (wsrep_sst_auth);
+}
+
+void wsrep_sst_auth_init (const char* value)
+{
+    if (wsrep_sst_auth == value) wsrep_sst_auth = NULL;
+    if (value) sst_auth_real_set (value);
+}
+
+bool  wsrep_sst_donor_check (sys_var *self, THD* thd, set_var* var)
+{
+  return 0;
+}
+
+bool wsrep_sst_donor_update (sys_var *self, THD* thd, enum_var_type type)
+{
+    return 0;
+}
+
+static wsrep_uuid_t cluster_uuid = WSREP_UUID_UNDEFINED;
+
+bool wsrep_before_SE()
+{
+  return (wsrep_provider != NULL
+          && strcmp (wsrep_provider,   WSREP_NONE)
+          && strcmp (wsrep_sst_method, WSREP_SST_SKIP)
+          && strcmp (wsrep_sst_method, WSREP_SST_MYSQLDUMP));
+}
+
+static bool            sst_complete = false;
+static bool            sst_needed   = false;
+
+void wsrep_sst_grab ()
+{
+  WSREP_INFO("wsrep_sst_grab()");
+  if (mysql_mutex_lock (&LOCK_wsrep_sst)) abort();
+  sst_complete = false;
+  mysql_mutex_unlock (&LOCK_wsrep_sst);
+}
+
+// Wait for end of SST
+bool wsrep_sst_wait ()
+{
+  if (mysql_mutex_lock (&LOCK_wsrep_sst)) abort();
+  while (!sst_complete)
+  {
+    WSREP_INFO("Waiting for SST to complete.");
+    mysql_cond_wait (&COND_wsrep_sst, &LOCK_wsrep_sst);
+  }
+
+  if (local_seqno >= 0)
+  {
+    WSREP_INFO("SST complete, seqno: %lld", (long long) local_seqno);
+  }
+  else
+  {
+    WSREP_ERROR("SST failed: %d (%s)",
+                int(-local_seqno), strerror(-local_seqno));
+  }
+
+  mysql_mutex_unlock (&LOCK_wsrep_sst);
+
+  return (local_seqno >= 0);
+}
+
+// Signal end of SST
+void wsrep_sst_complete (const wsrep_uuid_t* sst_uuid,
+                         wsrep_seqno_t       sst_seqno,
+                         bool                needed)
+{
+  if (mysql_mutex_lock (&LOCK_wsrep_sst)) abort();
+  if (!sst_complete)
+  {
+    sst_complete = true;
+    sst_needed   = needed;
+    local_uuid   = *sst_uuid;
+    local_seqno  = sst_seqno;
+    mysql_cond_signal (&COND_wsrep_sst);
+  }
+  else
+  {
+    /* This can happen when called from wsrep_synced_cb().
+       At the moment there is no way to check there
+       if main thread is still waiting for signal,
+       so wsrep_sst_complete() is called from there
+       each time wsrep_ready changes from FALSE -> TRUE.
+    */
+    WSREP_DEBUG("Nobody is waiting for SST.");
+  }
+  mysql_mutex_unlock (&LOCK_wsrep_sst);
+}
+
+void wsrep_sst_received (wsrep_t*            const wsrep,
+                         const wsrep_uuid_t* const uuid,
+                         wsrep_seqno_t       const seqno,
+                         const void*         const state,
+                         size_t              const state_len)
+{
+    int const rcode(seqno < 0 ? seqno : 0);
+    wsrep_gtid_t const state_id = {
+        *uuid, (rcode ? WSREP_SEQNO_UNDEFINED : seqno)
+    };
+#ifdef GTID_SUPPORT
+    wsrep_init_sidno(state_id.uuid);
+#endif /* GTID_SUPPORT */
+    wsrep->sst_received(wsrep, &state_id, state, state_len, rcode);
+}
+
+// Let applier threads to continue
+void wsrep_sst_continue ()
+{
+  if (sst_needed)
+  {
+    WSREP_INFO("Signalling provider to continue.");
+    wsrep_sst_received (wsrep, &local_uuid, local_seqno, NULL, 0);
+  }
+}
+
+struct sst_thread_arg
+{
+  const char*     cmd;
+  int             err;
+  char*           ret_str;
+  mysql_mutex_t   lock;
+  mysql_cond_t    cond;
+
+  sst_thread_arg (const char* c) : cmd(c), err(-1), ret_str(0)
+  {
+    mysql_mutex_init(key_LOCK_wsrep_sst_thread, &lock, MY_MUTEX_INIT_FAST);
+    mysql_cond_init(key_COND_wsrep_sst_thread, &cond, NULL);
+  }
+
+  ~sst_thread_arg()
+  {
+    mysql_cond_destroy  (&cond);
+    mysql_mutex_unlock  (&lock);
+    mysql_mutex_destroy (&lock);
+  }
+};
+
+static int sst_scan_uuid_seqno (const char* str,
+                                wsrep_uuid_t* uuid, wsrep_seqno_t* seqno)
+{
+  int offt = wsrep_uuid_scan (str, strlen(str), uuid);
+  if (offt > 0 && strlen(str) > (unsigned int)offt && ':' == str[offt])
+  {
+    *seqno = strtoll (str + offt + 1, NULL, 10);
+    if (*seqno != LLONG_MAX || errno != ERANGE)
+    {
+      return 0;
+    }
+  }
+
+  WSREP_ERROR("Failed to parse uuid:seqno pair: '%s'", str);
+  return EINVAL;
+}
+
+// get rid of trailing \n
+static char* my_fgets (char* buf, size_t buf_len, FILE* stream)
+{
+   char* ret= fgets (buf, buf_len, stream);
+
+   if (ret)
+   {
+       size_t len = strlen(ret);
+       if (len > 0 && ret[len - 1] == '\n') ret[len - 1] = '\0';
+   }
+
+   return ret;
+}
+
+/*
+  Generate opt_binlog_opt_val for sst_donate_other(), sst_prepare_other().
+
+  Returns zero on success, negative error code otherwise.
+
+  String containing binlog name is stored in param ret if binlog is enabled
+  and GTID mode is on, otherwise empty string. Returned string should be
+  freed with my_free().
+ */
+static int generate_binlog_opt_val(char** ret)
+{
+  DBUG_ASSERT(ret);
+  *ret= NULL;
+  if (opt_bin_log)
+  {
+    assert(opt_bin_logname);
+    *ret= strcmp(opt_bin_logname, "0") ?
+        my_strdup(opt_bin_logname, MYF(0)) : my_strdup("", MYF(0));
+  }
+  else
+  {
+    *ret= my_strdup("", MYF(0));
+  }
+  if (!*ret) return -ENOMEM;
+  return 0;
+}
+
+static void* sst_joiner_thread (void* a)
+{
+  sst_thread_arg* arg= (sst_thread_arg*) a;
+  int err= 1;
+
+  {
+    const char magic[] = "ready";
+    const size_t magic_len = sizeof(magic) - 1;
+    const size_t out_len = 512;
+    char out[out_len];
+
+    WSREP_INFO("Running: '%s'", arg->cmd);
+
+    wsp::process proc (arg->cmd, "r");
+
+    if (proc.pipe() && !proc.error())
+    {
+      const char* tmp= my_fgets (out, out_len, proc.pipe());
+
+      if (!tmp || strlen(tmp) < (magic_len + 2) ||
+          strncasecmp (tmp, magic, magic_len))
+      {
+        WSREP_ERROR("Failed to read '%s <addr>' from: %s\n\tRead: '%s'",
+                    magic, arg->cmd, tmp);
+        proc.wait();
+        if (proc.error()) err = proc.error();
+      }
+      else
+      {
+        err = 0;
+      }
+    }
+    else
+    {
+      err = proc.error();
+      WSREP_ERROR("Failed to execute: %s : %d (%s)",
+                  arg->cmd, err, strerror(err));
+    }
+
+    // signal sst_prepare thread with ret code,
+    // it will go on sending SST request
+    mysql_mutex_lock   (&arg->lock);
+    if (!err)
+    {
+      arg->ret_str = strdup (out + magic_len + 1);
+      if (!arg->ret_str) err = ENOMEM;
+    }
+    arg->err = -err;
+    mysql_cond_signal  (&arg->cond);
+    mysql_mutex_unlock (&arg->lock); //! @note arg is unusable after that.
+
+    if (err) return NULL; /* lp:808417 - return immediately, don't signal
+                           * initializer thread to ensure single thread of
+                           * shutdown. */
+
+    wsrep_uuid_t  ret_uuid  = WSREP_UUID_UNDEFINED;
+    wsrep_seqno_t ret_seqno = WSREP_SEQNO_UNDEFINED;
+
+    // in case of successfull receiver start, wait for SST completion/end
+    char* tmp = my_fgets (out, out_len, proc.pipe());
+
+    proc.wait();
+    err= EINVAL;
+
+    if (!tmp)
+    {
+      WSREP_ERROR("Failed to read uuid:seqno from joiner script.");
+      if (proc.error()) err = proc.error();
+    }
+    else
+    {
+      err= sst_scan_uuid_seqno (out, &ret_uuid, &ret_seqno);
+    }
+
+    if (err)
+    {
+      ret_uuid=  WSREP_UUID_UNDEFINED;
+      ret_seqno= -err;
+    }
+
+    // Tell initializer thread that SST is complete
+    wsrep_sst_complete (&ret_uuid, ret_seqno, true);
+  }
+
+  return NULL;
+}
+
+static ssize_t sst_prepare_other (const char*  method,
+                                  const char*  addr_in,
+                                  const char** addr_out)
+{
+  ssize_t cmd_len= 1024;
+  char    cmd_str[cmd_len];
+  const char* sst_dir= mysql_real_data_home;
+  const char* binlog_opt= "";
+  char* binlog_opt_val= NULL;
+
+  int ret;
+  if ((ret= generate_binlog_opt_val(&binlog_opt_val)))
+  {
+    WSREP_ERROR("sst_prepare_other(): generate_binlog_opt_val() failed: %d",
+                ret);
+    return ret;
+  }
+  if (strlen(binlog_opt_val)) binlog_opt= WSREP_SST_OPT_BINLOG;
+
+
+  ret= snprintf (cmd_str, cmd_len,
+                 "wsrep_sst_%s "
+                 WSREP_SST_OPT_ROLE" 'joiner' "
+                 WSREP_SST_OPT_ADDR" '%s' "
+                 WSREP_SST_OPT_AUTH" '%s' "
+                 WSREP_SST_OPT_DATA" '%s' "
+                 WSREP_SST_OPT_CONF" '%s' "
+                 WSREP_SST_OPT_PARENT" '%d'"
+                 " %s '%s' ",
+                 method, addr_in, (sst_auth_real) ? sst_auth_real : "",
+                 sst_dir, wsrep_defaults_file, (int)getpid(),
+                 binlog_opt, binlog_opt_val);
+  my_free(binlog_opt_val);
+
+  if (ret < 0 || ret >= cmd_len)
+  {
+    WSREP_ERROR("sst_prepare_other(): snprintf() failed: %d", ret);
+    return (ret < 0 ? ret : -EMSGSIZE);
+  }
+
+  pthread_t tmp;
+  sst_thread_arg arg(cmd_str);
+  mysql_mutex_lock (&arg.lock);
+  ret = pthread_create (&tmp, NULL, sst_joiner_thread, &arg);
+  if (ret)
+  {
+    WSREP_ERROR("sst_prepare_other(): pthread_create() failed: %d (%s)",
+                ret, strerror(ret));
+    return ret;
+  }
+  mysql_cond_wait (&arg.cond, &arg.lock);
+
+  *addr_out= arg.ret_str;
+
+  if (!arg.err)
+    ret = strlen(*addr_out);
+  else
+  {
+    assert (arg.err < 0);
+    ret = arg.err;
+  }
+
+  pthread_detach (tmp);
+
+  return ret;
+}
+
+extern uint  mysqld_port;
+
+/*! Just tells donor where to send mysqldump */
+static ssize_t sst_prepare_mysqldump (const char*  addr_in,
+                                      const char** addr_out)
+{
+  ssize_t ret = strlen (addr_in);
+
+  if (!strrchr(addr_in, ':'))
+  {
+    ssize_t s = ret + 7;
+    char* tmp = (char*) malloc (s);
+
+    if (tmp)
+    {
+      ret= snprintf (tmp, s, "%s:%u", addr_in, mysqld_port);
+
+      if (ret > 0 && ret < s)
+      {
+        *addr_out= tmp;
+        return ret;
+      }
+      if (ret > 0) /* buffer too short */ ret = -EMSGSIZE;
+      free (tmp);
+    }
+    else {
+      ret= -ENOMEM;
+    }
+
+    WSREP_ERROR ("Could not prepare state transfer request: "
+                 "adding default port failed: %zd.", ret);
+  }
+  else {
+    *addr_out= addr_in;
+  }
+
+  return ret;
+}
+
+static bool SE_initialized = false;
+
+ssize_t wsrep_sst_prepare (void** msg)
+{
+  const ssize_t ip_max= 256;
+  char ip_buf[ip_max];
+  const char* addr_in=  NULL;
+  const char* addr_out= NULL;
+
+  if (!strcmp(wsrep_sst_method, WSREP_SST_SKIP))
+  {
+    ssize_t ret = strlen(WSREP_STATE_TRANSFER_TRIVIAL) + 1;
+    *msg = strdup(WSREP_STATE_TRANSFER_TRIVIAL);
+    if (!msg)
+    {
+      WSREP_ERROR("Could not allocate %zd bytes for state request", ret);
+      unireg_abort(1);
+    }
+    return ret;
+  }
+
+  // Figure out SST address. Common for all SST methods
+  if (wsrep_sst_receive_address &&
+    strcmp (wsrep_sst_receive_address, WSREP_SST_ADDRESS_AUTO))
+  {
+    addr_in= wsrep_sst_receive_address;
+  }
+  else if (wsrep_node_address && strlen(wsrep_node_address))
+  {
+    const char* const colon= strchr (wsrep_node_address, ':');
+    if (colon)
+    {
+      ptrdiff_t const len= colon - wsrep_node_address;
+      strncpy (ip_buf, wsrep_node_address, len);
+      ip_buf[len]= '\0';
+      addr_in= ip_buf;
+    }
+    else
+    {
+      addr_in= wsrep_node_address;
+    }
+  }
+  else
+  {
+    ssize_t ret= wsrep_guess_ip (ip_buf, ip_max);
+
+    if (ret && ret < ip_max)
+    {
+      addr_in= ip_buf;
+    }
+    else
+    {
+      WSREP_ERROR("Could not prepare state transfer request: "
+                  "failed to guess address to accept state transfer at. "
+                  "wsrep_sst_receive_address must be set manually.");
+      unireg_abort(1);
+    }
+  }
+
+  ssize_t addr_len= -ENOSYS;
+  if (!strcmp(wsrep_sst_method, WSREP_SST_MYSQLDUMP))
+  {
+    addr_len= sst_prepare_mysqldump (addr_in, &addr_out);
+    if (addr_len < 0) unireg_abort(1);
+  }
+  else
+  {
+    /*! A heuristic workaround until we learn how to stop and start engines */
+    if (SE_initialized)
+    {
+      // we already did SST at initializaiton, now engines are running
+      // sql_print_information() is here because the message is too long
+      // for WSREP_INFO.
+      sql_print_information ("WSREP: "
+                 "You have configured '%s' state snapshot transfer method "
+                 "which cannot be performed on a running server. "
+                 "Wsrep provider won't be able to fall back to it "
+                 "if other means of state transfer are unavailable. "
+                 "In that case you will need to restart the server.",
+                 wsrep_sst_method);
+      *msg = 0;
+      return 0;
+    }
+
+    addr_len = sst_prepare_other (wsrep_sst_method, addr_in, &addr_out);
+    if (addr_len < 0)
+    {
+      WSREP_ERROR("Failed to prepare for '%s' SST. Unrecoverable.",
+                   wsrep_sst_method);
+      unireg_abort(1);
+    }
+  }
+
+  size_t const method_len(strlen(wsrep_sst_method));
+  size_t const msg_len   (method_len + addr_len + 2 /* + auth_len + 1*/);
+
+  *msg = malloc (msg_len);
+  if (NULL != *msg) {
+    char* const method_ptr(reinterpret_cast<char*>(*msg));
+    strcpy (method_ptr, wsrep_sst_method);
+    char* const addr_ptr(method_ptr + method_len + 1);
+    strcpy (addr_ptr, addr_out);
+
+    WSREP_INFO ("Prepared SST request: %s|%s", method_ptr, addr_ptr);
+  }
+  else {
+    WSREP_ERROR("Failed to allocate SST request of size %zu. Can't continue.",
+                msg_len);
+    unireg_abort(1);
+  }
+
+  if (addr_out != addr_in) /* malloc'ed */ free ((char*)addr_out);
+
+  return msg_len;
+}
+
+// helper method for donors
+static int sst_run_shell (const char* cmd_str, int max_tries)
+{
+  int ret = 0;
+
+  for (int tries=1; tries <= max_tries; tries++)
+  {
+    wsp::process proc (cmd_str, "r");
+
+    if (NULL != proc.pipe())
+    {
+      proc.wait();
+    }
+
+    if ((ret = proc.error()))
+    {
+      WSREP_ERROR("Try %d/%d: '%s' failed: %d (%s)",
+                  tries, max_tries, proc.cmd(), ret, strerror(ret));
+      sleep (1);
+    }
+    else
+    {
+      WSREP_DEBUG("SST script successfully completed.");
+      break;
+    }
+  }
+
+  return -ret;
+}
+
+static void sst_reject_queries(my_bool close_conn)
+{
+    wsrep_ready_set (FALSE); // this will be resotred when donor becomes synced
+    WSREP_INFO("Rejecting client queries for the duration of SST.");
+    if (TRUE == close_conn) wsrep_close_client_connections(FALSE);
+}
+
+static int sst_mysqldump_check_addr (const char* user, const char* pswd,
+                                     const char* host, const char* port)
+{
+  return 0;
+}
+
+static int sst_donate_mysqldump (const char*         addr,
+                                 const wsrep_uuid_t* uuid,
+                                 const char*         uuid_str,
+                                 wsrep_seqno_t       seqno,
+                                 bool                bypass)
+{
+  size_t host_len;
+  const char* port = strchr (addr, ':');
+
+  if (port)
+  {
+    port += 1;
+    host_len = port - addr;
+  }
+  else
+  {
+    port = "";
+    host_len = strlen (addr) + 1;
+  }
+
+  char host[host_len];
+
+  strncpy (host, addr, host_len - 1);
+  host[host_len - 1] = '\0';
+
+  const char* auth = sst_auth_real;
+  const char* pswd = (auth) ? strchr (auth, ':') : NULL;
+  size_t user_len;
+
+  if (pswd)
+  {
+    pswd += 1;
+    user_len = pswd - auth;
+  }
+  else
+  {
+    pswd = "";
+    user_len = (auth) ? strlen (auth) + 1 : 1;
+  }
+
+  char user[user_len];
+
+  strncpy (user, (auth) ? auth : "", user_len - 1);
+  user[user_len - 1] = '\0';
+
+  int ret = sst_mysqldump_check_addr (user, pswd, host, port);
+  if (!ret)
+  {
+    size_t cmd_len= 1024;
+    char   cmd_str[cmd_len];
+
+    if (!bypass && wsrep_sst_donor_rejects_queries) sst_reject_queries(TRUE);
+
+    snprintf (cmd_str, cmd_len,
+              "wsrep_sst_mysqldump "
+              WSREP_SST_OPT_USER" '%s' "
+              WSREP_SST_OPT_PSWD" '%s' "
+              WSREP_SST_OPT_HOST" '%s' "
+              WSREP_SST_OPT_PORT" '%s' "
+              WSREP_SST_OPT_LPORT" '%u' "
+              WSREP_SST_OPT_SOCKET" '%s' "
+              WSREP_SST_OPT_CONF" '%s' "
+              WSREP_SST_OPT_GTID" '%s:%lld'"
+              "%s",
+              user, pswd, host, port, mysqld_port, mysqld_unix_port,
+              wsrep_defaults_file, uuid_str,
+              (long long)seqno, bypass ? " "WSREP_SST_OPT_BYPASS : "");
+
+    WSREP_DEBUG("Running: '%s'", cmd_str);
+
+    ret= sst_run_shell (cmd_str, 3);
+  }
+
+  wsrep_gtid_t const state_id = { *uuid, (ret ? WSREP_SEQNO_UNDEFINED : seqno)};
+
+  wsrep->sst_sent (wsrep, &state_id, ret);
+
+  return ret;
+}
+
+wsrep_seqno_t wsrep_locked_seqno= WSREP_SEQNO_UNDEFINED;
+
+static int run_sql_command(THD *thd, const char *query)
+{
+  thd->set_query((char *)query, strlen(query));
+
+  Parser_state ps;
+  if (ps.init(thd, thd->query(), thd->query_length()))
+  {
+    WSREP_ERROR("SST query: %s failed", query);
+    return -1;
+  }
+
+  mysql_parse(thd, thd->query(), thd->query_length(), &ps);
+  if (thd->is_error())
+  {
+    int const err= thd->get_stmt_da()->sql_errno();
+    WSREP_WARN ("error executing '%s': %d (%s)%s",
+                query, err, thd->get_stmt_da()->message(),
+                err == ER_UNKNOWN_SYSTEM_VARIABLE ?
+                ". Was mysqld built with --with-innodb-disallow-writes ?" : "");
+    thd->clear_error();
+    return -1;
+  }
+  return 0;
+}
+
+static int sst_flush_tables(THD* thd)
+{
+  WSREP_INFO("Flushing tables for SST...");
+
+  int err;
+  int not_used;
+  CHARSET_INFO *current_charset;
+
+  current_charset = thd->variables.character_set_client;
+
+  if (!is_supported_parser_charset(current_charset))
+  {
+      /* Do not use non-supported parser character sets */
+      WSREP_WARN("Current client character set is non-supported parser character set: %s", current_charset->csname);
+      thd->variables.character_set_client = &my_charset_latin1;
+      WSREP_WARN("For SST temporally setting character set to : %s",
+	      my_charset_latin1.csname);
+  }
+
+  if (run_sql_command(thd, "FLUSH TABLES WITH READ LOCK"))
+  {
+    WSREP_ERROR("Failed to flush and lock tables");
+    err = -1;
+  }
+  else
+  {
+    /* make sure logs are flushed after global read lock acquired */
+    err= reload_acl_and_cache(thd, REFRESH_ENGINE_LOG | REFRESH_BINARY_LOG,
+			      (TABLE_LIST*) 0, &not_used);
+  }
+
+  thd->variables.character_set_client = current_charset;
+
+
+  if (err)
+  {
+    WSREP_ERROR("Failed to flush tables: %d (%s)", err, strerror(err));
+  }
+  else
+  {
+    WSREP_INFO("Tables flushed.");
+    const char base_name[]= "tables_flushed";
+    ssize_t const full_len= strlen(mysql_real_data_home) + strlen(base_name)+2;
+    char real_name[full_len];
+    sprintf(real_name, "%s/%s", mysql_real_data_home, base_name);
+    char tmp_name[full_len + 4];
+    sprintf(tmp_name, "%s.tmp", real_name);
+
+    FILE* file= fopen(tmp_name, "w+");
+    if (0 == file)
+    {
+      err= errno;
+      WSREP_ERROR("Failed to open '%s': %d (%s)", tmp_name, err,strerror(err));
+    }
+    else
+    {
+      fprintf(file, "%s:%lld\n",
+              wsrep_cluster_state_uuid, (long long)wsrep_locked_seqno);
+      fsync(fileno(file));
+      fclose(file);
+      if (rename(tmp_name, real_name) == -1)
+      {
+        err= errno;
+        WSREP_ERROR("Failed to rename '%s' to '%s': %d (%s)",
+                     tmp_name, real_name, err,strerror(err));
+      }
+    }
+  }
+
+  return err;
+}
+
+static void sst_disallow_writes (THD* thd, bool yes)
+{
+  char query_str[64] = { 0, };
+  ssize_t const query_max = sizeof(query_str) - 1;
+  CHARSET_INFO *current_charset;
+
+  current_charset = thd->variables.character_set_client;
+
+  if (!is_supported_parser_charset(current_charset))
+  {
+      /* Do not use non-supported parser character sets */
+      WSREP_WARN("Current client character set is non-supported parser character set: %s", current_charset->csname);
+      thd->variables.character_set_client = &my_charset_latin1;
+      WSREP_WARN("For SST temporally setting character set to : %s",
+	      my_charset_latin1.csname);
+  }
+
+  snprintf (query_str, query_max, "SET GLOBAL innodb_disallow_writes=%d",
+            yes ? 1 : 0);
+
+  if (run_sql_command(thd, query_str))
+  {
+    WSREP_ERROR("Failed to disallow InnoDB writes");
+  }
+  thd->variables.character_set_client = current_charset;
+}
+
+static void* sst_donor_thread (void* a)
+{
+  sst_thread_arg* arg= (sst_thread_arg*)a;
+
+  WSREP_INFO("Running: '%s'", arg->cmd);
+
+  int  err= 1;
+  bool locked= false;
+
+  const char*  out= NULL;
+  const size_t out_len= 128;
+  char         out_buf[out_len];
+
+  wsrep_uuid_t  ret_uuid= WSREP_UUID_UNDEFINED;
+  wsrep_seqno_t ret_seqno= WSREP_SEQNO_UNDEFINED; // seqno of complete SST
+
+  wsp::thd thd(FALSE); // we turn off wsrep_on for this THD so that it can
+                       // operate with wsrep_ready == OFF
+  wsp::process proc(arg->cmd, "r");
+
+  err= proc.error();
+
+/* Inform server about SST script startup and release TO isolation */
+  mysql_mutex_lock   (&arg->lock);
+  arg->err = -err;
+  mysql_cond_signal  (&arg->cond);
+  mysql_mutex_unlock (&arg->lock); //! @note arg is unusable after that.
+
+  if (proc.pipe() && !err)
+  {
+wait_signal:
+    out= my_fgets (out_buf, out_len, proc.pipe());
+
+    if (out)
+    {
+      const char magic_flush[]= "flush tables";
+      const char magic_cont[]= "continue";
+      const char magic_done[]= "done";
+
+      if (!strcasecmp (out, magic_flush))
+      {
+        err= sst_flush_tables (thd.ptr);
+        if (!err)
+        {
+          sst_disallow_writes (thd.ptr, true);
+          locked= true;
+          goto wait_signal;
+        }
+      }
+      else if (!strcasecmp (out, magic_cont))
+      {
+        if (locked)
+        {
+          sst_disallow_writes (thd.ptr, false);
+          thd.ptr->global_read_lock.unlock_global_read_lock (thd.ptr);
+          locked= false;
+        }
+        err=  0;
+        goto wait_signal;
+      }
+      else if (!strncasecmp (out, magic_done, strlen(magic_done)))
+      {
+        err= sst_scan_uuid_seqno (out + strlen(magic_done) + 1,
+                                  &ret_uuid, &ret_seqno);
+      }
+      else
+      {
+        WSREP_WARN("Received unknown signal: '%s'", out);
+      }
+    }
+    else
+    {
+      WSREP_ERROR("Failed to read from: %s", proc.cmd());
+      proc.wait();
+    }
+    if (!err && proc.error()) err= proc.error();
+  }
+  else
+  {
+    WSREP_ERROR("Failed to execute: %s : %d (%s)",
+                proc.cmd(), err, strerror(err));
+  }
+
+  if (locked) // don't forget to unlock server before return
+  {
+    sst_disallow_writes (thd.ptr, false);
+    thd.ptr->global_read_lock.unlock_global_read_lock (thd.ptr);
+  }
+
+  // signal to donor that SST is over
+  struct wsrep_gtid const state_id = {
+      ret_uuid, err ? WSREP_SEQNO_UNDEFINED : ret_seqno
+  };
+  wsrep->sst_sent (wsrep, &state_id, -err);
+  proc.wait();
+
+  return NULL;
+}
+
+
+
+static int sst_donate_other (const char*   method,
+                             const char*   addr,
+                             const char*   uuid,
+                             wsrep_seqno_t seqno,
+                             bool          bypass)
+{
+  ssize_t cmd_len = 4096;
+  char    cmd_str[cmd_len];
+  const char* binlog_opt= "";
+  char* binlog_opt_val= NULL;
+
+  int ret;
+  if ((ret= generate_binlog_opt_val(&binlog_opt_val)))
+  {
+    WSREP_ERROR("sst_donate_other(): generate_binlog_opt_val() failed: %d",ret);
+    return ret;
+  }
+  if (strlen(binlog_opt_val)) binlog_opt= WSREP_SST_OPT_BINLOG;
+
+  ret= snprintf (cmd_str, cmd_len,
+                 "wsrep_sst_%s "
+                 WSREP_SST_OPT_ROLE" 'donor' "
+                 WSREP_SST_OPT_ADDR" '%s' "
+                 WSREP_SST_OPT_AUTH" '%s' "
+                 WSREP_SST_OPT_SOCKET" '%s' "
+                 WSREP_SST_OPT_DATA" '%s' "
+                 WSREP_SST_OPT_CONF" '%s' "
+                 " %s '%s' "
+                 WSREP_SST_OPT_GTID" '%s:%lld'"
+                 "%s",
+                 method, addr, sst_auth_real, mysqld_unix_port,
+                 mysql_real_data_home, wsrep_defaults_file,
+                 binlog_opt, binlog_opt_val,
+                 uuid, (long long) seqno,
+                 bypass ? " "WSREP_SST_OPT_BYPASS : "");
+  my_free(binlog_opt_val);
+
+  if (ret < 0 || ret >= cmd_len)
+  {
+    WSREP_ERROR("sst_donate_other(): snprintf() failed: %d", ret);
+    return (ret < 0 ? ret : -EMSGSIZE);
+  }
+
+  if (!bypass && wsrep_sst_donor_rejects_queries) sst_reject_queries(FALSE);
+
+  pthread_t tmp;
+  sst_thread_arg arg(cmd_str);
+  mysql_mutex_lock (&arg.lock);
+  ret = pthread_create (&tmp, NULL, sst_donor_thread, &arg);
+  if (ret)
+  {
+    WSREP_ERROR("sst_donate_other(): pthread_create() failed: %d (%s)",
+                ret, strerror(ret));
+    return ret;
+  }
+  mysql_cond_wait (&arg.cond, &arg.lock);
+
+  WSREP_INFO("sst_donor_thread signaled with %d", arg.err);
+  return arg.err;
+}
+
+wsrep_cb_status_t wsrep_sst_donate_cb (void* app_ctx, void* recv_ctx,
+                                       const void* msg, size_t msg_len,
+                                       const wsrep_gtid_t* current_gtid,
+                                       const char* state, size_t state_len,
+                                       bool bypass)
+{
+  /* This will be reset when sync callback is called.
+   * Should we set wsrep_ready to FALSE here too? */
+//  wsrep_notify_status(WSREP_MEMBER_DONOR);
+  local_status.set(WSREP_MEMBER_DONOR);
+
+  const char* method = (char*)msg;
+  size_t method_len  = strlen (method);
+  const char* data   = method + method_len + 1;
+
+  char uuid_str[37];
+  wsrep_uuid_print (&current_gtid->uuid, uuid_str, sizeof(uuid_str));
+
+  int ret;
+  if (!strcmp (WSREP_SST_MYSQLDUMP, method))
+  {
+    ret = sst_donate_mysqldump(data, &current_gtid->uuid, uuid_str,
+                               current_gtid->seqno, bypass);
+  }
+  else
+  {
+    ret = sst_donate_other(method, data, uuid_str, current_gtid->seqno,bypass);
+  }
+
+  return (ret > 0 ? WSREP_CB_SUCCESS : WSREP_CB_FAILURE);
+}
+
+void wsrep_SE_init_grab()
+{
+  if (mysql_mutex_lock (&LOCK_wsrep_sst_init)) abort();
+}
+
+void wsrep_SE_init_wait()
+{
+  while (SE_initialized == false)
+  {
+    mysql_cond_wait (&COND_wsrep_sst_init, &LOCK_wsrep_sst_init);
+  }
+  mysql_mutex_unlock (&LOCK_wsrep_sst_init);
+}
+
+void wsrep_SE_init_done()
+{
+  mysql_cond_signal (&COND_wsrep_sst_init);
+  mysql_mutex_unlock (&LOCK_wsrep_sst_init);
+}
+
+void wsrep_SE_initialized()
+{
+  SE_initialized = true;
+}
diff --git a/sql/wsrep_sst.h b/sql/wsrep_sst.h
new file mode 100644
index 00000000000..d85fed97fca
--- /dev/null
+++ b/sql/wsrep_sst.h
@@ -0,0 +1,68 @@
+/* Copyright (C) 2013 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with this program; if not, write to the Free Software Foundation, Inc.,
+   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+
+#if !defined(WSREP_SST_H) && defined(WITH_WSREP)
+#define WSREP_SST_H
+
+#include <mysql.h>                    // my_bool
+
+#define WSREP_SST_OPT_ROLE     "--role"
+#define WSREP_SST_OPT_ADDR     "--address"
+#define WSREP_SST_OPT_AUTH     "--auth"
+#define WSREP_SST_OPT_DATA     "--datadir"
+#define WSREP_SST_OPT_CONF     "--defaults-file"
+#define WSREP_SST_OPT_PARENT   "--parent"
+#define WSREP_SST_OPT_BINLOG   "--binlog"
+
+// mysqldump-specific options
+#define WSREP_SST_OPT_USER     "--user"
+#define WSREP_SST_OPT_PSWD     "--password"
+#define WSREP_SST_OPT_HOST     "--host"
+#define WSREP_SST_OPT_PORT     "--port"
+#define WSREP_SST_OPT_LPORT    "--local-port"
+
+// donor-specific
+#define WSREP_SST_OPT_SOCKET   "--socket"
+#define WSREP_SST_OPT_GTID     "--gtid"
+#define WSREP_SST_OPT_BYPASS   "--bypass"
+
+#define WSREP_SST_MYSQLDUMP    "mysqldump"
+#define WSREP_SST_RSYNC        "rsync"
+#define WSREP_SST_SKIP         "skip"
+#define WSREP_SST_DEFAULT      WSREP_SST_RSYNC
+#define WSREP_SST_ADDRESS_AUTO "AUTO"
+#define WSREP_SST_AUTH_MASK    "********"
+
+/* system variables */
+extern const char* wsrep_sst_method;
+extern const char* wsrep_sst_receive_address;
+extern const char* wsrep_sst_donor;
+extern       char* wsrep_sst_auth;
+extern    my_bool  wsrep_sst_donor_rejects_queries;
+
+/*! Synchronizes applier thread start with init thread */
+extern void wsrep_sst_grab();
+/*! Init thread waits for SST completion */
+extern bool wsrep_sst_wait();
+/*! Signals wsrep that initialization is complete, writesets can be applied */
+extern void wsrep_sst_continue();
+extern void wsrep_sst_auth_free();
+
+extern void wsrep_SE_init_grab();   /*! grab init critical section */
+extern void wsrep_SE_init_wait();   /*! wait for SE init to complete */
+extern void wsrep_SE_init_done();   /*! signal that SE init is complte */
+extern void wsrep_SE_initialized(); /*! mark SE initialization complete */
+
+#endif /* WSREP_SST_H */
diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc
new file mode 100644
index 00000000000..fabced2d11a
--- /dev/null
+++ b/sql/wsrep_thd.cc
@@ -0,0 +1,602 @@
+/* Copyright (C) 2013 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with this program; if not, write to the Free Software Foundation, Inc.,
+   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+
+#include "wsrep_thd.h"
+
+#include "transaction.h"
+#include "rpl_rli.h"
+#include "log_event.h"
+#include "sql_parse.h"
+//#include "global_threads.h" // LOCK_thread_count, etc.
+#include "sql_base.h" // close_thread_tables()
+#include "mysqld.h"   // start_wsrep_THD();
+
+#include "slave.h"    // opt_log_slave_updates
+#include "rpl_filter.h"
+#include "rpl_rli.h"
+#include "rpl_mi.h"
+
+#if (__LP64__)
+static volatile int64 wsrep_bf_aborts_counter(0);
+#define WSREP_ATOMIC_LOAD_LONG my_atomic_load64
+#define WSREP_ATOMIC_ADD_LONG  my_atomic_add64
+#else
+static volatile int32 wsrep_bf_aborts_counter(0);
+#define WSREP_ATOMIC_LOAD_LONG my_atomic_load32
+#define WSREP_ATOMIC_ADD_LONG  my_atomic_add32
+#endif
+
+int wsrep_show_bf_aborts (THD *thd, SHOW_VAR *var, char *buff)
+{
+    wsrep_local_bf_aborts = WSREP_ATOMIC_LOAD_LONG(&wsrep_bf_aborts_counter);
+    var->type = SHOW_LONGLONG;
+    var->value = (char*)&wsrep_local_bf_aborts;
+    return 0;
+}
+
+/* must have (&thd->LOCK_wsrep_thd) */
+void wsrep_client_rollback(THD *thd)
+{
+  WSREP_DEBUG("client rollback due to BF abort for (%ld), query: %s",
+              thd->thread_id, thd->query());
+
+  WSREP_ATOMIC_ADD_LONG(&wsrep_bf_aborts_counter, 1);
+
+  thd->wsrep_conflict_state= ABORTING;
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  trans_rollback(thd);
+
+  if (thd->locked_tables_mode && thd->lock)
+  {
+    WSREP_DEBUG("unlocking tables for BF abort (%ld)", thd->thread_id);
+    thd->locked_tables_list.unlock_locked_tables(thd);
+    thd->variables.option_bits&= ~(OPTION_TABLE_LOCK);
+  }
+
+  if (thd->global_read_lock.is_acquired())
+  {
+    WSREP_DEBUG("unlocking GRL for BF abort (%ld)", thd->thread_id);
+    thd->global_read_lock.unlock_global_read_lock(thd);
+  }
+
+  /* Release transactional metadata locks. */
+  thd->mdl_context.release_transactional_locks();
+
+  /* release explicit MDL locks */
+  thd->mdl_context.release_explicit_locks();
+
+  if (thd->get_binlog_table_maps())
+  {
+    WSREP_DEBUG("clearing binlog table map for BF abort (%ld)", thd->thread_id);
+    thd->clear_binlog_table_maps();
+  }
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  thd->wsrep_conflict_state= ABORTED;
+}
+
+#define NUMBER_OF_FIELDS_TO_IDENTIFY_COORDINATOR 1
+#define NUMBER_OF_FIELDS_TO_IDENTIFY_WORKER 2
+//#include "rpl_info_factory.h"
+
+static Relay_log_info* wsrep_relay_log_init(const char* log_fname)
+{
+
+  /* MySQL 5.6 version has rli factory: */
+#ifdef MYSQL_56
+  uint rli_option = INFO_REPOSITORY_DUMMY;
+  Relay_log_info *rli= NULL;
+  rli = Rpl_info_factory::create_rli(rli_option, false);
+  rli->set_rli_description_event(
+      new Format_description_log_event(BINLOG_VERSION));
+#endif
+  Relay_log_info* rli= new Relay_log_info(false);
+  rli->sql_driver_thd= current_thd;
+ 
+  rli->no_storage= true;
+  rli->relay_log.description_event_for_exec=
+    new Format_description_log_event(4);
+
+  return rli;
+}
+
+class Master_info;
+
+static rpl_group_info* wsrep_relay_group_init(const char* log_fname)
+{
+  Relay_log_info* rli= new Relay_log_info(false);
+
+  rli->no_storage= true;
+  if (!rli->relay_log.description_event_for_exec)
+  {
+    rli->relay_log.description_event_for_exec=
+      new Format_description_log_event(4);
+  }
+  static LEX_STRING dbname= { C_STRING_WITH_LEN("mysql") };
+
+  rli->mi = new Master_info( &dbname,  false);
+  //rli->mi = new Master_info( &(C_STRING_WITH_LEN("wsrep")),  false);
+
+  rli->mi->rpl_filter = new Rpl_filter;
+  copy_filter_setting(rli->mi->rpl_filter, get_or_create_rpl_filter("", 0));
+
+  rli->sql_driver_thd= current_thd;
+
+  struct rpl_group_info *rgi= new rpl_group_info(rli);
+  rgi->thd= current_thd;
+
+  return rgi;
+}
+
+static void wsrep_prepare_bf_thd(THD *thd, struct wsrep_thd_shadow* shadow)
+{
+  shadow->options       = thd->variables.option_bits;
+  shadow->server_status = thd->server_status;
+  shadow->wsrep_exec_mode = thd->wsrep_exec_mode;
+  shadow->vio           = thd->net.vio;
+
+  if (opt_log_slave_updates)
+    thd->variables.option_bits|= OPTION_BIN_LOG;
+  else
+    thd->variables.option_bits&= ~(OPTION_BIN_LOG);
+
+  //if (!thd->wsrep_rli) thd->wsrep_rli= wsrep_relay_log_init("wsrep_relay");
+  if (!thd->wsrep_rgi) thd->wsrep_rgi= wsrep_relay_group_init("wsrep_relay");
+  //  thd->wsrep_rli->info_thd = thd;
+
+  thd->wsrep_exec_mode= REPL_RECV;
+  thd->net.vio= 0;
+  thd->clear_error();
+
+  shadow->tx_isolation        = thd->variables.tx_isolation;
+  thd->variables.tx_isolation = ISO_READ_COMMITTED;
+  thd->tx_isolation           = ISO_READ_COMMITTED;
+
+  shadow->db            = thd->db;
+  shadow->db_length     = thd->db_length;
+  thd->reset_db(NULL, 0);
+}
+
+static void wsrep_return_from_bf_mode(THD *thd, struct wsrep_thd_shadow* shadow)
+{
+  thd->variables.option_bits  = shadow->options;
+  thd->server_status          = shadow->server_status;
+  thd->wsrep_exec_mode        = shadow->wsrep_exec_mode;
+  thd->net.vio                = shadow->vio;
+  thd->variables.tx_isolation = shadow->tx_isolation;
+  thd->reset_db(shadow->db, shadow->db_length);
+
+  delete thd->wsrep_rgi->rli->mi->rpl_filter;
+  delete thd->wsrep_rgi->rli->mi;
+  delete thd->wsrep_rgi->rli;
+  delete thd->wsrep_rgi;
+  thd->wsrep_rgi = NULL;
+;
+}
+
+void wsrep_replay_transaction(THD *thd)
+{
+  /* checking if BF trx must be replayed */
+  if (thd->wsrep_conflict_state== MUST_REPLAY) {
+    DBUG_ASSERT(wsrep_thd_trx_seqno(thd));
+    if (thd->wsrep_exec_mode!= REPL_RECV) {
+      if (thd->get_stmt_da()->is_sent())
+      {
+        WSREP_ERROR("replay issue, thd has reported status already");
+      }
+      thd->get_stmt_da()->reset_diagnostics_area();
+
+      thd->wsrep_conflict_state= REPLAYING;
+      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+      mysql_reset_thd_for_next_command(thd);
+      thd->killed= NOT_KILLED;
+      close_thread_tables(thd);
+      if (thd->locked_tables_mode && thd->lock)
+      {
+        WSREP_DEBUG("releasing table lock for replaying (%ld)",
+                    thd->thread_id);
+        thd->locked_tables_list.unlock_locked_tables(thd);
+        thd->variables.option_bits&= ~(OPTION_TABLE_LOCK);
+      }
+      thd->mdl_context.release_transactional_locks();
+      /*
+        Replaying will call MYSQL_START_STATEMENT when handling
+        BEGIN Query_log_event so end statement must be called before
+        replaying.
+      */
+      MYSQL_END_STATEMENT(thd->m_statement_psi, thd->get_stmt_da());
+      thd->m_statement_psi= NULL;
+      thd_proc_info(thd, "wsrep replaying trx");
+      WSREP_DEBUG("replay trx: %s %lld",
+                  thd->query() ? thd->query() : "void",
+                  (long long)wsrep_thd_trx_seqno(thd));
+      struct wsrep_thd_shadow shadow;
+      wsrep_prepare_bf_thd(thd, &shadow);
+
+      /* From trans_begin() */
+      thd->variables.option_bits|= OPTION_BEGIN;
+      thd->server_status|= SERVER_STATUS_IN_TRANS;
+
+      int rcode = wsrep->replay_trx(wsrep,
+                                    &thd->wsrep_ws_handle,
+                                    (void *)thd);
+
+      wsrep_return_from_bf_mode(thd, &shadow);
+      if (thd->wsrep_conflict_state!= REPLAYING)
+        WSREP_WARN("lost replaying mode: %d", thd->wsrep_conflict_state );
+
+      mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+
+      switch (rcode)
+      {
+      case WSREP_OK:
+        thd->wsrep_conflict_state= NO_CONFLICT;
+        wsrep->post_commit(wsrep, &thd->wsrep_ws_handle);
+        WSREP_DEBUG("trx_replay successful for: %ld %llu",
+                    thd->thread_id, (long long)thd->real_id);
+        if (thd->get_stmt_da()->is_sent())
+        {
+          WSREP_WARN("replay ok, thd has reported status");
+        }
+        else if (thd->get_stmt_da()->is_set())
+        {
+          if (thd->get_stmt_da()->status() != Diagnostics_area::DA_OK)
+          {
+            WSREP_WARN("replay ok, thd has error status %d",
+                       thd->get_stmt_da()->status());
+          }
+        }
+        else
+        {
+          my_ok(thd);
+        }
+        break;
+      case WSREP_TRX_FAIL:
+        if (thd->get_stmt_da()->is_sent())
+        {
+          WSREP_ERROR("replay failed, thd has reported status");
+        }
+        else
+        {
+          WSREP_DEBUG("replay failed, rolling back");
+          //my_error(ER_LOCK_DEADLOCK, MYF(0), "wsrep aborted transaction");
+        }
+        thd->wsrep_conflict_state= ABORTED;
+        wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle);
+        break;
+      default:
+        WSREP_ERROR("trx_replay failed for: %d, query: %s",
+                    rcode, thd->query() ? thd->query() : "void");
+        /* we're now in inconsistent state, must abort */
+        unireg_abort(1);
+        break;
+      }
+
+      wsrep_cleanup_transaction(thd);
+
+      mysql_mutex_lock(&LOCK_wsrep_replaying);
+      wsrep_replaying--;
+      WSREP_DEBUG("replaying decreased: %d, thd: %lu",
+                  wsrep_replaying, thd->thread_id);
+      mysql_cond_broadcast(&COND_wsrep_replaying);
+      mysql_mutex_unlock(&LOCK_wsrep_replaying);
+    }
+  }
+}
+
+static void wsrep_replication_process(THD *thd)
+{
+  int rcode;
+  DBUG_ENTER("wsrep_replication_process");
+
+  struct wsrep_thd_shadow shadow;
+  wsrep_prepare_bf_thd(thd, &shadow);
+
+  /* From trans_begin() */
+  thd->variables.option_bits|= OPTION_BEGIN;
+  thd->server_status|= SERVER_STATUS_IN_TRANS;
+
+  rcode = wsrep->recv(wsrep, (void *)thd);
+  DBUG_PRINT("wsrep",("wsrep_repl returned: %d", rcode));
+
+  WSREP_INFO("applier thread exiting (code:%d)", rcode);
+
+  switch (rcode) {
+  case WSREP_OK:
+  case WSREP_NOT_IMPLEMENTED:
+  case WSREP_CONN_FAIL:
+    /* provider does not support slave operations / disconnected from group,
+     * just close applier thread */
+    break;
+  case WSREP_NODE_FAIL:
+    /* data inconsistency => SST is needed */
+    /* Note: we cannot just blindly restart replication here,
+     * SST might require server restart if storage engines must be
+     * initialized after SST */
+    WSREP_ERROR("node consistency compromised, aborting");
+    wsrep_kill_mysql(thd);
+    break;
+  case WSREP_WARNING:
+  case WSREP_TRX_FAIL:
+  case WSREP_TRX_MISSING:
+    /* these suggests a bug in provider code */
+    WSREP_WARN("bad return from recv() call: %d", rcode);
+    /* fall through to node shutdown */
+  case WSREP_FATAL:
+    /* Cluster connectivity is lost.
+     *
+     * If applier was killed on purpose (KILL_CONNECTION), we
+     * avoid mysql shutdown. This is because the killer will then handle
+     * shutdown processing (or replication restarting)
+     */
+    if (thd->killed != KILL_CONNECTION)
+    {
+      wsrep_kill_mysql(thd);
+    }
+    break;
+  }
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  wsrep_close_applier(thd);
+  mysql_cond_broadcast(&COND_thread_count);
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  TABLE *tmp;
+  while ((tmp = thd->temporary_tables))
+  {
+    WSREP_WARN("Applier %lu, has temporary tables at exit: %s.%s",
+                  thd->thread_id, 
+                  (tmp->s) ? tmp->s->db.str : "void",
+                  (tmp->s) ? tmp->s->table_name.str : "void");
+  }
+  wsrep_return_from_bf_mode(thd, &shadow);
+  DBUG_VOID_RETURN;
+}
+
+void wsrep_create_appliers(long threads)
+{
+  if (!wsrep_connected)
+  {
+    /* see wsrep_replication_start() for the logic */
+    if (wsrep_cluster_address && strlen(wsrep_cluster_address) &&
+        wsrep_provider && strcasecmp(wsrep_provider, "none"))
+    {
+      WSREP_ERROR("Trying to launch slave threads before creating "
+                  "connection at '%s'", wsrep_cluster_address);
+      assert(0);
+    }
+    return;
+  }
+
+  long wsrep_threads=0;
+  pthread_t hThread;
+  while (wsrep_threads++ < threads) {
+    if (pthread_create(
+      &hThread, &connection_attrib,
+      start_wsrep_THD, (void*)wsrep_replication_process))
+      WSREP_WARN("Can't create thread to manage wsrep replication");
+  }
+}
+
+static void wsrep_rollback_process(THD *thd)
+{
+  DBUG_ENTER("wsrep_rollback_process");
+
+  mysql_mutex_lock(&LOCK_wsrep_rollback);
+  wsrep_aborting_thd= NULL;
+
+  while (thd->killed == NOT_KILLED) {
+    thd_proc_info(thd, "wsrep aborter idle");
+    thd->mysys_var->current_mutex= &LOCK_wsrep_rollback;
+    thd->mysys_var->current_cond=  &COND_wsrep_rollback;
+
+    mysql_cond_wait(&COND_wsrep_rollback,&LOCK_wsrep_rollback);
+
+    WSREP_DEBUG("WSREP rollback thread wakes for signal");
+
+    mysql_mutex_lock(&thd->mysys_var->mutex);
+    thd_proc_info(thd, "wsrep aborter active");
+    thd->mysys_var->current_mutex= 0;
+    thd->mysys_var->current_cond=  0;
+    mysql_mutex_unlock(&thd->mysys_var->mutex);
+
+    /* check for false alarms */
+    if (!wsrep_aborting_thd)
+    {
+      WSREP_DEBUG("WSREP rollback thread has empty abort queue");
+    }
+    /* process all entries in the queue */
+    while (wsrep_aborting_thd) {
+      THD *aborting;
+      wsrep_aborting_thd_t next = wsrep_aborting_thd->next;
+      aborting = wsrep_aborting_thd->aborting_thd;
+      my_free(wsrep_aborting_thd);
+      wsrep_aborting_thd= next;
+      /*
+       * must release mutex, appliers my want to add more
+       * aborting thds in our work queue, while we rollback
+       */
+      mysql_mutex_unlock(&LOCK_wsrep_rollback);
+
+      mysql_mutex_lock(&aborting->LOCK_wsrep_thd);
+      if (aborting->wsrep_conflict_state== ABORTED)
+      {
+        WSREP_DEBUG("WSREP, thd already aborted: %llu state: %d",
+                    (long long)aborting->real_id,
+                    aborting->wsrep_conflict_state);
+
+        mysql_mutex_unlock(&aborting->LOCK_wsrep_thd);
+        mysql_mutex_lock(&LOCK_wsrep_rollback);
+        continue;
+      }
+      aborting->wsrep_conflict_state= ABORTING;
+
+      mysql_mutex_unlock(&aborting->LOCK_wsrep_thd);
+
+      set_current_thd(aborting); 
+      aborting->store_globals();
+
+      mysql_mutex_lock(&aborting->LOCK_wsrep_thd);
+      wsrep_client_rollback(aborting);
+      WSREP_DEBUG("WSREP rollbacker aborted thd: (%lu %llu)",
+                  aborting->thread_id, (long long)aborting->real_id);
+      mysql_mutex_unlock(&aborting->LOCK_wsrep_thd);
+
+      set_current_thd(thd); 
+      thd->store_globals();
+
+      mysql_mutex_lock(&LOCK_wsrep_rollback);
+    }
+  }
+
+  mysql_mutex_unlock(&LOCK_wsrep_rollback);
+  sql_print_information("WSREP: rollbacker thread exiting");
+
+  DBUG_PRINT("wsrep",("wsrep rollbacker thread exiting"));
+  DBUG_VOID_RETURN;
+}
+
+void wsrep_create_rollbacker()
+{
+  if (wsrep_provider && strcasecmp(wsrep_provider, "none"))
+  {
+    pthread_t hThread;
+    /* create rollbacker */
+    if (pthread_create( &hThread, &connection_attrib,
+                        start_wsrep_THD, (void*)wsrep_rollback_process))
+      WSREP_WARN("Can't create thread to manage wsrep rollback");
+  }
+}
+
+void wsrep_thd_set_PA_safe(void *thd_ptr, my_bool safe)
+{ 
+  if (thd_ptr) 
+  {
+    THD* thd = (THD*)thd_ptr;
+    thd->wsrep_PA_safe = safe;
+  }
+}
+
+int wsrep_thd_conflict_state(void *thd_ptr, my_bool sync)
+{ 
+  int state = -1;
+  if (thd_ptr) 
+  {
+    THD* thd = (THD*)thd_ptr;
+    if (sync) mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+    
+    state = thd->wsrep_conflict_state;
+    if (sync) mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  }
+  return state;
+}
+
+my_bool wsrep_thd_is_wsrep(void *thd_ptr)
+{
+  my_bool status = FALSE;
+  if (thd_ptr)
+  {
+    THD* thd = (THD*)thd_ptr;
+
+    status = (WSREP(thd) && WSREP_PROVIDER_EXISTS);
+  }
+  return status;
+}
+
+my_bool wsrep_thd_is_BF(void *thd_ptr, my_bool sync)
+{
+  my_bool status = FALSE;
+  if (thd_ptr)
+  {
+    THD* thd = (THD*)thd_ptr;
+    // THD can be BF only if provider exists
+    if (wsrep_thd_is_wsrep(thd_ptr))
+    {
+      if (sync)
+	mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+
+      status = ((thd->wsrep_exec_mode == REPL_RECV)    ||
+      	        (thd->wsrep_exec_mode == TOTAL_ORDER));
+      if (sync)
+        mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+    }
+  }
+  return status;
+}
+
+extern "C"
+my_bool wsrep_thd_is_BF_or_commit(void *thd_ptr, my_bool sync)
+{
+  bool status = FALSE;
+  if (thd_ptr) 
+  {
+    THD* thd = (THD*)thd_ptr;
+    if (sync) mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+    
+    status = ((thd->wsrep_exec_mode == REPL_RECV)    ||
+	      (thd->wsrep_exec_mode == TOTAL_ORDER)  ||
+	      (thd->wsrep_exec_mode == LOCAL_COMMIT));
+    if (sync) mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  }
+  return status;
+}
+
+extern "C"
+my_bool wsrep_thd_is_local(void *thd_ptr, my_bool sync)
+{
+  bool status = FALSE;
+  if (thd_ptr) 
+  {
+    THD* thd = (THD*)thd_ptr;
+    if (sync) mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+
+    status = (thd->wsrep_exec_mode == LOCAL_STATE);
+    if (sync) mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+  }
+  return status;
+}
+
+int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
+{
+  THD *victim_thd = (THD *) victim_thd_ptr;
+  THD *bf_thd     = (THD *) bf_thd_ptr;
+  DBUG_ENTER("wsrep_abort_thd");
+
+  if ( (WSREP(bf_thd) ||
+         ( (WSREP_ON || wsrep_OSU_method_options == WSREP_OSU_RSU) &&
+           bf_thd->wsrep_exec_mode == TOTAL_ORDER) )               &&
+       victim_thd)
+  {
+    WSREP_DEBUG("wsrep_abort_thd, by: %llu, victim: %llu", (bf_thd) ?
+                (long long)bf_thd->real_id : 0, (long long)victim_thd->real_id);
+    ha_abort_transaction(bf_thd, victim_thd, signal);
+  }
+  else
+  {
+    WSREP_DEBUG("wsrep_abort_thd not effective: %p %p", bf_thd, victim_thd);
+  }
+
+  DBUG_RETURN(1);
+}
+
+extern "C"
+int wsrep_thd_in_locking_session(void *thd_ptr)
+{
+  if (thd_ptr && ((THD *)thd_ptr)->in_lock_tables) {
+    return 1;
+  }
+  return 0;
+}
+
diff --git a/sql/wsrep_thd.h b/sql/wsrep_thd.h
new file mode 100644
index 00000000000..c5a497bb428
--- /dev/null
+++ b/sql/wsrep_thd.h
@@ -0,0 +1,40 @@
+/* Copyright (C) 2013 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with this program; if not, write to the Free Software Foundation, Inc.,
+   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+
+#if !defined(WSREP_THD_H) && defined(WITH_WSREP)
+#define WSREP_THD_H
+
+#include "sql_class.h"
+
+int wsrep_show_bf_aborts (THD *thd, SHOW_VAR *var, char *buff);
+void wsrep_client_rollback(THD *thd);
+void wsrep_replay_transaction(THD *thd);
+void wsrep_create_appliers(long threads);
+void wsrep_create_rollbacker();
+
+int  wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr,
+                                my_bool signal);
+
+extern void  wsrep_thd_set_PA_safe(void *thd_ptr, my_bool safe);
+extern my_bool  wsrep_thd_is_BF(void *thd_ptr, my_bool sync);
+extern my_bool wsrep_thd_is_wsrep(void *thd_ptr);
+
+extern int wsrep_thd_conflict_state(void *thd_ptr, my_bool sync);
+//extern "C" my_bool  wsrep_thd_is_BF(void *thd_ptr, my_bool sync);
+extern "C" my_bool  wsrep_thd_is_BF_or_commit(void *thd_ptr, my_bool sync);
+extern "C" my_bool  wsrep_thd_is_local(void *thd_ptr, my_bool sync);
+extern "C" int  wsrep_thd_in_locking_session(void *thd_ptr);
+
+#endif /* WSREP_THD_H */
diff --git a/sql/wsrep_utils.cc b/sql/wsrep_utils.cc
new file mode 100644
index 00000000000..cdee1c2cece
--- /dev/null
+++ b/sql/wsrep_utils.cc
@@ -0,0 +1,524 @@
+/* Copyright 2010 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+//! @file some utility functions and classes not directly related to replication
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE // POSIX_SPAWN_USEVFORK flag
+#endif
+
+#include "wsrep_utils.h"
+#include "wsrep_mysqld.h"
+
+#include <sql_class.h>
+
+#include <spawn.h>    // posix_spawn()
+#include <unistd.h>   // pipe()
+#include <errno.h>    // errno
+#include <string.h>   // strerror()
+#include <sys/wait.h> // waitpid()
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>    // getaddrinfo()
+
+extern char** environ; // environment variables
+
+static wsp::string wsrep_PATH;
+
+void
+wsrep_prepend_PATH (const char* path)
+{
+    int count = 0;
+
+    while (environ[count])
+    {
+        if (strncmp (environ[count], "PATH=", 5))
+        {
+            count++;
+            continue;
+        }
+
+        char* const old_path (environ[count]);
+
+        if (strstr (old_path, path)) return; // path already there
+
+        size_t const new_path_len(strlen(old_path) + strlen(":") +
+                                  strlen(path) + 1);
+
+        char* const new_path (reinterpret_cast<char*>(malloc(new_path_len)));
+
+        if (new_path)
+        {
+            snprintf (new_path, new_path_len, "PATH=%s:%s", path,
+                      old_path + strlen("PATH="));
+
+            wsrep_PATH.set (new_path);
+            environ[count] = new_path;
+        }
+        else
+        {
+            WSREP_ERROR ("Failed to allocate 'PATH' environment variable "
+                         "buffer of size %zu.", new_path_len);
+        }
+
+        return;
+    }
+
+    WSREP_ERROR ("Failed to find 'PATH' environment variable. "
+                 "State snapshot transfer may not be working.");
+}
+
+namespace wsp
+{
+
+#define PIPE_READ  0
+#define PIPE_WRITE 1
+#define STDIN_FD   0
+#define STDOUT_FD  1
+
+#ifndef POSIX_SPAWN_USEVFORK
+# define POSIX_SPAWN_USEVFORK 0
+#endif
+
+process::process (const char* cmd, const char* type)
+    : str_(cmd ? strdup(cmd) : strdup("")), io_(NULL), err_(EINVAL), pid_(0)
+{
+    if (0 == str_)
+    {
+        WSREP_ERROR ("Can't allocate command line of size: %zu", strlen(cmd));
+        err_ = ENOMEM;
+        return;
+    }
+
+    if (0 == strlen(str_))
+    {
+        WSREP_ERROR ("Can't start a process: null or empty command line.");
+        return;
+    }
+
+    if (NULL == type || (strcmp (type, "w") && strcmp(type, "r")))
+    {
+        WSREP_ERROR ("type argument should be either \"r\" or \"w\".");
+        return;
+    }
+
+    int pipe_fds[2] = { -1, };
+    if (::pipe(pipe_fds))
+    {
+        err_ = errno;
+        WSREP_ERROR ("pipe() failed: %d (%s)", err_, strerror(err_));
+        return;
+    }
+
+    // which end of pipe will be returned to parent
+    int const parent_end (strcmp(type,"w") ? PIPE_READ : PIPE_WRITE);
+    int const child_end  (parent_end == PIPE_READ ? PIPE_WRITE : PIPE_READ);
+    int const close_fd   (parent_end == PIPE_READ ? STDOUT_FD : STDIN_FD);
+
+    char* const pargv[4] = { strdup("sh"), strdup("-c"), strdup(str_), NULL };
+    if (!(pargv[0] && pargv[1] && pargv[2]))
+    {
+        err_ = ENOMEM;
+        WSREP_ERROR ("Failed to allocate pargv[] array.");
+        goto cleanup_pipe;
+    }
+
+    posix_spawnattr_t attr;
+    err_ = posix_spawnattr_init (&attr);
+    if (err_)
+    {
+        WSREP_ERROR ("posix_spawnattr_init() failed: %d (%s)",
+                     err_, strerror(err_));
+        goto cleanup_pipe;
+    }
+
+    err_ = posix_spawnattr_setflags (&attr, POSIX_SPAWN_SETSIGDEF |
+                                            POSIX_SPAWN_USEVFORK);
+    if (err_)
+    {
+        WSREP_ERROR ("posix_spawnattr_setflags() failed: %d (%s)",
+                     err_, strerror(err_));
+        goto cleanup_attr;
+    }
+
+    posix_spawn_file_actions_t fact;
+    err_ = posix_spawn_file_actions_init (&fact);
+    if (err_)
+    {
+        WSREP_ERROR ("posix_spawn_file_actions_init() failed: %d (%s)",
+                     err_, strerror(err_));
+        goto cleanup_attr;
+    }
+
+    // close child's stdout|stdin depending on what we returning
+    err_ = posix_spawn_file_actions_addclose (&fact, close_fd);
+    if (err_)
+    {
+        WSREP_ERROR ("posix_spawn_file_actions_addclose() failed: %d (%s)",
+                     err_, strerror(err_));
+        goto cleanup_fact;
+    }
+
+    // substitute our pipe descriptor in place of the closed one
+    err_ = posix_spawn_file_actions_adddup2 (&fact,
+                                             pipe_fds[child_end], close_fd);
+    if (err_)
+    {
+        WSREP_ERROR ("posix_spawn_file_actions_addup2() failed: %d (%s)",
+                     err_, strerror(err_));
+        goto cleanup_fact;
+    }
+
+    err_ = posix_spawnp (&pid_, pargv[0], &fact, &attr, pargv, environ);
+    if (err_)
+    {
+        WSREP_ERROR ("posix_spawnp(%s) failed: %d (%s)",
+                     pargv[2], err_, strerror(err_));
+        pid_ = 0; // just to make sure it was not messed up in the call
+        goto cleanup_fact;
+    }
+
+    io_ = fdopen (pipe_fds[parent_end], type);
+
+    if (io_)
+    {
+        pipe_fds[parent_end] = -1; // skip close on cleanup
+    }
+    else
+    {
+        err_ = errno;
+        WSREP_ERROR ("fdopen() failed: %d (%s)", err_, strerror(err_));
+    }
+
+cleanup_fact:
+    int err; // to preserve err_ code
+    err = posix_spawn_file_actions_destroy (&fact);
+    if (err)
+    {
+        WSREP_ERROR ("posix_spawn_file_actions_destroy() failed: %d (%s)\n",
+                     err, strerror(err));
+    }
+
+cleanup_attr:
+    err = posix_spawnattr_destroy (&attr);
+    if (err)
+    {
+        WSREP_ERROR ("posix_spawnattr_destroy() failed: %d (%s)",
+                     err, strerror(err));
+    }
+
+cleanup_pipe:
+    if (pipe_fds[0] >= 0) close (pipe_fds[0]);
+    if (pipe_fds[1] >= 0) close (pipe_fds[1]);
+
+    free (pargv[0]);
+    free (pargv[1]);
+    free (pargv[2]);
+}
+
+process::~process ()
+{
+    if (io_)
+    {
+        assert (pid_);
+        assert (str_);
+
+        WSREP_WARN("Closing pipe to child process: %s, PID(%ld) "
+                   "which might still be running.", str_, (long)pid_);
+
+        if (fclose (io_) == -1)
+        {
+            err_ = errno;
+            WSREP_ERROR("fclose() failed: %d (%s)", err_, strerror(err_));
+        }
+    }
+
+    if (str_) free (const_cast<char*>(str_));
+}
+
+int
+process::wait ()
+{
+  if (pid_)
+  {
+      int status;
+      if (-1 == waitpid(pid_, &status, 0))
+      {
+          err_ = errno; assert (err_);
+          WSREP_ERROR("Waiting for process failed: %s, PID(%ld): %d (%s)",
+                      str_, (long)pid_, err_, strerror (err_));
+      }
+      else
+      {                // command completed, check exit status
+          if (WIFEXITED (status)) {
+              err_ = WEXITSTATUS (status);
+          }
+          else {       // command didn't complete with exit()
+              WSREP_ERROR("Process was aborted.");
+              err_ = errno ? errno : ECHILD;
+          }
+
+          if (err_) {
+              switch (err_) /* Translate error codes to more meaningful */
+              {
+              case 126: err_ = EACCES; break; /* Permission denied */
+              case 127: err_ = ENOENT; break; /* No such file or directory */
+              }
+              WSREP_ERROR("Process completed with error: %s: %d (%s)",
+                          str_, err_, strerror(err_));
+          }
+
+          pid_ = 0;
+          if (io_) fclose (io_);
+          io_ = NULL;
+      }
+  }
+  else {
+      assert (NULL == io_);
+      WSREP_ERROR("Command did not run: %s", str_);
+  }
+
+  return err_;
+}
+
+thd::thd (my_bool won) : init(), ptr(new THD)
+{
+  if (ptr)
+  {
+    ptr->thread_stack= (char*) &ptr;
+    ptr->store_globals();
+    ptr->variables.option_bits&= ~OPTION_BIN_LOG; // disable binlog
+    ptr->variables.wsrep_on = won;
+    ptr->security_ctx->master_access= ~(ulong)0;
+    lex_start(ptr);
+  }
+}
+
+thd::~thd ()
+{
+  if (ptr)
+  {
+    delete ptr;
+    my_pthread_setspecific_ptr (THR_THD, 0);
+  }
+}
+
+} // namespace wsp
+
+/* Returns INADDR_NONE, INADDR_ANY, INADDR_LOOPBACK or something else */
+unsigned int wsrep_check_ip (const char* const addr)
+{
+#if 0
+  if (addr && 0 == strcasecmp(addr, MY_BIND_ALL_ADDRESSES)) return INADDR_ANY;
+#endif
+
+  unsigned int ret = INADDR_NONE;
+  struct addrinfo *res, hints;
+
+  memset (&hints, 0, sizeof(hints));
+  hints.ai_flags= AI_PASSIVE/*|AI_ADDRCONFIG*/;
+  hints.ai_socktype= SOCK_STREAM;
+  hints.ai_family= AF_UNSPEC;
+
+  int gai_ret = getaddrinfo(addr, NULL, &hints, &res);
+  if (0 == gai_ret)
+  {
+    if (AF_INET == res->ai_family) /* IPv4 */
+    {
+      struct sockaddr_in* a= (struct sockaddr_in*)res->ai_addr;
+      ret= htonl(a->sin_addr.s_addr);
+    }
+    else /* IPv6 */
+    {
+      struct sockaddr_in6* a= (struct sockaddr_in6*)res->ai_addr;
+      if (IN6_IS_ADDR_UNSPECIFIED(&a->sin6_addr))
+        ret= INADDR_ANY;
+      else if (IN6_IS_ADDR_LOOPBACK(&a->sin6_addr))
+        ret= INADDR_LOOPBACK;
+      else
+        ret= 0xdeadbeef;
+    }
+    freeaddrinfo (res);
+  }
+  else {
+    WSREP_ERROR ("getaddrinfo() failed on '%s': %d (%s)",
+                 addr, gai_ret, gai_strerror(gai_ret));
+  }
+
+  // uint8_t* b= (uint8_t*)&ret;
+  // fprintf (stderr, "########## wsrep_check_ip returning: %hhu.%hhu.%hhu.%hhu\n",
+  //          b[0], b[1], b[2], b[3]);
+
+  return ret;
+}
+
+extern char* my_bind_addr_str;
+
+size_t wsrep_guess_ip (char* buf, size_t buf_len)
+{
+  size_t ip_len = 0;
+
+  if (my_bind_addr_str && strlen(my_bind_addr_str))
+  {
+    unsigned int const ip_type= wsrep_check_ip(my_bind_addr_str);
+
+    if (INADDR_NONE == ip_type) {
+      WSREP_ERROR("Networking not configured, cannot receive state transfer.");
+      return 0;
+    }
+
+    if (INADDR_ANY != ip_type) {;
+      strncpy (buf, my_bind_addr_str, buf_len);
+      return strlen(buf);
+    }
+  }
+
+  // mysqld binds to all interfaces - try IP from wsrep_node_address
+  if (wsrep_node_address && wsrep_node_address[0] != '\0') {
+    const char* const colon_ptr = strchr(wsrep_node_address, ':');
+
+    if (colon_ptr)
+      ip_len = colon_ptr - wsrep_node_address;
+    else
+      ip_len = strlen(wsrep_node_address);
+
+    if (ip_len >= buf_len) {
+      WSREP_WARN("default_ip(): buffer too short: %zu <= %zd", buf_len, ip_len);
+      return 0;
+    }
+
+    memcpy (buf, wsrep_node_address, ip_len);
+    buf[ip_len] = '\0';
+    return ip_len;
+  }
+
+  // try to find the address of the first one
+#if (TARGET_OS_LINUX == 1)
+  const char cmd[] = "/sbin/ifconfig | "
+//      "grep -m1 -1 -E '^[a-z]?eth[0-9]' | tail -n 1 | "
+      "grep -E '^[[:space:]]+inet addr:' | grep -m1 -v 'inet addr:127' | "
+      "sed 's/:/ /' | awk '{ print $3 }'";
+#elif defined(__sun__)
+  const char cmd[] = "/sbin/ifconfig -a | "
+      "/usr/gnu/bin/grep -m1 -1 -E 'net[0-9]:' | tail -n 1 | awk '{ print $2 }'";
+#elif defined(__APPLE__) || defined(__FreeBSD__)
+  const char cmd[] = "/sbin/route -nv get 8.8.8.8 | tail -n1 | awk '{print $(NF)}'";
+#else
+  char *cmd;
+#error "OS not supported"
+#endif
+  wsp::process proc (cmd, "r");
+
+  if (NULL != proc.pipe()) {
+    char* ret;
+
+    ret = fgets (buf, buf_len, proc.pipe());
+
+    if (proc.wait()) return 0;
+
+    if (NULL == ret) {
+      WSREP_ERROR("Failed to read output of: '%s'", cmd);
+      return 0;
+    }
+  }
+  else {
+    WSREP_ERROR("Failed to execute: '%s'", cmd);
+    return 0;
+  }
+
+  // clear possible \n at the end of ip string left by fgets()
+  ip_len = strlen (buf);
+  if (ip_len > 0 && '\n' == buf[ip_len - 1]) {
+    ip_len--;
+    buf[ip_len] = '\0';
+  }
+
+  if (INADDR_NONE == inet_addr(buf)) {
+    if (strlen(buf) != 0) {
+      WSREP_WARN("Shell command returned invalid address: '%s'", buf);
+    }
+    return 0;
+  }
+
+  return ip_len;
+}
+
+size_t wsrep_guess_address(char* buf, size_t buf_len)
+{
+  size_t addr_len = wsrep_guess_ip (buf, buf_len);
+
+  if (addr_len && addr_len < buf_len) {
+    addr_len += snprintf (buf + addr_len, buf_len - addr_len,
+                          ":%u", mysqld_port);
+  }
+
+  return addr_len;
+}
+
+/*
+ * WSREPXid
+ */
+
+#define WSREP_XID_PREFIX "WSREPXid"
+#define WSREP_XID_PREFIX_LEN MYSQL_XID_PREFIX_LEN
+#define WSREP_XID_UUID_OFFSET 8
+#define WSREP_XID_SEQNO_OFFSET (WSREP_XID_UUID_OFFSET + sizeof(wsrep_uuid_t))
+#define WSREP_XID_GTRID_LEN (WSREP_XID_SEQNO_OFFSET + sizeof(wsrep_seqno_t))
+
+void wsrep_xid_init(XID* xid, const wsrep_uuid_t* uuid, wsrep_seqno_t seqno)
+{
+  xid->formatID= 1;
+  xid->gtrid_length= WSREP_XID_GTRID_LEN;
+  xid->bqual_length= 0;
+  memset(xid->data, 0, sizeof(xid->data));
+  memcpy(xid->data, WSREP_XID_PREFIX, WSREP_XID_PREFIX_LEN);
+  memcpy(xid->data + WSREP_XID_UUID_OFFSET, uuid, sizeof(wsrep_uuid_t));
+  memcpy(xid->data + WSREP_XID_SEQNO_OFFSET, &seqno, sizeof(wsrep_seqno_t));
+}
+
+const wsrep_uuid_t* wsrep_xid_uuid(const XID* xid)
+{
+  if (wsrep_is_wsrep_xid(xid))
+    return reinterpret_cast<const wsrep_uuid_t*>(xid->data
+                                                 + WSREP_XID_UUID_OFFSET);
+  else
+    return &WSREP_UUID_UNDEFINED;
+}
+
+wsrep_seqno_t wsrep_xid_seqno(const XID* xid)
+{
+
+  if (wsrep_is_wsrep_xid(xid))
+  {
+    wsrep_seqno_t seqno;
+    memcpy(&seqno, xid->data + WSREP_XID_SEQNO_OFFSET, sizeof(wsrep_seqno_t));
+    return seqno;
+  }
+  else
+  {
+    return WSREP_SEQNO_UNDEFINED;
+  }
+}
+
+extern
+int wsrep_is_wsrep_xid(const void* xid_ptr)
+{
+  const XID* xid= reinterpret_cast<const XID*>(xid_ptr);
+  return (xid->formatID      == 1                   &&
+          xid->gtrid_length  == WSREP_XID_GTRID_LEN &&
+          xid->bqual_length  == 0                   &&
+          !memcmp(xid->data, WSREP_XID_PREFIX, WSREP_XID_PREFIX_LEN));
+}
diff --git a/sql/wsrep_utils.h b/sql/wsrep_utils.h
new file mode 100644
index 00000000000..337678238f8
--- /dev/null
+++ b/sql/wsrep_utils.h
@@ -0,0 +1,208 @@
+/* Copyright (C) 2013 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with this program; if not, write to the Free Software Foundation, Inc.,
+   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+
+#ifndef WSREP_UTILS_H
+#define WSREP_UTILS_H
+
+#include "wsrep_priv.h"
+
+unsigned int wsrep_check_ip (const char* addr);
+size_t wsrep_guess_ip (char* buf, size_t buf_len);
+size_t wsrep_guess_address(char* buf, size_t buf_len);
+
+namespace wsp {
+class node_status
+{
+public:
+  node_status() : status(WSREP_MEMBER_UNDEFINED) {}
+  void set(wsrep_member_status_t new_status,
+           const wsrep_view_info_t* view = 0)
+  {
+    if (status != new_status || 0 != view)
+    {
+      wsrep_notify_status(new_status, view);
+      status = new_status;
+    }
+  }
+  wsrep_member_status_t get() const { return status; }
+private:
+  wsrep_member_status_t status;
+};
+} /* namespace wsp */
+
+extern wsp::node_status local_status;
+
+namespace wsp {
+/* A small class to run external programs. */
+class process
+{
+private:
+    const char* const str_;
+    FILE*       io_;
+    int         err_;
+    pid_t       pid_;
+
+public:
+/*! @arg type is a pointer to a null-terminated string which  must  contain
+         either  the  letter  'r'  for  reading  or the letter 'w' for writing.
+ */
+    process  (const char* cmd, const char* type);
+    ~process ();
+
+    FILE* pipe () { return io_;  }
+    int   error() { return err_; }
+    int   wait ();
+    const char* cmd() { return str_; }
+};
+
+class thd
+{
+  class thd_init
+  {
+  public:
+    thd_init()  { my_thread_init(); }
+    ~thd_init() { my_thread_end();  }
+  }
+  init;
+
+  thd (const thd&);
+  thd& operator= (const thd&);
+
+public:
+
+  thd(my_bool wsrep_on);
+  ~thd();
+  THD* const ptr;
+};
+
+class string
+{
+public:
+    string() : string_(0) {}
+    void set(char* str) { if (string_) free (string_); string_ = str; }
+    ~string() { set (0); }
+private:
+    char* string_;
+};
+
+#ifdef REMOVED
+class lock
+{
+  pthread_mutex_t* const mtx_;
+
+public:
+
+  lock (pthread_mutex_t* mtx) : mtx_(mtx)
+  {
+    int err = pthread_mutex_lock (mtx_);
+
+    if (err)
+    {
+      WSREP_ERROR("Mutex lock failed: %s", strerror(err));
+      abort();
+    }
+  }
+
+  virtual ~lock ()
+  {
+    int err = pthread_mutex_unlock (mtx_);
+
+    if (err)
+    {
+      WSREP_ERROR("Mutex unlock failed: %s", strerror(err));
+      abort();
+    }
+  }
+
+  inline void wait (pthread_cond_t* cond)
+  {
+    pthread_cond_wait (cond, mtx_);
+  }
+
+private:
+
+  lock (const lock&);
+  lock& operator=(const lock&);
+
+};
+
+class monitor
+{
+  int             mutable refcnt;
+  pthread_mutex_t mutable mtx;
+  pthread_cond_t  mutable cond;
+
+public:
+
+  monitor() : refcnt(0)
+  {
+    pthread_mutex_init (&mtx, NULL);
+    pthread_cond_init  (&cond, NULL);
+  }
+
+  ~monitor()
+  {
+    pthread_mutex_destroy (&mtx);
+    pthread_cond_destroy  (&cond);
+  }
+
+  void enter() const
+  {
+    lock l(&mtx);
+
+    while (refcnt)
+    {
+      l.wait(&cond);
+    }
+    refcnt++;
+  }
+
+  void leave() const
+  {
+    lock l(&mtx);
+
+    refcnt--;
+    if (refcnt == 0)
+    {
+      pthread_cond_signal (&cond);
+    }
+  }
+
+private:
+
+  monitor (const monitor&);
+  monitor& operator= (const monitor&);
+};
+
+class critical
+{
+  const monitor& mon;
+
+public:
+
+  critical(const monitor& m) : mon(m) { mon.enter(); }
+
+  ~critical() { mon.leave(); }
+
+private:
+
+  critical (const critical&);
+  critical& operator= (const critical&);
+};
+#endif
+
+} // namespace wsrep
+
+#endif /* WSREP_UTILS_H */
diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc
new file mode 100644
index 00000000000..2272945535d
--- /dev/null
+++ b/sql/wsrep_var.cc
@@ -0,0 +1,602 @@
+/* Copyright 2008 Codership Oy <http://www.codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include "wsrep_var.h"
+
+#include <mysqld.h>
+#include <sql_class.h>
+#include <sql_plugin.h>
+#include <set_var.h>
+#include <sql_acl.h>
+#include "wsrep_priv.h"
+#include "wsrep_thd.h"
+#include <my_dir.h>
+#include <cstdio>
+#include <cstdlib>
+
+const  char* wsrep_provider         = 0;
+const  char* wsrep_provider_options = 0;
+const  char* wsrep_cluster_address  = 0;
+const  char* wsrep_cluster_name     = 0;
+const  char* wsrep_node_name        = 0;
+const  char* wsrep_node_address     = 0;
+const  char* wsrep_node_incoming_address = 0;
+const  char* wsrep_start_position   = 0;
+ulong  wsrep_OSU_method_options;
+
+int wsrep_init_vars()
+{
+  wsrep_provider        = my_strdup(WSREP_NONE, MYF(MY_WME));
+  wsrep_provider_options= my_strdup("", MYF(MY_WME));
+  wsrep_cluster_address = my_strdup("", MYF(MY_WME));
+  wsrep_cluster_name    = my_strdup(WSREP_CLUSTER_NAME, MYF(MY_WME));
+  wsrep_node_name       = my_strdup("", MYF(MY_WME));
+  wsrep_node_address    = my_strdup("", MYF(MY_WME));
+  wsrep_node_incoming_address= my_strdup(WSREP_NODE_INCOMING_AUTO, MYF(MY_WME));
+  wsrep_start_position  = my_strdup(WSREP_START_POSITION_ZERO, MYF(MY_WME));
+
+  global_system_variables.binlog_format=BINLOG_FORMAT_ROW;
+  return 0;
+}
+
+bool wsrep_on_update (sys_var *self, THD* thd, enum_var_type var_type)
+{
+  if (var_type == OPT_GLOBAL) {
+    // FIXME: this variable probably should be changed only per session
+    thd->variables.wsrep_on = global_system_variables.wsrep_on;
+  }
+  return false;
+}
+
+bool wsrep_causal_reads_update (sys_var *self, THD* thd, enum_var_type var_type)
+{
+  // global setting should not affect session setting.
+  // if (var_type == OPT_GLOBAL) {
+  //   thd->variables.wsrep_causal_reads = global_system_variables.wsrep_causal_reads;
+  // }
+  if (thd->variables.wsrep_causal_reads) {
+    thd->variables.wsrep_sync_wait |= WSREP_SYNC_WAIT_BEFORE_READ;
+  } else {
+    thd->variables.wsrep_sync_wait &= ~WSREP_SYNC_WAIT_BEFORE_READ;
+  }
+  return false;
+}
+
+bool wsrep_sync_wait_update (sys_var* self, THD* thd, enum_var_type var_type)
+{
+  // global setting should not affect session setting.
+  // if (var_type == OPT_GLOBAL) {
+  //   thd->variables.wsrep_sync_wait = global_system_variables.wsrep_sync_wait;
+  // }
+  thd->variables.wsrep_causal_reads = thd->variables.wsrep_sync_wait &
+          WSREP_SYNC_WAIT_BEFORE_READ;
+  return false;
+}
+
+static int wsrep_start_position_verify (const char* start_str)
+{
+  size_t        start_len;
+  wsrep_uuid_t  uuid;
+  ssize_t       uuid_len;
+
+  start_len = strlen (start_str);
+  if (start_len < 34)
+    return 1;
+
+  uuid_len = wsrep_uuid_scan (start_str, start_len, &uuid);
+  if (uuid_len < 0 || (start_len - uuid_len) < 2)
+    return 1;
+
+  if (start_str[uuid_len] != ':') // separator should follow UUID
+    return 1;
+
+  char* endptr;
+  wsrep_seqno_t const seqno __attribute__((unused)) // to avoid GCC warnings
+    (strtoll(&start_str[uuid_len + 1], &endptr, 10));
+
+  if (*endptr == '\0') return 0; // remaining string was seqno
+
+  return 1;
+}
+
+bool wsrep_start_position_check (sys_var *self, THD* thd, set_var* var)
+{
+  char start_pos_buf[FN_REFLEN];
+
+  if ((! var->save_result.string_value.str) ||
+      (var->save_result.string_value.length > (FN_REFLEN - 1))) // safety
+    goto err;
+
+  memcpy(start_pos_buf, var->save_result.string_value.str,
+         var->save_result.string_value.length);
+  start_pos_buf[var->save_result.string_value.length]= 0;
+
+  if (!wsrep_start_position_verify(start_pos_buf)) return 0;
+
+err:
+  my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), var->var->name.str,
+           var->save_result.string_value.str ?
+           var->save_result.string_value.str : "NULL");
+  return 1;
+}
+
+void wsrep_set_local_position (const char* value)
+{
+  size_t value_len  = strlen (value);
+  size_t uuid_len   = wsrep_uuid_scan (value, value_len, &local_uuid);
+
+  local_seqno = strtoll (value + uuid_len + 1, NULL, 10);
+
+  XID xid;
+  wsrep_xid_init(&xid, &local_uuid, local_seqno);
+  wsrep_set_SE_checkpoint(&xid);
+  WSREP_INFO ("wsrep_start_position var submitted: '%s'", wsrep_start_position);
+}
+
+bool wsrep_start_position_update (sys_var *self, THD* thd, enum_var_type type)
+{
+  // since this value passed wsrep_start_position_check, don't check anything
+  // here
+  wsrep_set_local_position (wsrep_start_position);
+
+  if (wsrep) {
+    wsrep_sst_received (wsrep, &local_uuid, local_seqno, NULL, 0);
+  }
+
+  return 0;
+}
+
+void wsrep_start_position_init (const char* val)
+{
+  if (NULL == val || wsrep_start_position_verify (val))
+  {
+    WSREP_ERROR("Bad initial value for wsrep_start_position: %s", 
+                (val ? val : ""));
+    return;
+  }
+
+  wsrep_set_local_position (val);
+}
+
+static bool refresh_provider_options()
+{
+  WSREP_DEBUG("refresh_provider_options: %s", 
+              (wsrep_provider_options) ? wsrep_provider_options : "null");
+  char* opts= wsrep->options_get(wsrep);
+  if (opts)
+  {
+    if (wsrep_provider_options) my_free((void *)wsrep_provider_options);
+    wsrep_provider_options = (char*)my_memdup(opts, strlen(opts) + 1, 
+                                              MYF(MY_WME));
+  }
+  else
+  {
+    WSREP_ERROR("Failed to get provider options");
+    return true;
+  }
+  return false;
+}
+
+static int wsrep_provider_verify (const char* provider_str)
+{
+  MY_STAT   f_stat;
+  char path[FN_REFLEN];
+
+  if (!provider_str || strlen(provider_str)== 0)
+    return 1;
+
+  if (!strcmp(provider_str, WSREP_NONE))
+    return 0;
+
+  if (!unpack_filename(path, provider_str))
+    return 1;
+
+  /* check that provider file exists */
+  bzero(&f_stat, sizeof(MY_STAT));
+  if (!my_stat(path, &f_stat, MYF(0)))
+  {
+    return 1;
+  }
+  return 0;
+}
+
+bool wsrep_provider_check (sys_var *self, THD* thd, set_var* var)
+{
+  char wsrep_provider_buf[FN_REFLEN];
+
+  if ((! var->save_result.string_value.str) ||
+      (var->save_result.string_value.length > (FN_REFLEN - 1))) // safety
+    goto err;
+
+  memcpy(wsrep_provider_buf, var->save_result.string_value.str,
+         var->save_result.string_value.length);
+  wsrep_provider_buf[var->save_result.string_value.length]= 0;
+
+  if (!wsrep_provider_verify(wsrep_provider_buf)) return 0;
+
+err:
+  my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), var->var->name.str,
+           var->save_result.string_value.str ?
+           var->save_result.string_value.str : "NULL");
+  return 1;
+}
+
+bool wsrep_provider_update (sys_var *self, THD* thd, enum_var_type type)
+{
+  bool rcode= false;
+
+  bool wsrep_on_saved= thd->variables.wsrep_on;
+  thd->variables.wsrep_on= false;
+
+  WSREP_DEBUG("wsrep_provider_update: %s", wsrep_provider);
+
+  /* stop replication is heavy operation, and includes closing all client 
+     connections. Closing clients may need to get LOCK_global_system_variables
+     at least in MariaDB.
+
+     Note: releasing LOCK_global_system_variables may cause race condition, if 
+     there can be several concurrent clients changing wsrep_provider
+  */
+  mysql_mutex_unlock(&LOCK_global_system_variables);
+  wsrep_stop_replication(thd);
+  mysql_mutex_lock(&LOCK_global_system_variables);
+
+  if (wsrep_inited == 1)
+    wsrep_deinit(false);
+
+  char* tmp= strdup(wsrep_provider); // wsrep_init() rewrites provider 
+                                     //when fails
+  if (wsrep_init())
+  {
+    my_error(ER_CANT_OPEN_LIBRARY, MYF(0), tmp);
+    rcode = true;
+  }
+  free(tmp);
+
+  // we sure don't want to use old address with new provider
+  wsrep_cluster_address_init(NULL);
+  wsrep_provider_options_init(NULL);
+
+  thd->variables.wsrep_on= wsrep_on_saved;
+
+  refresh_provider_options();
+
+  return rcode;
+}
+
+void wsrep_provider_init (const char* value)
+{
+  WSREP_DEBUG("wsrep_provider_init: %s -> %s", 
+              (wsrep_provider) ? wsrep_provider : "null", 
+              (value) ? value : "null");
+  if (NULL == value || wsrep_provider_verify (value))
+  {
+    WSREP_ERROR("Bad initial value for wsrep_provider: %s",
+                (value ? value : ""));
+    return;
+  }
+
+  if (wsrep_provider) my_free((void *)wsrep_provider);
+  wsrep_provider = my_strdup(value, MYF(0));
+}
+
+bool wsrep_provider_options_check(sys_var *self, THD* thd, set_var* var)
+{
+  return 0;
+}
+
+bool wsrep_provider_options_update(sys_var *self, THD* thd, enum_var_type type)
+{
+  wsrep_status_t ret= wsrep->options_set(wsrep, wsrep_provider_options);
+  if (ret != WSREP_OK)
+  {
+    WSREP_ERROR("Set options returned %d", ret);
+    refresh_provider_options();
+    return true;
+  }
+  return refresh_provider_options();
+}
+
+void wsrep_provider_options_init(const char* value)
+{
+  if (wsrep_provider_options && wsrep_provider_options != value) 
+    my_free((void *)wsrep_provider_options);
+  wsrep_provider_options = (value) ? my_strdup(value, MYF(0)) : NULL;
+}
+
+static int wsrep_cluster_address_verify (const char* cluster_address_str)
+{
+  /* There is no predefined address format, it depends on provider. */
+  return 0;
+}
+
+bool wsrep_cluster_address_check (sys_var *self, THD* thd, set_var* var)
+{
+  char addr_buf[FN_REFLEN];
+
+  if ((! var->save_result.string_value.str) ||
+      (var->save_result.string_value.length > (FN_REFLEN - 1))) // safety
+    goto err;
+
+  memcpy(addr_buf, var->save_result.string_value.str,
+         var->save_result.string_value.length);
+  addr_buf[var->save_result.string_value.length]= 0;
+
+  if (!wsrep_cluster_address_verify(addr_buf)) return 0;
+
+ err:
+  my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), var->var->name.str,
+           var->save_result.string_value.str ?
+           var->save_result.string_value.str : "NULL");
+  return 1;
+}
+
+bool wsrep_cluster_address_update (sys_var *self, THD* thd, enum_var_type type)
+{
+  bool wsrep_on_saved= thd->variables.wsrep_on;
+  thd->variables.wsrep_on= false;
+
+  /* stop replication is heavy operation, and includes closing all client 
+     connections. Closing clients may need to get LOCK_global_system_variables
+     at least in MariaDB.
+
+     Note: releasing LOCK_global_system_variables may cause race condition, if 
+     there can be several concurrent clients changing wsrep_provider
+  */
+  mysql_mutex_unlock(&LOCK_global_system_variables);
+  wsrep_stop_replication(thd);
+  mysql_mutex_lock(&LOCK_global_system_variables);
+
+  if (wsrep_start_replication())
+  {
+    wsrep_create_rollbacker();
+    wsrep_create_appliers(wsrep_slave_threads);
+  }
+
+  thd->variables.wsrep_on= wsrep_on_saved;
+
+  return false;
+}
+
+void wsrep_cluster_address_init (const char* value)
+{
+  WSREP_DEBUG("wsrep_cluster_address_init: %s -> %s", 
+              (wsrep_cluster_address) ? wsrep_cluster_address : "null", 
+              (value) ? value : "null");
+
+  if (wsrep_cluster_address) my_free ((void*)wsrep_cluster_address);
+  wsrep_cluster_address = (value) ? my_strdup(value, MYF(0)) : NULL;
+}
+
+/* wsrep_cluster_name cannot be NULL or an empty string. */
+bool wsrep_cluster_name_check (sys_var *self, THD* thd, set_var* var)
+{
+  if (!var->save_result.string_value.str ||
+      (var->save_result.string_value.length == 0))
+  {
+    my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), var->var->name.str,
+             (var->save_result.string_value.str ?
+              var->save_result.string_value.str : "NULL"));
+    return 1;
+  }
+  return 0;
+}
+
+bool wsrep_cluster_name_update (sys_var *self, THD* thd, enum_var_type type)
+{
+  return 0;
+}
+
+bool wsrep_node_name_check (sys_var *self, THD* thd, set_var* var)
+{
+  // TODO: for now 'allow' 0-length string to be valid (default)
+  if (!var->save_result.string_value.str)
+  {
+    my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), var->var->name.str,
+             (var->save_result.string_value.str ?
+              var->save_result.string_value.str : "NULL"));
+    return 1;
+  }
+  return 0;
+}
+
+bool wsrep_node_name_update (sys_var *self, THD* thd, enum_var_type type)
+{
+  return 0;
+}
+
+// TODO: do something more elaborate, like checking connectivity
+bool wsrep_node_address_check (sys_var *self, THD* thd, set_var* var)
+{
+  char addr_buf[FN_REFLEN];
+
+  if ((! var->save_result.string_value.str) ||
+      (var->save_result.string_value.length > (FN_REFLEN - 1))) // safety
+    goto err;
+
+  memcpy(addr_buf, var->save_result.string_value.str,
+         var->save_result.string_value.length);
+  addr_buf[var->save_result.string_value.length]= 0;
+
+  // TODO: for now 'allow' 0-length string to be valid (default)
+  return 0;
+
+err:
+  my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), var->var->name.str,
+           var->save_result.string_value.str ?
+           var->save_result.string_value.str : "NULL");
+  return 1;
+}
+
+bool wsrep_node_address_update (sys_var *self, THD* thd, enum_var_type type)
+{
+  return 0;
+}
+
+void wsrep_node_address_init (const char* value)
+{
+  if (wsrep_node_address && strcmp(wsrep_node_address, value))
+    my_free ((void*)wsrep_node_address);
+
+  wsrep_node_address = (value) ? my_strdup(value, MYF(0)) : NULL;
+}
+
+bool wsrep_slave_threads_check (sys_var *self, THD* thd, set_var* var)
+{
+  mysql_mutex_lock(&LOCK_wsrep_slave_threads);
+  wsrep_slave_count_change += (var->save_result.ulonglong_value -
+                               wsrep_slave_threads);
+  mysql_mutex_unlock(&LOCK_wsrep_slave_threads);
+
+  return 0;
+}
+
+bool wsrep_slave_threads_update (sys_var *self, THD* thd, enum_var_type type)
+{
+  if (wsrep_slave_count_change > 0)
+  {
+    wsrep_create_appliers(wsrep_slave_count_change);
+    wsrep_slave_count_change = 0;
+  }
+  return false;
+}
+
+bool wsrep_desync_check (sys_var *self, THD* thd, set_var* var)
+{
+  bool new_wsrep_desync= (bool) var->save_result.ulonglong_value;
+  if (wsrep_desync == new_wsrep_desync) {
+    if (new_wsrep_desync) {
+      push_warning (thd, Sql_condition::WARN_LEVEL_WARN,
+                   ER_WRONG_VALUE_FOR_VAR,
+                   "'wsrep_desync' is already ON.");
+    } else {
+      push_warning (thd, Sql_condition::WARN_LEVEL_WARN,
+                   ER_WRONG_VALUE_FOR_VAR,
+                   "'wsrep_desync' is already OFF.");
+    }
+  }
+  return 0;
+}
+
+bool wsrep_desync_update (sys_var *self, THD* thd, enum_var_type type)
+{
+  wsrep_status_t ret(WSREP_WARNING);
+  if (wsrep_desync) {
+    ret = wsrep->desync (wsrep);
+    if (ret != WSREP_OK) {
+      WSREP_WARN ("SET desync failed %d for %s", ret, thd->query());
+      my_error (ER_CANNOT_USER, MYF(0), "'desync'", thd->query());
+      return true;
+    }
+  } else {
+    ret = wsrep->resync (wsrep);
+    if (ret != WSREP_OK) {
+      WSREP_WARN ("SET resync failed %d for %s", ret, thd->query());
+      my_error (ER_CANNOT_USER, MYF(0), "'resync'", thd->query());
+      return true;
+    }
+  }
+  return false;
+}
+
+/*
+ * Status variables stuff below
+ */
+static inline void
+wsrep_assign_to_mysql (SHOW_VAR* mysql, wsrep_stats_var* wsrep)
+{
+  mysql->name = wsrep->name;
+  switch (wsrep->type) {
+  case WSREP_VAR_INT64:
+    mysql->value = (char*) &wsrep->value._int64;
+    mysql->type  = SHOW_LONGLONG;
+    break;
+  case WSREP_VAR_STRING:
+    mysql->value = (char*) &wsrep->value._string;
+    mysql->type  = SHOW_CHAR_PTR;
+    break;
+  case WSREP_VAR_DOUBLE:
+    mysql->value = (char*) &wsrep->value._double;
+    mysql->type  = SHOW_DOUBLE;
+    break;
+  }
+}
+
+#if DYNAMIC
+// somehow this mysql status thing works only with statically allocated arrays.
+static SHOW_VAR*          mysql_status_vars = NULL;
+static int                mysql_status_len  = -1;
+#else
+static SHOW_VAR           mysql_status_vars[512 + 1];
+static const int          mysql_status_len  = 512;
+#endif
+
+static void export_wsrep_status_to_mysql(THD* thd)
+{
+  int wsrep_status_len, i;
+
+  thd->wsrep_status_vars = wsrep->stats_get(wsrep);
+
+  if (!thd->wsrep_status_vars) {
+    return;
+  }
+
+  for (wsrep_status_len = 0;
+       thd->wsrep_status_vars[wsrep_status_len].name != NULL;
+       wsrep_status_len++);
+
+#if DYNAMIC
+  if (wsrep_status_len != mysql_status_len) {
+    void* tmp = realloc (mysql_status_vars,
+                         (wsrep_status_len + 1) * sizeof(SHOW_VAR));
+    if (!tmp) {
+
+      sql_print_error ("Out of memory for wsrep status variables."
+                       "Number of variables: %d", wsrep_status_len);
+      return;
+    }
+
+    mysql_status_len  = wsrep_status_len;
+    mysql_status_vars = (SHOW_VAR*)tmp;
+  }
+  /* @TODO: fix this: */
+#else
+  if (mysql_status_len < wsrep_status_len) wsrep_status_len= mysql_status_len;
+#endif
+
+  for (i = 0; i < wsrep_status_len; i++)
+    wsrep_assign_to_mysql (mysql_status_vars + i, thd->wsrep_status_vars + i);
+
+  mysql_status_vars[wsrep_status_len].name  = NullS;
+  mysql_status_vars[wsrep_status_len].value = NullS;
+  mysql_status_vars[wsrep_status_len].type  = SHOW_LONG;
+}
+
+int wsrep_show_status (THD *thd, SHOW_VAR *var, char *buff)
+{
+  export_wsrep_status_to_mysql(thd);
+  var->type= SHOW_ARRAY;
+  var->value= (char *) &mysql_status_vars;
+  return 0;
+}
+
+void wsrep_free_status (THD* thd)
+{
+  if (thd->wsrep_status_vars)
+  {
+    wsrep->stats_free (wsrep, thd->wsrep_status_vars);
+    thd->wsrep_status_vars = 0;
+  }
+}
diff --git a/sql/wsrep_var.h b/sql/wsrep_var.h
new file mode 100644
index 00000000000..58d0374baa5
--- /dev/null
+++ b/sql/wsrep_var.h
@@ -0,0 +1,86 @@
+/* Copyright (C) 2013 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with this program; if not, write to the Free Software Foundation, Inc.,
+   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+
+#if !defined (WSREP_VAR_H) && defined(WITH_WSREP)
+#define WSREP_VAR_H
+
+#define WSREP_CLUSTER_NAME        "my_wsrep_cluster"
+#define WSREP_NODE_INCOMING_AUTO  "AUTO"
+#define WSREP_START_POSITION_ZERO "00000000-0000-0000-0000-000000000000:-1"
+
+// MySQL variables funcs
+
+#include "sql_priv.h"
+class sys_var;
+class set_var;
+class THD;
+
+int wsrep_init_vars();
+
+#define CHECK_ARGS   (sys_var *self, THD* thd, set_var *var)
+#define UPDATE_ARGS  (sys_var *self, THD* thd, enum_var_type type)
+#define DEFAULT_ARGS (THD* thd, enum_var_type var_type)
+#define INIT_ARGS    (const char* opt)
+
+extern bool wsrep_on_update                  UPDATE_ARGS;
+extern bool wsrep_causal_reads_update        UPDATE_ARGS;
+extern bool wsrep_sync_wait_update           UPDATE_ARGS;
+extern bool wsrep_start_position_check       CHECK_ARGS;
+extern bool wsrep_start_position_update      UPDATE_ARGS;
+extern void wsrep_start_position_init        INIT_ARGS;
+
+extern bool wsrep_provider_check             CHECK_ARGS;
+extern bool wsrep_provider_update            UPDATE_ARGS;
+extern void wsrep_provider_init              INIT_ARGS;
+
+extern bool wsrep_provider_options_check     CHECK_ARGS;
+extern bool wsrep_provider_options_update    UPDATE_ARGS;
+extern void wsrep_provider_options_init      INIT_ARGS;
+
+extern bool wsrep_cluster_address_check      CHECK_ARGS;
+extern bool wsrep_cluster_address_update     UPDATE_ARGS;
+extern void wsrep_cluster_address_init       INIT_ARGS;
+
+extern bool wsrep_cluster_name_check         CHECK_ARGS;
+extern bool wsrep_cluster_name_update        UPDATE_ARGS;
+
+extern bool wsrep_node_name_check            CHECK_ARGS;
+extern bool wsrep_node_name_update           UPDATE_ARGS;
+
+extern bool wsrep_node_address_check         CHECK_ARGS;
+extern bool wsrep_node_address_update        UPDATE_ARGS;
+extern void wsrep_node_address_init          INIT_ARGS;
+
+extern bool wsrep_sst_method_check           CHECK_ARGS;
+extern bool wsrep_sst_method_update          UPDATE_ARGS;
+extern void wsrep_sst_method_init            INIT_ARGS;
+
+extern bool wsrep_sst_receive_address_check  CHECK_ARGS;
+extern bool wsrep_sst_receive_address_update UPDATE_ARGS;
+
+extern bool wsrep_sst_auth_check             CHECK_ARGS;
+extern bool wsrep_sst_auth_update            UPDATE_ARGS;
+extern void wsrep_sst_auth_init              INIT_ARGS;
+
+extern bool wsrep_sst_donor_check            CHECK_ARGS;
+extern bool wsrep_sst_donor_update           UPDATE_ARGS;
+
+extern bool wsrep_slave_threads_check        CHECK_ARGS;
+extern bool wsrep_slave_threads_update       UPDATE_ARGS;
+
+extern bool wsrep_desync_check               CHECK_ARGS;
+extern bool wsrep_desync_update              UPDATE_ARGS;
+
+#endif /* WSREP_VAR_H */
diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt
index 622fff87536..12d43bba5bc 100644
--- a/storage/innobase/CMakeLists.txt
+++ b/storage/innobase/CMakeLists.txt
@@ -251,6 +251,27 @@ ENDIF()
 INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/storage/innobase/include
 		    ${CMAKE_SOURCE_DIR}/storage/innobase/handler)
 
+IF(WITH_WSREP)
+  # ssl include directory
+  INCLUDE_DIRECTORIES(${SSL_INCLUDE_DIRS}
+                      ${CMAKE_SOURCE_DIR}/storage/innobase/wsrep)
+
+  IF(SSL_DEFINES)
+    ADD_DEFINITIONS(${SSL_DEFINES})
+  ENDIF()
+
+  LINK_LIBRARIES(${SSL_LIBRARIES})
+
+  # We do RESTRICT_SYMBOL_EXPORTS(yassl) elsewhere.
+  # In order to get correct symbol visibility, these files
+  # must be compiled with "-fvisibility=hidden"
+  IF(WITH_SSL STREQUAL "bundled" AND HAVE_VISIBILITY_HIDDEN)
+    SET_SOURCE_FILES_PROPERTIES(
+      {CMAKE_CURRENT_SOURCE_DIR}/wsrep/md5.cc
+      PROPERTIES COMPILE_FLAGS "-fvisibility=hidden")
+  ENDIF()
+ENDIF()
+
 # Sun Studio bug with -xO2
 IF(CMAKE_CXX_COMPILER_ID MATCHES "SunPro"
 	AND CMAKE_CXX_FLAGS_RELEASE MATCHES "O2"
@@ -397,6 +418,10 @@ SET(INNOBASE_SOURCES
 	ut/ut0vec.cc
 	ut/ut0wqueue.cc)
 
+IF(WITH_WSREP)
+  SET(INNOBASE_SOURCES ${INNOBASE_SOURCES} wsrep/md5.cc)
+ENDIF()
+
 IF(WITH_INNODB)
   # Legacy option
   SET(WITH_INNOBASE_STORAGE_ENGINE TRUE)
diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc
index ec76c9923fe..9e81d010d0f 100644
--- a/storage/innobase/buf/buf0rea.cc
+++ b/storage/innobase/buf/buf0rea.cc
@@ -1,7 +1,6 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -185,14 +184,15 @@ buf_read_page_low(
 		*err = fil_io(OS_FILE_READ | wake_later
 			      | ignore_nonexistent_pages,
 			      sync, space, zip_size, offset, 0, zip_size,
-			      bpage->zip.data, bpage, &bpage->write_size);
+			bpage->zip.data, bpage, &bpage->write_size);
 	} else {
 		ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE);
 
 		*err = fil_io(OS_FILE_READ | wake_later
 			      | ignore_nonexistent_pages,
 			      sync, space, 0, offset, 0, UNIV_PAGE_SIZE,
-			      ((buf_block_t*) bpage)->frame, bpage, 0);
+			      ((buf_block_t*) bpage)->frame, bpage,
+			      &bpage->write_size);
 	}
 
 	if (sync) {
diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc
index 0c83089478a..72a0f39e19e 100644
--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@@ -3237,7 +3237,29 @@ dict_foreign_find_index(
 
 	return(NULL);
 }
-
+#ifdef WITH_WSREP
+dict_index_t*
+wsrep_dict_foreign_find_index(
+/*====================*/
+	dict_table_t*	table,	/*!< in: table */
+	const char**	col_names, /*!< in: column names, or NULL
+					to use table->col_names */
+	const char**	columns,/*!< in: array of column names */
+	ulint		n_cols,	/*!< in: number of columns */
+	dict_index_t*	types_idx, /*!< in: NULL or an index to whose types the
+				   column types must match */
+	ibool		check_charsets,
+				/*!< in: whether to check charsets.
+				only has an effect if types_idx != NULL */
+	ulint		check_null)
+				/*!< in: nonzero if none of the columns must
+				be declared NOT NULL */
+{
+	return dict_foreign_find_index(
+		table, col_names, columns, n_cols, types_idx, check_charsets,
+		check_null);
+}
+#endif /* WITH_WSREP */
 /**********************************************************************//**
 Report an error in a foreign key definition. */
 static
diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc
index a81a982db6c..f324631dff1 100644
--- a/storage/innobase/fil/fil0pagecompress.cc
+++ b/storage/innobase/fil/fil0pagecompress.cc
@@ -490,7 +490,9 @@ fil_decompress_page(
         ulint actual_size = 0;
 	ulint compression_alg = 0;
 	byte *in_buf;
+#ifdef HAVE_LZO
 	ulint olen=0;
+#endif
 	ulint ptype;
 
 	ut_ad(buf);
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index ead86fd3085..970127a7420 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -117,6 +117,46 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #  define MYSQL_PLUGIN_IMPORT /* nothing */
 # endif /* MYSQL_PLUGIN_IMPORT */
 
+#ifdef WITH_WSREP
+#include "dict0priv.h"
+#include "../storage/innobase/include/ut0byte.h"
+#include <wsrep_mysqld.h>
+#include <wsrep_md5.h>
+
+extern my_bool wsrep_certify_nonPK;
+class  binlog_trx_data;
+extern handlerton *binlog_hton;
+
+extern MYSQL_PLUGIN_IMPORT MYSQL_BIN_LOG mysql_bin_log;
+extern MYSQL_PLUGIN_IMPORT mysql_mutex_t LOCK_wsrep_rollback;
+extern MYSQL_PLUGIN_IMPORT mysql_cond_t COND_wsrep_rollback;
+extern MYSQL_PLUGIN_IMPORT wsrep_aborting_thd_t wsrep_aborting_thd;
+
+static inline wsrep_ws_handle_t*
+wsrep_ws_handle(THD* thd, const trx_t* trx) {
+	return wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd),
+				       (wsrep_trx_id_t)trx->id);
+}
+
+extern bool wsrep_prepare_key_for_innodb(const uchar *cache_key,
+					 size_t cache_key_len,
+                                         const uchar* row_id,
+                                         size_t row_id_len,
+                                         wsrep_buf_t* key,
+                                         size_t* key_len);
+
+extern handlerton * wsrep_hton;
+extern TC_LOG* tc_log;
+extern void wsrep_cleanup_transaction(THD *thd);
+static int
+wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd,
+			my_bool signal);
+static void
+wsrep_fake_trx_id(handlerton* hton, THD *thd);
+static int innobase_wsrep_set_checkpoint(handlerton* hton, const XID* xid);
+static int innobase_wsrep_get_checkpoint(handlerton* hton, XID* xid);
+#endif /* WITH_WSREP */
+
 /** to protect innobase_open_files */
 static mysql_mutex_t innobase_share_mutex;
 /** to force correct commit order in binlog */
@@ -1253,6 +1293,10 @@ innobase_srv_conc_enter_innodb(
 /*===========================*/
 	trx_t*	trx)	/*!< in: transaction handle */
 {
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd) && 
+	    wsrep_thd_is_BF(trx->mysql_thd, FALSE)) return;
+#endif /* WITH_WSREP */
 	if (srv_thread_concurrency) {
 		if (trx->n_tickets_to_enter_innodb > 0) {
 
@@ -1287,6 +1331,10 @@ innobase_srv_conc_exit_innodb(
 #ifdef UNIV_SYNC_DEBUG
 	ut_ad(!sync_thread_levels_nonempty_trx(trx->has_search_latch));
 #endif /* UNIV_SYNC_DEBUG */
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd) && 
+	    wsrep_thd_is_BF(trx->mysql_thd, FALSE)) return;
+#endif /* WITH_WSREP */
 
 	/* This is to avoid making an unnecessary function call. */
 	if (trx->declared_to_be_inside_innodb
@@ -1407,6 +1455,15 @@ thd_to_trx(
 {
 	return(*(trx_t**) thd_ha_data(thd, innodb_hton_ptr));
 }
+#ifdef WITH_WSREP
+ulonglong
+thd_to_trx_id(
+/*=======*/
+	THD*	thd)	/*!< in: MySQL thread */
+{
+	return(thd_to_trx(thd)->id);
+}
+#endif /* WITH_WSREP */
 
 /********************************************************************//**
 Call this function when mysqld passes control to the client. That is to
@@ -1892,6 +1949,9 @@ int
 innobase_mysql_tmpfile(void)
 /*========================*/
 {
+#ifdef WITH_INNODB_DISALLOW_WRITES
+	os_event_wait(srv_allow_writes_event);
+#endif /* WITH_INNODB_DISALLOW_WRITES */
 	int	fd2 = -1;
 	File	fd;
 
@@ -2957,6 +3017,12 @@ innobase_init(
 
 	innobase_hton->release_temporary_latches =
 		innobase_release_temporary_latches;
+#ifdef WITH_WSREP
+        innobase_hton->abort_transaction=wsrep_abort_transaction;
+        innobase_hton->set_checkpoint=innobase_wsrep_set_checkpoint;
+        innobase_hton->get_checkpoint=innobase_wsrep_get_checkpoint;
+        innobase_hton->fake_trx_id=wsrep_fake_trx_id;
+#endif /* WITH_WSREP */
 	innobase_hton->kill_query = innobase_kill_query;
 
         if (srv_file_per_table)
@@ -3615,10 +3681,30 @@ innobase_commit_low(
 /*================*/
 	trx_t*	trx)	/*!< in: transaction handle */
 {
+#ifdef WITH_WSREP
+	THD* thd = (THD*)trx->mysql_thd;
+	const char* tmp = 0;
+	if (wsrep_on((void*)thd)) {
+#ifdef WSREP_PROC_INFO
+		char info[64];
+		info[sizeof(info) - 1] = '\0';
+		snprintf(info, sizeof(info) - 1,
+			 "innobase_commit_low():trx_commit_for_mysql(%lld)",
+			 (long long) wsrep_thd_trx_seqno(thd));
+		tmp = thd_proc_info(thd, info);
+
+#else
+		tmp = thd_proc_info(thd, "innobase_commit_low()");
+#endif /* WSREP_PROC_INFO */
+	}
+#endif /* WITH_WSREP */
 	if (trx_is_started(trx)) {
 
 		trx_commit_for_mysql(trx);
 	}
+#ifdef WITH_WSREP
+	if (wsrep_on((void*)thd)) { thd_proc_info(thd, tmp); }
+#endif /* WITH_WSREP */
 }
 
 /*****************************************************************//**
@@ -4343,6 +4429,20 @@ innobase_kill_query(
 	DBUG_ENTER("innobase_kill_query");
 	DBUG_ASSERT(hton == innodb_hton_ptr);
 
+#ifdef WITH_WSREP
+	wsrep_thd_LOCK(thd);
+	if (wsrep_thd_conflict_state(thd) != NO_CONFLICT) {
+		/* if victim has been signaled by BF thread and/or aborting
+		   is already progressing, following query aborting is not necessary
+		   any more.
+		   Also, BF thread should own trx mutex for the victim, which would
+		   conflict with trx_mutex_enter() below
+		*/
+		wsrep_thd_UNLOCK(thd);
+		DBUG_VOID_RETURN;
+	}
+	wsrep_thd_UNLOCK(thd);
+#endif /* WITH_WSREP */
 	trx = thd_to_trx(thd);
 
         if (trx)
@@ -4513,7 +4613,11 @@ ha_innobase::max_supported_key_length() const
 	case 8192:
 		return(1536);
 	default:
+#ifdef WITH_WSREP
 		return(3500);
+#else
+		return(3500);
+#endif
 	}
 }
 
@@ -5620,6 +5724,117 @@ get_field_offset(
 	return((uint) (field->ptr - table->record[0]));
 }
 
+#ifdef WITH_WSREP
+UNIV_INTERN
+int
+wsrep_innobase_mysql_sort(
+/*===============*/
+					/* out: str contains sort string */
+	int		mysql_type,	/* in: MySQL type */
+	uint		charset_number,	/* in: number of the charset */
+	unsigned char*	str,		/* in: data field */
+	unsigned int	str_length,	/* in: data field length,
+					not UNIV_SQL_NULL */
+	unsigned int	buf_length)	/* in: total str buffer length */
+
+{
+	CHARSET_INFO*		charset;
+	enum_field_types	mysql_tp;
+	int ret_length =	str_length;
+
+	DBUG_ASSERT(str_length != UNIV_SQL_NULL);
+
+	mysql_tp = (enum_field_types) mysql_type;
+
+	switch (mysql_tp) {
+
+	case MYSQL_TYPE_BIT:
+	case MYSQL_TYPE_STRING:
+	case MYSQL_TYPE_VAR_STRING:
+	case MYSQL_TYPE_TINY_BLOB:
+	case MYSQL_TYPE_MEDIUM_BLOB:
+	case MYSQL_TYPE_BLOB:
+	case MYSQL_TYPE_LONG_BLOB:
+	case MYSQL_TYPE_VARCHAR:
+	{
+		uchar tmp_str[REC_VERSION_56_MAX_INDEX_COL_LEN] = {'\0'};
+		uint tmp_length = REC_VERSION_56_MAX_INDEX_COL_LEN;
+
+		/* Use the charset number to pick the right charset struct for
+		the comparison. Since the MySQL function get_charset may be
+		slow before Bar removes the mutex operation there, we first
+		look at 2 common charsets directly. */
+
+		if (charset_number == default_charset_info->number) {
+			charset = default_charset_info;
+		} else if (charset_number == my_charset_latin1.number) {
+			charset = &my_charset_latin1;
+		} else {
+			charset = get_charset(charset_number, MYF(MY_WME));
+
+			if (charset == NULL) {
+			  sql_print_error("InnoDB needs charset %lu for doing "
+					  "a comparison, but MySQL cannot "
+					  "find that charset.",
+					  (ulong) charset_number);
+				ut_a(0);
+			}
+		}
+
+		ut_a(str_length <= tmp_length);
+		memcpy(tmp_str, str, str_length);
+
+		tmp_length = charset->coll->strnxfrm(charset, str, str_length,
+                                             str_length, tmp_str,
+                                             tmp_length, 0);
+		DBUG_ASSERT(tmp_length <= str_length);
+		if (wsrep_protocol_version < 3) {
+			tmp_length = charset->coll->strnxfrm(
+				charset, str, str_length,
+				str_length, tmp_str, tmp_length, 0);
+			DBUG_ASSERT(tmp_length <= str_length);
+		} else {
+			/* strnxfrm will expand the destination string,
+			   protocols < 3 truncated the sorted sring
+			   protocols >= 3 gets full sorted sring
+			*/
+			tmp_length = charset->coll->strnxfrm(
+				charset, str, buf_length,
+				str_length, tmp_str, str_length, 0);
+			DBUG_ASSERT(tmp_length <= buf_length);
+			ret_length = tmp_length;
+		}
+ 
+		break;
+	}
+	case MYSQL_TYPE_DECIMAL :
+	case MYSQL_TYPE_TINY :
+	case MYSQL_TYPE_SHORT :
+	case MYSQL_TYPE_LONG :
+	case MYSQL_TYPE_FLOAT :
+	case MYSQL_TYPE_DOUBLE :
+	case MYSQL_TYPE_NULL :
+	case MYSQL_TYPE_TIMESTAMP :
+	case MYSQL_TYPE_LONGLONG :
+	case MYSQL_TYPE_INT24 :
+	case MYSQL_TYPE_DATE :
+	case MYSQL_TYPE_TIME :
+	case MYSQL_TYPE_DATETIME :
+	case MYSQL_TYPE_YEAR :
+	case MYSQL_TYPE_NEWDATE :
+	case MYSQL_TYPE_NEWDECIMAL :
+	case MYSQL_TYPE_ENUM :
+	case MYSQL_TYPE_SET :
+	case MYSQL_TYPE_GEOMETRY :
+		break;
+	default:
+		break;
+	}
+
+	return ret_length;
+}
+#endif /* WITH_WSREP */
+
 /*************************************************************//**
 InnoDB uses this function to compare two data fields for which the data type
 is such that we must use MySQL code to compare them. NOTE that the prototype
@@ -6120,11 +6335,313 @@ innobase_read_from_2_little_endian(
 	return((uint) ((ulint)(buf[0]) + 256 * ((ulint)(buf[1]))));
 }
 
+#ifdef WITH_WSREP
 /*******************************************************************//**
 Stores a key value for a row to a buffer.
 @return	key value length as stored in buff */
 UNIV_INTERN
 uint
+wsrep_store_key_val_for_row(
+/*===============================*/
+	THD* 		thd,
+	TABLE*		table,
+	uint		keynr,	/*!< in: key number */
+	char*		buff,	/*!< in/out: buffer for the key value (in MySQL
+				format) */
+	uint		buff_len,/*!< in: buffer length */
+	const uchar*	record,
+	ibool*          key_is_null)/*!< out: full key was null */
+{
+	KEY*		key_info	= table->key_info + keynr;
+	KEY_PART_INFO*	key_part	= key_info->key_part;
+	KEY_PART_INFO*	end		= key_part + key_info->user_defined_key_parts;
+	char*		buff_start	= buff;
+	enum_field_types mysql_type;
+	Field*		field;
+	uint buff_space = buff_len;
+
+	DBUG_ENTER("wsrep_store_key_val_for_row");
+
+	memset(buff, 0, buff_len);
+	*key_is_null = TRUE;
+
+	for (; key_part != end; key_part++) {
+
+		uchar sorted[REC_VERSION_56_MAX_INDEX_COL_LEN] = {'\0'};
+		ibool part_is_null = FALSE;
+
+		if (key_part->null_bit) {
+			if (buff_space > 0) {
+				if (record[key_part->null_offset] 
+				    & key_part->null_bit) {
+					*buff = 1;
+					part_is_null = TRUE;
+				} else {
+					*buff = 0;
+				}
+				buff++;
+				buff_space--;
+			} else {
+				fprintf (stderr, "WSREP: key truncated: %s\n",
+					 wsrep_thd_query(thd));
+			}
+		}
+		if (!part_is_null)  *key_is_null = FALSE;
+
+		field = key_part->field;
+		mysql_type = field->type();
+
+		if (mysql_type == MYSQL_TYPE_VARCHAR) {
+						/* >= 5.0.3 true VARCHAR */
+			ulint		lenlen;
+			ulint		len;
+			const byte*	data;
+			ulint		key_len;
+			ulint		true_len;
+			const CHARSET_INFO* cs;
+			int		error=0;
+
+			key_len = key_part->length;
+
+			if (part_is_null) {
+				true_len = key_len + 2;
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+				buff       += true_len;
+				buff_space -= true_len;
+				continue;
+			}
+			cs = field->charset();
+
+			lenlen = (ulint)
+				(((Field_varstring*)field)->length_bytes);
+
+			data = row_mysql_read_true_varchar(&len,
+				(byte*) (record
+				+ (ulint)get_field_offset(table, field)),
+				lenlen);
+
+			true_len = len;
+
+			/* For multi byte character sets we need to calculate
+			the true length of the key */
+
+			if (len > 0 && cs->mbmaxlen > 1) {
+				true_len = (ulint) cs->cset->well_formed_len(cs,
+						(const char *) data,
+						(const char *) data + len,
+                                                (uint) (key_len /
+                                                        cs->mbmaxlen),
+						&error);
+			}
+
+			/* In a column prefix index, we may need to truncate
+			the stored value: */
+
+			if (true_len > key_len) {
+				true_len = key_len;
+			}
+
+			memcpy(sorted, data, true_len);
+			true_len = wsrep_innobase_mysql_sort(
+				mysql_type, cs->number, sorted, true_len, 
+				REC_VERSION_56_MAX_INDEX_COL_LEN);
+
+			if (wsrep_protocol_version > 1) {
+			/* Note that we always reserve the maximum possible
+			   length of the true VARCHAR in the key value, though
+			   only len first bytes after the 2 length bytes contain
+			   actual data. The rest of the space was reset to zero
+			   in the bzero() call above. */
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+ 				memcpy(buff, sorted, true_len);
+                                buff       += true_len;
+				buff_space -= true_len;
+                        } else {
+                                buff += key_len;
+                        }
+		} else if (mysql_type == MYSQL_TYPE_TINY_BLOB
+			|| mysql_type == MYSQL_TYPE_MEDIUM_BLOB
+			|| mysql_type == MYSQL_TYPE_BLOB
+			|| mysql_type == MYSQL_TYPE_LONG_BLOB
+			/* MYSQL_TYPE_GEOMETRY data is treated
+			as BLOB data in innodb. */
+			|| mysql_type == MYSQL_TYPE_GEOMETRY) {
+
+			const CHARSET_INFO* cs;
+			ulint		key_len;
+			ulint		true_len;
+			int		error=0;
+			ulint		blob_len;
+			const byte*	blob_data;
+
+			ut_a(key_part->key_part_flag & HA_PART_KEY_SEG);
+
+			key_len = key_part->length;
+
+			if (part_is_null) {
+				true_len = key_len + 2;
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+				buff       += true_len;
+				buff_space -= true_len;
+
+				continue;
+			}
+
+			cs = field->charset();
+
+			blob_data = row_mysql_read_blob_ref(&blob_len,
+				(byte*) (record
+				+ (ulint)get_field_offset(table, field)),
+					(ulint) field->pack_length());
+
+			true_len = blob_len;
+
+			ut_a(get_field_offset(table, field)
+				== key_part->offset);
+
+			/* For multi byte character sets we need to calculate
+			the true length of the key */
+
+			if (blob_len > 0 && cs->mbmaxlen > 1) {
+				true_len = (ulint) cs->cset->well_formed_len(cs,
+						(const char *) blob_data,
+						(const char *) blob_data
+							+ blob_len,
+                                                (uint) (key_len /
+                                                        cs->mbmaxlen),
+						&error);
+			}
+
+			/* All indexes on BLOB and TEXT are column prefix
+			indexes, and we may need to truncate the data to be
+			stored in the key value: */
+
+			if (true_len > key_len) {
+				true_len = key_len;
+			}
+
+			memcpy(sorted, blob_data, true_len);
+			true_len = wsrep_innobase_mysql_sort(
+				mysql_type, cs->number, sorted, true_len,
+				REC_VERSION_56_MAX_INDEX_COL_LEN);
+
+
+			/* Note that we always reserve the maximum possible
+			length of the BLOB prefix in the key value. */
+                        if (wsrep_protocol_version > 1) {
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+				buff       += true_len;
+				buff_space -= true_len;
+			} else {
+				buff += key_len;
+			}
+			memcpy(buff, sorted, true_len);
+		} else {
+			/* Here we handle all other data types except the
+			true VARCHAR, BLOB and TEXT. Note that the column
+			value we store may be also in a column prefix
+			index. */
+
+			const CHARSET_INFO*	cs = NULL;
+			ulint			true_len;
+			ulint			key_len;
+			const uchar*		src_start;
+			int			error=0;
+			enum_field_types	real_type;
+
+			key_len = key_part->length;
+
+			if (part_is_null) {
+				true_len = key_len;
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+				buff       += true_len;
+				buff_space -= true_len;
+
+				continue;
+			}
+
+			src_start = record + key_part->offset;
+			real_type = field->real_type();
+			true_len = key_len;
+
+			/* Character set for the field is defined only
+			to fields whose type is string and real field
+			type is not enum or set. For these fields check
+			if character set is multi byte. */
+
+			if (real_type != MYSQL_TYPE_ENUM
+				&& real_type != MYSQL_TYPE_SET
+				&& ( mysql_type == MYSQL_TYPE_VAR_STRING
+					|| mysql_type == MYSQL_TYPE_STRING)) {
+
+				cs = field->charset();
+
+				/* For multi byte character sets we need to
+				calculate the true length of the key */
+
+				if (key_len > 0 && cs->mbmaxlen > 1) {
+
+					true_len = (ulint)
+						cs->cset->well_formed_len(cs,
+							(const char *)src_start,
+							(const char *)src_start
+								+ key_len,
+                                                        (uint) (key_len /
+                                                                cs->mbmaxlen),
+							&error);
+				}
+				memcpy(sorted, src_start, true_len);
+				true_len = wsrep_innobase_mysql_sort(
+					mysql_type, cs->number, sorted, true_len,
+					REC_VERSION_56_MAX_INDEX_COL_LEN);
+
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len   = buff_space;
+				}
+				memcpy(buff, sorted, true_len);
+			} else {
+				memcpy(buff, src_start, true_len);
+			}
+			buff       += true_len;
+			buff_space -= true_len;
+		}
+	}
+
+	ut_a(buff <= buff_start + buff_len);
+
+	DBUG_RETURN((uint)(buff - buff_start));
+}
+#endif /* WITH_WSREP */
+UNIV_INTERN
+uint
 ha_innobase::store_key_val_for_row(
 /*===============================*/
 	uint		keynr,	/*!< in: key number */
@@ -6965,6 +7482,9 @@ ha_innobase::write_row(
 	dberr_t		error;
 	int		error_result= 0;
 	ibool		auto_inc_used= FALSE;
+#ifdef WITH_WSREP
+	ibool           auto_inc_inserted= FALSE; /* if NULL was inserted */
+#endif
 	ulint		sql_command;
 	trx_t*		trx = thd_to_trx(user_thd);
 
@@ -6998,8 +7518,20 @@ ha_innobase::write_row(
 	if ((sql_command == SQLCOM_ALTER_TABLE
 	     || sql_command == SQLCOM_OPTIMIZE
 	     || sql_command == SQLCOM_CREATE_INDEX
+#ifdef WITH_WSREP
+	     || (wsrep_on(user_thd) && wsrep_load_data_splitting &&
+		 sql_command == SQLCOM_LOAD                      &&
+		 !thd_test_options(
+			user_thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))
+#endif /* WITH_WSREP */
 	     || sql_command == SQLCOM_DROP_INDEX)
 	    && num_write_row >= 10000) {
+#ifdef WITH_WSREP
+		if (wsrep_on(user_thd) && sql_command == SQLCOM_LOAD) {
+			WSREP_DEBUG("forced trx split for LOAD: %s", 
+				    wsrep_thd_query(user_thd));
+		}
+#endif /* WITH_WSREP */
 		/* ALTER TABLE is COMMITted at every 10000 copied rows.
 		The IX table lock for the original table has to be re-issued.
 		As this method will be called on a temporary table where the
@@ -7033,6 +7565,21 @@ no_commit:
 			*/
 			;
 		} else if (src_table == prebuilt->table) {
+#ifdef WITH_WSREP
+			switch (wsrep_run_wsrep_commit(user_thd, wsrep_hton, 1))
+			{
+			case WSREP_TRX_OK:
+				break;
+			case WSREP_TRX_SIZE_EXCEEDED:
+			case WSREP_TRX_CERT_FAIL:
+			case WSREP_TRX_ERROR:
+				DBUG_RETURN(1);
+			}
+
+			if (binlog_hton->commit(binlog_hton, user_thd, 1))
+                                DBUG_RETURN(1);
+                        wsrep_post_commit(user_thd, TRUE);
+#endif /* WITH_WSREP */
 			/* Source table is not in InnoDB format:
 			no need to re-acquire locks on it. */
 
@@ -7043,6 +7590,21 @@ no_commit:
 			/* We will need an IX lock on the destination table. */
 			prebuilt->sql_stat_start = TRUE;
 		} else {
+#ifdef WITH_WSREP
+			switch (wsrep_run_wsrep_commit(user_thd, wsrep_hton, 1))
+			{
+			case WSREP_TRX_OK:
+				break;
+			case WSREP_TRX_SIZE_EXCEEDED:
+			case WSREP_TRX_CERT_FAIL:
+			case WSREP_TRX_ERROR:
+				DBUG_RETURN(1);
+			}
+
+			if (binlog_hton->commit(binlog_hton, user_thd, 1))
+                                DBUG_RETURN(1);
+                        wsrep_post_commit(user_thd, TRUE);
+#endif /* WITH_WSREP */
 			/* Ensure that there are no other table locks than
 			LOCK_IX and LOCK_AUTO_INC on the destination table. */
 
@@ -7072,6 +7634,10 @@ no_commit:
 		innobase_get_auto_increment(). */
 		prebuilt->autoinc_error = DB_SUCCESS;
 
+#ifdef WITH_WSREP
+		auto_inc_inserted= (table->next_number_field->val_int() == 0);
+#endif
+
 		if ((error_result = update_auto_increment())) {
 			/* We don't want to mask autoinc overflow errors. */
 
@@ -7150,6 +7716,33 @@ no_commit:
 			case SQLCOM_REPLACE_SELECT:
 				goto set_max_autoinc;
 
+#ifdef WITH_WSREP
+			/* workaround for LP bug #355000, retrying the insert */
+			case SQLCOM_INSERT:
+                               if (wsrep_on(current_thd)                     &&
+                                   auto_inc_inserted                         &&
+                                   wsrep_drupal_282555_workaround            &&
+                                   wsrep_thd_retry_counter(current_thd) == 0 &&
+				    !thd_test_options(current_thd, 
+						      OPTION_NOT_AUTOCOMMIT | 
+						      OPTION_BEGIN)) {
+					WSREP_DEBUG(
+					    "retrying insert: %s",
+					    (*wsrep_thd_query(current_thd)) ? 
+						wsrep_thd_query(current_thd) : 
+						(char *)"void");
+					error= DB_SUCCESS;
+					wsrep_thd_set_conflict_state(
+						current_thd, MUST_ABORT);
+                                        innobase_srv_conc_exit_innodb(
+						prebuilt->trx);
+                                        /* jump straight to func exit over
+                                         * later wsrep hooks */
+                                        goto func_exit;
+				}
+                                break;
+#endif /* WITH_WSREP */
+
 			default:
 				break;
 			}
@@ -7209,6 +7802,21 @@ report_error:
 						   prebuilt->table->flags,
 						   user_thd);
 
+#ifdef WITH_WSREP
+	if (!error_result && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE &&
+	    wsrep_on(user_thd) && !wsrep_consistency_check(user_thd) &&
+	    (sql_command != SQLCOM_LOAD || 
+	     thd_binlog_format(user_thd) == BINLOG_FORMAT_ROW)) {
+
+		if (wsrep_append_keys(user_thd, false, record, NULL)) {
+ 			DBUG_PRINT("wsrep", ("row key failed"));
+ 			error_result = HA_ERR_INTERNAL_ERROR;
+			goto wsrep_error;
+		}
+	}
+wsrep_error:
+#endif /* WITH_WSREP */
+
 	if (error_result == HA_FTS_INVALID_DOCID) {
 		my_error(HA_FTS_INVALID_DOCID, MYF(0));
 	}
@@ -7496,6 +8104,87 @@ calc_row_difference(
 	return(DB_SUCCESS);
 }
 
+#ifdef WITH_WSREP
+static
+int
+wsrep_calc_row_hash(
+/*================*/
+	byte*		digest,		/*!< in/out: md5 sum */
+	const uchar*	row,		/*!< in: row in MySQL format */
+	TABLE*		table,		/*!< in: table in MySQL data
+					dictionary */
+	row_prebuilt_t*	prebuilt,	/*!< in: InnoDB prebuilt struct */
+	THD*		thd)		/*!< in: user thread */
+{
+	Field*		field;
+	enum_field_types field_mysql_type;
+	uint		n_fields;
+	ulint		len;
+	const byte*	ptr;
+	ulint		col_type;
+	uint		i;
+
+	void *ctx = wsrep_md5_init();
+
+	n_fields = table->s->fields;
+
+	for (i = 0; i < n_fields; i++) {
+		byte null_byte=0;
+		byte true_byte=1;
+
+		field = table->field[i];
+
+		ptr = (const byte*) row + get_field_offset(table, field);
+		len = field->pack_length();
+
+		field_mysql_type = field->type();
+
+		col_type = prebuilt->table->cols[i].mtype;
+
+		switch (col_type) {
+
+		case DATA_BLOB:
+			ptr = row_mysql_read_blob_ref(&len, ptr, len);
+
+			break;
+
+		case DATA_VARCHAR:
+		case DATA_BINARY:
+		case DATA_VARMYSQL:
+			if (field_mysql_type == MYSQL_TYPE_VARCHAR) {
+				/* This is a >= 5.0.3 type true VARCHAR where
+				the real payload data length is stored in
+				1 or 2 bytes */
+
+				ptr = row_mysql_read_true_varchar(
+					&len, ptr,
+					(ulint)
+					(((Field_varstring*)field)->length_bytes));
+
+			}
+
+			break;
+		default:
+			;
+		}
+		/*
+		if (field->null_ptr &&
+		    field_in_record_is_null(table, field, (char*) row)) {
+		*/
+
+		if (field->is_null_in_record(row)) {
+			wsrep_md5_update(ctx, (char*)&null_byte, 1);
+		} else {
+			wsrep_md5_update(ctx, (char*)&true_byte, 1);
+			wsrep_md5_update(ctx, (char*)ptr, len);
+		}
+	}
+
+	wsrep_compute_md5_hash((char*)digest, ctx);
+
+	return(0);
+}
+#endif /* WITH_WSREP */
 /**********************************************************************//**
 Updates a row given as a parameter to a new value. Note that we are given
 whole rows, not just the fields which are updated: this incurs some
@@ -7633,6 +8322,24 @@ func_exit:
 
 	innobase_active_small();
 
+#ifdef WITH_WSREP
+	if (error == DB_SUCCESS                          && 
+	    wsrep_thd_exec_mode(user_thd) == LOCAL_STATE &&
+            wsrep_on(user_thd)) {
+
+		DBUG_PRINT("wsrep", ("update row key"));
+
+		if (wsrep_append_keys(user_thd, false, old_row, new_row)) {
+			WSREP_DEBUG("WSREP: UPDATE_ROW_KEY FAILED");
+			DBUG_PRINT("wsrep", ("row key failed"));
+			err = HA_ERR_INTERNAL_ERROR;
+			goto wsrep_error;
+		}
+	}
+wsrep_error:
+#endif /* WITH_WSREP */
+
+
 	DBUG_RETURN(err);
 }
 
@@ -7680,6 +8387,19 @@ ha_innobase::delete_row(
 
 	innobase_active_small();
 
+#ifdef WITH_WSREP
+	if (error == DB_SUCCESS                          && 
+	    wsrep_thd_exec_mode(user_thd) == LOCAL_STATE &&
+            wsrep_on(user_thd)) {
+
+		if (wsrep_append_keys(user_thd, false, record, NULL)) {
+			DBUG_PRINT("wsrep", ("delete fail"));
+			error = (dberr_t)HA_ERR_INTERNAL_ERROR;
+			goto wsrep_error;
+		}
+	}
+wsrep_error:
+#endif
 	DBUG_RETURN(convert_error_code_to_mysql(
 			    error, prebuilt->table->flags, user_thd));
 }
@@ -8840,6 +9560,394 @@ ha_innobase::ft_end()
 
 	rnd_end();
 }
+#ifdef WITH_WSREP
+extern dict_index_t*
+wsrep_dict_foreign_find_index(
+	dict_table_t*	table,
+	const char**	col_names,
+	const char**	columns,
+	ulint		n_cols,
+	dict_index_t*	types_idx,
+	ibool		check_charsets,
+	ulint		check_null);
+
+
+extern dberr_t
+wsrep_append_foreign_key(
+/*===========================*/
+	trx_t*		trx,		/*!< in: trx */
+	dict_foreign_t*	foreign,	/*!< in: foreign key constraint */
+	const rec_t*	rec,		/*!<in: clustered index record */
+	dict_index_t*	index,		/*!<in: clustered index */
+	ibool		referenced,	/*!<in: is check for referenced table */
+	ibool		shared)		/*!<in: is shared access */
+{
+	ut_a(trx);
+	THD*  thd = (THD*)trx->mysql_thd;
+	ulint rcode = DB_SUCCESS;
+	char  cache_key[513] = {'\0'};
+	int   cache_key_len;
+    bool const copy = true;
+
+	if (!wsrep_on(trx->mysql_thd) ||
+	    wsrep_thd_exec_mode(thd) != LOCAL_STATE)
+		return DB_SUCCESS;
+
+	if (!thd || !foreign ||
+	    (!foreign->referenced_table && !foreign->foreign_table))
+	{
+		WSREP_INFO("FK: %s missing in: %s",
+			(!thd)      ?  "thread"     :
+			((!foreign) ?  "constraint" :
+			((!foreign->referenced_table) ?
+			     "referenced table" : "foreign table")),
+			   (thd && wsrep_thd_query(thd)) ?
+			   wsrep_thd_query(thd) : "void");
+		return DB_ERROR;
+	}
+
+	if ( !((referenced) ?
+		foreign->referenced_table : foreign->foreign_table))
+	{
+		WSREP_DEBUG("pulling %s table into cache",
+			    (referenced) ? "referenced" : "foreign");
+		mutex_enter(&(dict_sys->mutex));
+		if (referenced)
+		{
+			foreign->referenced_table =
+				dict_table_get_low(
+					foreign->referenced_table_name_lookup);
+			if (foreign->referenced_table)
+			{
+				foreign->referenced_index =
+					wsrep_dict_foreign_find_index(
+						foreign->referenced_table, NULL,
+						foreign->referenced_col_names,
+						foreign->n_fields, 
+						foreign->foreign_index,
+						TRUE, FALSE);
+			}
+		}
+		else
+		{
+	  		foreign->foreign_table =
+				dict_table_get_low(
+					foreign->foreign_table_name_lookup);
+			if (foreign->foreign_table)
+			{
+				foreign->foreign_index =
+					wsrep_dict_foreign_find_index(
+						foreign->foreign_table, NULL,
+						foreign->foreign_col_names,
+						foreign->n_fields,
+						foreign->referenced_index, 
+						TRUE, FALSE);
+			}
+		}
+		mutex_exit(&(dict_sys->mutex));
+	}
+
+	if ( !((referenced) ?
+		foreign->referenced_table : foreign->foreign_table))
+	{
+		WSREP_WARN("FK: %s missing in query: %s",
+			   (!foreign->referenced_table) ?
+			   "referenced table" : "foreign table",
+			   (wsrep_thd_query(thd)) ?
+			   wsrep_thd_query(thd) : "void");
+		return DB_ERROR;
+	}
+	byte  key[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+	ulint len = WSREP_MAX_SUPPORTED_KEY_LENGTH;
+
+	dict_index_t *idx_target = (referenced) ?
+		foreign->referenced_index : index;
+	dict_index_t *idx = (referenced) ?
+		UT_LIST_GET_FIRST(foreign->referenced_table->indexes) :
+		UT_LIST_GET_FIRST(foreign->foreign_table->indexes);
+	int i = 0;
+	while (idx != NULL && idx != idx_target) {
+		if (innobase_strcasecmp (idx->name, innobase_index_reserve_name) != 0) {
+			i++;
+		}
+		idx = UT_LIST_GET_NEXT(indexes, idx);
+	}
+	ut_a(idx);
+	key[0] = (char)i;
+
+	rcode = wsrep_rec_get_foreign_key(
+		&key[1], &len, rec, index, idx,
+		wsrep_protocol_version > 1);
+	if (rcode != DB_SUCCESS) {
+		WSREP_ERROR(
+			"FK key set failed: %lu (%lu %lu), index: %s %s, %s",
+			rcode, referenced, shared,
+			(index && index->name)       ? index->name :
+				"void index",
+			(index && index->table_name) ? index->table_name :
+				"void table",
+			wsrep_thd_query(thd));
+		return DB_ERROR;
+	}
+	strncpy(cache_key,
+		(wsrep_protocol_version > 1) ?
+		((referenced) ?
+			foreign->referenced_table->name :
+			foreign->foreign_table->name) :
+		foreign->foreign_table->name, sizeof(cache_key) - 1);
+	cache_key_len = strlen(cache_key);
+#ifdef WSREP_DEBUG_PRINT
+	ulint j;
+	fprintf(stderr, "FK parent key, table: %s %s len: %lu ",
+		cache_key, (shared) ? "shared" : "exclusive", len+1);
+	for (j=0; j<len+1; j++) {
+		fprintf(stderr, " %hhX, ", key[j]);
+	}
+	fprintf(stderr, "\n");
+#endif
+	char *p = strchr(cache_key, '/');
+	if (p) {
+		*p = '\0';
+	} else {
+		WSREP_WARN("unexpected foreign key table %s %s",
+			   foreign->referenced_table->name,
+			   foreign->foreign_table->name);
+	}
+
+	wsrep_buf_t wkey_part[3];
+        wsrep_key_t wkey = {wkey_part, 3};
+	if (!wsrep_prepare_key_for_innodb(
+		(const uchar*)cache_key,
+		cache_key_len +  1,
+		(const uchar*)key, len+1,
+		wkey_part,
+		(size_t*)&wkey.key_parts_num)) {
+		WSREP_WARN("key prepare failed for cascaded FK: %s",
+			   (wsrep_thd_query(thd)) ?
+			    wsrep_thd_query(thd) : "void");
+		return DB_ERROR;
+	}
+	rcode = (int)wsrep->append_key(
+		wsrep,
+		wsrep_ws_handle(thd, trx),
+		&wkey,
+		1,
+		shared ? WSREP_KEY_SHARED : WSREP_KEY_EXCLUSIVE,
+                copy);
+	if (rcode) {
+		DBUG_PRINT("wsrep", ("row key failed: %lu", rcode));
+		WSREP_ERROR("Appending cascaded fk row key failed: %s, %lu",
+			    (wsrep_thd_query(thd)) ?
+			    wsrep_thd_query(thd) : "void", rcode);
+		return DB_ERROR;
+	}
+
+	return DB_SUCCESS;
+}
+
+static int
+wsrep_append_key(
+/*==================*/
+	THD		*thd,
+	trx_t 		*trx,
+	TABLE_SHARE 	*table_share,
+	TABLE 		*table,
+	const char*	key,
+	uint16_t        key_len,
+	bool            shared
+)
+{
+	DBUG_ENTER("wsrep_append_key");
+	bool const copy = true;
+#ifdef WSREP_DEBUG_PRINT
+	fprintf(stderr, "%s conn %ld, trx %llu, keylen %d, table %s ",
+		(shared) ? "Shared" : "Exclusive",
+		wsrep_thd_thread_id(thd), (long long)trx->id, key_len,
+		table_share->table_name.str);
+	for (int i=0; i<key_len; i++) {
+		fprintf(stderr, "%hhX, ", key[i]);
+	}
+	fprintf(stderr, "\n");
+#endif
+	wsrep_buf_t wkey_part[3];
+        wsrep_key_t wkey = {wkey_part, 3};
+	if (!wsrep_prepare_key_for_innodb(
+			(const uchar*)table_share->table_cache_key.str,
+			table_share->table_cache_key.length,
+			(const uchar*)key, key_len,
+			wkey_part,
+			(size_t*)&wkey.key_parts_num)) {
+		WSREP_WARN("key prepare failed for: %s",
+			   (wsrep_thd_query(thd)) ?
+			   wsrep_thd_query(thd) : "void");
+		DBUG_RETURN(HA_ERR_INTERNAL_ERROR);
+	}
+
+	int rcode = (int)wsrep->append_key(
+			       wsrep,
+			       wsrep_ws_handle(thd, trx),
+			       &wkey,
+			       1,
+			       shared ? WSREP_KEY_SHARED : WSREP_KEY_EXCLUSIVE,
+                               copy);
+	if (rcode) {
+		DBUG_PRINT("wsrep", ("row key failed: %d", rcode));
+		WSREP_WARN("Appending row key failed: %s, %d",
+			   (wsrep_thd_query(thd)) ?
+			   wsrep_thd_query(thd) : "void", rcode);
+		DBUG_RETURN(HA_ERR_INTERNAL_ERROR);
+	}
+	DBUG_RETURN(0);
+}
+
+extern void compute_md5_hash(char *digest, const char *buf, int len);
+#define MD5_HASH compute_md5_hash
+
+int
+ha_innobase::wsrep_append_keys(
+/*==================*/
+	THD 		*thd,
+	bool		shared,
+	const uchar*	record0,	/* in: row in MySQL format */
+	const uchar*	record1)	/* in: row in MySQL format */
+{
+	int rcode;
+	DBUG_ENTER("wsrep_append_keys");
+
+	bool key_appended = false;
+	trx_t *trx = thd_to_trx(thd);
+
+	if (table_share && table_share->tmp_table  != NO_TMP_TABLE) {
+		WSREP_DEBUG("skipping tmp table DML: THD: %lu tmp: %d SQL: %s", 
+			    wsrep_thd_thread_id(thd),
+			    table_share->tmp_table,
+			    (wsrep_thd_query(thd)) ? 
+			    wsrep_thd_query(thd) : "void");
+		DBUG_RETURN(0);
+	}
+
+	if (wsrep_protocol_version == 0) {
+		uint	len;
+		char 	keyval[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+		char 	*key 		= &keyval[0];
+		ibool    is_null;
+
+		len = wsrep_store_key_val_for_row(
+			thd, table, 0, key, WSREP_MAX_SUPPORTED_KEY_LENGTH, 
+			record0, &is_null);
+
+		if (!is_null) {
+			rcode = wsrep_append_key(
+				thd, trx, table_share, table, keyval, 
+				len, shared);
+			if (rcode) DBUG_RETURN(rcode);
+		}
+		else
+		{
+			WSREP_DEBUG("NULL key skipped (proto 0): %s", 
+				    wsrep_thd_query(thd));
+		}
+	} else {
+		ut_a(table->s->keys <= 256);
+		uint i;
+                bool hasPK= false;
+
+		for (i=0; i<table->s->keys; ++i) {
+			KEY*  key_info	= table->key_info + i;
+			if (key_info->flags & HA_NOSAME) {
+				hasPK = true;
+			}
+		}
+
+		for (i=0; i<table->s->keys; ++i) {
+			uint  len;
+			char  keyval0[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+			char  keyval1[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+			char* key0 		= &keyval0[1];
+			char* key1 		= &keyval1[1];
+			KEY*  key_info	= table->key_info + i;
+			ibool is_null;
+
+			dict_index_t* idx  = innobase_get_index(i);
+			dict_table_t* tab  = (idx) ? idx->table : NULL;
+
+			keyval0[0] = (char)i;
+			keyval1[0] = (char)i;
+
+			if (!tab) {
+				WSREP_WARN("MySQL-InnoDB key mismatch %s %s",
+					   table->s->table_name.str, 
+					   key_info->name);
+			}
+			/* !hasPK == table with no PK, must append all non-unique keys */
+			if (!hasPK || key_info->flags & HA_NOSAME ||
+			    ((tab &&
+			      dict_table_get_referenced_constraint(tab, idx)) ||
+			     (!tab && referenced_by_foreign_key()))) {
+
+				len = wsrep_store_key_val_for_row(
+					thd, table, i, key0, 
+					WSREP_MAX_SUPPORTED_KEY_LENGTH, 
+					record0, &is_null);
+				if (!is_null) {
+					rcode = wsrep_append_key(
+						thd, trx, table_share, table, 
+						keyval0, len+1, shared);
+					if (rcode) DBUG_RETURN(rcode);
+
+				if (key_info->flags & HA_NOSAME || shared)
+			  		key_appended = true;
+				}
+				else
+				{
+					WSREP_DEBUG("NULL key skipped: %s", 
+						    wsrep_thd_query(thd));
+				}
+				if (record1) {
+					len = wsrep_store_key_val_for_row(
+						thd, table, i, key1, 
+						WSREP_MAX_SUPPORTED_KEY_LENGTH,
+						record1, &is_null);
+					if (!is_null && memcmp(key0, key1, len)) {
+						rcode = wsrep_append_key(
+							thd, trx, table_share, 
+							table, 
+							keyval1, len+1, shared);
+						if (rcode) DBUG_RETURN(rcode);
+					}
+				}
+			}
+		}
+	}
+
+	/* if no PK, calculate hash of full row, to be the key value */
+	if (!key_appended && wsrep_certify_nonPK) {
+		uchar digest[16];
+		int rcode;
+
+		wsrep_calc_row_hash(digest, record0, table, prebuilt, thd);
+		if ((rcode = wsrep_append_key(thd, trx, table_share, table, 
+					      (const char*) digest, 16, 
+					      shared))) {
+			DBUG_RETURN(rcode);
+		}
+
+		if (record1) {
+			wsrep_calc_row_hash(
+				digest, record1, table, prebuilt, thd);
+			if ((rcode = wsrep_append_key(thd, trx, table_share, 
+						      table,
+						      (const char*) digest, 
+						      16, shared))) {
+				DBUG_RETURN(rcode);
+			}
+		}
+		DBUG_RETURN(0);
+	}
+
+	DBUG_RETURN(0);
+}
+#endif /* WITH_WSREP */
 
 /*********************************************************************//**
 Stores a reference to the current row to 'ref' field of the handle. Note
@@ -12699,11 +13807,18 @@ ha_innobase::external_lock(
 		/* used by test case */
 		DBUG_EXECUTE_IF("no_innodb_binlog_errors", skip = true;);
 		if (!skip) {
+#ifdef WITH_WSREP
+		  if (!wsrep_on(thd) || wsrep_thd_exec_mode(thd) == LOCAL_STATE)
+			{
+#endif /* WITH_WSREP */
 			my_error(ER_BINLOG_STMT_MODE_AND_ROW_ENGINE, MYF(0),
 			         " InnoDB is limited to row-logging when "
 			         "transaction isolation level is "
 			         "READ COMMITTED or READ UNCOMMITTED.");
 			DBUG_RETURN(HA_ERR_LOGGING_IMPOSSIBLE);
+#ifdef WITH_WSREP
+			}
+#endif /* WITH_WSREP */
 		}
 	}
 
@@ -14157,6 +15272,9 @@ innobase_xa_prepare(
 	to the session variable take effect only in the next transaction */
 	if (!trx->support_xa) {
 
+#ifdef WITH_WSREP
+                thd_get_xid(thd, (MYSQL_XID*) &trx->xid);
+#endif // WITH_WSREP
 		return(0);
 	}
 
@@ -16225,6 +17343,305 @@ static SHOW_VAR innodb_status_variables_export[]= {
 static struct st_mysql_storage_engine innobase_storage_engine=
 { MYSQL_HANDLERTON_INTERFACE_VERSION };
 
+#ifdef WITH_WSREP
+void
+wsrep_abort_slave_trx(wsrep_seqno_t bf_seqno, wsrep_seqno_t victim_seqno)
+{
+	WSREP_ERROR("Trx %lld tries to abort slave trx %lld. This could be "
+		"caused by:\n\t"
+		"1) unsupported configuration options combination, please check documentation.\n\t"
+		"2) a bug in the code.\n\t"
+		"3) a database corruption.\n Node consistency compromized, "
+		"need to abort. Restart the node to resync with cluster.",
+		(long long)bf_seqno, (long long)victim_seqno);
+	abort();
+}
+/*******************************************************************//**
+This function is used to kill one transaction in BF. */
+
+int
+wsrep_innobase_kill_one_trx(void * const bf_thd_ptr,
+                            const trx_t * const bf_trx,
+                            trx_t *victim_trx, ibool signal)
+{
+        ut_ad(lock_mutex_own());
+        ut_ad(trx_mutex_own(victim_trx));
+        ut_ad(bf_thd_ptr);
+        ut_ad(victim_trx);
+
+	DBUG_ENTER("wsrep_innobase_kill_one_trx");
+	THD *bf_thd       = bf_thd_ptr ? (THD*) bf_thd_ptr : NULL;
+	THD *thd          = (THD *) victim_trx->mysql_thd;
+	int64_t bf_seqno  = (bf_thd) ? wsrep_thd_trx_seqno(bf_thd) : 0;
+
+	if (!thd) {
+		DBUG_PRINT("wsrep", ("no thd for conflicting lock"));
+		WSREP_WARN("no THD for trx: %lu", victim_trx->id);
+		DBUG_RETURN(1);
+	}
+	if (!bf_thd) {
+		DBUG_PRINT("wsrep", ("no BF thd for conflicting lock"));
+		WSREP_WARN("no BF THD for trx: %lu", (bf_trx) ? bf_trx->id : 0);
+		DBUG_RETURN(1);
+	}
+
+	WSREP_LOG_CONFLICT(bf_thd, thd, TRUE);
+
+	WSREP_DEBUG("BF kill (%lu, seqno: %lld), victim: (%lu) trx: %lu",
+ 		    signal, (long long)bf_seqno,
+ 		    wsrep_thd_thread_id(thd),
+		    victim_trx->id);
+
+	WSREP_DEBUG("Aborting query: %s",
+		  (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void");
+
+	wsrep_thd_LOCK(thd);
+
+	if (wsrep_thd_query_state(thd) == QUERY_EXITING) {
+		WSREP_DEBUG("kill trx EXITING for %lu", victim_trx->id);
+		wsrep_thd_UNLOCK(thd);
+		DBUG_RETURN(0);
+	}
+	if(wsrep_thd_exec_mode(thd) != LOCAL_STATE) {
+		WSREP_DEBUG("withdraw for BF trx: %lu, state: %d",
+			    victim_trx->id,
+		wsrep_thd_conflict_state(thd));
+	}
+
+	switch (wsrep_thd_conflict_state(thd)) {
+	case NO_CONFLICT:
+		wsrep_thd_set_conflict_state(thd, MUST_ABORT);
+		break;
+        case MUST_ABORT:
+		WSREP_DEBUG("victim %lu in MUST ABORT state",
+			    victim_trx->id);
+		wsrep_thd_UNLOCK(thd);
+		wsrep_thd_awake(thd, signal);
+		DBUG_RETURN(0);
+		break;
+	case ABORTED:
+	case ABORTING: // fall through
+	default:
+		WSREP_DEBUG("victim %lu in state %d",
+			    victim_trx->id, wsrep_thd_conflict_state(thd));
+		wsrep_thd_UNLOCK(thd);
+		DBUG_RETURN(0);
+		break;
+	}
+
+	switch (wsrep_thd_query_state(thd)) {
+	case QUERY_COMMITTING:
+		enum wsrep_status rcode;
+
+		WSREP_DEBUG("kill query for: %ld",
+			    wsrep_thd_thread_id(thd));
+		WSREP_DEBUG("kill trx QUERY_COMMITTING for %lu",
+			    victim_trx->id);
+
+		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
+			wsrep_abort_slave_trx(bf_seqno,
+					      wsrep_thd_trx_seqno(thd));
+		} else {
+			rcode = wsrep->abort_pre_commit(
+				wsrep, bf_seqno,
+				(wsrep_trx_id_t)victim_trx->id
+			);
+
+			switch (rcode) {
+			case WSREP_WARNING:
+				WSREP_DEBUG("cancel commit warning: %lu",
+					    victim_trx->id);
+				wsrep_thd_UNLOCK(thd);
+				wsrep_thd_awake(thd, signal);
+				DBUG_RETURN(1);
+				break;
+			case WSREP_OK:
+				break;
+			default:
+				WSREP_ERROR(
+					"cancel commit bad exit: %d %lu",
+					rcode,
+					victim_trx->id);
+				/* unable to interrupt, must abort */
+				/* note: kill_mysql() will block, if we cannot.
+				 * kill the lock holder first.
+				 */
+				abort();
+				break;
+			}
+		}
+		wsrep_thd_UNLOCK(thd);
+		wsrep_thd_awake(thd, signal);
+		break;
+	case QUERY_EXEC:
+		/* it is possible that victim trx is itself waiting for some
+		 * other lock. We need to cancel this waiting
+		 */
+		WSREP_DEBUG("kill trx QUERY_EXEC for %lu", victim_trx->id);
+
+		victim_trx->lock.was_chosen_as_deadlock_victim= TRUE;
+		if (victim_trx->lock.wait_lock) {
+			WSREP_DEBUG("victim has wait flag: %ld",
+				wsrep_thd_thread_id(thd));
+			lock_t*  wait_lock = victim_trx->lock.wait_lock;
+			if (wait_lock) {
+				WSREP_DEBUG("canceling wait lock");
+				victim_trx->lock.was_chosen_as_deadlock_victim= TRUE;
+				lock_cancel_waiting_and_release(wait_lock);
+			}
+
+			wsrep_thd_UNLOCK(thd);
+			wsrep_thd_awake(thd, signal);
+		} else {
+			/* abort currently executing query */
+			DBUG_PRINT("wsrep",("sending KILL_QUERY to: %ld",
+                                            wsrep_thd_thread_id(thd)));
+			WSREP_DEBUG("kill query for: %ld",
+				wsrep_thd_thread_id(thd));
+			/* Note that innobase_kill_connection will take lock_mutex
+			and trx_mutex */
+			wsrep_thd_UNLOCK(thd);
+			wsrep_thd_awake(thd, signal);
+
+			/* for BF thd, we need to prevent him from committing */
+			if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
+				wsrep_abort_slave_trx(bf_seqno,
+						    wsrep_thd_trx_seqno(thd));
+			}
+		}
+		break;
+	case QUERY_IDLE:
+	{
+		bool skip_abort= false;
+		wsrep_aborting_thd_t abortees;
+
+		WSREP_DEBUG("kill IDLE for %lu", victim_trx->id);
+
+		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
+			WSREP_DEBUG("kill BF IDLE, seqno: %lld",
+				    (long long)wsrep_thd_trx_seqno(thd));
+			wsrep_thd_UNLOCK(thd);
+			wsrep_abort_slave_trx(bf_seqno,
+					      wsrep_thd_trx_seqno(thd));
+			DBUG_RETURN(0);
+		}
+                /* This will lock thd from proceeding after net_read() */
+		wsrep_thd_set_conflict_state(thd, ABORTING);
+
+		mysql_mutex_lock(&LOCK_wsrep_rollback);
+
+		abortees = wsrep_aborting_thd;
+		while (abortees && !skip_abort) {
+			/* check if we have a kill message for this already */
+			if (abortees->aborting_thd == thd) {
+				skip_abort = true;
+				WSREP_WARN("duplicate thd aborter %lu",
+					  wsrep_thd_thread_id(thd));
+			}
+			abortees = abortees->next;
+		}
+		if (!skip_abort) {
+			wsrep_aborting_thd_t aborting = (wsrep_aborting_thd_t)
+				my_malloc(sizeof(struct wsrep_aborting_thd),
+					  MYF(0));
+			aborting->aborting_thd  = thd;
+			aborting->next          = wsrep_aborting_thd;
+			wsrep_aborting_thd      = aborting;
+ 			DBUG_PRINT("wsrep",("enqueuing trx abort for %lu",
+                                             wsrep_thd_thread_id(thd)));
+			WSREP_DEBUG("enqueuing trx abort for (%lu)",
+				    wsrep_thd_thread_id(thd));
+		}
+
+		DBUG_PRINT("wsrep",("signalling wsrep rollbacker"));
+		WSREP_DEBUG("signaling aborter");
+		mysql_cond_signal(&COND_wsrep_rollback);
+		mysql_mutex_unlock(&LOCK_wsrep_rollback);
+		wsrep_thd_UNLOCK(thd);
+
+		break;
+	}
+	default:
+		WSREP_WARN("bad wsrep query state: %d",
+			  wsrep_thd_query_state(thd));
+		wsrep_thd_UNLOCK(thd);
+		break;
+	}
+
+	DBUG_RETURN(0);
+}
+
+static
+int
+wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd,
+			my_bool signal)
+{
+	DBUG_ENTER("wsrep_innobase_abort_thd");
+	trx_t* victim_trx = thd_to_trx(victim_thd);
+	trx_t* bf_trx     = (bf_thd) ? thd_to_trx(bf_thd) : NULL;
+	WSREP_DEBUG("abort transaction: BF: %s victim: %s",
+		    wsrep_thd_query(bf_thd),
+		    wsrep_thd_query(victim_thd));
+
+	if (victim_trx)
+	{
+                lock_mutex_enter();
+                trx_mutex_enter(victim_trx);
+		int rcode = wsrep_innobase_kill_one_trx(bf_thd, bf_trx,
+                                                        victim_trx, signal);
+                trx_mutex_exit(victim_trx);
+                lock_mutex_exit();
+		wsrep_srv_conc_cancel_wait(victim_trx);
+
+ 		DBUG_RETURN(rcode);
+	} else {
+		WSREP_DEBUG("victim does not have transaction");
+		wsrep_thd_LOCK(victim_thd);
+		wsrep_thd_set_conflict_state(victim_thd, MUST_ABORT);
+		wsrep_thd_UNLOCK(victim_thd);
+		wsrep_thd_awake(victim_thd, signal); 
+	}
+	DBUG_RETURN(-1);
+}
+
+static int innobase_wsrep_set_checkpoint(handlerton* hton, const XID* xid)
+{
+	DBUG_ASSERT(hton == innodb_hton_ptr);
+        if (wsrep_is_wsrep_xid(xid)) {
+                mtr_t mtr;
+                mtr_start(&mtr);
+                trx_sysf_t* sys_header = trx_sysf_get(&mtr);
+                trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr);
+                mtr_commit(&mtr);
+                innobase_flush_logs(hton);
+                return 0;
+        } else {
+                return 1;
+        }
+}
+
+static int innobase_wsrep_get_checkpoint(handlerton* hton, XID* xid)
+{
+	DBUG_ASSERT(hton == innodb_hton_ptr);
+        trx_sys_read_wsrep_checkpoint(xid);
+        return 0;
+}
+
+static void
+wsrep_fake_trx_id(
+/*==================*/
+	handlerton	*hton,
+	THD		*thd)	/*!< in: user thread handle */
+{
+	mutex_enter(&trx_sys->mutex);
+	trx_id_t trx_id = trx_sys_get_new_trx_id();
+	mutex_exit(&trx_sys->mutex);
+
+	(void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id);
+}
+
+#endif /* WITH_WSREP */
+
 /* plugin options */
 
 static MYSQL_SYSVAR_ENUM(checksum_algorithm, srv_checksum_algorithm,
@@ -16957,6 +18374,40 @@ static MYSQL_SYSVAR_BOOL(disable_background_merge,
   NULL, NULL, FALSE);
 #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
 
+#ifdef WITH_INNODB_DISALLOW_WRITES
+/*******************************************************
+ *    innobase_disallow_writes variable definition     *
+ *******************************************************/
+ 
+/* Must always init to FALSE. */
+static my_bool	innobase_disallow_writes	= FALSE;
+
+/**************************************************************************
+An "update" method for innobase_disallow_writes variable. */
+static
+void
+innobase_disallow_writes_update(
+/*============================*/
+	THD*			thd,		/* in: thread handle */
+	st_mysql_sys_var*	var,		/* in: pointer to system
+						variable */
+	void*			var_ptr,	/* out: pointer to dynamic
+						variable */
+	const void*		save)		/* in: temporary storage */
+{
+	*(my_bool*)var_ptr = *(my_bool*)save;
+	ut_a(srv_allow_writes_event);
+	if (*(my_bool*)var_ptr)
+		os_event_reset(srv_allow_writes_event);
+	else
+		os_event_set(srv_allow_writes_event);
+}
+
+static MYSQL_SYSVAR_BOOL(disallow_writes, innobase_disallow_writes,
+  PLUGIN_VAR_NOCMDOPT,
+  "Tell InnoDB to stop any writes to disk",
+  NULL, innobase_disallow_writes_update, FALSE);
+#endif /* WITH_INNODB_DISALLOW_WRITES */
 static MYSQL_SYSVAR_BOOL(random_read_ahead, srv_random_read_ahead,
   PLUGIN_VAR_NOCMDARG,
   "Whether to use read ahead for random access within an extent.",
@@ -17218,6 +18669,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
   MYSQL_SYSVAR(change_buffering_debug),
   MYSQL_SYSVAR(disable_background_merge),
 #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
+#ifdef WITH_INNODB_DISALLOW_WRITES
+  MYSQL_SYSVAR(disallow_writes),
+#endif /* WITH_INNODB_DISALLOW_WRITES */
   MYSQL_SYSVAR(random_read_ahead),
   MYSQL_SYSVAR(read_ahead_threshold),
   MYSQL_SYSVAR(read_only),
diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h
index 912be30c0ec..1fa9fe139fb 100644
--- a/storage/innobase/handler/ha_innodb.h
+++ b/storage/innobase/handler/ha_innodb.h
@@ -119,6 +119,10 @@ class ha_innobase: public handler
 	void innobase_initialize_autoinc();
 	dict_index_t* innobase_get_index(uint keynr);
 
+#ifdef WITH_WSREP
+	int wsrep_append_keys(THD *thd, bool shared,
+			      const uchar* record0, const uchar* record1);
+#endif
 	/* Init values for the class: */
  public:
 	ha_innobase(handlerton *hton, TABLE_SHARE *table_arg);
@@ -466,7 +470,40 @@ __attribute__((nonnull));
  */
 extern void mysql_bin_log_commit_pos(THD *thd, ulonglong *out_pos, const char **out_file);
 
-struct trx_t;
+#ifdef WITH_WSREP
+#include <wsrep_mysqld.h>
+//extern "C" int wsrep_trx_order_before(void *thd1, void *thd2);
+
+extern "C" bool wsrep_thd_is_wsrep_on(THD *thd);
+
+extern "C" enum wsrep_exec_mode wsrep_thd_exec_mode(THD *thd);
+extern "C" enum wsrep_conflict_state wsrep_thd_conflict_state(THD *thd);
+extern "C" enum wsrep_query_state wsrep_thd_query_state(THD *thd);
+extern "C" const char * wsrep_thd_exec_mode_str(THD *thd);
+extern "C" const char * wsrep_thd_conflict_state_str(THD *thd);
+extern "C" const char * wsrep_thd_query_state_str(THD *thd);
+extern "C" wsrep_ws_handle_t* wsrep_thd_ws_handle(THD *thd);
+
+extern "C" void wsrep_thd_set_exec_mode(THD *thd, enum wsrep_exec_mode mode);
+extern "C" void wsrep_thd_set_query_state(
+	THD *thd, enum wsrep_query_state state);
+extern "C" void wsrep_thd_set_conflict_state(
+	THD *thd, enum wsrep_conflict_state state);
+
+extern "C" void wsrep_thd_set_trx_to_replay(THD *thd, uint64 trx_id);
+
+extern "C"void wsrep_thd_LOCK(THD *thd);
+extern "C"void wsrep_thd_UNLOCK(THD *thd);
+extern "C" uint32 wsrep_thd_wsrep_rand(THD *thd);
+extern "C" time_t wsrep_thd_query_start(THD *thd);
+extern "C" my_thread_id wsrep_thd_thread_id(THD *thd);
+extern "C" int64_t wsrep_thd_trx_seqno(THD *thd);
+extern "C" query_id_t wsrep_thd_query_id(THD *thd);
+extern "C" char * wsrep_thd_query(THD *thd);
+extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd);
+extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id);
+extern "C" void wsrep_thd_awake(THD* thd, my_bool signal);
+#endif
 
 extern const struct _ft_vft ft_vft_result;
 
@@ -504,6 +541,9 @@ innobase_index_name_is_reserved(
 	__attribute__((nonnull, warn_unused_result));
 
 /*****************************************************************//**
+#ifdef WITH_WSREP
+extern "C" int wsrep_trx_is_aborting(void *thd_ptr);
+#endif
 Determines InnoDB table flags.
 @retval true if successful, false if error */
 UNIV_INTERN
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index a621a59042e..0b13573abc6 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -49,6 +49,10 @@ Smart ALTER TABLE
 #include "pars0pars.h"
 #include "row0sel.h"
 #include "ha_innodb.h"
+#ifdef WITH_WSREP
+//#include "wsrep_api.h"
+#include <sql_acl.h>	// PROCESS_ACL
+#endif
 
 /** Operations for creating secondary indexes (no rebuild needed) */
 static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_ONLINE_CREATE
diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h
index b026210b214..b5e62e53a72 100644
--- a/storage/innobase/include/dict0mem.h
+++ b/storage/innobase/include/dict0mem.h
@@ -518,6 +518,9 @@ be REC_VERSION_56_MAX_INDEX_COL_LEN (3072) bytes */
 
 /** Defines the maximum fixed length column size */
 #define DICT_MAX_FIXED_COL_LEN		DICT_ANTELOPE_MAX_INDEX_COL_LEN
+#ifdef WITH_WSREP
+#define WSREP_MAX_SUPPORTED_KEY_LENGTH 3500
+#endif /* WITH_WSREP */
 
 /** Data structure for a field in an index */
 struct dict_field_t{
diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h
index a02b8f1893a..563a01f747d 100644
--- a/storage/innobase/include/ha_prototypes.h
+++ b/storage/innobase/include/ha_prototypes.h
@@ -286,6 +286,23 @@ innobase_casedn_str(
 /*================*/
 	char*	a);	/*!< in/out: string to put in lower case */
 
+#ifdef WITH_WSREP
+UNIV_INTERN
+int
+wsrep_innobase_kill_one_trx(void *thd_ptr,
+                            const trx_t *bf_trx, trx_t *victim_trx, ibool signal);
+my_bool wsrep_thd_set_PA_safe(void *thd_ptr, my_bool safe);
+int wsrep_thd_conflict_state(void *thd_ptr, my_bool sync);
+my_bool wsrep_thd_is_BF(void *thd_ptr, my_bool sync);
+my_bool wsrep_thd_is_wsrep(void *thd_ptr);
+int wsrep_trx_order_before(void *thd1, void *thd2);
+int wsrep_innobase_mysql_sort(int mysql_type, uint charset_number,
+			      unsigned char* str, unsigned int str_length,
+			      unsigned int buf_length);
+int 
+wsrep_on(void *thd_ptr);
+extern "C" int wsrep_is_wsrep_xid(const void*);
+#endif /* WITH_WSREP */
 /**********************************************************************//**
 Determines the connection character set.
 @return	connection character set */
diff --git a/storage/innobase/include/hash0hash.h b/storage/innobase/include/hash0hash.h
index 6f9a628df5d..9a4077befb1 100644
--- a/storage/innobase/include/hash0hash.h
+++ b/storage/innobase/include/hash0hash.h
@@ -144,6 +144,33 @@ do {\
 	}\
 } while (0)
 
+#ifdef WITH_WSREP
+/*******************************************************************//**
+Inserts a struct to the head of hash table. */
+
+#define HASH_PREPEND(TYPE, NAME, TABLE, FOLD, DATA)	\
+do {							\
+	hash_cell_t*	cell3333;			\
+	TYPE*		struct3333;			\
+							\
+	HASH_ASSERT_OWN(TABLE, FOLD)			\
+							\
+	(DATA)->NAME = NULL;				\
+							\
+	cell3333 = hash_get_nth_cell(TABLE, hash_calc_hash(FOLD, TABLE));\
+							\
+	if (cell3333->node == NULL) {			\
+		cell3333->node = DATA;			\
+		DATA->NAME = NULL;			\
+	} else {					\
+		struct3333 = (TYPE*) cell3333->node;	\
+							\
+		DATA->NAME = struct3333;		\
+							\
+		cell3333->node = DATA;			\
+	}						\
+} while (0)
+#endif /*WITH_WSREP */
 #ifdef UNIV_HASH_DEBUG
 # define HASH_ASSERT_VALID(DATA) ut_a((void*) (DATA) != (void*) -1)
 # define HASH_INVALIDATE(DATA, NAME) *(void**) (&DATA->NAME) = (void*) -1
diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h
index 6d5ed35d5d8..385853bdb68 100644
--- a/storage/innobase/include/lock0lock.h
+++ b/storage/innobase/include/lock0lock.h
@@ -972,6 +972,16 @@ extern lock_sys_t*	lock_sys;
 	mutex_exit(&lock_sys->wait_mutex);	\
 } while (0)
 
+#ifdef WITH_WSREP
+/*********************************************************************//**
+Cancels a waiting lock request and releases possible other transactions
+waiting behind it. */
+UNIV_INTERN
+void
+lock_cancel_waiting_and_release(
+/*============================*/
+	lock_t*	lock);	/*!< in/out: waiting lock request */
+#endif /* WITH_WSREP */
 #ifndef UNIV_NONINL
 #include "lock0lock.ic"
 #endif
diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h
index 8e7d5ff2d48..238cb04e1f8 100644
--- a/storage/innobase/include/rem0rec.h
+++ b/storage/innobase/include/rem0rec.h
@@ -981,6 +981,15 @@ are given in one byte (resp. two byte) format. */
 two upmost bits in a two byte offset for special purposes */
 #define REC_MAX_DATA_SIZE	(16 * 1024)
 
+#ifdef WITH_WSREP
+int wsrep_rec_get_foreign_key(
+	byte 		*buf,     /* out: extracted key */
+	ulint 		*buf_len, /* in/out: length of buf */
+	const rec_t*	rec,	  /* in: physical record */
+	dict_index_t*	index_for,  /* in: index for foreign table */
+	dict_index_t*	index_ref,  /* in: index for referenced table */
+	ibool		new_protocol); /* in: protocol > 1 */
+#endif /* WITH_WSREP */
 #ifndef UNIV_NONINL
 #include "rem0rec.ic"
 #endif
diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h
index 905d4a0afa7..a1da2ee03b1 100644
--- a/storage/innobase/include/srv0srv.h
+++ b/storage/innobase/include/srv0srv.h
@@ -304,6 +304,10 @@ extern ulong	srv_flush_log_at_trx_commit;
 extern uint	srv_flush_log_at_timeout;
 extern char	srv_adaptive_flushing;
 
+#ifdef WITH_INNODB_DISALLOW_WRITES
+/* When this event is reset we do not allow any file writes to take place. */
+extern os_event_t	srv_allow_writes_event;
+#endif /* WITH_INNODB_DISALLOW_WRITES */
 /* If this flag is TRUE, then we will load the indexes' (and tables') metadata
 even if they are marked as "corrupted". Mostly it is for DBA to process
 corrupted index and table */
@@ -955,5 +959,13 @@ struct srv_slot_t{
 # define srv_start_raw_disk_in_use		0
 # define srv_file_per_table			1
 #endif /* !UNIV_HOTBACKUP */
+#ifdef WITH_WSREP
+UNIV_INTERN
+void
+wsrep_srv_conc_cancel_wait(
+/*==================*/
+	trx_t*	trx);	/*!< in: transaction object associated with the
+			thread */
+#endif /* WITH_WSREP */
 
 #endif
diff --git a/storage/innobase/include/sync0sync.ic b/storage/innobase/include/sync0sync.ic
index cb375cb4a4f..f34f3f90b63 100644
--- a/storage/innobase/include/sync0sync.ic
+++ b/storage/innobase/include/sync0sync.ic
@@ -204,7 +204,10 @@ mutex_enter_func(
 	ulint		line)		/*!< in: line where locked */
 {
 	ut_ad(mutex_validate(mutex));
+#ifndef WITH_WSREP
+	/* this cannot be be granted when BF trx kills a trx in lock wait state */
 	ut_ad(!mutex_own(mutex));
+#endif /* WITH_WSREP */
 
 	/* Note that we do not peek at the value of lock_word before trying
 	the atomic test_and_set; we could peek, and possibly save time. */
diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h
index 70f214d1ac7..9ffc8d99a7f 100644
--- a/storage/innobase/include/trx0sys.h
+++ b/storage/innobase/include/trx0sys.h
@@ -42,6 +42,9 @@ Created 3/26/1996 Heikki Tuuri
 #include "read0types.h"
 #include "page0types.h"
 #include "ut0bh.h"
+#ifdef WITH_WSREP
+#include "trx0xa.h"
+#endif /* WITH_WSREP */
 
 typedef UT_LIST_BASE_NODE_T(trx_t) trx_list_t;
 
@@ -293,6 +296,9 @@ trx_sys_update_mysql_binlog_offset(
 	ib_int64_t	offset,	/*!< in: position in that log file */
 	ulint		field,	/*!< in: offset of the MySQL log info field in
 				the trx sys header */
+#ifdef WITH_WSREP
+        trx_sysf_t*     sys_header, /*!< in: trx sys header */
+#endif /* WITH_WSREP */
 	mtr_t*		mtr);	/*!< in: mtr */
 /*****************************************************************//**
 Prints to stderr the MySQL binlog offset info in the trx system header if
@@ -301,6 +307,19 @@ UNIV_INTERN
 void
 trx_sys_print_mysql_binlog_offset(void);
 /*===================================*/
+#ifdef WITH_WSREP
+/** Update WSREP checkpoint XID in sys header. */
+void
+trx_sys_update_wsrep_checkpoint(
+        const XID*      xid,         /*!< in: WSREP XID */
+        trx_sysf_t*     sys_header,  /*!< in: sys_header */
+        mtr_t*          mtr);        /*!< in: mtr       */
+
+void
+/** Read WSREP checkpoint XID from sys header. */
+trx_sys_read_wsrep_checkpoint(
+        XID* xid); /*!< out: WSREP XID */
+#endif /* WITH_WSREP */
 /*****************************************************************//**
 Prints to stderr the MySQL master log offset info in the trx system header if
 the magic number shows it valid. */
@@ -529,6 +548,20 @@ this contains the same fields as TRX_SYS_MYSQL_LOG_INFO below */
 						within that file */
 #define TRX_SYS_MYSQL_LOG_NAME		12	/*!< MySQL log file name */
 
+#ifdef WITH_WSREP
+/* The offset to WSREP XID headers */
+#define TRX_SYS_WSREP_XID_INFO (UNIV_PAGE_SIZE - 3500)
+#define TRX_SYS_WSREP_XID_MAGIC_N_FLD 0
+#define TRX_SYS_WSREP_XID_MAGIC_N 0x77737265
+
+/* XID field: formatID, gtrid_len, bqual_len, xid_data */
+#define TRX_SYS_WSREP_XID_LEN        (4 + 4 + 4 + XIDDATASIZE)
+#define TRX_SYS_WSREP_XID_FORMAT     4
+#define TRX_SYS_WSREP_XID_GTRID_LEN  8
+#define TRX_SYS_WSREP_XID_BQUAL_LEN 12
+#define TRX_SYS_WSREP_XID_DATA      16
+#endif /* WITH_WSREP*/
+
 /** Doublewrite buffer */
 /* @{ */
 /** The offset of the doublewrite buffer header on the trx system header page */
diff --git a/storage/innobase/include/trx0sys.ic b/storage/innobase/include/trx0sys.ic
index e097e29b551..7265a97ae25 100644
--- a/storage/innobase/include/trx0sys.ic
+++ b/storage/innobase/include/trx0sys.ic
@@ -445,7 +445,10 @@ trx_id_t
 trx_sys_get_new_trx_id(void)
 /*========================*/
 {
+#ifndef WITH_WSREP
+	/* wsrep_fake_trx_id  violates this assert */
 	ut_ad(mutex_own(&trx_sys->mutex));
+#endif /* WITH_WSREP */
 
 	/* VERY important: after the database is started, max_trx_id value is
 	divisible by TRX_SYS_TRX_ID_WRITE_MARGIN, and the following if
diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h
index 34e4c0067e2..a30bbdbebb2 100644
--- a/storage/innobase/include/trx0trx.h
+++ b/storage/innobase/include/trx0trx.h
@@ -1004,6 +1004,9 @@ struct trx_t{
 	/*------------------------------*/
 	char detailed_error[256];	/*!< detailed error message for last
 					error, or empty. */
+#ifdef WITH_WSREP
+	os_event_t	wsrep_event;	/* event waited for in srv_conc_slot */
+#endif /* WITH_WSREP */
 };
 
 /* Transaction isolation levels (trx->isolation_level) */
diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc
index e4db2c30751..ce107dda077 100644
--- a/storage/innobase/lock/lock0lock.cc
+++ b/storage/innobase/lock/lock0lock.cc
@@ -50,6 +50,11 @@ Created 5/7/1996 Heikki Tuuri
 #include "dict0boot.h"
 #include <set>
 
+#ifdef WITH_WSREP
+extern my_bool wsrep_debug;
+extern my_bool wsrep_log_conflicts;
+#include "ha_prototypes.h"
+#endif
 /* Restricts the length of search we will do in the waits-for
 graph of transactions */
 #define LOCK_MAX_N_STEPS_IN_DEADLOCK_CHECK 1000000
@@ -945,6 +950,9 @@ UNIV_INLINE
 ibool
 lock_rec_has_to_wait(
 /*=================*/
+#ifdef WITH_WSREP
+	ibool		for_locking, /*!< is caller locking or releasing */
+#endif /* WITH_WSREP */
 	const trx_t*	trx,	/*!< in: trx of new lock */
 	ulint		type_mode,/*!< in: precise mode of the new lock
 				to set: LOCK_S or LOCK_X, possibly
@@ -1015,6 +1023,50 @@ lock_rec_has_to_wait(
 			return(FALSE);
 		}
 
+#ifdef WITH_WSREP
+		/* if BF thread is locking and has conflict with another BF
+		   thread, we need to look at trx ordering and lock types */
+		if (for_locking                                    &&
+		    wsrep_thd_is_BF(trx->mysql_thd, FALSE)         &&
+		    wsrep_thd_is_BF(lock2->trx->mysql_thd, TRUE)) {
+
+			if (wsrep_debug) {
+				fprintf(stderr, "\n BF-BF lock conflict \n");
+				lock_rec_print(stderr, lock2);
+			}
+
+			if (wsrep_trx_order_before(trx->mysql_thd,
+						   lock2->trx->mysql_thd) &&
+			    (type_mode & LOCK_MODE_MASK) == LOCK_X        &&
+			    (lock2->type_mode & LOCK_MODE_MASK) == LOCK_X)
+			{
+				/* exclusive lock conflicts are not accepted */
+				fprintf(stderr, "BF-BF X lock conflict,"
+					"type_mode: %lu supremum: %lu\n",
+					type_mode, lock_is_on_supremum);
+				fprintf(stderr, "conflicts states: my %d locked %d\n",
+					wsrep_thd_conflict_state(trx->mysql_thd, FALSE),
+					wsrep_thd_conflict_state(lock2->trx->mysql_thd, FALSE) );
+				lock_rec_print(stderr, lock2);
+				return FALSE;
+				//abort();
+			} else {
+				/* if lock2->index->n_uniq <=
+				   lock2->index->n_user_defined_cols
+				   operation is on uniq index
+				*/
+				if (wsrep_debug) fprintf(stderr,
+					"BF conflict, modes: %lu %lu, "
+					"idx: %s-%s n_uniq %u n_user %u\n",
+					type_mode, lock2->type_mode,
+					lock2->index->name, 
+					lock2->index->table_name,
+					lock2->index->n_uniq,
+					lock2->index->n_user_defined_cols);
+				return FALSE;
+			}
+		}
+#endif /* WITH_WSREP */
 		return(TRUE);
 	}
 
@@ -1045,7 +1097,11 @@ lock_has_to_wait(
 			/* If this lock request is for a supremum record
 			then the second bit on the lock bitmap is set */
 
+#ifdef WITH_WSREP
+			return(lock_rec_has_to_wait(FALSE, lock1->trx,
+#else
 			return(lock_rec_has_to_wait(lock1->trx,
+#endif /* WITH_WSREP */
 						    lock1->type_mode, lock2,
 						    lock_rec_get_nth_bit(
 							    lock1, 1)));
@@ -1514,6 +1570,11 @@ lock_rec_has_expl(
 	return(NULL);
 }
 
+#ifdef WITH_WSREP
+static
+void
+lock_rec_discard(lock_t*	in_lock);
+#endif
 #ifdef UNIV_DEBUG
 /*********************************************************************//**
 Checks if some other transaction has a lock request in the queue.
@@ -1562,6 +1623,69 @@ lock_rec_other_has_expl_req(
 }
 #endif /* UNIV_DEBUG */
 
+#ifdef WITH_WSREP
+static
+void
+wsrep_kill_victim(
+	const trx_t * const trx,
+	const lock_t *lock)
+{
+        ut_ad(lock_mutex_own());
+        ut_ad(trx_mutex_own(lock->trx));
+	my_bool bf_this  = wsrep_thd_is_BF(trx->mysql_thd, FALSE);
+	my_bool bf_other = wsrep_thd_is_BF(lock->trx->mysql_thd, TRUE);
+
+	if ((bf_this && !bf_other) ||
+		(bf_this && bf_other && wsrep_trx_order_before(
+			trx->mysql_thd, lock->trx->mysql_thd))) {
+
+		if (lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT) {
+			if (wsrep_debug) {
+				fprintf(stderr, "WSREP: BF victim waiting\n");
+			}
+			/* cannot release lock, until our lock
+			is in the queue*/
+		} else if (lock->trx != trx) {
+			if (wsrep_log_conflicts) {
+				mutex_enter(&trx_sys->mutex);
+				if (bf_this) {
+					fputs("\n*** Priority TRANSACTION:\n",
+					      stderr);
+				} else {
+					fputs("\n*** Victim TRANSACTION:\n",
+					      stderr);
+				}
+
+				trx_print_latched(stderr, trx, 3000);
+
+				if (bf_other) {
+					fputs("\n*** Priority TRANSACTION:\n",
+					      stderr);
+				} else {
+					fputs("\n*** Victim TRANSACTION:\n",
+					      stderr);
+				}
+
+				trx_print_latched(stderr, lock->trx, 3000);
+
+				mutex_exit(&trx_sys->mutex);
+
+				fputs("*** WAITING FOR THIS LOCK TO BE GRANTED:\n",
+				      stderr);
+
+				if (lock_get_type(lock) == LOCK_REC) {
+					lock_rec_print(stderr, lock);
+				} else {
+					lock_table_print(stderr, lock);
+				}
+			}
+
+			wsrep_innobase_kill_one_trx(trx->mysql_thd,
+				(const trx_t*) trx, lock->trx, TRUE);
+		}
+	}
+}
+#endif
 /*********************************************************************//**
 Checks if some other transaction has a conflicting explicit lock request
 in the queue, so that we have to wait.
@@ -1590,7 +1714,15 @@ lock_rec_other_has_conflicting(
 	     lock != NULL;
 	     lock = lock_rec_get_next_const(heap_no, lock)) {
 
-		if (lock_rec_has_to_wait(trx, mode, lock, is_supremum)) {
+#ifdef WITH_WSREP
+		if (lock_rec_has_to_wait(TRUE, trx, mode, lock, is_supremum)) {
+			trx_mutex_enter(lock->trx);
+			wsrep_kill_victim(trx, lock);
+			trx_mutex_exit(lock->trx);
+#else
+ 		if (lock_rec_has_to_wait(trx, mode, lock, is_supremum)) {
+#endif /* WITH_WSREP */
+
 			return(lock);
 		}
 	}
@@ -1771,6 +1903,28 @@ lock_number_of_rows_locked(
 
 /*============== RECORD LOCK CREATION AND QUEUE MANAGEMENT =============*/
 
+#ifdef WITH_WSREP
+static
+void
+wsrep_print_wait_locks(
+/*============*/
+	lock_t*		c_lock) /* conflicting lock to print */
+{
+	if (wsrep_debug &&  c_lock->trx->lock.wait_lock != c_lock) {
+		fprintf(stderr, "WSREP: c_lock != wait lock\n");
+		if (lock_get_type_low(c_lock) & LOCK_TABLE)
+			lock_table_print(stderr, c_lock);
+		else
+			lock_rec_print(stderr, c_lock);
+
+		if (lock_get_type_low(c_lock->trx->lock.wait_lock) & LOCK_TABLE)
+			lock_table_print(stderr, c_lock->trx->lock.wait_lock);
+		else
+			lock_rec_print(stderr, c_lock->trx->lock.wait_lock);
+	}
+}
+#endif /* WITH_WSREP */
+
 /*********************************************************************//**
 Creates a new record lock and inserts it to the lock queue. Does NOT check
 for deadlocks or lock compatibility!
@@ -1779,6 +1933,10 @@ static
 lock_t*
 lock_rec_create(
 /*============*/
+#ifdef WITH_WSREP
+	lock_t*			const c_lock,   /* conflicting lock */
+	que_thr_t*		thr,
+#endif
 	ulint			type_mode,/*!< in: lock mode and wait
 					flag, type is ignored and
 					replaced by LOCK_REC */
@@ -1850,8 +2008,91 @@ lock_rec_create(
 
 	ut_ad(index->table->n_ref_count > 0 || !index->table->can_be_evicted);
 
+#ifdef WITH_WSREP
+	if (c_lock && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+		lock_t *hash	= (lock_t *)c_lock->hash;
+		lock_t *prev	= NULL;
+
+		while (hash 						       &&
+		       wsrep_thd_is_BF(((lock_t *)hash)->trx->mysql_thd, TRUE) &&
+		       wsrep_trx_order_before(
+				((lock_t *)hash)->trx->mysql_thd,
+				trx->mysql_thd)) {
+			prev = hash;
+			hash = (lock_t *)hash->hash;
+		}
+		lock->hash = hash;
+		if (prev) {
+			prev->hash = lock;
+		} else {
+			c_lock->hash = lock;
+		}
+		/*
+		 * delayed conflict resolution '...kill_one_trx' was not called,
+		 * if victim was waiting for some other lock
+		 */
+		trx_mutex_enter(c_lock->trx);
+		if (c_lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT) {
+
+			c_lock->trx->lock.was_chosen_as_deadlock_victim = TRUE;
+
+			if (wsrep_debug) {
+				wsrep_print_wait_locks(c_lock);
+			}
+
+			trx->lock.que_state = TRX_QUE_LOCK_WAIT;
+			lock_set_lock_and_trx_wait(lock, trx);
+			UT_LIST_ADD_LAST(trx_locks, trx->lock.trx_locks, lock);
+
+			ut_ad(thr != NULL);
+			trx->lock.wait_thr = thr;
+			thr->state = QUE_THR_LOCK_WAIT;
+
+			/* have to release trx mutex for the duration of
+			   victim lock release. This will eventually call
+			   lock_grant, which wants to grant trx mutex again
+			*/
+			if (caller_owns_trx_mutex) {
+				trx_mutex_exit(trx);
+			}
+			lock_cancel_waiting_and_release(
+				c_lock->trx->lock.wait_lock);
+
+			if (caller_owns_trx_mutex) {
+				trx_mutex_enter(trx);
+			}
+
+			/* trx might not wait for c_lock, but some other lock
+			   does not matter if wait_lock was released above
+			 */
+			if (c_lock->trx->lock.wait_lock == c_lock) {
+				lock_reset_lock_and_trx_wait(lock);
+			}
+
+			trx_mutex_exit(c_lock->trx);
+
+			if (wsrep_debug) {
+				fprintf(
+					stderr,
+					"WSREP: c_lock canceled %llu\n",
+					(ulonglong) c_lock->trx->id);
+			}
+
+			/* have to bail out here to avoid lock_set_lock... */
+			return(lock);
+		}
+		trx_mutex_exit(c_lock->trx);
+	} else if (wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+		HASH_PREPEND(lock_t, hash, lock_sys->rec_hash,
+			    lock_rec_fold(space, page_no), lock);
+	} else {
+		HASH_INSERT(lock_t, hash, lock_sys->rec_hash,
+			    lock_rec_fold(space, page_no), lock);
+	}
+#else
 	HASH_INSERT(lock_t, hash, lock_sys->rec_hash,
 		    lock_rec_fold(space, page_no), lock);
+#endif /* WITH_WSREP */
 
 	if (!caller_owns_trx_mutex) {
 		trx_mutex_enter(trx);
@@ -1859,7 +2100,6 @@ lock_rec_create(
 	ut_ad(trx_mutex_own(trx));
 
 	if (type_mode & LOCK_WAIT) {
-
 		lock_set_lock_and_trx_wait(lock, trx);
 	}
 
@@ -1871,7 +2111,6 @@ lock_rec_create(
 
 	MONITOR_INC(MONITOR_RECLOCK_CREATED);
 	MONITOR_INC(MONITOR_NUM_RECLOCK);
-
 	return(lock);
 }
 
@@ -1886,6 +2125,9 @@ static
 dberr_t
 lock_rec_enqueue_waiting(
 /*=====================*/
+#ifdef WITH_WSREP
+	lock_t*			c_lock,   /* conflicting lock */
+#endif
 	ulint			type_mode,/*!< in: lock mode this
 					transaction is requesting:
 					LOCK_S or LOCK_X, possibly
@@ -1943,6 +2185,9 @@ lock_rec_enqueue_waiting(
 	/* Enqueue the lock request that will wait to be granted, note that
 	we already own the trx mutex. */
 	lock = lock_rec_create(
+#ifdef WITH_WSREP
+                c_lock, thr,
+#endif /* WITH_WSREP */
 		type_mode | LOCK_WAIT, block, heap_no, index, trx, TRUE);
 
 	/* Release the mutex to obey the latching order.
@@ -2043,7 +2288,19 @@ lock_rec_add_to_queue(
 		const lock_t*	other_lock
 			= lock_rec_other_has_expl_req(mode, 0, LOCK_WAIT,
 						      block, heap_no, trx);
+#ifdef WITH_WSREP
+		/* this can potentionally assert with wsrep */
+		if (wsrep_thd_is_wsrep(trx->mysql_thd)) {
+			if (wsrep_debug && other_lock) {
+				fprintf(stderr,
+					"WSREP: InnoDB assert ignored\n");
+			}
+		} else {
+			ut_a(!other_lock);
+		}
+#else
 		ut_a(!other_lock);
+#endif /* WITH_WSREP */
 	}
 #endif /* UNIV_DEBUG */
 
@@ -2071,7 +2328,16 @@ lock_rec_add_to_queue(
 
 		if (lock_get_wait(lock)
 		    && lock_rec_get_nth_bit(lock, heap_no)) {
-
+#ifdef WITH_WSREP
+			if (wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+				if (wsrep_debug) {
+					fprintf(stderr,
+						"BF skipping wait: %lu\n",
+						trx->id);
+					lock_rec_print(stderr, lock);
+				}
+		  } else
+#endif
 			goto somebody_waits;
 		}
 	}
@@ -2094,9 +2360,15 @@ lock_rec_add_to_queue(
 	}
 
 somebody_waits:
-	return(lock_rec_create(
+#ifdef WITH_WSREP
+	return(lock_rec_create(NULL, NULL,
 			type_mode, block, heap_no, index, trx,
 			caller_owns_trx_mutex));
+#else
+ 	return(lock_rec_create(
+                        type_mode, block, heap_no, index, trx,
+                        caller_owns_trx_mutex));
+#endif /* WITH_WSREP */
 }
 
 /** Record locking request status */
@@ -2159,9 +2431,13 @@ lock_rec_lock_fast(
 	if (lock == NULL) {
 		if (!impl) {
 			/* Note that we don't own the trx mutex. */
+#ifdef WITH_WSREP
+			lock = lock_rec_create(NULL, thr,
+				mode, block, heap_no, index, trx, FALSE);
+#else
 			lock = lock_rec_create(
 				mode, block, heap_no, index, trx, FALSE);
-
+#endif /* WITH_WSREP */
 		}
 		status = LOCK_REC_SUCCESS_CREATED;
 	} else {
@@ -2214,6 +2490,9 @@ lock_rec_lock_slow(
 	que_thr_t*		thr)	/*!< in: query thread */
 {
 	trx_t*			trx;
+#ifdef WITH_WSREP
+	lock_t*			c_lock(NULL);
+#endif
 	dberr_t			err = DB_SUCCESS;
 
 	ut_ad(lock_mutex_own());
@@ -2237,18 +2516,31 @@ lock_rec_lock_slow(
 
 		/* The trx already has a strong enough lock on rec: do
 		nothing */
-
+#ifdef WITH_WSREP
+	} else if ((c_lock = (ib_lock_t*)lock_rec_other_has_conflicting(
+                        static_cast<enum lock_mode>(mode),
+                        block, heap_no, trx))) {
+#else
 	} else if (lock_rec_other_has_conflicting(
 			static_cast<enum lock_mode>(mode),
 			block, heap_no, trx)) {
+#endif /* WITH_WSREP */
 
 		/* If another transaction has a non-gap conflicting
 		request in the queue, as this transaction does not
 		have a lock strong enough already granted on the
 		record, we have to wait. */
 
+#ifdef WITH_WSREP
+		/* c_lock is NULL here if jump to enqueue_waiting happened
+		but it's ok because lock is not NULL in that case and c_lock
+		is not used. */
+		err = lock_rec_enqueue_waiting(c_lock,
+			mode, block, heap_no, index, thr);
+#else
 		err = lock_rec_enqueue_waiting(
 			mode, block, heap_no, index, thr);
+#endif /* WITH_WSREP */
 
 	} else if (!impl) {
 		/* Set the requested lock on the record, note that
@@ -2354,7 +2646,13 @@ lock_rec_has_to_wait_in_queue(
 		if (heap_no < lock_rec_get_n_bits(lock)
 		    && (p[bit_offset] & bit_mask)
 		    && lock_has_to_wait(wait_lock, lock)) {
-
+#ifdef WITH_WSREP
+			if (wsrep_thd_is_BF(wait_lock->trx->mysql_thd, FALSE) &&
+			    wsrep_thd_is_BF(lock->trx->mysql_thd, TRUE)) {
+				/* don't wait for another BF lock */
+				continue;
+			}
+#endif
 			return(lock);
 		}
 	}
@@ -3738,10 +4036,22 @@ lock_deadlock_select_victim(
 		/* The joining  transaction is 'smaller',
 		choose it as the victim and roll it back. */
 
-		return(ctx->start);
+#ifdef WITH_WSREP
+		if (wsrep_thd_is_BF(ctx->start->mysql_thd, TRUE)) {
+			return(ctx->wait_lock->trx);
+		}
+		else
+#endif /* WITH_WSREP */
+			return(ctx->start);
 	}
 
-	return(ctx->wait_lock->trx);
+#ifdef WITH_WSREP
+	if (wsrep_thd_is_BF(ctx->wait_lock->trx->mysql_thd, TRUE)) {
+		return(ctx->start);
+	}
+	else
+#endif /* WITH_WSREP */
+		return(ctx->wait_lock->trx);
 }
 
 /********************************************************************//**
@@ -3870,8 +4180,14 @@ lock_deadlock_search(
 
 			ctx->too_deep = TRUE;
 
+#ifdef WITH_WSREP
+			if (wsrep_thd_is_BF(ctx->start->mysql_thd, TRUE)) {
+				return(ctx->wait_lock->trx->id);
+			}
+			else
+#endif /* WITH_WSREP */
 			/* Select the joining transaction as the victim. */
-			return(ctx->start->id);
+				return(ctx->start->id);
 
 		} else if (lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT) {
 
@@ -3888,7 +4204,12 @@ lock_deadlock_search(
 
 				ctx->too_deep = TRUE;
 
-				return(ctx->start->id);
+#ifdef WITH_WSREP
+				if (wsrep_thd_is_BF(ctx->start->mysql_thd, TRUE))
+					return(lock->trx->id);
+				else
+#endif /* WITH_WSREP */
+					return(ctx->start->id);
 			}
 
 			ctx->wait_lock = lock->trx->lock.wait_lock;
@@ -4006,9 +4327,18 @@ lock_deadlock_check_and_resolve(
 			ut_a(trx == ctx.start);
 			ut_a(victim_trx_id == trx->id);
 
-			if (!srv_read_only_mode) {
-				lock_deadlock_joining_trx_print(trx, lock);
+#ifdef WITH_WSREP
+			if (!wsrep_thd_is_BF(ctx.start->mysql_thd, TRUE))
+			{
+#endif /* WITH_WSREP */
+				if (!srv_read_only_mode) {
+					lock_deadlock_joining_trx_print(trx, lock);
+				}
+#ifdef WITH_WSREP
+			} else {
+			  /* BF processor */;
 			}
+#endif /* WITH_WSREP */
 
 			MONITOR_INC(MONITOR_DEADLOCK);
 
@@ -4046,6 +4376,9 @@ UNIV_INLINE
 lock_t*
 lock_table_create(
 /*==============*/
+#ifdef WITH_WSREP
+	lock_t*		c_lock, /*!< in: conflicting lock */
+#endif
 	dict_table_t*	table,	/*!< in/out: database table
 				in dictionary cache */
 	ulint		type_mode,/*!< in: lock mode possibly ORed with
@@ -4089,7 +4422,59 @@ lock_table_create(
 	ut_ad(table->n_ref_count > 0 || !table->can_be_evicted);
 
 	UT_LIST_ADD_LAST(trx_locks, trx->lock.trx_locks, lock);
+
+#ifdef WITH_WSREP
+	if (wsrep_thd_is_wsrep(trx->mysql_thd)) {
+		if (c_lock && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+			UT_LIST_INSERT_AFTER(
+				un_member.tab_lock.locks, table->locks, c_lock, lock);
+		} else {
+			UT_LIST_ADD_LAST(un_member.tab_lock.locks, table->locks, lock);
+		}
+
+		if (c_lock) {
+			trx_mutex_enter(c_lock->trx);
+		}
+
+		if (c_lock && c_lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT) {
+
+			c_lock->trx->lock.was_chosen_as_deadlock_victim = TRUE;
+
+			if (wsrep_debug) {
+				wsrep_print_wait_locks(c_lock);
+				wsrep_print_wait_locks(c_lock->trx->lock.wait_lock);
+			}
+
+			/* have to release trx mutex for the duration of
+			victim lock release. This will eventually call
+			lock_grant, which wants to grant trx mutex again
+			*/
+			/* caller has trx_mutex, have to release for lock cancel */
+			trx_mutex_exit(trx);
+			lock_cancel_waiting_and_release(c_lock->trx->lock.wait_lock);
+			trx_mutex_enter(trx);
+
+			/* trx might not wait for c_lock, but some other lock
+			does not matter if wait_lock was released above
+			*/
+			if (c_lock->trx->lock.wait_lock == c_lock) {
+				lock_reset_lock_and_trx_wait(lock);
+			}
+
+			if (wsrep_debug) {
+				fprintf(stderr, "WSREP: c_lock canceled %llu\n",
+					(ulonglong) c_lock->trx->id);
+			}
+		}
+		if (c_lock) {
+			trx_mutex_exit(c_lock->trx);
+		}
+	} else {
+		UT_LIST_ADD_LAST(un_member.tab_lock.locks, table->locks, lock);
+	}
+#else
 	UT_LIST_ADD_LAST(un_member.tab_lock.locks, table->locks, lock);
+#endif /* WITH_WSREP */
 
 	if (UNIV_UNLIKELY(type_mode & LOCK_WAIT)) {
 
@@ -4246,6 +4631,9 @@ static
 dberr_t
 lock_table_enqueue_waiting(
 /*=======================*/
+#ifdef WITH_WSREP
+	lock_t*		c_lock, /*!< in: conflicting lock */
+#endif
 	ulint		mode,	/*!< in: lock mode this transaction is
 				requesting */
 	dict_table_t*	table,	/*!< in/out: table */
@@ -4290,7 +4678,14 @@ lock_table_enqueue_waiting(
 
 	/* Enqueue the lock request that will wait to be granted */
 
-	lock = lock_table_create(table, mode | LOCK_WAIT, trx);
+#ifdef WITH_WSREP
+	if (trx->lock.was_chosen_as_deadlock_victim) {
+		return(DB_DEADLOCK);
+	}
+	lock = lock_table_create(c_lock, table, mode | LOCK_WAIT, trx);
+#else
+ 	lock = lock_table_create(table, mode | LOCK_WAIT, trx);
+#endif /* WITH_WSREP */
 
 	/* Release the mutex to obey the latching order.
 	This is safe, because lock_deadlock_check_and_resolve()
@@ -4362,6 +4757,18 @@ lock_table_other_has_incompatible(
 		    && !lock_mode_compatible(lock_get_mode(lock), mode)
 		    && (wait || !lock_get_wait(lock))) {
 
+#ifdef WITH_WSREP
+			if(wsrep_thd_is_wsrep(trx->mysql_thd)) {
+				if (wsrep_debug) {
+					fprintf(stderr, "WSREP: trx %ld table lock abort\n",
+						trx->id);
+				}
+				trx_mutex_enter(lock->trx);
+				wsrep_kill_victim((trx_t *)trx, (lock_t *)lock);
+				trx_mutex_exit(lock->trx);
+			}
+#endif
+
 			return(lock);
 		}
 	}
@@ -4384,6 +4791,9 @@ lock_table(
 	enum lock_mode	mode,	/*!< in: lock mode */
 	que_thr_t*	thr)	/*!< in: query thread */
 {
+#ifdef WITH_WSREP
+	lock_t *c_lock = NULL;
+#endif
 	trx_t*		trx;
 	dberr_t		err;
 	const lock_t*	wait_for;
@@ -4414,8 +4824,13 @@ lock_table(
 	/* We have to check if the new lock is compatible with any locks
 	other transactions have in the table lock queue. */
 
+#ifdef WITH_WSREP
+	wait_for = lock_table_other_has_incompatible(
+                trx, LOCK_WAIT, table, mode);
+#else
 	wait_for = lock_table_other_has_incompatible(
 		trx, LOCK_WAIT, table, mode);
+#endif
 
 	trx_mutex_enter(trx);
 
@@ -4423,9 +4838,17 @@ lock_table(
 	mode: this trx may have to wait */
 
 	if (wait_for != NULL) {
+#ifdef WITH_WSREP
+                err = lock_table_enqueue_waiting((ib_lock_t*)wait_for, mode | flags, table, thr);
+#else
 		err = lock_table_enqueue_waiting(mode | flags, table, thr);
+#endif
 	} else {
+#ifdef WITH_WSREP
+	        lock_table_create(c_lock, table, mode | flags, trx);
+#else
 		lock_table_create(table, mode | flags, trx);
+#endif
 
 		ut_a(!flags || mode == LOCK_S || mode == LOCK_X);
 
@@ -4463,7 +4886,11 @@ lock_table_ix_resurrect(
 		      trx, LOCK_WAIT, table, LOCK_IX));
 
 	trx_mutex_enter(trx);
+#ifdef WITH_WSREP
+	lock_table_create(NULL, table, LOCK_IX, trx);
+#else
 	lock_table_create(table, LOCK_IX, trx);
+#endif
 	lock_mutex_exit();
 	trx_mutex_exit(trx);
 }
@@ -5594,15 +6021,19 @@ lock_rec_queue_validate(
 
 		if (!lock_rec_get_gap(lock) && !lock_get_wait(lock)) {
 
-			enum lock_mode	mode;
+#ifndef WITH_WSREP
+			if (wsrep_thd_is_wsrep(lock->trx->mysql_thd)) {
+				enum lock_mode	mode;
 
-			if (lock_get_mode(lock) == LOCK_S) {
-				mode = LOCK_X;
-			} else {
-				mode = LOCK_S;
+				if (lock_get_mode(lock) == LOCK_S) {
+					mode = LOCK_X;
+				} else {
+					mode = LOCK_S;
+				}
+				ut_a(!lock_rec_other_has_expl_req(
+						mode, 0, 0, block, heap_no, lock->trx));
 			}
-			ut_a(!lock_rec_other_has_expl_req(
-				     mode, 0, 0, block, heap_no, lock->trx));
+#endif /* WITH_WSREP */
 
 		} else if (lock_get_wait(lock) && !lock_rec_get_gap(lock)) {
 
@@ -5906,6 +6337,9 @@ lock_rec_insert_check_and_lock(
 	lock_t*		lock;
 	dberr_t		err;
 	ulint		next_rec_heap_no;
+#ifdef WITH_WSREP
+	lock_t*		c_lock=NULL;
+#endif
 
 	ut_ad(block->frame == page_align(rec));
 	ut_ad(!dict_index_is_online_ddl(index)
@@ -5962,17 +6396,30 @@ lock_rec_insert_check_and_lock(
 	had to wait for their insert. Both had waiting gap type lock requests
 	on the successor, which produced an unnecessary deadlock. */
 
+#ifdef WITH_WSREP
+	if ((c_lock = (ib_lock_t*)lock_rec_other_has_conflicting(
+		    static_cast<enum lock_mode>(
+                            LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION),
+		    block, next_rec_heap_no, trx))) {
+#else
 	if (lock_rec_other_has_conflicting(
 		    static_cast<enum lock_mode>(
 			    LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION),
 		    block, next_rec_heap_no, trx)) {
+#endif /* WITH_WSREP */
 
 		/* Note that we may get DB_SUCCESS also here! */
 		trx_mutex_enter(trx);
 
+#ifdef WITH_WSREP
+		err = lock_rec_enqueue_waiting(c_lock,
+                        LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION,
+			block, next_rec_heap_no, index, thr);
+#else
 		err = lock_rec_enqueue_waiting(
 			LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION,
 			block, next_rec_heap_no, index, thr);
+#endif /* WITH_WSREP */
 
 		trx_mutex_exit(trx);
 	} else {
diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc
index 45ca07bc7c4..fd9e60d123b 100644
--- a/storage/innobase/os/os0file.cc
+++ b/storage/innobase/os/os0file.cc
@@ -104,6 +104,12 @@ UNIV_INTERN os_ib_mutex_t	os_file_seek_mutexes[OS_FILE_N_SEEK_MUTEXES];
 /* In simulated aio, merge at most this many consecutive i/os */
 #define OS_AIO_MERGE_N_CONSECUTIVE	64
 
+#ifdef WITH_INNODB_DISALLOW_WRITES
+#define WAIT_ALLOW_WRITES() os_event_wait(srv_allow_writes_event)
+#else
+#define WAIT_ALLOW_WRITES() do { } while (0)
+#endif /* WITH_INNODB_DISALLOW_WRITES */
+
 /**********************************************************************
 
 InnoDB AIO Implementation:
@@ -927,7 +933,9 @@ os_file_create_tmpfile(void)
 /*========================*/
 {
 	FILE*	file	= NULL;
-	int	fd	= innobase_mysql_tmpfile();
+	int	fd;
+	WAIT_ALLOW_WRITES();
+	fd	= innobase_mysql_tmpfile();
 
 	ut_ad(!srv_read_only_mode);
 
@@ -1253,6 +1261,7 @@ os_file_create_directory(
 	return(TRUE);
 #else
 	int	rcode;
+	WAIT_ALLOW_WRITES();
 
 	rcode = mkdir(pathname, 0770);
 
@@ -1379,6 +1388,8 @@ os_file_create_simple_func(
 
 #else /* __WIN__ */
 	int		create_flag;
+	if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW)
+		WAIT_ALLOW_WRITES();
 
 	ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT));
 	ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT));
@@ -1566,6 +1577,8 @@ os_file_create_simple_no_error_handling_func(
 	int		create_flag;
 
 	ut_a(name);
+	if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW)
+		WAIT_ALLOW_WRITES();
 
 	ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT));
 	ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT));
@@ -1894,6 +1907,8 @@ os_file_create_func(
 #else /* __WIN__ */
 	int		create_flag;
 	const char*	mode_str	= NULL;
+	if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW)
+		WAIT_ALLOW_WRITES();
 
 	on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT
 		? TRUE : FALSE;
@@ -2089,6 +2104,7 @@ loop:
 	goto loop;
 #else
 	int	ret;
+	WAIT_ALLOW_WRITES();
 
 	ret = unlink(name);
 
@@ -2153,6 +2169,7 @@ loop:
 	goto loop;
 #else
 	int	ret;
+	WAIT_ALLOW_WRITES();
 
 	ret = unlink(name);
 
@@ -2206,6 +2223,7 @@ os_file_rename_func(
 	return(FALSE);
 #else
 	int	ret;
+	WAIT_ALLOW_WRITES();
 
 	ret = rename(oldpath, newpath);
 
@@ -2440,6 +2458,7 @@ os_file_set_eof(
 	HANDLE h = (HANDLE) _get_osfhandle(fileno(file));
 	return(SetEndOfFile(h));
 #else /* __WIN__ */
+	WAIT_ALLOW_WRITES();
 	return(!ftruncate(fileno(file), ftell(file)));
 #endif /* __WIN__ */
 }
@@ -2534,6 +2553,7 @@ os_file_flush_func(
 	return(FALSE);
 #else
 	int	ret;
+	WAIT_ALLOW_WRITES();
 
 #if defined(HAVE_DARWIN_THREADS)
 # ifndef F_FULLFSYNC
@@ -3251,6 +3271,7 @@ retry:
 	return(FALSE);
 #else
 	ssize_t	ret;
+	WAIT_ALLOW_WRITES();
 
 	ret = os_file_pwrite(file, buf, n, offset);
 
diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc
index 0d7b7c16785..3ff71d5c59e 100644
--- a/storage/innobase/rem/rem0rec.cc
+++ b/storage/innobase/rem/rem0rec.cc
@@ -33,6 +33,9 @@ Created 5/30/1994 Heikki Tuuri
 #include "mtr0mtr.h"
 #include "mtr0log.h"
 #include "fts0fts.h"
+#ifdef WITH_WSREP
+#include <ha_prototypes.h>
+#endif /* WITH_WSREP */
 
 /*			PHYSICAL RECORD (OLD STYLE)
 			===========================
@@ -1961,3 +1964,134 @@ rec_get_trx_id(
 }
 # endif /* UNIV_DEBUG */
 #endif /* !UNIV_HOTBACKUP */
+
+#ifdef WITH_WSREP
+int
+wsrep_rec_get_foreign_key(
+	byte 		*buf,     /* out: extracted key */
+	ulint 		*buf_len, /* in/out: length of buf */
+	const rec_t*	rec,	  /* in: physical record */
+	dict_index_t*	index_for,  /* in: index in foreign table */
+	dict_index_t*	index_ref,  /* in: index in referenced table */
+	ibool		new_protocol) /* in: protocol > 1 */
+{
+	const byte*	data;
+	ulint		len;
+	ulint		key_len = 0;
+	ulint		i;
+	uint            key_parts;
+	mem_heap_t*	heap	= NULL;
+	ulint		offsets_[REC_OFFS_NORMAL_SIZE];
+        const ulint*    offsets;
+
+	ut_ad(index_for);
+	ut_ad(index_ref);
+
+        rec_offs_init(offsets_);
+	offsets = rec_get_offsets(rec, index_for, offsets_, 
+				  ULINT_UNDEFINED, &heap);
+
+	ut_ad(rec_offs_validate(rec, NULL, offsets));
+
+	ut_ad(rec);
+
+	key_parts = dict_index_get_n_unique_in_tree(index_for);
+	for (i = 0; 
+	     i < key_parts && 
+	       (index_for->type & DICT_CLUSTERED || i < key_parts - 1); 
+	     i++) {
+		dict_field_t*	  field_f = 
+			dict_index_get_nth_field(index_for, i);
+		const dict_col_t* col_f = dict_field_get_col(field_f);
+                dict_field_t*	  field_r = 
+			dict_index_get_nth_field(index_ref, i);
+		const dict_col_t* col_r = dict_field_get_col(field_r);
+
+		data = rec_get_nth_field(rec, offsets, i, &len);
+		if (key_len + ((len != UNIV_SQL_NULL) ? len + 1 : 1) > 
+		    *buf_len) {
+			fprintf (stderr, 
+				 "WSREP: FK key len exceeded %lu %lu %lu\n", 
+				 key_len, len, *buf_len);
+			goto err_out;
+		}
+
+		if (len == UNIV_SQL_NULL) {
+			ut_a(!(col_f->prtype & DATA_NOT_NULL));
+			*buf++ = 1;
+			key_len++;
+		} else if (!new_protocol) {
+			if (!(col_r->prtype & DATA_NOT_NULL)) {
+				*buf++ = 0;
+				key_len++;
+			}
+			memcpy(buf, data, len);
+			*buf_len = wsrep_innobase_mysql_sort(
+				(int)(col_f->prtype & DATA_MYSQL_TYPE_MASK),
+				(uint)dtype_get_charset_coll(col_f->prtype),
+				buf, len, *buf_len);
+		} else { /* new protocol */
+			if (!(col_r->prtype & DATA_NOT_NULL)) {
+				*buf++ = 0;
+				key_len++;
+			}
+			switch (col_f->mtype) {
+			case DATA_INT: {
+				byte* ptr = buf+len;
+				for (;;) {
+					ptr--;
+					*ptr = *data;
+					if (ptr == buf) {
+						break;
+					}
+					data++;
+				}
+		
+				if (!(col_f->prtype & DATA_UNSIGNED)) {
+					buf[len-1] = (byte) (buf[len-1] ^ 128);
+				}
+
+				break;
+			}
+			case DATA_VARCHAR:
+			case DATA_VARMYSQL:
+			case DATA_CHAR:
+			case DATA_MYSQL:
+				/* Copy the actual data */
+				ut_memcpy(buf, data, len);
+				len = wsrep_innobase_mysql_sort(
+					(int)
+					(col_f->prtype & DATA_MYSQL_TYPE_MASK),
+					(uint)
+					dtype_get_charset_coll(col_f->prtype),
+					buf, len, *buf_len);
+				break;
+			case DATA_BLOB:
+			case DATA_BINARY:
+				memcpy(buf, data, len);
+				break;
+			default: 
+				break;
+			}
+
+			key_len += len;
+			buf 	+= len;
+		}
+	}
+
+	rec_validate(rec, offsets);
+
+	if (UNIV_LIKELY_NULL(heap)) {
+		mem_heap_free(heap);
+	}
+
+	*buf_len = key_len;
+	return DB_SUCCESS;
+
+ err_out:
+	if (UNIV_LIKELY_NULL(heap)) {
+		mem_heap_free(heap);
+	}
+	return DB_ERROR;
+}
+#endif // WITH_WSREP
diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc
index 65a6c433f5c..e6487730a77 100644
--- a/storage/innobase/row/row0ins.cc
+++ b/storage/innobase/row/row0ins.cc
@@ -918,6 +918,14 @@ row_ins_invalidate_query_cache(
 	innobase_invalidate_query_cache(thr_get_trx(thr), buf, len);
 	mem_free(buf);
 }
+#ifdef WITH_WSREP
+dberr_t wsrep_append_foreign_key(trx_t *trx,  
+			       dict_foreign_t*	foreign,
+			       const rec_t*	clust_rec,
+			       dict_index_t*	clust_index,
+			       ibool		referenced,
+			       ibool            shared);
+#endif /* WITH_WSREP */
 
 /*********************************************************************//**
 Perform referential actions or checks when a parent row is deleted or updated
@@ -1269,7 +1277,19 @@ row_ins_foreign_check_on_constraint(
 
 	cascade->state = UPD_NODE_UPDATE_CLUSTERED;
 
-	err = row_update_cascade_for_mysql(thr, cascade,
+#ifdef WITH_WSREP
+	err = wsrep_append_foreign_key(
+				       thr_get_trx(thr),
+				       foreign,
+				       clust_rec,
+				       clust_index,
+				       FALSE, FALSE);
+	if (err != DB_SUCCESS) {
+		fprintf(stderr,
+			"WSREP: foreign key append failed: %d\n", err);
+	} else
+#endif /* WITH_WSREP */
+		err = row_update_cascade_for_mysql(thr, cascade,
 					   foreign->foreign_table);
 
 	if (foreign->foreign_table->n_foreign_key_checks_running == 0) {
@@ -1601,7 +1621,14 @@ run_again:
 
 				if (check_ref) {
 					err = DB_SUCCESS;
-
+#ifdef WITH_WSREP
+					err = wsrep_append_foreign_key(
+						thr_get_trx(thr),
+						foreign,
+						rec, 
+						check_index, 
+						check_ref, TRUE);
+#endif /* WITH_WSREP */
 					goto end_scan;
 				} else if (foreign->type != 0) {
 					/* There is an ON UPDATE or ON DELETE
diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc
index fb626519132..adced7059e4 100644
--- a/storage/innobase/row/row0upd.cc
+++ b/storage/innobase/row/row0upd.cc
@@ -51,6 +51,9 @@ Created 12/27/1996 Heikki Tuuri
 #include "pars0sym.h"
 #include "eval0eval.h"
 #include "buf0lru.h"
+#ifdef WITH_WSREP
+extern my_bool wsrep_debug;
+#endif
 
 
 /* What kind of latch and lock can we assume when the control comes to
@@ -170,6 +173,50 @@ func_exit:
 	return(is_referenced);
 }
 
+#ifdef WITH_WSREP
+static
+ibool
+wsrep_row_upd_index_is_foreign(
+/*========================*/
+	dict_index_t*	index,	/*!< in: index */
+	trx_t*		trx)	/*!< in: transaction */
+{
+	dict_table_t*	table		= index->table;
+	dict_foreign_t*	foreign;
+	ibool		froze_data_dict	= FALSE;
+	ibool		is_referenced	= FALSE;
+
+	if (!UT_LIST_GET_FIRST(table->foreign_list)) {
+
+		return(FALSE);
+	}
+
+	if (trx->dict_operation_lock_mode == 0) {
+		row_mysql_freeze_data_dictionary(trx);
+		froze_data_dict = TRUE;
+	}
+
+	foreign = UT_LIST_GET_FIRST(table->foreign_list);
+
+	while (foreign) {
+		if (foreign->foreign_index == index) {
+
+			is_referenced = TRUE;
+			goto func_exit;
+		}
+
+		foreign = UT_LIST_GET_NEXT(foreign_list, foreign);
+	}
+
+func_exit:
+	if (froze_data_dict) {
+		row_mysql_unfreeze_data_dictionary(trx);
+	}
+
+	return(is_referenced);
+}
+#endif /* WITH_WSREP */
+
 /*********************************************************************//**
 Checks if possible foreign key constraints hold after a delete of the record
 under pcur.
@@ -287,7 +334,123 @@ run_again:
 	}
 
 	err = DB_SUCCESS;
+func_exit:
+	if (got_s_lock) {
+		row_mysql_unfreeze_data_dictionary(trx);
+	}
+
+	mem_heap_free(heap);
 
+	return(err);
+}
+#ifdef WITH_WSREP
+static
+dberr_t
+wsrep_row_upd_check_foreign_constraints(
+/*=================================*/
+	upd_node_t*	node,	/*!< in: row update node */
+	btr_pcur_t*	pcur,	/*!< in: cursor positioned on a record; NOTE: the
+				cursor position is lost in this function! */
+	dict_table_t*	table,	/*!< in: table in question */
+	dict_index_t*	index,	/*!< in: index of the cursor */
+	ulint*		offsets,/*!< in/out: rec_get_offsets(pcur.rec, index) */
+	que_thr_t*	thr,	/*!< in: query thread */
+	mtr_t*		mtr)	/*!< in: mtr */
+{
+	dict_foreign_t*	foreign;
+	mem_heap_t*	heap;
+	dtuple_t*	entry;
+	trx_t*		trx;
+	const rec_t*	rec;
+	ulint		n_ext;
+	dberr_t		err;
+	ibool		got_s_lock	= FALSE;
+	ibool		opened     	= FALSE;
+
+	if (UT_LIST_GET_FIRST(table->foreign_list) == NULL) {
+
+		return(DB_SUCCESS);
+	}
+
+	trx = thr_get_trx(thr);
+
+        /* TODO: make native slave thread bail out here */
+
+	rec = btr_pcur_get_rec(pcur);
+	ut_ad(rec_offs_validate(rec, index, offsets));
+
+	heap = mem_heap_create(500);
+
+	entry = row_rec_to_index_entry(rec, index, offsets,
+				       &n_ext, heap);
+
+	mtr_commit(mtr);
+
+	mtr_start(mtr);
+
+	if (trx->dict_operation_lock_mode == 0) {
+		got_s_lock = TRUE;
+
+		row_mysql_freeze_data_dictionary(trx);
+	}
+
+	foreign = UT_LIST_GET_FIRST(table->foreign_list);
+
+	while (foreign) {
+		/* Note that we may have an update which updates the index
+		record, but does NOT update the first fields which are
+		referenced in a foreign key constraint. Then the update does
+		NOT break the constraint. */
+
+		if (foreign->foreign_index == index
+		    && (node->is_delete
+			|| row_upd_changes_first_fields_binary(
+				entry, index, node->update,
+				foreign->n_fields))) {
+
+			if (foreign->referenced_table == NULL) {
+				foreign->referenced_table =
+					dict_table_open_on_name(
+					  foreign->referenced_table_name_lookup,
+					  FALSE, FALSE, DICT_ERR_IGNORE_NONE);
+				opened = TRUE;
+			}
+
+			if (foreign->referenced_table) {
+				os_inc_counter(dict_sys->mutex,
+					       foreign->referenced_table
+					       ->n_foreign_key_checks_running);
+			}
+
+			/* NOTE that if the thread ends up waiting for a lock
+			we will release dict_operation_lock temporarily!
+			But the counter on the table protects 'foreign' from
+			being dropped while the check is running. */
+
+			err = row_ins_check_foreign_constraint(
+				TRUE, foreign, table, entry, thr);
+
+			if (foreign->referenced_table) {
+				os_dec_counter(dict_sys->mutex,
+					       foreign->referenced_table
+					       ->n_foreign_key_checks_running);
+
+				if (opened == TRUE) {
+					dict_table_close(foreign->referenced_table, TRUE, FALSE);
+					opened = FALSE;
+				}
+			}
+
+			if (err != DB_SUCCESS) {
+
+				goto func_exit;
+			}
+		}
+
+		foreign = UT_LIST_GET_NEXT(foreign_list, foreign);
+	}
+
+	err = DB_SUCCESS;
 func_exit:
 	if (got_s_lock) {
 		row_mysql_unfreeze_data_dictionary(trx);
@@ -299,6 +462,7 @@ func_exit:
 
 	return(err);
 }
+#endif /* WITH_WSREP */
 
 /*********************************************************************//**
 Creates an update node for a query graph.
@@ -1673,6 +1837,9 @@ row_upd_sec_index_entry(
 	index = node->index;
 
 	referenced = row_upd_index_is_referenced(index, trx);
+#ifdef WITH_WSREP
+	ibool foreign = wsrep_row_upd_index_is_foreign(index, trx);
+#endif /* WITH_WSREP */
 
 	heap = mem_heap_create(1024);
 
@@ -1800,6 +1967,9 @@ row_upd_sec_index_entry(
 		row_ins_sec_index_entry() below */
 		if (!rec_get_deleted_flag(
 			    rec, dict_table_is_comp(index->table))) {
+#ifdef WITH_WSREP
+			que_node_t *parent = que_node_get_parent(node);
+#endif /* WITH_WSREP */
 			err = btr_cur_del_mark_set_sec_rec(
 				0, btr_cur, TRUE, thr, &mtr);
 
@@ -1817,6 +1987,37 @@ row_upd_sec_index_entry(
 					node, &pcur, index->table,
 					index, offsets, thr, &mtr);
 			}
+#ifdef WITH_WSREP
+			if (err == DB_SUCCESS && !referenced                  &&
+			    !(parent && que_node_get_type(parent) ==
+				QUE_NODE_UPDATE                               &&
+			      ((upd_node_t*)parent)->cascade_node == node)    &&
+			    foreign
+			) {
+				ulint*	offsets =
+					rec_get_offsets(
+						rec, index, NULL, ULINT_UNDEFINED,
+						&heap);
+				err = wsrep_row_upd_check_foreign_constraints(
+					node, &pcur, index->table,
+					index, offsets, thr, &mtr);
+				switch (err) {
+				case DB_SUCCESS:
+				case DB_NO_REFERENCED_ROW:
+					err = DB_SUCCESS;
+					break;
+				case DB_DEADLOCK:
+					if (wsrep_debug) fprintf (stderr, 
+						"WSREP: sec index FK check fail for deadlock");
+					break;
+				default:
+					fprintf (stderr, 
+						 "WSREP: referenced FK check fail: %d", 
+						 (int)err);
+					break;
+				}
+			}
+#endif /* WITH_WSREP */
 		}
 		break;
 	}
@@ -1971,6 +2172,9 @@ row_upd_clust_rec_by_insert(
 	que_thr_t*	thr,	/*!< in: query thread */
 	ibool		referenced,/*!< in: TRUE if index may be referenced in
 				a foreign key constraint */
+#ifdef WITH_WSREP
+	ibool		foreign, /*!< in: TRUE if index is foreign key index */
+#endif /* WITH_WSREP */
 	mtr_t*		mtr)	/*!< in/out: mtr; gets committed here */
 {
 	mem_heap_t*	heap;
@@ -1984,6 +2188,9 @@ row_upd_clust_rec_by_insert(
 	rec_t*		rec;
 	ulint*		offsets			= NULL;
 
+#ifdef WITH_WSREP
+	que_node_t *parent = que_node_get_parent(node);
+#endif /* WITH_WSREP */
 	ut_ad(node);
 	ut_ad(dict_index_is_clust(index));
 
@@ -2066,6 +2273,34 @@ err_exit:
 				goto err_exit;
 			}
 		}
+#ifdef WITH_WSREP
+		if (!referenced                                              &&
+		    !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE &&
+		      ((upd_node_t*)parent)->cascade_node == node)           &&
+		    foreign
+		) {
+			err = wsrep_row_upd_check_foreign_constraints(
+				node, pcur, table, index, offsets, thr, mtr);
+			switch (err) {
+			case DB_SUCCESS:
+			case DB_NO_REFERENCED_ROW:
+				err = DB_SUCCESS;
+				break;
+			case DB_DEADLOCK:
+				if (wsrep_debug) fprintf (stderr, 
+					"WSREP: insert FK check fail for deadlock");
+				break;
+			default:
+				fprintf (stderr, 
+					"WSREP: referenced FK check fail: %d", 
+					 (int)err);
+				break;
+			}
+			if (err != DB_SUCCESS) {
+				goto err_exit;
+			}
+		}
+#endif /* WITH_WSREP */
 	}
 
 	mtr_commit(mtr);
@@ -2258,11 +2493,18 @@ row_upd_del_mark_clust_rec(
 	ibool		referenced,
 				/*!< in: TRUE if index may be referenced in
 				a foreign key constraint */
+#ifdef WITH_WSREP
+	ibool		foreign,/*!< in: TRUE if index is foreign key index */
+#endif /* WITH_WSREP */
 	mtr_t*		mtr)	/*!< in: mtr; gets committed here */
 {
 	btr_pcur_t*	pcur;
 	btr_cur_t*	btr_cur;
 	dberr_t		err;
+#ifdef WITH_WSREP
+	rec_t*		rec;
+	que_node_t *parent = que_node_get_parent(node);
+#endif /* WITH_WSREP */
 
 	ut_ad(node);
 	ut_ad(dict_index_is_clust(index));
@@ -2279,8 +2521,16 @@ row_upd_del_mark_clust_rec(
 	/* Mark the clustered index record deleted; we do not have to check
 	locks, because we assume that we have an x-lock on the record */
 
+#ifdef WITH_WSREP
+	rec = btr_cur_get_rec(btr_cur);
+#endif /* WITH_WSREP */
+
 	err = btr_cur_del_mark_set_clust_rec(
+#ifdef WITH_WSREP
+		btr_cur_get_block(btr_cur), rec,
+#else
 		btr_cur_get_block(btr_cur), btr_cur_get_rec(btr_cur),
+#endif /* WITH_WSREP */
 		index, offsets, thr, mtr);
 	if (err == DB_SUCCESS && referenced) {
 		/* NOTE that the following call loses the position of pcur ! */
@@ -2288,6 +2538,32 @@ row_upd_del_mark_clust_rec(
 		err = row_upd_check_references_constraints(
 			node, pcur, index->table, index, offsets, thr, mtr);
 	}
+#ifdef WITH_WSREP
+	if (err == DB_SUCCESS && !referenced                         &&
+	    !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE &&
+	      ((upd_node_t*)parent)->cascade_node == node)           &&
+	    thr_get_trx(thr)                                         &&
+	    foreign
+	) {
+		err = wsrep_row_upd_check_foreign_constraints(
+			node, pcur, index->table, index, offsets, thr, mtr);
+		switch (err) {
+		case DB_SUCCESS:
+		case DB_NO_REFERENCED_ROW:
+			err = DB_SUCCESS;
+			break;
+		case DB_DEADLOCK:
+			if (wsrep_debug) fprintf (stderr, 
+				"WSREP: clust rec FK check fail for deadlock");
+			break;
+		default:
+			fprintf (stderr, 
+				"WSREP: clust rec referenced FK check fail: %d", 
+				 (int)err);
+			break;
+		}
+	}
+#endif /* WITH_WSREP */
 
 	mtr_commit(mtr);
 
@@ -2320,6 +2596,10 @@ row_upd_clust_step(
 	index = dict_table_get_first_index(node->table);
 
 	referenced = row_upd_index_is_referenced(index, thr_get_trx(thr));
+#ifdef WITH_WSREP
+	ibool foreign = wsrep_row_upd_index_is_foreign(
+		index, thr_get_trx(thr));
+#endif /* WITH_WSREP */
 
 	pcur = node->pcur;
 
@@ -2414,7 +2694,11 @@ row_upd_clust_step(
 
 	if (node->is_delete) {
 		err = row_upd_del_mark_clust_rec(
+#ifdef WITH_WSREP
+			node, index, offsets, thr, referenced, foreign, &mtr);
+#else
 			node, index, offsets, thr, referenced, &mtr);
+#endif /* WITH_WSREP */
 
 		if (err == DB_SUCCESS) {
 			node->state = UPD_NODE_UPDATE_ALL_SEC;
@@ -2459,7 +2743,11 @@ row_upd_clust_step(
 		externally! */
 
 		err = row_upd_clust_rec_by_insert(
+#ifdef WITH_WSREP
+			node, index, thr, referenced, foreign, &mtr);
+#else
 			node, index, thr, referenced, &mtr);
+#endif /* WITH_WSREP */
 
 		if (err != DB_SUCCESS) {
 
diff --git a/storage/innobase/srv/srv0conc.cc b/storage/innobase/srv/srv0conc.cc
index dc3c0b1dd88..cbf81f8e8f7 100644
--- a/storage/innobase/srv/srv0conc.cc
+++ b/storage/innobase/srv/srv0conc.cc
@@ -42,6 +42,10 @@ Created 2011/04/18 Sunny Bains
 #include "trx0trx.h"
 
 #include "mysql/plugin.h"
+#ifdef WITH_WSREP
+extern "C" int wsrep_trx_is_aborting(void *thd_ptr);
+extern my_bool wsrep_debug;
+#endif
 
 /** Number of times a thread is allowed to enter InnoDB within the same
 SQL query after it has once got the ticket. */
@@ -86,6 +90,9 @@ struct srv_conc_slot_t{
 					reserved may still be TRUE at that
 					point */
 	srv_conc_node_t	srv_conc_queue;	/*!< queue node */
+#ifdef WITH_WSREP
+	void				*thd;		/*!< to see priority */
+#endif
 };
 
 /** Queue of threads waiting to get in */
@@ -145,6 +152,9 @@ srv_conc_init(void)
 
 		conc_slot->event = os_event_create();
 		ut_a(conc_slot->event);
+#ifdef WITH_WSREP
+		conc_slot->thd = NULL;
+#endif /* WITH_WSREP */
 	}
 #endif /* !HAVE_ATOMIC_BUILTINS */
 }
@@ -202,6 +212,16 @@ srv_conc_enter_innodb_with_atomics(
 
 	for (;;) {
 		ulint	sleep_in_us;
+#ifdef WITH_WSREP
+		if (wsrep_on(trx->mysql_thd) && 
+		    wsrep_trx_is_aborting(trx->mysql_thd)) {
+			if (wsrep_debug)
+		  		fprintf(stderr,	
+					"srv_conc_enter due to MUST_ABORT");
+			srv_conc_force_enter_innodb(trx);
+			return;
+		}
+#endif /* WITH_WSREP */
 
 		if (srv_conc.n_active < (lint) srv_thread_concurrency) {
 			ulint	n_active;
@@ -319,6 +339,9 @@ srv_conc_exit_innodb_without_atomics(
 	slot = NULL;
 
 	if (srv_conc.n_active < (lint) srv_thread_concurrency) {
+#ifdef WITH_WSREP
+		srv_conc_slot_t*  wsrep_slot;
+#endif
 		/* Look for a slot where a thread is waiting and no other
 		thread has yet released the thread */
 
@@ -329,6 +352,19 @@ srv_conc_exit_innodb_without_atomics(
 			/* No op */
 		}
 
+#ifdef WITH_WSREP
+		/* look for aborting trx, they must be released asap */
+		wsrep_slot= slot;
+		while (wsrep_slot && (wsrep_slot->wait_ended == TRUE || 
+		    !wsrep_trx_is_aborting(wsrep_slot->thd))) {
+			wsrep_slot = UT_LIST_GET_NEXT(srv_conc_queue, wsrep_slot);
+		}
+		if (wsrep_slot) {
+			slot = wsrep_slot;
+			if (wsrep_debug)
+			    fprintf(stderr, "WSREP: releasing aborting thd\n");
+		}
+#endif
 		if (slot != NULL) {
 			slot->wait_ended = TRUE;
 
@@ -384,6 +420,13 @@ retry:
 
 		return;
 	}
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd) && 
+	    wsrep_thd_is_brute_force(trx->mysql_thd)) {
+		srv_conc_force_enter_innodb(trx);
+		return;
+	}
+#endif
 
 	/* If the transaction is not holding resources, let it sleep
 	for srv_thread_sleep_delay microseconds, and try again then */
@@ -450,6 +493,9 @@ retry:
 	/* Add to the queue */
 	slot->reserved = TRUE;
 	slot->wait_ended = FALSE;
+#ifdef WITH_WSREP
+	slot->thd = trx->mysql_thd;
+#endif
 
 	UT_LIST_ADD_LAST(srv_conc_queue, srv_conc_queue, slot);
 
@@ -457,6 +503,18 @@ retry:
 
 	srv_conc.n_waiting++;
 
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd) && 
+	    wsrep_trx_is_aborting(trx->mysql_thd)) {
+		os_fast_mutex_unlock(&srv_conc_mutex);
+		if (wsrep_debug)
+			fprintf(stderr, "srv_conc_enter due to MUST_ABORT");
+		trx->declared_to_be_inside_innodb = TRUE;
+		trx->n_tickets_to_enter_innodb = srv_n_free_tickets_to_enter;
+		return;
+	}
+	trx->wsrep_event = slot->event;
+#endif /* WITH_WSREP */
 	os_fast_mutex_unlock(&srv_conc_mutex);
 
 	/* Go to wait for the event; when a thread leaves InnoDB it will
@@ -472,6 +530,9 @@ retry:
 
 	os_event_wait(slot->event);
 	thd_wait_end(trx->mysql_thd);
+#ifdef WITH_WSREP
+	trx->wsrep_event = NULL;
+#endif /* WITH_WSREP */
 
 	trx->op_info = "";
 
@@ -483,6 +544,9 @@ retry:
 	incremented the thread counter on behalf of this thread */
 
 	slot->reserved = FALSE;
+#ifdef WITH_WSREP
+	slot->thd = NULL;
+#endif
 
 	UT_LIST_REMOVE(srv_conc_queue, srv_conc_queue, slot);
 
@@ -593,5 +657,32 @@ srv_conc_get_active_threads(void)
 /*==============================*/
 {
 	return(srv_conc.n_active);
- }
+}
+
+#ifdef WITH_WSREP
+UNIV_INTERN
+void
+wsrep_srv_conc_cancel_wait(
+/*==================*/
+	trx_t*	trx)	/*!< in: transaction object associated with the
+			thread */
+{
+#ifdef HAVE_ATOMIC_BUILTINS
+	/* aborting transactions will enter innodb by force in 
+	   srv_conc_enter_innodb_with_atomics(). No need to cancel here,
+	   thr will wake up after os_sleep and let to enter innodb
+	*/
+	if (wsrep_debug)
+		fprintf(stderr, "WSREP: conc slot cancel, no atomics\n");
+#else
+	os_fast_mutex_lock(&srv_conc_mutex);
+	if (trx->wsrep_event) {
+		if (wsrep_debug) 
+			fprintf(stderr, "WSREP: conc slot cancel\n");
+		os_event_set(trx->wsrep_event);
+	}
+	os_fast_mutex_unlock(&srv_conc_mutex);
+#endif
+}
+#endif /* WITH_WSREP */
 
diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc
index b9cfb3544b9..ba742c7238c 100644
--- a/storage/innobase/srv/srv0srv.cc
+++ b/storage/innobase/srv/srv0srv.cc
@@ -73,6 +73,10 @@ Created 10/8/1995 Heikki Tuuri
 #include "mysql/service_thd_wait.h"
 #include "fil0pagecompress.h"
 
+#ifdef WITH_WSREP
+extern int wsrep_debug;
+extern int wsrep_trx_is_aborting(void *thd_ptr);
+#endif
 /* The following is the maximum allowed duration of a lock wait. */
 UNIV_INTERN ulint	srv_fatal_semaphore_wait_threshold = 600;
 
@@ -223,6 +227,10 @@ srv_printf_innodb_monitor() will request mutex acquisition
 with mutex_enter(), which will wait until it gets the mutex. */
 #define MUTEX_NOWAIT(mutex_skipped)	((mutex_skipped) < MAX_MUTEX_NOWAIT)
 
+#ifdef WITH_INNODB_DISALLOW_WRITES
+UNIV_INTERN os_event_t	srv_allow_writes_event;
+#endif /* WITH_INNODB_DISALLOW_WRITES */
+
 /** The sort order table of the MySQL latin1_swedish_ci character set
 collation */
 UNIV_INTERN const byte*	srv_latin1_ordering;
@@ -1004,6 +1012,14 @@ srv_init(void)
 	dict_ind_init();
 
 	srv_conc_init();
+#ifdef WITH_INNODB_DISALLOW_WRITES
+	/* Writes have to be enabled on init or else we hang. Thus, we
+	always set the event here regardless of innobase_disallow_writes.
+	That flag will always be 0 at this point because it isn't settable
+	via my.cnf or command line arg. */
+	srv_allow_writes_event = os_event_create();
+	os_event_set(srv_allow_writes_event);
+#endif /* WITH_INNODB_DISALLOW_WRITES */
 
 	/* Initialize some INFORMATION SCHEMA internal structures */
 	trx_i_s_cache_init(trx_i_s_cache);
@@ -1774,7 +1790,20 @@ loop:
 
 	if (sync_array_print_long_waits(&waiter, &sema)
 	    && sema == old_sema && os_thread_eq(waiter, old_waiter)) {
+#if defined(WITH_WSREP) && defined(WITH_INNODB_DISALLOW_WRITES)
+	  if (srv_allow_writes_event->is_set) {
+#endif /* WITH_WSREP */
 		fatal_cnt++;
+#if defined(WITH_WSREP) && defined(WITH_INNODB_DISALLOW_WRITES)
+	  } else {
+		fprintf(stderr,
+			"WSREP: avoiding InnoDB self crash due to long "
+			"semaphore wait of  > %lu seconds\n"
+			"Server is processing SST donor operation, "
+			"fatal_cnt now: %lu",
+			(ulong) srv_fatal_semaphore_wait_threshold, fatal_cnt);
+	  }
+#endif /* WITH_WSREP */
 		if (fatal_cnt > 10) {
 
 			fprintf(stderr,
diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc
index fcf1c1cedf4..4af61d4869b 100644
--- a/storage/innobase/trx/trx0sys.cc
+++ b/storage/innobase/trx/trx0sys.cc
@@ -44,6 +44,10 @@ Created 3/26/1996 Heikki Tuuri
 #include "os0file.h"
 #include "read0read.h"
 
+#ifdef WITH_WSREP
+#include "ha_prototypes.h" /* wsrep_is_wsrep_xid() */
+#endif /* */
+
 /** The file format tag structure with id and name. */
 struct file_format_t {
 	ulint		id;		/*!< id of the file format */
@@ -174,7 +178,12 @@ trx_sys_flush_max_trx_id(void)
 	mtr_t		mtr;
 	trx_sysf_t*	sys_header;
 
+#ifndef WITH_WSREP
+       /* wsrep_fake_trx_id  violates this assert
+        * Copied from trx_sys_get_new_trx_id
+        */
 	ut_ad(mutex_own(&trx_sys->mutex));
+#endif /* WITH_WSREP */
 
 	if (!srv_read_only_mode) {
 		mtr_start(&mtr);
@@ -202,9 +211,14 @@ trx_sys_update_mysql_binlog_offset(
 	ib_int64_t	offset,	/*!< in: position in that log file */
 	ulint		field,	/*!< in: offset of the MySQL log info field in
 				the trx sys header */
+#ifdef WITH_WSREP
+        trx_sysf_t*     sys_header, /*!< in: trx sys header */
+#endif /* WITH_WSREP */
 	mtr_t*		mtr)	/*!< in: mtr */
 {
+#ifndef WITH_WSREP
 	trx_sysf_t*	sys_header;
+#endif /* !WITH_WSREP */
 
 	if (ut_strlen(file_name) >= TRX_SYS_MYSQL_LOG_NAME_LEN) {
 
@@ -213,7 +227,9 @@ trx_sys_update_mysql_binlog_offset(
 		return;
 	}
 
+#ifndef WITH_WSREP
 	sys_header = trx_sysf_get(mtr);
+#endif /* !WITH_WSREP */
 
 	if (mach_read_from_4(sys_header + field
 			     + TRX_SYS_MYSQL_LOG_MAGIC_N_FLD)
@@ -300,6 +316,124 @@ trx_sys_print_mysql_binlog_offset(void)
 	mtr_commit(&mtr);
 }
 
+#ifdef WITH_WSREP
+
+#ifdef UNIV_DEBUG
+static long long trx_sys_cur_xid_seqno = -1;
+static unsigned char trx_sys_cur_xid_uuid[16];
+
+long long read_wsrep_xid_seqno(const XID* xid)
+{
+    long long seqno;
+    memcpy(&seqno, xid->data + 24, sizeof(long long));
+    return seqno;
+}
+
+void read_wsrep_xid_uuid(const XID* xid, unsigned char* buf)
+{
+    memcpy(buf, xid->data + 8, 16);
+}
+
+#endif /* UNIV_DEBUG */
+
+void
+trx_sys_update_wsrep_checkpoint(
+        const XID*      xid,        /*!< in: transaction XID */
+        trx_sysf_t*     sys_header, /*!< in: sys_header */
+        mtr_t*          mtr)        /*!< in: mtr */
+{
+#ifdef UNIV_DEBUG
+        {
+            /* Check that seqno is monotonically increasing */
+            unsigned char xid_uuid[16];
+            long long xid_seqno = read_wsrep_xid_seqno(xid);
+            read_wsrep_xid_uuid(xid, xid_uuid);
+            if (!memcmp(xid_uuid, trx_sys_cur_xid_uuid, 8))
+            {
+                ut_ad(xid_seqno > trx_sys_cur_xid_seqno);
+                trx_sys_cur_xid_seqno = xid_seqno;
+            }
+            else
+            {
+                memcpy(trx_sys_cur_xid_uuid, xid_uuid, 16);
+            }
+            trx_sys_cur_xid_seqno = xid_seqno;
+        }
+#endif /* UNIV_DEBUG */
+
+        ut_ad(xid && mtr);
+        ut_a(xid->formatID == -1 || wsrep_is_wsrep_xid(xid));
+
+        if (mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
+                             + TRX_SYS_WSREP_XID_MAGIC_N_FLD)
+            != TRX_SYS_WSREP_XID_MAGIC_N) {
+                mlog_write_ulint(sys_header + TRX_SYS_WSREP_XID_INFO
+                                 + TRX_SYS_WSREP_XID_MAGIC_N_FLD,
+                                 TRX_SYS_WSREP_XID_MAGIC_N,
+                                 MLOG_4BYTES, mtr);
+        }
+
+        mlog_write_ulint(sys_header + TRX_SYS_WSREP_XID_INFO
+                         + TRX_SYS_WSREP_XID_FORMAT,
+                         (int)xid->formatID,
+                         MLOG_4BYTES, mtr);
+        mlog_write_ulint(sys_header + TRX_SYS_WSREP_XID_INFO
+                         + TRX_SYS_WSREP_XID_GTRID_LEN,
+                         (int)xid->gtrid_length,
+                         MLOG_4BYTES, mtr);
+        mlog_write_ulint(sys_header + TRX_SYS_WSREP_XID_INFO
+                         + TRX_SYS_WSREP_XID_BQUAL_LEN,
+                         (int)xid->bqual_length,
+                         MLOG_4BYTES, mtr);
+        mlog_write_string(sys_header + TRX_SYS_WSREP_XID_INFO
+                          + TRX_SYS_WSREP_XID_DATA,
+                          (const unsigned char*) xid->data,
+                          XIDDATASIZE, mtr);
+
+}
+
+void
+trx_sys_read_wsrep_checkpoint(XID* xid)
+/*===================================*/
+{
+        trx_sysf_t* sys_header;
+	mtr_t	    mtr;
+        ulint       magic;
+
+        ut_ad(xid);
+
+	mtr_start(&mtr);
+
+	sys_header = trx_sysf_get(&mtr);
+
+        if ((magic = mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
+                                      + TRX_SYS_WSREP_XID_MAGIC_N_FLD))
+            != TRX_SYS_WSREP_XID_MAGIC_N) {
+                memset(xid, 0, sizeof(*xid));
+                xid->formatID = -1;
+                trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr);
+                mtr_commit(&mtr);
+                return;
+        }
+
+        xid->formatID     = (int)mach_read_from_4(
+                sys_header
+                + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_FORMAT);
+        xid->gtrid_length = (int)mach_read_from_4(
+                sys_header
+                + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_GTRID_LEN);
+        xid->bqual_length = (int)mach_read_from_4(
+                sys_header
+                + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_BQUAL_LEN);
+        ut_memcpy(xid->data,
+                  sys_header + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_DATA,
+                  XIDDATASIZE);
+
+	mtr_commit(&mtr);
+}
+
+#endif /* WITH_WSREP */
+
 /*****************************************************************//**
 Prints to stderr the MySQL master log offset info in the trx system header if
 the magic number shows it valid. */
diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc
index a07167168fc..48073667b2f 100644
--- a/storage/innobase/trx/trx0trx.cc
+++ b/storage/innobase/trx/trx0trx.cc
@@ -159,6 +159,9 @@ trx_create(void)
 	trx->lock.table_locks = ib_vector_create(
 		heap_alloc, sizeof(void**), 32);
 
+#ifdef WITH_WSREP
+	trx->wsrep_event = NULL;
+#endif /* WITH_WSREP */
 	return(trx);
 }
 
@@ -854,6 +857,11 @@ trx_start_low(
 			srv_undo_logs, srv_undo_tablespaces);
 	}
 
+#ifdef WITH_WSREP
+        memset(&trx->xid, 0, sizeof(trx->xid));
+        trx->xid.formatID = -1;
+#endif /* WITH_WSREP */
+
 	/* The initial value for trx->no: TRX_ID_MAX is used in
 	read_view_open_now: */
 
@@ -968,6 +976,9 @@ trx_write_serialisation_history(
 	trx_t*		trx,	/*!< in/out: transaction */
 	mtr_t*		mtr)	/*!< in/out: mini-transaction */
 {
+#ifdef WITH_WSREP
+        trx_sysf_t* sys_header;
+#endif /* WITH_WSREP */
 	trx_rseg_t*	rseg;
 
 	rseg = trx->rseg;
@@ -1014,6 +1025,15 @@ trx_write_serialisation_history(
 
 	MONITOR_INC(MONITOR_TRX_COMMIT_UNDO);
 
+#ifdef WITH_WSREP
+	sys_header = trx_sysf_get(mtr);
+	/* Update latest MySQL wsrep XID in trx sys header. */
+	if (wsrep_is_wsrep_xid((const void *)&trx->xid))
+	{
+		trx_sys_update_wsrep_checkpoint(&trx->xid, sys_header, mtr);
+	}
+#endif /* WITH_WSREP */
+
 	/* Update the latest MySQL binlog name and offset info
 	in trx sys header if MySQL binlogging is on or the database
 	server is a MySQL replication slave */
@@ -1024,7 +1044,11 @@ trx_write_serialisation_history(
 		trx_sys_update_mysql_binlog_offset(
 			trx->mysql_log_file_name,
 			trx->mysql_log_offset,
-			TRX_SYS_MYSQL_LOG_INFO, mtr);
+			TRX_SYS_MYSQL_LOG_INFO, 
+#ifdef WITH_WSREP
+                        sys_header,
+#endif /* WITH_WSREP */
+			mtr);
 
 		trx->mysql_log_file_name = NULL;
 	}
@@ -1312,6 +1336,11 @@ trx_commit_in_memory(
 	ut_ad(!trx->in_ro_trx_list);
 	ut_ad(!trx->in_rw_trx_list);
 
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd)) {
+		trx->lock.was_chosen_as_deadlock_victim = FALSE;
+	}
+#endif
 	trx->dict_operation = TRX_DICT_OP_NONE;
 
 	trx->error_state = DB_SUCCESS;
@@ -1496,6 +1525,10 @@ trx_commit_or_rollback_prepare(
 
 	switch (trx->state) {
 	case TRX_STATE_NOT_STARTED:
+#ifdef WITH_WSREP
+		ut_d(trx->start_file = __FILE__);
+		ut_d(trx->start_line = __LINE__);
+#endif /* WITH_WSREP */
 		trx_start_low(trx);
 		/* fall through */
 	case TRX_STATE_ACTIVE:
diff --git a/storage/innobase/wsrep/md5.cc b/storage/innobase/wsrep/md5.cc
new file mode 100644
index 00000000000..30be6ab7dd3
--- /dev/null
+++ b/storage/innobase/wsrep/md5.cc
@@ -0,0 +1,74 @@
+/*
+   Copyright (c) 2014 SkySQL AB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifdef WITH_WSREP
+
+#if defined(HAVE_YASSL)
+#include "my_config.h"
+#include "md5.hpp"
+#elif defined(HAVE_OPENSSL)
+#include <openssl/md5.h>
+#endif /* HAVE_YASSL */
+
+/* Initialize md5 object. */
+void *wsrep_md5_init()
+{
+#if defined(HAVE_YASSL)
+  TaoCrypt::MD5 *hasher= new TaoCrypt::MD5;
+  return (void*)hasher;
+#elif defined(HAVE_OPENSSL)
+  MD5_CTX *ctx = new MD5_CTX();
+  MD5_Init (ctx);
+  return (void *)ctx;
+#endif /* HAVE_YASSL */
+}
+
+/**
+  Supply message to be hashed.
+
+  @param ctx [IN]    Pointer to MD5 context
+  @param buf [IN]    Message to be computed.
+  @param len [IN]    Length of the message.
+*/
+void wsrep_md5_update(void *ctx, char* buf, int len)
+{
+#if defined(HAVE_YASSL)
+  ((TaoCrypt::MD5 *)ctx)->Update((TaoCrypt::byte *) buf, len);
+#elif defined(HAVE_OPENSSL)
+  MD5_Update((MD5_CTX*)(ctx), buf, len);
+#endif /* HAVE_YASSL */
+}
+
+/**
+  Place computed MD5 digest into the given buffer.
+
+  @param digest [OUT] Computed MD5 digest
+  @param ctx    [IN]  Pointer to MD5 context
+*/
+void wsrep_compute_md5_hash(char *digest, void *ctx)
+{
+#if defined(HAVE_YASSL)
+  ((TaoCrypt::MD5*)ctx)->Final((TaoCrypt::byte *) digest);
+  delete (TaoCrypt::MD5*)ctx;
+#elif defined(HAVE_OPENSSL)
+  MD5_Final ((unsigned char*)digest, (MD5_CTX*)ctx);
+  delete (MD5_CTX*)ctx;
+#endif /* HAVE_YASSL */
+}
+
+#endif /* WITH_WSREP */
+
diff --git a/storage/innobase/wsrep/wsrep_md5.h b/storage/innobase/wsrep/wsrep_md5.h
new file mode 100644
index 00000000000..1339f7f4b86
--- /dev/null
+++ b/storage/innobase/wsrep/wsrep_md5.h
@@ -0,0 +1,26 @@
+#ifndef WSREP_MD5_INCLUDED
+#define WSREP_MD5_INCLUDED
+
+/* Copyright (c) 2014 SkySQL AB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+*/
+
+#ifdef WITH_WSREP
+void *wsrep_md5_init();
+void wsrep_md5_update(void *ctx, char* buf, int len);
+void wsrep_compute_md5_hash(char *digest, void *ctx);
+#endif /* WITH_WSREP */
+
+#endif /* WSREP_MD5_INCLUDED */
diff --git a/storage/perfschema/CMakeLists.txt b/storage/perfschema/CMakeLists.txt
index 718baa8296b..aaa1142a180 100644
--- a/storage/perfschema/CMakeLists.txt
+++ b/storage/perfschema/CMakeLists.txt
@@ -13,6 +13,10 @@
 # along with this program; if not, write to the Free Software Foundation,
 # 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
 
+IF (WITH_WSREP)
+  BUILD_WITH_WSREP()
+ENDIF()
+
 INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}
                     ${CMAKE_SOURCE_DIR}/include
                     ${CMAKE_SOURCE_DIR}/sql
diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt
index 528c6f87fcc..6d9bd1a500a 100644
--- a/storage/xtradb/CMakeLists.txt
+++ b/storage/xtradb/CMakeLists.txt
@@ -268,6 +268,27 @@ ENDIF()
 INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/storage/xtradb/include
 		    ${CMAKE_SOURCE_DIR}/storage/xtradb/handler)
 
+IF(WITH_WSREP)
+  # ssl include directory
+  INCLUDE_DIRECTORIES(${SSL_INCLUDE_DIRS}
+                      ${CMAKE_SOURCE_DIR}/storage/xtradb/wsrep)
+
+  IF(SSL_DEFINES)
+    ADD_DEFINITIONS(${SSL_DEFINES})
+  ENDIF()
+
+  LINK_LIBRARIES(${SSL_LIBRARIES})
+
+  # We do RESTRICT_SYMBOL_EXPORTS(yassl) elsewhere.
+  # In order to get correct symbol visibility, these files
+  # must be compiled with "-fvisibility=hidden"
+  IF(WITH_SSL STREQUAL "bundled" AND HAVE_VISIBILITY_HIDDEN)
+    SET_SOURCE_FILES_PROPERTIES(
+      {CMAKE_CURRENT_SOURCE_DIR}/wsrep/md5.cc
+      PROPERTIES COMPILE_FLAGS "-fvisibility=hidden")
+  ENDIF()
+ENDIF()
+
 # Sun Studio bug with -xO2
 IF(CMAKE_CXX_COMPILER_ID MATCHES "SunPro"
 	AND CMAKE_CXX_FLAGS_RELEASE MATCHES "O2"
@@ -407,6 +428,10 @@ SET(INNOBASE_SOURCES
 	ut/ut0vec.cc
 	ut/ut0wqueue.cc)
 
+IF(WITH_WSREP)
+  SET(INNOBASE_SOURCES ${INNOBASE_SOURCES} wsrep/md5.cc)
+ENDIF()
+
 IF(NOT XTRADB_OK)
   MESSAGE(FATAL_ERROR "Percona XtraDB is not supported on this platform")
 ENDIF()
diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc
index af3518337d4..c340590b4bb 100644
--- a/storage/xtradb/dict/dict0dict.cc
+++ b/storage/xtradb/dict/dict0dict.cc
@@ -3241,7 +3241,29 @@ dict_foreign_find_index(
 
 	return(NULL);
 }
-
+#ifdef WITH_WSREP
+dict_index_t*
+wsrep_dict_foreign_find_index(
+/*====================*/
+	dict_table_t*	table,	/*!< in: table */
+	const char**	col_names, /*!< in: column names, or NULL
+					to use table->col_names */
+	const char**	columns,/*!< in: array of column names */
+	ulint		n_cols,	/*!< in: number of columns */
+	dict_index_t*	types_idx, /*!< in: NULL or an index to whose types the
+				   column types must match */
+	ibool		check_charsets,
+				/*!< in: whether to check charsets.
+				only has an effect if types_idx != NULL */
+	ulint		check_null)
+				/*!< in: nonzero if none of the columns must
+				be declared NOT NULL */
+{
+	return dict_foreign_find_index(
+		table, col_names, columns, n_cols, types_idx, check_charsets,
+		check_null);
+}
+#endif /* WITH_WSREP */
 /**********************************************************************//**
 Report an error in a foreign key definition. */
 static
diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc
index d40cfa57bd5..343cc0f47c4 100644
--- a/storage/xtradb/fil/fil0fil.cc
+++ b/storage/xtradb/fil/fil0fil.cc
@@ -6833,36 +6833,36 @@ fil_get_page_type_name(
 {
 	switch(page_type) {
 	case FIL_PAGE_PAGE_COMPRESSED:
-		return "PAGE_COMPRESSED";
+		return (char *)"PAGE_COMPRESSED";
 	case FIL_PAGE_INDEX:
-		return "INDEX";
+		return (char *)"INDEX";
 	case FIL_PAGE_UNDO_LOG:
-		return "UNDO LOG";
+		return (char *)"UNDO LOG";
 	case FIL_PAGE_INODE:
-		return "INODE";
+		return (char *)"INODE";
 	case FIL_PAGE_IBUF_FREE_LIST:
-		return "IBUF_FREE_LIST";
+		return (char *)"IBUF_FREE_LIST";
 	case FIL_PAGE_TYPE_ALLOCATED:
-		return "ALLOCATED";
+		return (char *)"ALLOCATED";
 	case FIL_PAGE_IBUF_BITMAP:
-		return "IBUF_BITMAP";
+		return (char *)"IBUF_BITMAP";
 	case FIL_PAGE_TYPE_SYS:
-		return "SYS";
+		return (char *)"SYS";
 	case FIL_PAGE_TYPE_TRX_SYS:
-		return "TRX_SYS";
+		return (char *)"TRX_SYS";
 	case FIL_PAGE_TYPE_FSP_HDR:
-		return "FSP_HDR";
+		return (char *)"FSP_HDR";
 	case FIL_PAGE_TYPE_XDES:
-		return "XDES";
+		return (char *)"XDES";
 	case FIL_PAGE_TYPE_BLOB:
-		return "BLOB";
+		return (char *)"BLOB";
 	case FIL_PAGE_TYPE_ZBLOB:
-		return "ZBLOB";
+		return (char *)"ZBLOB";
 	case FIL_PAGE_TYPE_ZBLOB2:
-		return "ZBLOB2";
+		return (char *)"ZBLOB2";
 	case FIL_PAGE_TYPE_COMPRESSED:
-		return "ORACLE PAGE COMPRESSED";
+		return (char *)"ORACLE PAGE COMPRESSED";
 	default:
-		return "PAGE TYPE CORRUPTED";
+		return (char *)"PAGE TYPE CORRUPTED";
 	}
 }
diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc
index 854b094ea81..ec19a4ef4b7 100644
--- a/storage/xtradb/fil/fil0pagecompress.cc
+++ b/storage/xtradb/fil/fil0pagecompress.cc
@@ -487,7 +487,9 @@ fil_decompress_page(
         ulint actual_size = 0;
 	ulint compression_alg = 0;
 	byte *in_buf;
+#ifdef HAVE_LZO
 	ulint olen=0;
+#endif
 	ulint ptype;
 
 	ut_ad(buf);
diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc
index fb3e097491d..4c4bccd15e1 100644
--- a/storage/xtradb/handler/ha_innodb.cc
+++ b/storage/xtradb/handler/ha_innodb.cc
@@ -120,6 +120,44 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #  define MYSQL_PLUGIN_IMPORT /* nothing */
 # endif /* MYSQL_PLUGIN_IMPORT */
 
+#ifdef WITH_WSREP
+#include "dict0priv.h"
+#include "../storage/innobase/include/ut0byte.h"
+#include <wsrep_mysqld.h>
+#include <wsrep_md5.h>
+
+extern my_bool wsrep_certify_nonPK;
+class  binlog_trx_data;
+extern handlerton *binlog_hton;
+
+extern MYSQL_PLUGIN_IMPORT mysql_mutex_t LOCK_wsrep_rollback;
+extern MYSQL_PLUGIN_IMPORT mysql_cond_t COND_wsrep_rollback;
+extern MYSQL_PLUGIN_IMPORT wsrep_aborting_thd_t wsrep_aborting_thd;
+
+static inline wsrep_ws_handle_t*
+wsrep_ws_handle(THD* thd, const trx_t* trx) {
+	return wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd),
+				       (wsrep_trx_id_t)trx->id);
+}
+
+extern bool wsrep_prepare_key_for_innodb(const uchar *cache_key,
+					 size_t cache_key_len,
+                                         const uchar* row_id,
+                                         size_t row_id_len,
+                                         wsrep_buf_t* key,
+                                         size_t* key_len);
+
+extern handlerton * wsrep_hton;
+extern TC_LOG* tc_log;
+extern void wsrep_cleanup_transaction(THD *thd);
+static int
+wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd,
+			my_bool signal);
+static void
+wsrep_fake_trx_id(handlerton* hton, THD *thd);
+static int innobase_wsrep_set_checkpoint(handlerton* hton, const XID* xid);
+static int innobase_wsrep_get_checkpoint(handlerton* hton, XID* xid);
+#endif /* WITH_WSREP */
 /** to protect innobase_open_files */
 static mysql_mutex_t innobase_share_mutex;
 /** to force correct commit order in binlog */
@@ -1483,6 +1521,10 @@ innobase_srv_conc_enter_innodb(
 /*===========================*/
 	trx_t*	trx)	/*!< in: transaction handle */
 {
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd) && 
+	    wsrep_thd_is_BF(trx->mysql_thd, FALSE)) return;
+#endif /* WITH_WSREP */
 	if (srv_thread_concurrency) {
 		if (trx->n_tickets_to_enter_innodb > 0) {
 
@@ -1517,6 +1559,10 @@ innobase_srv_conc_exit_innodb(
 #ifdef UNIV_SYNC_DEBUG
 	ut_ad(!sync_thread_levels_nonempty_trx(trx->has_search_latch));
 #endif /* UNIV_SYNC_DEBUG */
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd) && 
+	    wsrep_thd_is_BF(trx->mysql_thd, FALSE)) return;
+#endif /* WITH_WSREP */
 
 	/* This is to avoid making an unnecessary function call. */
 	if (trx->declared_to_be_inside_innodb
@@ -1649,6 +1695,16 @@ thd_to_trx(
 	return(*(trx_t**) thd_ha_data(thd, innodb_hton_ptr));
 }
 
+#ifdef WITH_WSREP
+ulonglong
+thd_to_trx_id(
+/*=======*/
+	THD*	thd)	/*!< in: MySQL thread */
+{
+	return(thd_to_trx(thd)->id);
+}
+#endif /* WITH_WSREP */
+
 my_bool
 ha_innobase::is_fake_change_enabled(THD* thd)
 {
@@ -2149,6 +2205,9 @@ int
 innobase_mysql_tmpfile(void)
 /*========================*/
 {
+#ifdef WITH_INNODB_DISALLOW_WRITES
+	os_event_wait(srv_allow_writes_event);
+#endif /* WITH_INNODB_DISALLOW_WRITES */
 	int	fd2 = -1;
 	File	fd;
 
@@ -3313,6 +3372,12 @@ innobase_init(
           innobase_hton->tablefile_extensions = ha_innobase_exts;
 
 	innobase_hton->table_options = innodb_table_option_list;
+#ifdef WITH_WSREP
+        innobase_hton->abort_transaction=wsrep_abort_transaction;
+        innobase_hton->set_checkpoint=innobase_wsrep_set_checkpoint;
+        innobase_hton->get_checkpoint=innobase_wsrep_get_checkpoint;
+        innobase_hton->fake_trx_id=wsrep_fake_trx_id;
+#endif /* WITH_WSREP */
 
 	ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR);
 
@@ -4050,10 +4115,30 @@ innobase_commit_low(
 /*================*/
 	trx_t*	trx)	/*!< in: transaction handle */
 {
+#ifdef WITH_WSREP
+	THD* thd = (THD*)trx->mysql_thd;
+	const char* tmp = 0;
+	if (wsrep_on((void*)thd)) {
+#ifdef WSREP_PROC_INFO
+		char info[64];
+		info[sizeof(info) - 1] = '\0';
+		snprintf(info, sizeof(info) - 1,
+			 "innobase_commit_low():trx_commit_for_mysql(%lld)",
+			 (long long) wsrep_thd_trx_seqno(thd));
+		tmp = thd_proc_info(thd, info);
+
+#else
+		tmp = thd_proc_info(thd, "innobase_commit_low()");
+#endif /* WSREP_PROC_INFO */
+	}
+#endif /* WITH_WSREP */
 	if (trx_is_started(trx)) {
 
 		trx_commit_for_mysql(trx);
 	}
+#ifdef WITH_WSREP
+	if (wsrep_on((void*)thd)) { thd_proc_info(thd, tmp); }
+#endif /* WITH_WSREP */
 }
 
 /*****************************************************************//**
@@ -4828,22 +4913,32 @@ innobase_kill_connection(
 	DBUG_ENTER("innobase_kill_connection");
 	DBUG_ASSERT(hton == innodb_hton_ptr);
 
-	lock_mutex_enter();
-
+#ifdef WITH_WSREP
+	wsrep_thd_LOCK(thd);
+	if (wsrep_thd_conflict_state(thd) != NO_CONFLICT) {
+		/* if victim has been signaled by BF thread and/or aborting
+		   is already progressing, following query aborting is not necessary
+		   any more.
+		   Also, BF thread should own trx mutex for the victim, which would
+		   conflict with trx_mutex_enter() below
+		*/
+		wsrep_thd_UNLOCK(thd);
+		DBUG_VOID_RETURN;
+	}
+	wsrep_thd_UNLOCK(thd);
+#endif /* WITH_WSREP */
 	trx = thd_to_trx(thd);
 
 	if (trx)
 	{
-		trx_mutex_enter(trx);
-
-		/* Cancel a pending lock request. */
-		if (trx->lock.wait_lock)
-			lock_cancel_waiting_and_release(trx->lock.wait_lock);
-
-		trx_mutex_exit(trx);
-	}
-
-	lock_mutex_exit();
+          /* Cancel a pending lock request. */
+          lock_mutex_enter();
+          trx_mutex_enter(trx);
+          if (trx->lock.wait_lock)
+            lock_cancel_waiting_and_release(trx->lock.wait_lock);
+          trx_mutex_exit(trx);
+          lock_mutex_exit();
+        }
 
 	DBUG_VOID_RETURN;
 }
@@ -4956,7 +5051,11 @@ ha_innobase::max_supported_key_length() const
 	case 8192:
 		return(1536);
 	default:
+#ifdef WITH_WSREP
+		return(3500);
+#else
 		return(3500);
+#endif
 	}
 }
 
@@ -6072,6 +6171,117 @@ get_field_offset(
 	return((uint) (field->ptr - table->record[0]));
 }
 
+#ifdef WITH_WSREP
+UNIV_INTERN
+int
+wsrep_innobase_mysql_sort(
+/*===============*/
+					/* out: str contains sort string */
+	int		mysql_type,	/* in: MySQL type */
+	uint		charset_number,	/* in: number of the charset */
+	unsigned char*	str,		/* in: data field */
+	unsigned int	str_length,	/* in: data field length,
+					not UNIV_SQL_NULL */
+	unsigned int	buf_length)	/* in: total str buffer length */
+
+{
+	CHARSET_INFO*		charset;
+	enum_field_types	mysql_tp;
+	int ret_length =	str_length;
+
+	DBUG_ASSERT(str_length != UNIV_SQL_NULL);
+
+	mysql_tp = (enum_field_types) mysql_type;
+
+	switch (mysql_tp) {
+
+	case MYSQL_TYPE_BIT:
+	case MYSQL_TYPE_STRING:
+	case MYSQL_TYPE_VAR_STRING:
+	case MYSQL_TYPE_TINY_BLOB:
+	case MYSQL_TYPE_MEDIUM_BLOB:
+	case MYSQL_TYPE_BLOB:
+	case MYSQL_TYPE_LONG_BLOB:
+	case MYSQL_TYPE_VARCHAR:
+	{
+		uchar tmp_str[REC_VERSION_56_MAX_INDEX_COL_LEN] = {'\0'};
+		uint tmp_length = REC_VERSION_56_MAX_INDEX_COL_LEN;
+
+		/* Use the charset number to pick the right charset struct for
+		the comparison. Since the MySQL function get_charset may be
+		slow before Bar removes the mutex operation there, we first
+		look at 2 common charsets directly. */
+
+		if (charset_number == default_charset_info->number) {
+			charset = default_charset_info;
+		} else if (charset_number == my_charset_latin1.number) {
+			charset = &my_charset_latin1;
+		} else {
+			charset = get_charset(charset_number, MYF(MY_WME));
+
+			if (charset == NULL) {
+			  sql_print_error("InnoDB needs charset %lu for doing "
+					  "a comparison, but MySQL cannot "
+					  "find that charset.",
+					  (ulong) charset_number);
+				ut_a(0);
+			}
+		}
+
+		ut_a(str_length <= tmp_length);
+		memcpy(tmp_str, str, str_length);
+
+		tmp_length = charset->coll->strnxfrm(charset, str, str_length,
+                                             str_length, tmp_str,
+                                             tmp_length, 0);
+		DBUG_ASSERT(tmp_length <= str_length);
+		if (wsrep_protocol_version < 3) {
+			tmp_length = charset->coll->strnxfrm(
+				charset, str, str_length,
+				str_length, tmp_str, tmp_length, 0);
+			DBUG_ASSERT(tmp_length <= str_length);
+		} else {
+			/* strnxfrm will expand the destination string,
+			   protocols < 3 truncated the sorted sring
+			   protocols >= 3 gets full sorted sring
+			*/
+			tmp_length = charset->coll->strnxfrm(
+				charset, str, buf_length,
+				str_length, tmp_str, str_length, 0);
+			DBUG_ASSERT(tmp_length <= buf_length);
+			ret_length = tmp_length;
+		}
+ 
+		break;
+	}
+	case MYSQL_TYPE_DECIMAL :
+	case MYSQL_TYPE_TINY :
+	case MYSQL_TYPE_SHORT :
+	case MYSQL_TYPE_LONG :
+	case MYSQL_TYPE_FLOAT :
+	case MYSQL_TYPE_DOUBLE :
+	case MYSQL_TYPE_NULL :
+	case MYSQL_TYPE_TIMESTAMP :
+	case MYSQL_TYPE_LONGLONG :
+	case MYSQL_TYPE_INT24 :
+	case MYSQL_TYPE_DATE :
+	case MYSQL_TYPE_TIME :
+	case MYSQL_TYPE_DATETIME :
+	case MYSQL_TYPE_YEAR :
+	case MYSQL_TYPE_NEWDATE :
+	case MYSQL_TYPE_NEWDECIMAL :
+	case MYSQL_TYPE_ENUM :
+	case MYSQL_TYPE_SET :
+	case MYSQL_TYPE_GEOMETRY :
+		break;
+	default:
+		break;
+	}
+
+	return ret_length;
+}
+#endif /* WITH_WSREP */
+
 /*************************************************************//**
 InnoDB uses this function to compare two data fields for which the data type
 is such that we must use MySQL code to compare them. NOTE that the prototype
@@ -6575,6 +6785,312 @@ innobase_read_from_2_little_endian(
 /*******************************************************************//**
 Stores a key value for a row to a buffer.
 @return	key value length as stored in buff */
+#ifdef WITH_WSREP
+UNIV_INTERN
+uint
+wsrep_store_key_val_for_row(
+/*===============================*/
+	THD* 		thd,
+	TABLE*		table,
+	uint		keynr,	/*!< in: key number */
+	char*		buff,	/*!< in/out: buffer for the key value (in MySQL
+				format) */
+	uint		buff_len,/*!< in: buffer length */
+	const uchar*	record,
+	ibool*          key_is_null)/*!< out: full key was null */
+{
+	KEY*		key_info	= table->key_info + keynr;
+	KEY_PART_INFO*	key_part	= key_info->key_part;
+	KEY_PART_INFO*	end		= key_part + key_info->user_defined_key_parts;
+	char*		buff_start	= buff;
+	enum_field_types mysql_type;
+	Field*		field;
+	uint buff_space = buff_len;
+
+	DBUG_ENTER("wsrep_store_key_val_for_row");
+
+	memset(buff, 0, buff_len);
+	*key_is_null = TRUE;
+
+	for (; key_part != end; key_part++) {
+
+		uchar sorted[REC_VERSION_56_MAX_INDEX_COL_LEN] = {'\0'};
+		ibool part_is_null = FALSE;
+
+		if (key_part->null_bit) {
+			if (buff_space > 0) {
+				if (record[key_part->null_offset] 
+				    & key_part->null_bit) {
+					*buff = 1;
+					part_is_null = TRUE;
+				} else {
+					*buff = 0;
+				}
+				buff++;
+				buff_space--;
+			} else {
+				fprintf (stderr, "WSREP: key truncated: %s\n",
+					 wsrep_thd_query(thd));
+			}
+		}
+		if (!part_is_null)  *key_is_null = FALSE;
+
+		field = key_part->field;
+		mysql_type = field->type();
+
+		if (mysql_type == MYSQL_TYPE_VARCHAR) {
+						/* >= 5.0.3 true VARCHAR */
+			ulint		lenlen;
+			ulint		len;
+			const byte*	data;
+			ulint		key_len;
+			ulint		true_len;
+			const CHARSET_INFO* cs;
+			int		error=0;
+
+			key_len = key_part->length;
+
+			if (part_is_null) {
+				true_len = key_len + 2;
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+				buff       += true_len;
+				buff_space -= true_len;
+				continue;
+			}
+			cs = field->charset();
+
+			lenlen = (ulint)
+				(((Field_varstring*)field)->length_bytes);
+
+			data = row_mysql_read_true_varchar(&len,
+				(byte*) (record
+				+ (ulint)get_field_offset(table, field)),
+				lenlen);
+
+			true_len = len;
+
+			/* For multi byte character sets we need to calculate
+			the true length of the key */
+
+			if (len > 0 && cs->mbmaxlen > 1) {
+				true_len = (ulint) cs->cset->well_formed_len(cs,
+						(const char *) data,
+						(const char *) data + len,
+                                                (uint) (key_len /
+                                                        cs->mbmaxlen),
+						&error);
+			}
+
+			/* In a column prefix index, we may need to truncate
+			the stored value: */
+
+			if (true_len > key_len) {
+				true_len = key_len;
+			}
+
+			memcpy(sorted, data, true_len);
+			true_len = wsrep_innobase_mysql_sort(
+				mysql_type, cs->number, sorted, true_len, 
+				REC_VERSION_56_MAX_INDEX_COL_LEN);
+
+			if (wsrep_protocol_version > 1) {
+			/* Note that we always reserve the maximum possible
+			   length of the true VARCHAR in the key value, though
+			   only len first bytes after the 2 length bytes contain
+			   actual data. The rest of the space was reset to zero
+			   in the bzero() call above. */
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+ 				memcpy(buff, sorted, true_len);
+                                buff       += true_len;
+				buff_space -= true_len;
+                        } else {
+                                buff += key_len;
+                        }
+		} else if (mysql_type == MYSQL_TYPE_TINY_BLOB
+			|| mysql_type == MYSQL_TYPE_MEDIUM_BLOB
+			|| mysql_type == MYSQL_TYPE_BLOB
+			|| mysql_type == MYSQL_TYPE_LONG_BLOB
+			/* MYSQL_TYPE_GEOMETRY data is treated
+			as BLOB data in innodb. */
+			|| mysql_type == MYSQL_TYPE_GEOMETRY) {
+
+			const CHARSET_INFO* cs;
+			ulint		key_len;
+			ulint		true_len;
+			int		error=0;
+			ulint		blob_len;
+			const byte*	blob_data;
+
+			ut_a(key_part->key_part_flag & HA_PART_KEY_SEG);
+
+			key_len = key_part->length;
+
+			if (part_is_null) {
+				true_len = key_len + 2;
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+				buff       += true_len;
+				buff_space -= true_len;
+
+				continue;
+			}
+
+			cs = field->charset();
+
+			blob_data = row_mysql_read_blob_ref(&blob_len,
+				(byte*) (record
+				+ (ulint)get_field_offset(table, field)),
+					(ulint) field->pack_length());
+
+			true_len = blob_len;
+
+			ut_a(get_field_offset(table, field)
+				== key_part->offset);
+
+			/* For multi byte character sets we need to calculate
+			the true length of the key */
+
+			if (blob_len > 0 && cs->mbmaxlen > 1) {
+				true_len = (ulint) cs->cset->well_formed_len(cs,
+						(const char *) blob_data,
+						(const char *) blob_data
+							+ blob_len,
+                                                (uint) (key_len /
+                                                        cs->mbmaxlen),
+						&error);
+			}
+
+			/* All indexes on BLOB and TEXT are column prefix
+			indexes, and we may need to truncate the data to be
+			stored in the key value: */
+
+			if (true_len > key_len) {
+				true_len = key_len;
+			}
+
+			memcpy(sorted, blob_data, true_len);
+			true_len = wsrep_innobase_mysql_sort(
+				mysql_type, cs->number, sorted, true_len,
+				REC_VERSION_56_MAX_INDEX_COL_LEN);
+
+
+			/* Note that we always reserve the maximum possible
+			length of the BLOB prefix in the key value. */
+                        if (wsrep_protocol_version > 1) {
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+				buff       += true_len;
+				buff_space -= true_len;
+			} else {
+				buff += key_len;
+			}
+			memcpy(buff, sorted, true_len);
+		} else {
+			/* Here we handle all other data types except the
+			true VARCHAR, BLOB and TEXT. Note that the column
+			value we store may be also in a column prefix
+			index. */
+
+			const CHARSET_INFO*	cs = NULL;
+			ulint			true_len;
+			ulint			key_len;
+			const uchar*		src_start;
+			int			error=0;
+			enum_field_types	real_type;
+
+			key_len = key_part->length;
+
+			if (part_is_null) {
+				true_len = key_len;
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len = buff_space;
+				}
+				buff       += true_len;
+				buff_space -= true_len;
+
+				continue;
+			}
+
+			src_start = record + key_part->offset;
+			real_type = field->real_type();
+			true_len = key_len;
+
+			/* Character set for the field is defined only
+			to fields whose type is string and real field
+			type is not enum or set. For these fields check
+			if character set is multi byte. */
+
+			if (real_type != MYSQL_TYPE_ENUM
+				&& real_type != MYSQL_TYPE_SET
+				&& ( mysql_type == MYSQL_TYPE_VAR_STRING
+					|| mysql_type == MYSQL_TYPE_STRING)) {
+
+				cs = field->charset();
+
+				/* For multi byte character sets we need to
+				calculate the true length of the key */
+
+				if (key_len > 0 && cs->mbmaxlen > 1) {
+
+					true_len = (ulint)
+						cs->cset->well_formed_len(cs,
+							(const char *)src_start,
+							(const char *)src_start
+								+ key_len,
+                                                        (uint) (key_len /
+                                                                cs->mbmaxlen),
+							&error);
+				}
+				memcpy(sorted, src_start, true_len);
+				true_len = wsrep_innobase_mysql_sort(
+					mysql_type, cs->number, sorted, true_len,
+					REC_VERSION_56_MAX_INDEX_COL_LEN);
+
+				if (true_len > buff_space) {
+					fprintf (stderr,
+						 "WSREP: key truncated: %s\n",
+						 wsrep_thd_query(thd));
+					true_len   = buff_space;
+				}
+				memcpy(buff, sorted, true_len);
+			} else {
+				memcpy(buff, src_start, true_len);
+			}
+			buff       += true_len;
+			buff_space -= true_len;
+		}
+	}
+
+	ut_a(buff <= buff_start + buff_len);
+
+	DBUG_RETURN((uint)(buff - buff_start));
+}
+#endif /* WITH_WSREP */
+
+/*******************************************************************//**
+Stores a key value for a row to a buffer.
+@return	key value length as stored in buff */
 UNIV_INTERN
 uint
 ha_innobase::store_key_val_for_row(
@@ -7419,6 +7935,9 @@ ha_innobase::write_row(
 	ibool		auto_inc_used= FALSE;
 	ulint		sql_command;
 	trx_t*		trx = thd_to_trx(user_thd);
+#ifdef WITH_WSREP
+	ibool           auto_inc_inserted= FALSE; /* if NULL was inserted */
+#endif
 
 	DBUG_ENTER("ha_innobase::write_row");
 
@@ -7454,8 +7973,18 @@ ha_innobase::write_row(
 	if ((sql_command == SQLCOM_ALTER_TABLE
 	     || sql_command == SQLCOM_OPTIMIZE
 	     || sql_command == SQLCOM_CREATE_INDEX
+#ifdef WITH_WSREP
+	     || (wsrep_on(user_thd) && wsrep_load_data_splitting &&
+		 sql_command == SQLCOM_LOAD)
+#endif /* WITH_WSREP */
 	     || sql_command == SQLCOM_DROP_INDEX)
 	    && num_write_row >= 10000) {
+#ifdef WITH_WSREP
+		if (wsrep_on(user_thd) && sql_command == SQLCOM_LOAD) {
+			WSREP_DEBUG("forced trx split for LOAD: %s", 
+				    wsrep_thd_query(user_thd));
+		}
+#endif /* WITH_WSREP */
 		/* ALTER TABLE is COMMITted at every 10000 copied rows.
 		The IX table lock for the original table has to be re-issued.
 		As this method will be called on a temporary table where the
@@ -7489,6 +8018,21 @@ no_commit:
 			*/
 			;
 		} else if (src_table == prebuilt->table) {
+#ifdef WITH_WSREP
+			switch (wsrep_run_wsrep_commit(user_thd, wsrep_hton, 1))
+			{
+			case WSREP_TRX_OK:
+				break;
+			case WSREP_TRX_SIZE_EXCEEDED:
+			case WSREP_TRX_CERT_FAIL:
+			case WSREP_TRX_ERROR:
+				DBUG_RETURN(1);
+			}
+
+			if (binlog_hton->commit(binlog_hton, user_thd, 1))
+                                DBUG_RETURN(1);
+                        wsrep_post_commit(user_thd, TRUE);
+#endif /* WITH_WSREP */
 			/* Source table is not in InnoDB format:
 			no need to re-acquire locks on it. */
 
@@ -7499,6 +8043,21 @@ no_commit:
 			/* We will need an IX lock on the destination table. */
 			prebuilt->sql_stat_start = TRUE;
 		} else {
+#ifdef WITH_WSREP
+			switch (wsrep_run_wsrep_commit(user_thd, wsrep_hton, 1))
+			{
+			case WSREP_TRX_OK:
+				break;
+			case WSREP_TRX_SIZE_EXCEEDED:
+			case WSREP_TRX_CERT_FAIL:
+			case WSREP_TRX_ERROR:
+				DBUG_RETURN(1);
+			}
+
+			if (binlog_hton->commit(binlog_hton, user_thd, 1))
+                                DBUG_RETURN(1);
+                        wsrep_post_commit(user_thd, TRUE);
+#endif /* WITH_WSREP */
 			/* Ensure that there are no other table locks than
 			LOCK_IX and LOCK_AUTO_INC on the destination table. */
 
@@ -7528,6 +8087,10 @@ no_commit:
 		innobase_get_auto_increment(). */
 		prebuilt->autoinc_error = DB_SUCCESS;
 
+#ifdef WITH_WSREP
+		auto_inc_inserted= (table->next_number_field->val_int() == 0);
+#endif
+
 		if ((error_result = update_auto_increment())) {
 			/* We don't want to mask autoinc overflow errors. */
 
@@ -7615,6 +8178,32 @@ no_commit:
 			case SQLCOM_REPLACE_SELECT:
 				goto set_max_autoinc;
 
+#ifdef WITH_WSREP
+			/* workaround for LP bug #355000, retrying the insert */
+			case SQLCOM_INSERT:
+                               if (wsrep_on(current_thd)                     &&
+                                   auto_inc_inserted                         &&
+                                   wsrep_drupal_282555_workaround            &&
+                                   wsrep_thd_retry_counter(current_thd) == 0 &&
+				    !thd_test_options(current_thd, 
+						      OPTION_NOT_AUTOCOMMIT | 
+						      OPTION_BEGIN)) {
+					WSREP_DEBUG(
+					    "retrying insert: %s",
+					    (*wsrep_thd_query(current_thd)) ? 
+						wsrep_thd_query(current_thd) : 
+						(char *)"void");
+					error= DB_SUCCESS;
+					wsrep_thd_set_conflict_state(
+						current_thd, MUST_ABORT);
+                                        innobase_srv_conc_exit_innodb(prebuilt->trx);
+                                        /* jump straight to func exit over
+                                         * later wsrep hooks */
+                                        goto func_exit;
+				}
+                                break;
+#endif /* WITH_WSREP */
+
 			default:
 				break;
 			}
@@ -7674,6 +8263,21 @@ report_error:
 						   prebuilt->table->flags,
 						   user_thd);
 
+#ifdef WITH_WSREP
+	if (!error_result && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE &&
+	    wsrep_on(user_thd) && !wsrep_consistency_check(user_thd) &&
+	    (sql_command != SQLCOM_LOAD || 
+	     thd_binlog_format(user_thd) == BINLOG_FORMAT_ROW)) {
+
+		if (wsrep_append_keys(user_thd, false, record, NULL)) {
+ 			DBUG_PRINT("wsrep", ("row key failed"));
+ 			error_result = HA_ERR_INTERNAL_ERROR;
+			goto wsrep_error;
+		}
+	}
+wsrep_error:
+#endif /* WITH_WSREP */
+
 	if (error_result == HA_FTS_INVALID_DOCID) {
 		my_error(HA_FTS_INVALID_DOCID, MYF(0));
 	}
@@ -7963,6 +8567,87 @@ calc_row_difference(
 	return(DB_SUCCESS);
 }
 
+#ifdef WITH_WSREP
+static
+int
+wsrep_calc_row_hash(
+/*================*/
+	byte*		digest,		/*!< in/out: md5 sum */
+	const uchar*	row,		/*!< in: row in MySQL format */
+	TABLE*		table,		/*!< in: table in MySQL data
+					dictionary */
+	row_prebuilt_t*	prebuilt,	/*!< in: InnoDB prebuilt struct */
+	THD*		thd)		/*!< in: user thread */
+{
+	Field*		field;
+	enum_field_types field_mysql_type;
+	uint		n_fields;
+	ulint		len;
+	const byte*	ptr;
+	ulint		col_type;
+	uint		i;
+
+	void *ctx = wsrep_md5_init();
+
+	n_fields = table->s->fields;
+
+	for (i = 0; i < n_fields; i++) {
+		byte null_byte=0;
+		byte true_byte=1;
+
+		field = table->field[i];
+
+		ptr = (const byte*) row + get_field_offset(table, field);
+		len = field->pack_length();
+
+		field_mysql_type = field->type();
+
+		col_type = prebuilt->table->cols[i].mtype;
+
+		switch (col_type) {
+
+		case DATA_BLOB:
+			ptr = row_mysql_read_blob_ref(&len, ptr, len);
+
+			break;
+
+		case DATA_VARCHAR:
+		case DATA_BINARY:
+		case DATA_VARMYSQL:
+			if (field_mysql_type == MYSQL_TYPE_VARCHAR) {
+				/* This is a >= 5.0.3 type true VARCHAR where
+				the real payload data length is stored in
+				1 or 2 bytes */
+
+				ptr = row_mysql_read_true_varchar(
+					&len, ptr,
+					(ulint)
+					(((Field_varstring*)field)->length_bytes));
+
+			}
+
+			break;
+		default:
+			;
+		}
+		/*
+		if (field->null_ptr &&
+		    field_in_record_is_null(table, field, (char*) row)) {
+		*/
+
+		if (field->is_null_in_record(row)) {
+			wsrep_md5_update(ctx, (char*)&null_byte, 1);
+		} else {
+			wsrep_md5_update(ctx, (char*)&true_byte, 1);
+			wsrep_md5_update(ctx, (char*)ptr, len);
+		}
+	}
+
+	wsrep_compute_md5_hash((char*)digest, ctx);
+
+	return(0);
+}
+#endif /* WITH_WSREP */
 /**********************************************************************//**
 Updates a row given as a parameter to a new value. Note that we are given
 whole rows, not just the fields which are updated: this incurs some
@@ -8109,6 +8794,23 @@ func_exit:
 
 	innobase_active_small();
 
+#ifdef WITH_WSREP
+	if (error == DB_SUCCESS                          && 
+	    wsrep_thd_exec_mode(user_thd) == LOCAL_STATE &&
+            wsrep_on(user_thd)) {
+
+		DBUG_PRINT("wsrep", ("update row key"));
+
+		if (wsrep_append_keys(user_thd, false, old_row, new_row)) {
+			WSREP_DEBUG("WSREP: UPDATE_ROW_KEY FAILED");
+			DBUG_PRINT("wsrep", ("row key failed"));
+			err = HA_ERR_INTERNAL_ERROR;
+			goto wsrep_error;
+		}
+	}
+wsrep_error:
+#endif /* WITH_WSREP */
+
 	if (UNIV_UNLIKELY(share->ib_table->is_corrupt)) {
 		DBUG_RETURN(HA_ERR_CRASHED);
 	}
@@ -8171,6 +8873,19 @@ ha_innobase::delete_row(
 
 	innobase_active_small();
 
+#ifdef WITH_WSREP
+	if (error == DB_SUCCESS && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE &&
+            wsrep_on(user_thd)) {
+
+		if (wsrep_append_keys(user_thd, false, record, NULL)) {
+			DBUG_PRINT("wsrep", ("delete fail"));
+			error = DB_ERROR;
+			goto wsrep_error;
+		}
+	}
+wsrep_error:
+#endif /* WITH_WSREP */
+
 	if (UNIV_UNLIKELY(share && share->ib_table
 			  && share->ib_table->is_corrupt)) {
 		DBUG_RETURN(HA_ERR_CRASHED);
@@ -9360,6 +10075,394 @@ ha_innobase::ft_end()
 
 	rnd_end();
 }
+#ifdef WITH_WSREP
+extern dict_index_t*
+wsrep_dict_foreign_find_index(
+	dict_table_t*	table,
+	const char**	col_names,
+	const char**	columns,
+	ulint		n_cols,
+	dict_index_t*	types_idx,
+	ibool		check_charsets,
+	ulint		check_null);
+
+
+extern dberr_t
+wsrep_append_foreign_key(
+/*===========================*/
+	trx_t*		trx,		/*!< in: trx */
+	dict_foreign_t*	foreign,	/*!< in: foreign key constraint */
+	const rec_t*	rec,		/*!<in: clustered index record */
+	dict_index_t*	index,		/*!<in: clustered index */
+	ibool		referenced,	/*!<in: is check for referenced table */
+	ibool		shared)		/*!<in: is shared access */
+{
+	ut_a(trx);
+	THD*  thd = (THD*)trx->mysql_thd;
+	ulint rcode = DB_SUCCESS;
+	char  cache_key[513] = {'\0'};
+	int   cache_key_len;
+    bool const copy = true;
+
+	if (!wsrep_on(trx->mysql_thd) ||
+	    wsrep_thd_exec_mode(thd) != LOCAL_STATE)
+		return DB_SUCCESS;
+
+	if (!thd || !foreign ||
+	    (!foreign->referenced_table && !foreign->foreign_table))
+	{
+		WSREP_INFO("FK: %s missing in: %s",
+			(!thd)      ?  "thread"     :
+			((!foreign) ?  "constraint" :
+			((!foreign->referenced_table) ?
+			     "referenced table" : "foreign table")),
+			   (thd && wsrep_thd_query(thd)) ?
+			   wsrep_thd_query(thd) : "void");
+		return DB_ERROR;
+	}
+
+	if ( !((referenced) ?
+		foreign->referenced_table : foreign->foreign_table))
+	{
+		WSREP_DEBUG("pulling %s table into cache",
+			    (referenced) ? "referenced" : "foreign");
+		mutex_enter(&(dict_sys->mutex));
+		if (referenced)
+		{
+			foreign->referenced_table =
+				dict_table_get_low(
+					foreign->referenced_table_name_lookup);
+			if (foreign->referenced_table)
+			{
+				foreign->referenced_index =
+					wsrep_dict_foreign_find_index(
+						foreign->referenced_table, NULL,
+						foreign->referenced_col_names,
+						foreign->n_fields, 
+						foreign->foreign_index,
+						TRUE, FALSE);
+			}
+		}
+		else
+		{
+	  		foreign->foreign_table =
+				dict_table_get_low(
+					foreign->foreign_table_name_lookup);
+			if (foreign->foreign_table)
+			{
+				foreign->foreign_index =
+					wsrep_dict_foreign_find_index(
+						foreign->foreign_table, NULL,
+						foreign->foreign_col_names,
+						foreign->n_fields,
+						foreign->referenced_index, 
+						TRUE, FALSE);
+			}
+		}
+		mutex_exit(&(dict_sys->mutex));
+	}
+
+	if ( !((referenced) ?
+		foreign->referenced_table : foreign->foreign_table))
+	{
+		WSREP_WARN("FK: %s missing in query: %s",
+			   (!foreign->referenced_table) ?
+			   "referenced table" : "foreign table",
+			   (wsrep_thd_query(thd)) ?
+			   wsrep_thd_query(thd) : "void");
+		return DB_ERROR;
+	}
+	byte  key[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+	ulint len = WSREP_MAX_SUPPORTED_KEY_LENGTH;
+
+	dict_index_t *idx_target = (referenced) ?
+		foreign->referenced_index : index;
+	dict_index_t *idx = (referenced) ?
+		UT_LIST_GET_FIRST(foreign->referenced_table->indexes) :
+		UT_LIST_GET_FIRST(foreign->foreign_table->indexes);
+	int i = 0;
+	while (idx != NULL && idx != idx_target) {
+		if (innobase_strcasecmp (idx->name, innobase_index_reserve_name) != 0) {
+			i++;
+		}
+		idx = UT_LIST_GET_NEXT(indexes, idx);
+	}
+	ut_a(idx);
+	key[0] = (char)i;
+
+	rcode = wsrep_rec_get_foreign_key(
+		&key[1], &len, rec, index, idx,
+		wsrep_protocol_version > 1);
+	if (rcode != DB_SUCCESS) {
+		WSREP_ERROR(
+			"FK key set failed: %lu (%lu %lu), index: %s %s, %s",
+			rcode, referenced, shared,
+			(index && index->name)       ? index->name :
+				"void index",
+			(index && index->table_name) ? index->table_name :
+				"void table",
+			wsrep_thd_query(thd));
+		return DB_ERROR;
+	}
+	strncpy(cache_key,
+		(wsrep_protocol_version > 1) ?
+		((referenced) ?
+			foreign->referenced_table->name :
+			foreign->foreign_table->name) :
+		foreign->foreign_table->name, sizeof(cache_key) - 1);
+	cache_key_len = strlen(cache_key);
+#ifdef WSREP_DEBUG_PRINT
+	ulint j;
+	fprintf(stderr, "FK parent key, table: %s %s len: %lu ",
+		cache_key, (shared) ? "shared" : "exclusive", len+1);
+	for (j=0; j<len+1; j++) {
+		fprintf(stderr, " %hhX, ", key[j]);
+	}
+	fprintf(stderr, "\n");
+#endif
+	char *p = strchr(cache_key, '/');
+	if (p) {
+		*p = '\0';
+	} else {
+		WSREP_WARN("unexpected foreign key table %s %s",
+			   foreign->referenced_table->name,
+			   foreign->foreign_table->name);
+	}
+
+	wsrep_buf_t wkey_part[3];
+        wsrep_key_t wkey = {wkey_part, 3};
+	if (!wsrep_prepare_key_for_innodb(
+		(const uchar*)cache_key,
+		cache_key_len +  1,
+		(const uchar*)key, len+1,
+		wkey_part,
+		(size_t*)&wkey.key_parts_num)) {
+		WSREP_WARN("key prepare failed for cascaded FK: %s",
+			   (wsrep_thd_query(thd)) ?
+			    wsrep_thd_query(thd) : "void");
+		return DB_ERROR;
+	}
+	rcode = (int)wsrep->append_key(
+		wsrep,
+		wsrep_ws_handle(thd, trx),
+		&wkey,
+		1,
+		shared ? WSREP_KEY_SHARED : WSREP_KEY_EXCLUSIVE,
+                copy);
+	if (rcode) {
+		DBUG_PRINT("wsrep", ("row key failed: %lu", rcode));
+		WSREP_ERROR("Appending cascaded fk row key failed: %s, %lu",
+			    (wsrep_thd_query(thd)) ?
+			    wsrep_thd_query(thd) : "void", rcode);
+		return DB_ERROR;
+	}
+
+	return DB_SUCCESS;
+}
+
+static int
+wsrep_append_key(
+/*==================*/
+	THD		*thd,
+	trx_t 		*trx,
+	TABLE_SHARE 	*table_share,
+	TABLE 		*table,
+	const char*	key,
+	uint16_t        key_len,
+	bool            shared
+)
+{
+	DBUG_ENTER("wsrep_append_key");
+	bool const copy = true;
+#ifdef WSREP_DEBUG_PRINT
+	fprintf(stderr, "%s conn %ld, trx %lu, keylen %d, table %s ",
+		(shared) ? "Shared" : "Exclusive",
+		wsrep_thd_thread_id(thd), (long long)trx->id, key_len,
+		table_share->table_name.str);
+	for (int i=0; i<key_len; i++) {
+		fprintf(stderr, "%hhX, ", key[i]);
+	}
+	fprintf(stderr, "\n");
+#endif
+	wsrep_buf_t wkey_part[3];
+        wsrep_key_t wkey = {wkey_part, 3};
+	if (!wsrep_prepare_key_for_innodb(
+			(const uchar*)table_share->table_cache_key.str,
+			table_share->table_cache_key.length,
+			(const uchar*)key, key_len,
+			wkey_part,
+			(size_t*)&wkey.key_parts_num)) {
+		WSREP_WARN("key prepare failed for: %s",
+			   (wsrep_thd_query(thd)) ?
+			   wsrep_thd_query(thd) : "void");
+		DBUG_RETURN(HA_ERR_INTERNAL_ERROR);
+	}
+
+	int rcode = (int)wsrep->append_key(
+			       wsrep,
+			       wsrep_ws_handle(thd, trx),
+			       &wkey,
+			       1,
+			       shared ? WSREP_KEY_SHARED : WSREP_KEY_EXCLUSIVE,
+                               copy);
+	if (rcode) {
+		DBUG_PRINT("wsrep", ("row key failed: %d", rcode));
+		WSREP_WARN("Appending row key failed: %s, %d",
+			   (wsrep_thd_query(thd)) ?
+			   wsrep_thd_query(thd) : "void", rcode);
+		DBUG_RETURN(HA_ERR_INTERNAL_ERROR);
+	}
+	DBUG_RETURN(0);
+}
+
+extern void compute_md5_hash(char *digest, const char *buf, int len);
+#define MD5_HASH compute_md5_hash
+
+int
+ha_innobase::wsrep_append_keys(
+/*==================*/
+	THD 		*thd,
+	bool		shared,
+	const uchar*	record0,	/* in: row in MySQL format */
+	const uchar*	record1)	/* in: row in MySQL format */
+{
+	int rcode;
+	DBUG_ENTER("wsrep_append_keys");
+
+	bool key_appended = false;
+	trx_t *trx = thd_to_trx(thd);
+
+	if (table_share && table_share->tmp_table  != NO_TMP_TABLE) {
+		WSREP_DEBUG("skipping tmp table DML: THD: %lu tmp: %d SQL: %s", 
+			    wsrep_thd_thread_id(thd),
+			    table_share->tmp_table,
+			    (wsrep_thd_query(thd)) ? 
+			    wsrep_thd_query(thd) : "void");
+		DBUG_RETURN(0);
+	}
+
+	if (wsrep_protocol_version == 0) {
+		uint	len;
+		char 	keyval[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+		char 	*key 		= &keyval[0];
+		ibool    is_null;
+
+		len = wsrep_store_key_val_for_row(
+			thd, table, 0, key, WSREP_MAX_SUPPORTED_KEY_LENGTH,
+			record0, &is_null);
+
+		if (!is_null) {
+			rcode = wsrep_append_key(
+				thd, trx, table_share, table, keyval, 
+				len, shared);
+			if (rcode) DBUG_RETURN(rcode);
+		}
+		else
+		{
+			WSREP_DEBUG("NULL key skipped (proto 0): %s", 
+				    wsrep_thd_query(thd));
+		}
+	} else {
+		ut_a(table->s->keys <= 256);
+		uint i;
+                bool hasPK= false;
+
+		for (i=0; i<table->s->keys; ++i) {
+			KEY*  key_info	= table->key_info + i;
+			if (key_info->flags & HA_NOSAME) {
+				hasPK = true;
+			}
+		}
+
+		for (i=0; i<table->s->keys; ++i) {
+			uint  len;
+			char  keyval0[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+			char  keyval1[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+			char* key0 		= &keyval0[1];
+			char* key1 		= &keyval1[1];
+			KEY*  key_info	= table->key_info + i;
+			ibool is_null;
+
+			dict_index_t* idx  = innobase_get_index(i);
+			dict_table_t* tab  = (idx) ? idx->table : NULL;
+
+			keyval0[0] = (char)i;
+			keyval1[0] = (char)i;
+
+			if (!tab) {
+				WSREP_WARN("MySQL-InnoDB key mismatch %s %s",
+					   table->s->table_name.str, 
+					   key_info->name);
+			}
+			/* !hasPK == table with no PK, must append all non-unique keys */
+			if (!hasPK || key_info->flags & HA_NOSAME ||
+			    ((tab &&
+			      dict_table_get_referenced_constraint(tab, idx)) ||
+			     (!tab && referenced_by_foreign_key()))) {
+
+				len = wsrep_store_key_val_for_row(
+					thd, table, i, key0, 
+					WSREP_MAX_SUPPORTED_KEY_LENGTH, 
+					record0, &is_null);
+				if (!is_null) {
+					rcode = wsrep_append_key(
+						thd, trx, table_share, table, 
+						keyval0, len+1, shared);
+					if (rcode) DBUG_RETURN(rcode);
+
+				if (key_info->flags & HA_NOSAME || shared)
+			  		key_appended = true;
+				}
+				else
+				{
+					WSREP_DEBUG("NULL key skipped: %s", 
+						    wsrep_thd_query(thd));
+				}
+				if (record1) {
+					len = wsrep_store_key_val_for_row(
+						thd, table, i, key1, 
+						WSREP_MAX_SUPPORTED_KEY_LENGTH,
+						record1, &is_null);
+					if (!is_null && memcmp(key0, key1, len)) {
+						rcode = wsrep_append_key(
+							thd, trx, table_share, 
+							table, 
+							keyval1, len+1, shared);
+						if (rcode) DBUG_RETURN(rcode);
+					}
+				}
+			}
+		}
+	}
+
+	/* if no PK, calculate hash of full row, to be the key value */
+	if (!key_appended && wsrep_certify_nonPK) {
+		uchar digest[16];
+		int rcode;
+
+		wsrep_calc_row_hash(digest, record0, table, prebuilt, thd);
+		if ((rcode = wsrep_append_key(thd, trx, table_share, table, 
+					      (const char*) digest, 16, 
+					      shared))) {
+			DBUG_RETURN(rcode);
+		}
+
+		if (record1) {
+			wsrep_calc_row_hash(
+				digest, record1, table, prebuilt, thd);
+			if ((rcode = wsrep_append_key(thd, trx, table_share, 
+						      table,
+						      (const char*) digest, 
+						      16, shared))) {
+				DBUG_RETURN(rcode);
+			}
+		}
+		DBUG_RETURN(0);
+	}
+
+	DBUG_RETURN(0);
+}
+#endif /* WITH_WSREP */
 
 /*********************************************************************//**
 Stores a reference to the current row to 'ref' field of the handle. Note
@@ -13276,11 +14379,18 @@ ha_innobase::external_lock(
 		/* used by test case */
 		DBUG_EXECUTE_IF("no_innodb_binlog_errors", skip = true;);
 		if (!skip) {
-			my_error(ER_BINLOG_STMT_MODE_AND_ROW_ENGINE, MYF(0),
-			         " InnoDB is limited to row-logging when "
-			         "transaction isolation level is "
-			         "READ COMMITTED or READ UNCOMMITTED.");
-			DBUG_RETURN(HA_ERR_LOGGING_IMPOSSIBLE);
+#ifdef WITH_WSREP
+			if (!wsrep_on(thd)
+			    || wsrep_thd_exec_mode(thd) == LOCAL_STATE) {
+#endif /* WITH_WSREP */
+				my_error(ER_BINLOG_STMT_MODE_AND_ROW_ENGINE, MYF(0),
+					" InnoDB is limited to row-logging when "
+					"transaction isolation level is "
+					"READ COMMITTED or READ UNCOMMITTED.");
+				DBUG_RETURN(HA_ERR_LOGGING_IMPOSSIBLE);
+#ifdef WITH_WSREP
+			}
+#endif /* WITH_WSREP */
 		}
 	}
 
@@ -14754,6 +15864,9 @@ innobase_xa_prepare(
 	time, not the current session variable value. Any possible changes
 	to the session variable take effect only in the next transaction */
 	if (!trx->support_xa) {
+#ifdef WITH_WSREP
+                thd_get_xid(thd, (MYSQL_XID*) &trx->xid);
+#endif // WITH_WSREP
 
 		return(0);
 	}
@@ -17155,6 +18268,303 @@ static SHOW_VAR innodb_status_variables_export[]= {
 static struct st_mysql_storage_engine innobase_storage_engine=
 { MYSQL_HANDLERTON_INTERFACE_VERSION };
 
+#ifdef WITH_WSREP
+void
+wsrep_abort_slave_trx(wsrep_seqno_t bf_seqno, wsrep_seqno_t victim_seqno)
+{
+	WSREP_ERROR("Trx %lld tries to abort slave trx %lld. This could be "
+		"caused by:\n\t"
+		"1) unsupported configuration options combination, please check documentation.\n\t"
+		"2) a bug in the code.\n\t"
+		"3) a database corruption.\n Node consistency compromized, "
+		"need to abort. Restart the node to resync with cluster.",
+		(long long)bf_seqno, (long long)victim_seqno);
+	abort();
+}
+/*******************************************************************//**
+This function is used to kill one transaction in BF. */
+
+int
+wsrep_innobase_kill_one_trx(void * const bf_thd_ptr,
+                            const trx_t * const bf_trx,
+                            trx_t *victim_trx, ibool signal)
+{
+        ut_ad(lock_mutex_own());
+        ut_ad(trx_mutex_own(victim_trx));
+        ut_ad(bf_thd_ptr);
+        ut_ad(victim_trx);
+
+	DBUG_ENTER("wsrep_innobase_kill_one_trx");
+	THD *bf_thd       = bf_thd_ptr ? (THD*) bf_thd_ptr : NULL;
+	THD *thd          = (THD *) victim_trx->mysql_thd;
+	int64_t bf_seqno  = (bf_thd) ? wsrep_thd_trx_seqno(bf_thd) : 0;
+
+	if (!thd) {
+		DBUG_PRINT("wsrep", ("no thd for conflicting lock"));
+		WSREP_WARN("no THD for trx: %lu", victim_trx->id);
+		DBUG_RETURN(1);
+	}
+	if (!bf_thd) {
+		DBUG_PRINT("wsrep", ("no BF thd for conflicting lock"));
+		WSREP_WARN("no BF THD for trx: %lu", (bf_trx) ? bf_trx->id : 0);
+		DBUG_RETURN(1);
+	}
+
+	WSREP_LOG_CONFLICT(bf_thd, thd, TRUE);
+
+	WSREP_DEBUG("BF kill (%lu, seqno: %lld), victim: (%lu) trx: %lu",
+ 		    signal, (long long)bf_seqno,
+ 		    wsrep_thd_thread_id(thd),
+		    victim_trx->id);
+
+	WSREP_DEBUG("Aborting query: %s",
+		  (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void");
+
+	wsrep_thd_LOCK(thd);
+
+	if (wsrep_thd_query_state(thd) == QUERY_EXITING) {
+		WSREP_DEBUG("kill trx EXITING for %lu", victim_trx->id);
+		wsrep_thd_UNLOCK(thd);
+		DBUG_RETURN(0);
+	}
+	if(wsrep_thd_exec_mode(thd) != LOCAL_STATE) {
+		WSREP_DEBUG("withdraw for BF trx: %lu, state: %d",
+			    victim_trx->id,
+		wsrep_thd_conflict_state(thd));
+	}
+
+	switch (wsrep_thd_conflict_state(thd)) {
+	case NO_CONFLICT:
+		wsrep_thd_set_conflict_state(thd, MUST_ABORT);
+		break;
+        case MUST_ABORT:
+		WSREP_DEBUG("victim %lu in MUST ABORT state",
+			    victim_trx->id);
+		wsrep_thd_UNLOCK(thd);
+		wsrep_thd_awake(thd, signal);
+		DBUG_RETURN(0);
+		break;
+	case ABORTED:
+	case ABORTING: // fall through
+	default:
+		WSREP_DEBUG("victim %lu in state %d",
+			    victim_trx->id, wsrep_thd_conflict_state(thd));
+		wsrep_thd_UNLOCK(thd);
+		DBUG_RETURN(0);
+		break;
+	}
+
+	switch (wsrep_thd_query_state(thd)) {
+	case QUERY_COMMITTING:
+		enum wsrep_status rcode;
+
+		WSREP_DEBUG("kill query for: %ld",
+			    wsrep_thd_thread_id(thd));
+		WSREP_DEBUG("kill trx QUERY_COMMITTING for %lu",
+			    victim_trx->id);
+
+		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
+			wsrep_abort_slave_trx(bf_seqno,
+					      wsrep_thd_trx_seqno(thd));
+		} else {
+			rcode = wsrep->abort_pre_commit(
+				wsrep, bf_seqno,
+				(wsrep_trx_id_t)victim_trx->id
+			);
+
+			switch (rcode) {
+			case WSREP_WARNING:
+				WSREP_DEBUG("cancel commit warning: %lu",
+					    victim_trx->id);
+				wsrep_thd_UNLOCK(thd);
+				wsrep_thd_awake(thd, signal);
+				DBUG_RETURN(1);
+				break;
+			case WSREP_OK:
+				break;
+			default:
+				WSREP_ERROR(
+					"cancel commit bad exit: %d %lu",
+					rcode,
+					victim_trx->id);
+				/* unable to interrupt, must abort */
+				/* note: kill_mysql() will block, if we cannot.
+				 * kill the lock holder first.
+				 */
+				abort();
+				break;
+			}
+		}
+		wsrep_thd_UNLOCK(thd);
+		wsrep_thd_awake(thd, signal);
+		break;
+	case QUERY_EXEC:
+		/* it is possible that victim trx is itself waiting for some
+		 * other lock. We need to cancel this waiting
+		 */
+		WSREP_DEBUG("kill trx QUERY_EXEC for %lu", victim_trx->id);
+
+		victim_trx->lock.was_chosen_as_deadlock_victim= TRUE;
+		if (victim_trx->lock.wait_lock) {
+			WSREP_DEBUG("victim has wait flag: %ld",
+				wsrep_thd_thread_id(thd));
+			lock_t*  wait_lock = victim_trx->lock.wait_lock;
+			if (wait_lock) {
+				WSREP_DEBUG("canceling wait lock");
+				victim_trx->lock.was_chosen_as_deadlock_victim= TRUE;
+				lock_cancel_waiting_and_release(wait_lock);
+			}
+
+			wsrep_thd_UNLOCK(thd);
+			wsrep_thd_awake(thd, signal);
+		} else {
+			/* abort currently executing query */
+			DBUG_PRINT("wsrep",("sending KILL_QUERY to: %ld",
+                                            wsrep_thd_thread_id(thd)));
+			WSREP_DEBUG("kill query for: %ld",
+				wsrep_thd_thread_id(thd));
+			/* Note that innobase_kill_connection will take lock_mutex
+			and trx_mutex */
+			wsrep_thd_UNLOCK(thd);
+			wsrep_thd_awake(thd, signal);
+
+			/* for BF thd, we need to prevent him from committing */
+			if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
+				wsrep_abort_slave_trx(bf_seqno,
+						    wsrep_thd_trx_seqno(thd));
+			}
+		}
+		break;
+	case QUERY_IDLE:
+	{
+		bool skip_abort= false;
+		wsrep_aborting_thd_t abortees;
+
+		WSREP_DEBUG("kill IDLE for %lu", victim_trx->id);
+
+		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
+			WSREP_DEBUG("kill BF IDLE, seqno: %lld",
+				    (long long)wsrep_thd_trx_seqno(thd));
+			wsrep_thd_UNLOCK(thd);
+			wsrep_abort_slave_trx(bf_seqno,
+					      wsrep_thd_trx_seqno(thd));
+			DBUG_RETURN(0);
+		}
+                /* This will lock thd from proceeding after net_read() */
+		wsrep_thd_set_conflict_state(thd, ABORTING);
+
+		mysql_mutex_lock(&LOCK_wsrep_rollback);
+
+		abortees = wsrep_aborting_thd;
+		while (abortees && !skip_abort) {
+			/* check if we have a kill message for this already */
+			if (abortees->aborting_thd == thd) {
+				skip_abort = true;
+				WSREP_WARN("duplicate thd aborter %lu",
+					  wsrep_thd_thread_id(thd));
+			}
+			abortees = abortees->next;
+		}
+		if (!skip_abort) {
+			wsrep_aborting_thd_t aborting = (wsrep_aborting_thd_t)
+				my_malloc(sizeof(struct wsrep_aborting_thd),
+					  MYF(0));
+			aborting->aborting_thd  = thd;
+			aborting->next          = wsrep_aborting_thd;
+			wsrep_aborting_thd      = aborting;
+ 			DBUG_PRINT("wsrep",("enqueuing trx abort for %lu",
+                                             wsrep_thd_thread_id(thd)));
+			WSREP_DEBUG("enqueuing trx abort for (%lu)",
+				    wsrep_thd_thread_id(thd));
+		}
+
+		DBUG_PRINT("wsrep",("signalling wsrep rollbacker"));
+		WSREP_DEBUG("signaling aborter");
+		mysql_cond_signal(&COND_wsrep_rollback);
+		mysql_mutex_unlock(&LOCK_wsrep_rollback);
+		wsrep_thd_UNLOCK(thd);
+
+		break;
+	}
+	default:
+		WSREP_WARN("bad wsrep query state: %d",
+			  wsrep_thd_query_state(thd));
+		wsrep_thd_UNLOCK(thd);
+		break;
+	}
+
+	DBUG_RETURN(0);
+}
+static int
+wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd,
+			my_bool signal)
+{
+	DBUG_ENTER("wsrep_innobase_abort_thd");
+	trx_t* victim_trx = thd_to_trx(victim_thd);
+	trx_t* bf_trx     = (bf_thd) ? thd_to_trx(bf_thd) : NULL;
+	WSREP_DEBUG("abort transaction: BF: %s victim: %s", 
+		    wsrep_thd_query(bf_thd),
+		    wsrep_thd_query(victim_thd));
+
+	if (victim_trx)
+	{
+                lock_mutex_enter();
+                trx_mutex_enter(victim_trx);
+		int rcode = wsrep_innobase_kill_one_trx(bf_thd, bf_trx,
+                                                        victim_trx, signal);
+                trx_mutex_exit(victim_trx);
+                lock_mutex_exit();
+		wsrep_srv_conc_cancel_wait(victim_trx);
+
+ 		DBUG_RETURN(rcode);
+	} else {
+		WSREP_DEBUG("victim does not have transaction");
+		wsrep_thd_LOCK(victim_thd);
+		wsrep_thd_set_conflict_state(victim_thd, MUST_ABORT);
+		wsrep_thd_UNLOCK(victim_thd);
+		wsrep_thd_awake(victim_thd, signal);
+	}
+	DBUG_RETURN(-1);
+}
+
+static int innobase_wsrep_set_checkpoint(handlerton* hton, const XID* xid)
+{
+	DBUG_ASSERT(hton == innodb_hton_ptr);
+        if (wsrep_is_wsrep_xid((const void *)xid)) {
+                mtr_t mtr;
+                mtr_start(&mtr);
+                trx_sysf_t* sys_header = trx_sysf_get(&mtr);
+                trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr);
+                mtr_commit(&mtr);
+                innobase_flush_logs(hton);
+                return 0;
+        } else {
+                return 1;
+        }
+}
+
+static int innobase_wsrep_get_checkpoint(handlerton* hton, XID* xid)
+{
+	DBUG_ASSERT(hton == innodb_hton_ptr);
+        trx_sys_read_wsrep_checkpoint(xid);
+        return 0;
+}
+
+static void
+wsrep_fake_trx_id(
+/*==================*/
+	handlerton	*hton,
+	THD		*thd)	/*!< in: user thread handle */
+{
+	mutex_enter(&trx_sys->mutex);
+	trx_id_t trx_id = trx_sys_get_new_trx_id();
+	mutex_exit(&trx_sys->mutex);
+
+	(void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id);
+}
+
+#endif /* WITH_WSREP */
+
 /* plugin options */
 
 static MYSQL_SYSVAR_ENUM(checksum_algorithm, srv_checksum_algorithm,
@@ -18092,6 +19502,40 @@ static MYSQL_SYSVAR_BOOL(disable_background_merge,
   NULL, NULL, FALSE);
 #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
 
+#ifdef WITH_INNODB_DISALLOW_WRITES
+/*******************************************************
+ *    innobase_disallow_writes variable definition     *
+ *******************************************************/
+ 
+/* Must always init to FALSE. */
+static my_bool	innobase_disallow_writes	= FALSE;
+
+/**************************************************************************
+An "update" method for innobase_disallow_writes variable. */
+static
+void
+innobase_disallow_writes_update(
+/*============================*/
+	THD*			thd,		/* in: thread handle */
+	st_mysql_sys_var*	var,		/* in: pointer to system
+						variable */
+	void*			var_ptr,	/* out: pointer to dynamic
+						variable */
+	const void*		save)		/* in: temporary storage */
+{
+	*(my_bool*)var_ptr = *(my_bool*)save;
+	ut_a(srv_allow_writes_event);
+	if (*(my_bool*)var_ptr)
+		os_event_reset(srv_allow_writes_event);
+	else
+		os_event_set(srv_allow_writes_event);
+}
+
+static MYSQL_SYSVAR_BOOL(disallow_writes, innobase_disallow_writes,
+  PLUGIN_VAR_NOCMDOPT,
+  "Tell InnoDB to stop any writes to disk",
+  NULL, innobase_disallow_writes_update, FALSE);
+#endif /* WITH_INNODB_DISALLOW_WRITES */
 static MYSQL_SYSVAR_BOOL(random_read_ahead, srv_random_read_ahead,
   PLUGIN_VAR_NOCMDARG,
   "Whether to use read ahead for random access within an extent.",
@@ -18403,6 +19847,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
   MYSQL_SYSVAR(change_buffering_debug),
   MYSQL_SYSVAR(disable_background_merge),
 #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
+#ifdef WITH_INNODB_DISALLOW_WRITES
+  MYSQL_SYSVAR(disallow_writes),
+#endif /* WITH_INNODB_DISALLOW_WRITES */
   MYSQL_SYSVAR(random_read_ahead),
   MYSQL_SYSVAR(read_ahead_threshold),
   MYSQL_SYSVAR(read_only),
diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h
index b4df711356c..ff4ab88335a 100644
--- a/storage/xtradb/handler/ha_innodb.h
+++ b/storage/xtradb/handler/ha_innodb.h
@@ -119,6 +119,10 @@ class ha_innobase: public handler
 	void innobase_initialize_autoinc();
 	dict_index_t* innobase_get_index(uint keynr);
 
+#ifdef WITH_WSREP
+	int wsrep_append_keys(THD *thd, bool shared,
+			      const uchar* record0, const uchar* record1);
+#endif
 	/* Init values for the class: */
  public:
 	ha_innobase(handlerton *hton, TABLE_SHARE *table_arg);
@@ -472,6 +476,40 @@ __attribute__((nonnull));
 extern void mysql_bin_log_commit_pos(THD *thd, ulonglong *out_pos, const char **out_file);
 
 struct trx_t;
+#ifdef WITH_WSREP
+#include <wsrep_mysqld.h>
+//extern "C" int wsrep_trx_order_before(void *thd1, void *thd2);
+
+extern "C" bool wsrep_thd_is_wsrep_on(THD *thd);
+
+extern "C" enum wsrep_exec_mode wsrep_thd_exec_mode(THD *thd);
+extern "C" enum wsrep_conflict_state wsrep_thd_conflict_state(THD *thd);
+extern "C" enum wsrep_query_state wsrep_thd_query_state(THD *thd);
+extern "C" const char * wsrep_thd_exec_mode_str(THD *thd);
+extern "C" const char * wsrep_thd_conflict_state_str(THD *thd);
+extern "C" const char * wsrep_thd_query_state_str(THD *thd);
+extern "C" wsrep_ws_handle_t* wsrep_thd_ws_handle(THD *thd);
+
+extern "C" void wsrep_thd_set_exec_mode(THD *thd, enum wsrep_exec_mode mode);
+extern "C" void wsrep_thd_set_query_state(
+	THD *thd, enum wsrep_query_state state);
+extern "C" void wsrep_thd_set_conflict_state(
+	THD *thd, enum wsrep_conflict_state state);
+
+extern "C" void wsrep_thd_set_trx_to_replay(THD *thd, uint64 trx_id);
+
+extern "C"void wsrep_thd_LOCK(THD *thd);
+extern "C"void wsrep_thd_UNLOCK(THD *thd);
+extern "C" uint32 wsrep_thd_wsrep_rand(THD *thd);
+extern "C" time_t wsrep_thd_query_start(THD *thd);
+extern "C" my_thread_id wsrep_thd_thread_id(THD *thd);
+extern "C" int64_t wsrep_thd_trx_seqno(THD *thd);
+extern "C" query_id_t wsrep_thd_query_id(THD *thd);
+extern "C" char * wsrep_thd_query(THD *thd);
+extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd);
+extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id);
+extern "C" void wsrep_thd_awake(THD* thd, my_bool signal);
+#endif
 
 extern const struct _ft_vft ft_vft_result;
 
@@ -509,6 +547,9 @@ innobase_index_name_is_reserved(
 	__attribute__((nonnull, warn_unused_result));
 
 /*****************************************************************//**
+#ifdef WITH_WSREP
+extern "C" int wsrep_trx_is_aborting(void *thd_ptr);
+#endif
 Determines InnoDB table flags.
 @retval true if successful, false if error */
 UNIV_INTERN
diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc
index 34048e7c987..2cd0500007a 100644
--- a/storage/xtradb/handler/handler0alter.cc
+++ b/storage/xtradb/handler/handler0alter.cc
@@ -49,6 +49,10 @@ Smart ALTER TABLE
 #include "pars0pars.h"
 #include "row0sel.h"
 #include "ha_innodb.h"
+#ifdef WITH_WSREP
+//#include "wsrep_api.h"
+#include <sql_acl.h>	// PROCESS_ACL
+#endif
 
 /** Operations for creating secondary indexes (no rebuild needed) */
 static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_ONLINE_CREATE
diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h
index a347a75ea42..93fe2efa830 100644
--- a/storage/xtradb/include/dict0mem.h
+++ b/storage/xtradb/include/dict0mem.h
@@ -527,6 +527,9 @@ be REC_VERSION_56_MAX_INDEX_COL_LEN (3072) bytes */
 
 /** Defines the maximum fixed length column size */
 #define DICT_MAX_FIXED_COL_LEN		DICT_ANTELOPE_MAX_INDEX_COL_LEN
+#ifdef WITH_WSREP
+#define WSREP_MAX_SUPPORTED_KEY_LENGTH 3500
+#endif /* WITH_WSREP */
 
 /** Data structure for a field in an index */
 struct dict_field_t{
diff --git a/storage/xtradb/include/ha_prototypes.h b/storage/xtradb/include/ha_prototypes.h
index 66a96282b69..4ebaee3c89e 100644
--- a/storage/xtradb/include/ha_prototypes.h
+++ b/storage/xtradb/include/ha_prototypes.h
@@ -285,6 +285,23 @@ innobase_casedn_str(
 /*================*/
 	char*	a);	/*!< in/out: string to put in lower case */
 
+#ifdef WITH_WSREP
+UNIV_INTERN
+int
+wsrep_innobase_kill_one_trx(void *thd_ptr,
+                            const trx_t *bf_trx, trx_t *victim_trx, ibool signal);
+my_bool wsrep_thd_set_PA_safe(void *thd_ptr, my_bool safe);
+int wsrep_thd_conflict_state(void *thd_ptr, my_bool sync);
+my_bool wsrep_thd_is_BF(void *thd_ptr, my_bool sync);
+my_bool wsrep_thd_is_wsrep(void *thd_ptr);
+int wsrep_trx_order_before(void *thd1, void *thd2);
+int wsrep_innobase_mysql_sort(int mysql_type, uint charset_number,
+			      unsigned char* str, unsigned int str_length,
+			      unsigned int buf_length);
+int 
+wsrep_on(void *thd_ptr);
+extern "C" int wsrep_is_wsrep_xid(const void*);
+#endif /* WITH_WSREP */
 /**********************************************************************//**
 Determines the connection character set.
 @return	connection character set */
diff --git a/storage/xtradb/include/hash0hash.h b/storage/xtradb/include/hash0hash.h
index a6fe4e680a1..68d3c6ace4e 100644
--- a/storage/xtradb/include/hash0hash.h
+++ b/storage/xtradb/include/hash0hash.h
@@ -144,6 +144,33 @@ do {\
 	}\
 } while (0)
 
+#ifdef WITH_WSREP
+/*******************************************************************//**
+Inserts a struct to the head of hash table. */
+
+#define HASH_PREPEND(TYPE, NAME, TABLE, FOLD, DATA)	\
+do {							\
+	hash_cell_t*	cell3333;			\
+	TYPE*		struct3333;			\
+							\
+	HASH_ASSERT_OWN(TABLE, FOLD)			\
+							\
+	(DATA)->NAME = NULL;				\
+							\
+	cell3333 = hash_get_nth_cell(TABLE, hash_calc_hash(FOLD, TABLE));\
+							\
+	if (cell3333->node == NULL) {			\
+		cell3333->node = DATA;			\
+		DATA->NAME = NULL;			\
+	} else {					\
+		struct3333 = (TYPE*) cell3333->node;	\
+							\
+		DATA->NAME = struct3333;		\
+							\
+		cell3333->node = DATA;			\
+	}						\
+} while (0)
+#endif /*WITH_WSREP */
 #ifdef UNIV_HASH_DEBUG
 # define HASH_ASSERT_VALID(DATA) ut_a((void*) (DATA) != (void*) -1)
 # define HASH_INVALIDATE(DATA, NAME) *(void**) (&DATA->NAME) = (void*) -1
diff --git a/storage/xtradb/include/rem0rec.h b/storage/xtradb/include/rem0rec.h
index 8e7d5ff2d48..238cb04e1f8 100644
--- a/storage/xtradb/include/rem0rec.h
+++ b/storage/xtradb/include/rem0rec.h
@@ -981,6 +981,15 @@ are given in one byte (resp. two byte) format. */
 two upmost bits in a two byte offset for special purposes */
 #define REC_MAX_DATA_SIZE	(16 * 1024)
 
+#ifdef WITH_WSREP
+int wsrep_rec_get_foreign_key(
+	byte 		*buf,     /* out: extracted key */
+	ulint 		*buf_len, /* in/out: length of buf */
+	const rec_t*	rec,	  /* in: physical record */
+	dict_index_t*	index_for,  /* in: index for foreign table */
+	dict_index_t*	index_ref,  /* in: index for referenced table */
+	ibool		new_protocol); /* in: protocol > 1 */
+#endif /* WITH_WSREP */
 #ifndef UNIV_NONINL
 #include "rem0rec.ic"
 #endif
diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h
index a02c8a96e1a..a54198f4e00 100644
--- a/storage/xtradb/include/srv0srv.h
+++ b/storage/xtradb/include/srv0srv.h
@@ -323,6 +323,10 @@ extern uint	srv_flush_log_at_timeout;
 extern char	srv_use_global_flush_log_at_trx_commit;
 extern char	srv_adaptive_flushing;
 
+#ifdef WITH_INNODB_DISALLOW_WRITES
+/* When this event is reset we do not allow any file writes to take place. */
+extern os_event_t	srv_allow_writes_event;
+#endif /* WITH_INNODB_DISALLOW_WRITES */
 /* If this flag is TRUE, then we will load the indexes' (and tables') metadata
 even if they are marked as "corrupted". Mostly it is for DBA to process
 corrupted index and table */
@@ -1167,4 +1171,13 @@ struct srv_slot_t{
 # define srv_file_per_table			1
 #endif /* !UNIV_HOTBACKUP */
 
+#ifdef WITH_WSREP
+UNIV_INTERN
+void
+wsrep_srv_conc_cancel_wait(
+/*==================*/
+	trx_t*	trx);	/*!< in: transaction object associated with the
+			thread */
+#endif /* WITH_WSREP */
+
 #endif
diff --git a/storage/xtradb/include/sync0sync.ic b/storage/xtradb/include/sync0sync.ic
index c3529af5262..a302e1473a5 100644
--- a/storage/xtradb/include/sync0sync.ic
+++ b/storage/xtradb/include/sync0sync.ic
@@ -255,7 +255,10 @@ mutex_enter_func(
 	ulint		line)		/*!< in: line where locked */
 {
 	ut_ad(mutex_validate(mutex));
+#ifndef WITH_WSREP
+	/* this cannot be be granted when BF trx kills a trx in lock wait state */
 	ut_ad(!mutex_own(mutex));
+#endif /* WITH_WSREP */
 
 	/* Note that we do not peek at the value of lock_word before trying
 	the atomic test_and_set; we could peek, and possibly save time. */
diff --git a/storage/xtradb/include/trx0sys.h b/storage/xtradb/include/trx0sys.h
index 7b97c6e99cd..7892fdd0bf1 100644
--- a/storage/xtradb/include/trx0sys.h
+++ b/storage/xtradb/include/trx0sys.h
@@ -42,6 +42,9 @@ Created 3/26/1996 Heikki Tuuri
 #include "read0types.h"
 #include "page0types.h"
 #include "ut0bh.h"
+#ifdef WITH_WSREP
+#include "trx0xa.h"
+#endif /* WITH_WSREP */
 
 typedef UT_LIST_BASE_NODE_T(trx_t) trx_list_t;
 
@@ -314,6 +317,9 @@ trx_sys_update_mysql_binlog_offset(
 	ib_int64_t	offset,	/*!< in: position in that log file */
 	ulint		field,	/*!< in: offset of the MySQL log info field in
 				the trx sys header */
+#ifdef WITH_WSREP
+        trx_sysf_t*     sys_header, /*!< in: trx sys header */
+#endif /* WITH_WSREP */
 	mtr_t*		mtr);	/*!< in: mtr */
 /*****************************************************************//**
 Prints to stderr the MySQL binlog offset info in the trx system header if
@@ -322,6 +328,19 @@ UNIV_INTERN
 void
 trx_sys_print_mysql_binlog_offset(void);
 /*===================================*/
+#ifdef WITH_WSREP
+/** Update WSREP checkpoint XID in sys header. */
+void
+trx_sys_update_wsrep_checkpoint(
+        const XID*      xid,         /*!< in: WSREP XID */
+        trx_sysf_t*     sys_header,  /*!< in: sys_header */
+        mtr_t*          mtr);        /*!< in: mtr       */
+
+void
+/** Read WSREP checkpoint XID from sys header. */
+trx_sys_read_wsrep_checkpoint(
+        XID* xid); /*!< out: WSREP XID */
+#endif /* WITH_WSREP */
 /*****************************************************************//**
 Prints to stderr the MySQL master log offset info in the trx system header if
 the magic number shows it valid. */
@@ -550,6 +569,20 @@ this contains the same fields as TRX_SYS_MYSQL_LOG_INFO below */
 						within that file */
 #define TRX_SYS_MYSQL_LOG_NAME		12	/*!< MySQL log file name */
 
+#ifdef WITH_WSREP
+/* The offset to WSREP XID headers */
+#define TRX_SYS_WSREP_XID_INFO (UNIV_PAGE_SIZE - 3500)
+#define TRX_SYS_WSREP_XID_MAGIC_N_FLD 0
+#define TRX_SYS_WSREP_XID_MAGIC_N 0x77737265
+
+/* XID field: formatID, gtrid_len, bqual_len, xid_data */
+#define TRX_SYS_WSREP_XID_LEN        (4 + 4 + 4 + XIDDATASIZE)
+#define TRX_SYS_WSREP_XID_FORMAT     4
+#define TRX_SYS_WSREP_XID_GTRID_LEN  8
+#define TRX_SYS_WSREP_XID_BQUAL_LEN 12
+#define TRX_SYS_WSREP_XID_DATA      16
+#endif /* WITH_WSREP*/
+
 /** Doublewrite buffer */
 /* @{ */
 /** The offset of the doublewrite buffer header on the trx system header page */
diff --git a/storage/xtradb/include/trx0sys.ic b/storage/xtradb/include/trx0sys.ic
index 699148cff6d..6024c1dc94e 100644
--- a/storage/xtradb/include/trx0sys.ic
+++ b/storage/xtradb/include/trx0sys.ic
@@ -474,7 +474,10 @@ trx_id_t
 trx_sys_get_new_trx_id(void)
 /*========================*/
 {
+#ifndef WITH_WSREP
+	/* wsrep_fake_trx_id  violates this assert */
 	ut_ad(mutex_own(&trx_sys->mutex));
+#endif /* WITH_WSREP */
 
 	/* VERY important: after the database is started, max_trx_id value is
 	divisible by TRX_SYS_TRX_ID_WRITE_MARGIN, and the following if
diff --git a/storage/xtradb/include/trx0trx.h b/storage/xtradb/include/trx0trx.h
index aaa74724a14..be13c48fdfc 100644
--- a/storage/xtradb/include/trx0trx.h
+++ b/storage/xtradb/include/trx0trx.h
@@ -1031,6 +1031,9 @@ struct trx_t{
 	/*------------------------------*/
 	char detailed_error[256];	/*!< detailed error message for last
 					error, or empty. */
+#ifdef WITH_WSREP
+	os_event_t	wsrep_event;	/* event waited for in srv_conc_slot */
+#endif /* WITH_WSREP */
 	/*------------------------------*/
 	ulint		io_reads;
 	ib_uint64_t	io_read;
diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc
index 4f9395e27d8..8f7724daad9 100644
--- a/storage/xtradb/lock/lock0lock.cc
+++ b/storage/xtradb/lock/lock0lock.cc
@@ -50,6 +50,11 @@ Created 5/7/1996 Heikki Tuuri
 #include "dict0boot.h"
 #include <set>
 
+#ifdef WITH_WSREP
+extern my_bool wsrep_debug;
+extern my_bool wsrep_log_conflicts;
+#include "ha_prototypes.h"
+#endif
 /* Restricts the length of search we will do in the waits-for
 graph of transactions */
 #define LOCK_MAX_N_STEPS_IN_DEADLOCK_CHECK 1000000
@@ -946,6 +951,9 @@ UNIV_INLINE
 ibool
 lock_rec_has_to_wait(
 /*=================*/
+#ifdef WITH_WSREP
+	ibool		for_locking, /*!< is caller locking or releasing */
+#endif /* WITH_WSREP */
 	const trx_t*	trx,	/*!< in: trx of new lock */
 	ulint		type_mode,/*!< in: precise mode of the new lock
 				to set: LOCK_S or LOCK_X, possibly
@@ -1016,6 +1024,50 @@ lock_rec_has_to_wait(
 			return(FALSE);
 		}
 
+#ifdef WITH_WSREP
+		/* if BF thread is locking and has conflict with another BF
+		   thread, we need to look at trx ordering and lock types */
+		if (for_locking                                    &&
+		    wsrep_thd_is_BF(trx->mysql_thd, FALSE)         &&
+		    wsrep_thd_is_BF(lock2->trx->mysql_thd, TRUE)) {
+
+			if (wsrep_debug) {
+				fprintf(stderr, "\n BF-BF lock conflict \n");
+				lock_rec_print(stderr, lock2);
+			}
+
+			if (wsrep_trx_order_before(trx->mysql_thd,
+						   lock2->trx->mysql_thd) &&
+			    (type_mode & LOCK_MODE_MASK) == LOCK_X        &&
+			    (lock2->type_mode & LOCK_MODE_MASK) == LOCK_X)
+			{
+				/* exclusive lock conflicts are not accepted */
+				fprintf(stderr, "BF-BF X lock conflict,"
+					"type_mode: %lu supremum: %lu\n",
+					type_mode, lock_is_on_supremum);
+				fprintf(stderr, "conflicts states: my %d locked %d\n",
+					wsrep_thd_conflict_state(trx->mysql_thd, FALSE),
+					wsrep_thd_conflict_state(lock2->trx->mysql_thd, FALSE) );
+				lock_rec_print(stderr, lock2);
+				return FALSE;
+				//abort();
+			} else {
+				/* if lock2->index->n_uniq <=
+				   lock2->index->n_user_defined_cols
+				   operation is on uniq index
+				*/
+				if (wsrep_debug) fprintf(stderr,
+					"BF conflict, modes: %lu %lu, "
+					"idx: %s-%s n_uniq %u n_user %u\n",
+					type_mode, lock2->type_mode,
+					lock2->index->name, 
+					lock2->index->table_name,
+					lock2->index->n_uniq,
+					lock2->index->n_user_defined_cols);
+				return FALSE;
+			}
+		}
+#endif /* WITH_WSREP */
 		return(TRUE);
 	}
 
@@ -1046,7 +1098,11 @@ lock_has_to_wait(
 			/* If this lock request is for a supremum record
 			then the second bit on the lock bitmap is set */
 
+#ifdef WITH_WSREP
+			return(lock_rec_has_to_wait(FALSE, lock1->trx,
+#else
 			return(lock_rec_has_to_wait(lock1->trx,
+#endif /* WITH_WSREP */
 						    lock1->type_mode, lock2,
 						    lock_rec_get_nth_bit(
 							    lock1, 1)));
@@ -1515,6 +1571,11 @@ lock_rec_has_expl(
 	return(NULL);
 }
 
+#ifdef WITH_WSREP
+static
+void
+lock_rec_discard(lock_t*	in_lock);
+#endif
 #ifdef UNIV_DEBUG
 /*********************************************************************//**
 Checks if some other transaction has a lock request in the queue.
@@ -1561,6 +1622,69 @@ lock_rec_other_has_expl_req(
 }
 #endif /* UNIV_DEBUG */
 
+#ifdef WITH_WSREP
+static
+void
+wsrep_kill_victim(
+	const trx_t * const trx,
+	const lock_t *lock)
+{
+        ut_ad(lock_mutex_own());
+        ut_ad(trx_mutex_own(lock->trx));
+	my_bool bf_this  = wsrep_thd_is_BF(trx->mysql_thd, FALSE);
+	my_bool bf_other = wsrep_thd_is_BF(lock->trx->mysql_thd, TRUE);
+
+	if ((bf_this && !bf_other) ||
+		(bf_this && bf_other && wsrep_trx_order_before(
+			trx->mysql_thd, lock->trx->mysql_thd))) {
+
+		if (lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT) {
+			if (wsrep_debug) {
+				fprintf(stderr, "WSREP: BF victim waiting\n");
+			}
+			/* cannot release lock, until our lock
+			is in the queue*/
+		} else if (lock->trx != trx) {
+			if (wsrep_log_conflicts) {
+				mutex_enter(&trx_sys->mutex);
+				if (bf_this) {
+					fputs("\n*** Priority TRANSACTION:\n",
+					      stderr);
+				} else {
+					fputs("\n*** Victim TRANSACTION:\n",
+					      stderr);
+				}
+
+				trx_print_latched(stderr, trx, 3000);
+
+				if (bf_other) {
+					fputs("\n*** Priority TRANSACTION:\n",
+					      stderr);
+				} else {
+					fputs("\n*** Victim TRANSACTION:\n",
+					      stderr);
+				}
+
+				trx_print_latched(stderr, lock->trx, 3000);
+
+				mutex_exit(&trx_sys->mutex);
+
+				fputs("*** WAITING FOR THIS LOCK TO BE GRANTED:\n",
+				      stderr);
+
+				if (lock_get_type(lock) == LOCK_REC) {
+					lock_rec_print(stderr, lock);
+				} else {
+					lock_table_print(stderr, lock);
+				}
+			}
+
+			wsrep_innobase_kill_one_trx(trx->mysql_thd,
+				(const trx_t*) trx, lock->trx, TRUE);
+		}
+	}
+}
+#endif
 /*********************************************************************//**
 Checks if some other transaction has a conflicting explicit lock request
 in the queue, so that we have to wait.
@@ -1589,7 +1713,15 @@ lock_rec_other_has_conflicting(
 	     lock != NULL;
 	     lock = lock_rec_get_next_const(heap_no, lock)) {
 
-		if (lock_rec_has_to_wait(trx, mode, lock, is_supremum)) {
+#ifdef WITH_WSREP
+		if (lock_rec_has_to_wait(TRUE, trx, mode, lock, is_supremum)) {
+			trx_mutex_enter(lock->trx);
+			wsrep_kill_victim((trx_t *)trx, (lock_t *)lock);
+			trx_mutex_exit(lock->trx);
+#else
+ 		if (lock_rec_has_to_wait(trx, mode, lock, is_supremum)) {
+#endif /* WITH_WSREP */
+
 			return(lock);
 		}
 	}
@@ -1782,6 +1914,28 @@ lock_number_of_rows_locked(
 
 /*============== RECORD LOCK CREATION AND QUEUE MANAGEMENT =============*/
 
+#ifdef WITH_WSREP
+static
+void
+wsrep_print_wait_locks(
+/*============*/
+	lock_t*		c_lock) /* conflicting lock to print */
+{
+	if (wsrep_debug &&  c_lock->trx->lock.wait_lock != c_lock) {
+		fprintf(stderr, "WSREP: c_lock != wait lock\n");
+		if (lock_get_type_low(c_lock) & LOCK_TABLE)
+			lock_table_print(stderr, c_lock);
+		else
+			lock_rec_print(stderr, c_lock);
+
+		if (lock_get_type_low(c_lock->trx->lock.wait_lock) & LOCK_TABLE)
+			lock_table_print(stderr, c_lock->trx->lock.wait_lock);
+		else
+			lock_rec_print(stderr, c_lock->trx->lock.wait_lock);
+	}
+}
+#endif /* WITH_WSREP */
+
 /*********************************************************************//**
 Creates a new record lock and inserts it to the lock queue. Does NOT check
 for deadlocks or lock compatibility!
@@ -1790,6 +1944,10 @@ static
 lock_t*
 lock_rec_create(
 /*============*/
+#ifdef WITH_WSREP
+	lock_t*			const c_lock,   /* conflicting lock */
+	que_thr_t*		thr,
+#endif
 	ulint			type_mode,/*!< in: lock mode and wait
 					flag, type is ignored and
 					replaced by LOCK_REC */
@@ -1861,8 +2019,91 @@ lock_rec_create(
 
 	ut_ad(index->table->n_ref_count > 0 || !index->table->can_be_evicted);
 
+#ifdef WITH_WSREP
+	if (c_lock && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+		lock_t *hash	= (lock_t *)c_lock->hash;
+		lock_t *prev	= NULL;
+
+		while (hash 						       &&
+		       wsrep_thd_is_BF(((lock_t *)hash)->trx->mysql_thd, TRUE) &&
+		       wsrep_trx_order_before(
+				((lock_t *)hash)->trx->mysql_thd,
+				trx->mysql_thd)) {
+			prev = hash;
+			hash = (lock_t *)hash->hash;
+		}
+		lock->hash = hash;
+		if (prev) {
+			prev->hash = lock;
+		} else {
+			c_lock->hash = lock;
+		}
+		/*
+		 * delayed conflict resolution '...kill_one_trx' was not called,
+		 * if victim was waiting for some other lock
+		 */
+		trx_mutex_enter(c_lock->trx);
+		if (c_lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT) {
+
+			c_lock->trx->lock.was_chosen_as_deadlock_victim = TRUE;
+
+			if (wsrep_debug) {
+				wsrep_print_wait_locks(c_lock);
+			}
+
+			trx->lock.que_state = TRX_QUE_LOCK_WAIT;
+			lock_set_lock_and_trx_wait(lock, trx);
+			UT_LIST_ADD_LAST(trx_locks, trx->lock.trx_locks, lock);
+
+			ut_ad(thr != NULL);
+			trx->lock.wait_thr = thr;
+			thr->state = QUE_THR_LOCK_WAIT;
+
+			/* have to release trx mutex for the duration of
+			   victim lock release. This will eventually call
+			   lock_grant, which wants to grant trx mutex again
+			*/
+			if (caller_owns_trx_mutex) {
+				trx_mutex_exit(trx);
+			}
+			lock_cancel_waiting_and_release(
+				c_lock->trx->lock.wait_lock);
+
+			if (caller_owns_trx_mutex) {
+				trx_mutex_enter(trx);
+			}
+
+			/* trx might not wait for c_lock, but some other lock
+			   does not matter if wait_lock was released above
+			 */
+			if (c_lock->trx->lock.wait_lock == c_lock) {
+				lock_reset_lock_and_trx_wait(lock);
+			}
+
+			trx_mutex_exit(c_lock->trx);
+
+			if (wsrep_debug) {
+				fprintf(
+					stderr,
+					"WSREP: c_lock canceled %llu\n",
+					(ulonglong) c_lock->trx->id);
+			}
+
+			/* have to bail out here to avoid lock_set_lock... */
+			return(lock);
+		}
+		trx_mutex_exit(c_lock->trx);
+	} else if (wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+		HASH_PREPEND(lock_t, hash, lock_sys->rec_hash,
+			    lock_rec_fold(space, page_no), lock);
+	} else {
+		HASH_INSERT(lock_t, hash, lock_sys->rec_hash,
+			    lock_rec_fold(space, page_no), lock);
+	}
+#else
 	HASH_INSERT(lock_t, hash, lock_sys->rec_hash,
 		    lock_rec_fold(space, page_no), lock);
+#endif /* WITH_WSREP */
 
 	lock_sys->rec_num++;
 
@@ -1872,7 +2113,6 @@ lock_rec_create(
 	ut_ad(trx_mutex_own(trx));
 
 	if (type_mode & LOCK_WAIT) {
-
 		lock_set_lock_and_trx_wait(lock, trx);
 	}
 
@@ -1884,7 +2124,6 @@ lock_rec_create(
 
 	MONITOR_INC(MONITOR_RECLOCK_CREATED);
 	MONITOR_INC(MONITOR_NUM_RECLOCK);
-
 	return(lock);
 }
 
@@ -1899,6 +2138,9 @@ static
 dberr_t
 lock_rec_enqueue_waiting(
 /*=====================*/
+#ifdef WITH_WSREP
+	lock_t*			c_lock,	/* conflicting lock */
+#endif
 	ulint			type_mode,/*!< in: lock mode this
 					transaction is requesting:
 					LOCK_S or LOCK_X, possibly
@@ -1959,7 +2201,10 @@ lock_rec_enqueue_waiting(
 	/* Enqueue the lock request that will wait to be granted, note that
 	we already own the trx mutex. */
 	lock = lock_rec_create(
-		type_mode | LOCK_WAIT, block, heap_no, index, trx, TRUE);
+#ifdef WITH_WSREP
+                c_lock, thr,
+#endif /* WITH_WSREP */
+                type_mode | LOCK_WAIT, block, heap_no, index, trx, TRUE);
 
 	/* Release the mutex to obey the latching order.
 	This is safe, because lock_deadlock_check_and_resolve()
@@ -2064,7 +2309,19 @@ lock_rec_add_to_queue(
 		const lock_t*	other_lock
 			= lock_rec_other_has_expl_req(mode, 0, LOCK_WAIT,
 						      block, heap_no, trx->id);
+#ifdef WITH_WSREP
+		/* this can potentionally assert with wsrep */
+		if (wsrep_thd_is_wsrep(trx->mysql_thd)) {
+			if (wsrep_debug && other_lock) {
+				fprintf(stderr,
+					"WSREP: InnoDB assert ignored\n");
+			}
+		} else {
+			ut_a(!other_lock);
+		}
+#else
 		ut_a(!other_lock);
+#endif /* WITH_WSREP */
 	}
 #endif /* UNIV_DEBUG */
 
@@ -2092,7 +2349,16 @@ lock_rec_add_to_queue(
 
 		if (lock_get_wait(lock)
 		    && lock_rec_get_nth_bit(lock, heap_no)) {
-
+#ifdef WITH_WSREP
+			if (wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+				if (wsrep_debug) {
+					fprintf(stderr,
+						"BF skipping wait: %lu\n",
+						trx->id);
+					lock_rec_print(stderr, lock);
+				}
+		  } else
+#endif
 			goto somebody_waits;
 		}
 	}
@@ -2115,9 +2381,15 @@ lock_rec_add_to_queue(
 	}
 
 somebody_waits:
+#ifdef WITH_WSREP
+	return(lock_rec_create(NULL, NULL,
+			type_mode, block, heap_no, index, trx,
+			caller_owns_trx_mutex));
+#else
 	return(lock_rec_create(
 			type_mode, block, heap_no, index, trx,
 			caller_owns_trx_mutex));
+#endif /* WITH_WSREP */
 }
 
 /** Record locking request status */
@@ -2180,8 +2452,13 @@ lock_rec_lock_fast(
 	if (lock == NULL) {
 		if (!impl) {
 			/* Note that we don't own the trx mutex. */
+#ifdef WITH_WSREP
+			lock = lock_rec_create(NULL, thr,
+				mode, block, heap_no, index, trx, FALSE);
+#else
 			lock = lock_rec_create(
 				mode, block, heap_no, index, trx, FALSE);
+#endif
 
 		}
 		status = LOCK_REC_SUCCESS_CREATED;
@@ -2235,6 +2512,9 @@ lock_rec_lock_slow(
 	que_thr_t*		thr)	/*!< in: query thread */
 {
 	trx_t*			trx;
+#ifdef WITH_WSREP
+	lock_t*			c_lock(NULL);
+#endif
 	dberr_t			err = DB_SUCCESS;
 
 	ut_ad(lock_mutex_own());
@@ -2259,17 +2539,31 @@ lock_rec_lock_slow(
 		/* The trx already has a strong enough lock on rec: do
 		nothing */
 
+#ifdef WITH_WSREP
+	} else if ((c_lock = (lock_t *)lock_rec_other_has_conflicting(
+			static_cast<enum lock_mode>(mode),
+			block, heap_no, trx))) {
+#else
 	} else if (lock_rec_other_has_conflicting(
 			static_cast<enum lock_mode>(mode),
 			block, heap_no, trx)) {
+#endif /* WITH_WSREP */
 
 		/* If another transaction has a non-gap conflicting
 		request in the queue, as this transaction does not
 		have a lock strong enough already granted on the
 		record, we have to wait. */
 
+#ifdef WITH_WSREP
+		/* c_lock is NULL here if jump to enqueue_waiting happened
+		but it's ok because lock is not NULL in that case and c_lock
+		is not used. */
+		err = lock_rec_enqueue_waiting(c_lock,
+			mode, block, heap_no, index, thr);
+#else
 		err = lock_rec_enqueue_waiting(
 			mode, block, heap_no, index, thr);
+#endif /* WITH_WSREP */
 
 	} else if (!impl) {
 		/* Set the requested lock on the record, note that
@@ -2321,6 +2615,7 @@ lock_rec_lock(
 	ut_ad(mode - (LOCK_MODE_MASK & mode) == LOCK_GAP
 	      || mode - (LOCK_MODE_MASK & mode) == LOCK_REC_NOT_GAP
 	      || mode - (LOCK_MODE_MASK & mode) == 0);
+
 	ut_ad(dict_index_is_clust(index) || !dict_index_is_online_ddl(index));
 
 	/* We try a simplified and faster subroutine for the most
@@ -2375,7 +2670,13 @@ lock_rec_has_to_wait_in_queue(
 		if (heap_no < lock_rec_get_n_bits(lock)
 		    && (p[bit_offset] & bit_mask)
 		    && lock_has_to_wait(wait_lock, lock)) {
-
+#ifdef WITH_WSREP
+			if (wsrep_thd_is_BF(wait_lock->trx->mysql_thd, FALSE) &&
+			    wsrep_thd_is_BF(lock->trx->mysql_thd, TRUE)) {
+				/* don't wait for another BF lock */
+				continue;
+			}
+#endif
 			return(lock);
 		}
 	}
@@ -3761,10 +4062,22 @@ lock_deadlock_select_victim(
 		/* The joining  transaction is 'smaller',
 		choose it as the victim and roll it back. */
 
-		return(ctx->start);
+#ifdef WITH_WSREP
+		if (wsrep_thd_is_BF(ctx->start->mysql_thd, TRUE)) {
+			return(ctx->wait_lock->trx);
+		}
+		else
+#endif /* WITH_WSREP */
+			return(ctx->start);
 	}
 
-	return(ctx->wait_lock->trx);
+#ifdef WITH_WSREP
+	if (wsrep_thd_is_BF(ctx->wait_lock->trx->mysql_thd, TRUE)) {
+		return(ctx->start);
+	}
+	else
+#endif /* WITH_WSREP */
+		return(ctx->wait_lock->trx);
 }
 
 /********************************************************************//**
@@ -3893,8 +4206,14 @@ lock_deadlock_search(
 
 			ctx->too_deep = TRUE;
 
+#ifdef WITH_WSREP
+			if (wsrep_thd_is_BF(ctx->start->mysql_thd, TRUE)) {
+				return(ctx->wait_lock->trx->id);
+			}
+			else
+#endif /* WITH_WSREP */
 			/* Select the joining transaction as the victim. */
-			return(ctx->start->id);
+				return(ctx->start->id);
 
 		} else if (lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT) {
 
@@ -3911,7 +4230,12 @@ lock_deadlock_search(
 
 				ctx->too_deep = TRUE;
 
-				return(ctx->start->id);
+#ifdef WITH_WSREP
+				if (wsrep_thd_is_BF(ctx->start->mysql_thd, TRUE))
+					return(lock->trx->id);
+				else
+#endif /* WITH_WSREP */
+					return(ctx->start->id);
 			}
 
 			ctx->wait_lock = lock->trx->lock.wait_lock;
@@ -4029,9 +4353,18 @@ lock_deadlock_check_and_resolve(
 			ut_a(trx == ctx.start);
 			ut_a(victim_trx_id == trx->id);
 
-			if (!srv_read_only_mode) {
-				lock_deadlock_joining_trx_print(trx, lock);
+#ifdef WITH_WSREP
+			if (!wsrep_thd_is_BF(ctx.start->mysql_thd, TRUE))
+			{
+#endif /* WITH_WSREP */
+				if (!srv_read_only_mode) {
+					lock_deadlock_joining_trx_print(trx, lock);
+				}
+#ifdef WITH_WSREP
+			} else {
+			  /* BF processor */;
 			}
+#endif /* WITH_WSREP */
 
 			MONITOR_INC(MONITOR_DEADLOCK);
 
@@ -4071,6 +4404,9 @@ UNIV_INLINE
 lock_t*
 lock_table_create(
 /*==============*/
+#ifdef WITH_WSREP
+	lock_t*		c_lock, /*!< in: conflicting lock */
+#endif
 	dict_table_t*	table,	/*!< in/out: database table
 				in dictionary cache */
 	ulint		type_mode,/*!< in: lock mode possibly ORed with
@@ -4114,7 +4450,59 @@ lock_table_create(
 	ut_ad(table->n_ref_count > 0 || !table->can_be_evicted);
 
 	UT_LIST_ADD_LAST(trx_locks, trx->lock.trx_locks, lock);
+
+#ifdef WITH_WSREP
+	if (wsrep_thd_is_wsrep(trx->mysql_thd)) {
+		if (c_lock && wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+			UT_LIST_INSERT_AFTER(
+				un_member.tab_lock.locks, table->locks, c_lock, lock);
+		} else {
+			UT_LIST_ADD_LAST(un_member.tab_lock.locks, table->locks, lock);
+		}
+
+		if (c_lock) {
+			trx_mutex_enter(c_lock->trx);
+		}
+
+		if (c_lock && c_lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT) {
+
+			c_lock->trx->lock.was_chosen_as_deadlock_victim = TRUE;
+
+			if (wsrep_debug) {
+				wsrep_print_wait_locks(c_lock);
+				wsrep_print_wait_locks(c_lock->trx->lock.wait_lock);
+			}
+
+			/* have to release trx mutex for the duration of
+			victim lock release. This will eventually call
+			lock_grant, which wants to grant trx mutex again
+			*/
+			/* caller has trx_mutex, have to release for lock cancel */
+			trx_mutex_exit(trx);
+			lock_cancel_waiting_and_release(c_lock->trx->lock.wait_lock);
+			trx_mutex_enter(trx);
+
+			/* trx might not wait for c_lock, but some other lock
+			does not matter if wait_lock was released above
+			*/
+			if (c_lock->trx->lock.wait_lock == c_lock) {
+				lock_reset_lock_and_trx_wait(lock);
+			}
+
+			if (wsrep_debug) {
+				fprintf(stderr, "WSREP: c_lock canceled %llu\n",
+					(ulonglong) c_lock->trx->id);
+			}
+		}
+		if (c_lock) {
+			trx_mutex_exit(c_lock->trx);
+		}
+	} else {
+		UT_LIST_ADD_LAST(un_member.tab_lock.locks, table->locks, lock);
+	}
+#else
 	UT_LIST_ADD_LAST(un_member.tab_lock.locks, table->locks, lock);
+#endif /* WITH_WSREP */
 
 	if (UNIV_UNLIKELY(type_mode & LOCK_WAIT)) {
 
@@ -4271,6 +4659,9 @@ static
 dberr_t
 lock_table_enqueue_waiting(
 /*=======================*/
+#ifdef WITH_WSREP
+	lock_t*		c_lock, /*!< in: conflicting lock */
+#endif
 	ulint		mode,	/*!< in: lock mode this transaction is
 				requesting */
 	dict_table_t*	table,	/*!< in/out: table */
@@ -4317,7 +4708,14 @@ lock_table_enqueue_waiting(
 
 	/* Enqueue the lock request that will wait to be granted */
 
+#ifdef WITH_WSREP
+	if (trx->lock.was_chosen_as_deadlock_victim) {
+		return(DB_DEADLOCK);
+	}
+	lock = lock_table_create(c_lock, table, mode | LOCK_WAIT, trx);
+#else
 	lock = lock_table_create(table, mode | LOCK_WAIT, trx);
+#endif /* WITH_WSREP */
 
 	/* Release the mutex to obey the latching order.
 	This is safe, because lock_deadlock_check_and_resolve()
@@ -4394,6 +4792,18 @@ lock_table_other_has_incompatible(
 		    && !lock_mode_compatible(lock_get_mode(lock), mode)
 		    && (wait || !lock_get_wait(lock))) {
 
+#ifdef WITH_WSREP
+			if(wsrep_thd_is_wsrep(trx->mysql_thd)) {
+				if (wsrep_debug) {
+					fprintf(stderr, "WSREP: trx %ld table lock abort\n",
+						trx->id);
+				}
+				trx_mutex_enter(lock->trx);
+				wsrep_kill_victim((trx_t *)trx, (lock_t *)lock);
+				trx_mutex_exit(lock->trx);
+			}
+#endif
+
 			return(lock);
 		}
 	}
@@ -4416,6 +4826,9 @@ lock_table(
 	enum lock_mode	mode,	/*!< in: lock mode */
 	que_thr_t*	thr)	/*!< in: query thread */
 {
+#ifdef WITH_WSREP
+	lock_t *c_lock = NULL;
+#endif
 	trx_t*		trx;
 	dberr_t		err;
 	const lock_t*	wait_for;
@@ -4450,8 +4863,13 @@ lock_table(
 	/* We have to check if the new lock is compatible with any locks
 	other transactions have in the table lock queue. */
 
+#ifdef WITH_WSREP
+	wait_for = lock_table_other_has_incompatible(
+                trx, LOCK_WAIT, table, mode);
+#else
 	wait_for = lock_table_other_has_incompatible(
 		trx, LOCK_WAIT, table, mode);
+#endif
 
 	trx_mutex_enter(trx);
 
@@ -4459,9 +4877,17 @@ lock_table(
 	mode: this trx may have to wait */
 
 	if (wait_for != NULL) {
+#ifdef WITH_WSREP
+                err = lock_table_enqueue_waiting((ib_lock_t*)wait_for, mode | flags, table, thr);
+#else
 		err = lock_table_enqueue_waiting(mode | flags, table, thr);
+#endif
 	} else {
+#ifdef WITH_WSREP
+	        lock_table_create(c_lock, table, mode | flags, trx);
+#else
 		lock_table_create(table, mode | flags, trx);
+#endif
 
 		ut_a(!flags || mode == LOCK_S || mode == LOCK_X);
 
@@ -4499,7 +4925,11 @@ lock_table_ix_resurrect(
 		      trx, LOCK_WAIT, table, LOCK_IX));
 
 	trx_mutex_enter(trx);
+#ifdef WITH_WSREP
+	lock_table_create(NULL, table, LOCK_IX, trx);
+#else
 	lock_table_create(table, LOCK_IX, trx);
+#endif
 	lock_mutex_exit();
 	trx_mutex_exit(trx);
 }
@@ -5640,16 +6070,20 @@ lock_rec_queue_validate(
 
 		if (!lock_rec_get_gap(lock) && !lock_get_wait(lock)) {
 
-			enum lock_mode	mode;
+#ifndef WITH_WSREP
+			if (wsrep_thd_is_wsrep(lock->trx->mysql_thd)) {
+				enum lock_mode	mode;
 
-			if (lock_get_mode(lock) == LOCK_S) {
-				mode = LOCK_X;
-			} else {
-				mode = LOCK_S;
+
+				if (lock_get_mode(lock) == LOCK_S) {
+					mode = LOCK_X;
+				} else {
+					mode = LOCK_S;
+				}
+				ut_a(!lock_rec_other_has_expl_req(
+				     mode, 0, 0, block, heap_no, lock->trx->id));
 			}
-			ut_a(!lock_rec_other_has_expl_req(
-				     mode, 0, 0, block, heap_no,
-				     lock->trx->id));
+#endif /* WITH_WSREP */
 
 		} else if (lock_get_wait(lock) && !lock_rec_get_gap(lock)) {
 
@@ -5953,6 +6387,9 @@ lock_rec_insert_check_and_lock(
 	lock_t*		lock;
 	dberr_t		err;
 	ulint		next_rec_heap_no;
+#ifdef WITH_WSREP
+	lock_t*		c_lock=NULL;
+#endif
 
 	ut_ad(block->frame == page_align(rec));
 	ut_ad(!dict_index_is_online_ddl(index)
@@ -6014,17 +6451,30 @@ lock_rec_insert_check_and_lock(
 	had to wait for their insert. Both had waiting gap type lock requests
 	on the successor, which produced an unnecessary deadlock. */
 
+#ifdef WITH_WSREP
+	if ((c_lock = (ib_lock_t*)lock_rec_other_has_conflicting(
+		    static_cast<enum lock_mode>(
+                            LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION),
+		    block, next_rec_heap_no, trx))) {
+#else
 	if (lock_rec_other_has_conflicting(
 		    static_cast<enum lock_mode>(
 			    LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION),
 		    block, next_rec_heap_no, trx)) {
+#endif /* WITH_WSREP */
 
 		/* Note that we may get DB_SUCCESS also here! */
 		trx_mutex_enter(trx);
 
+#ifdef WITH_WSREP
+		err = lock_rec_enqueue_waiting(c_lock,
+                        LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION,
+			block, next_rec_heap_no, index, thr);
+#else
 		err = lock_rec_enqueue_waiting(
 			LOCK_X | LOCK_GAP | LOCK_INSERT_INTENTION,
 			block, next_rec_heap_no, index, thr);
+#endif /* WITH_WSREP */
 
 		trx_mutex_exit(trx);
 	} else {
diff --git a/storage/xtradb/lock/lock0wait.cc b/storage/xtradb/lock/lock0wait.cc
index a1c35e20ead..388c847f580 100644
--- a/storage/xtradb/lock/lock0wait.cc
+++ b/storage/xtradb/lock/lock0wait.cc
@@ -184,6 +184,28 @@ lock_wait_table_reserve_slot(
 	return(NULL);
 }
 
+#ifdef WITH_WSREP
+/*********************************************************************//**
+check if lock timeout was for priority thread, 
+as a side effect trigger lock monitor
+@return        false for regular lock timeout */
+static ibool
+wsrep_is_BF_lock_timeout(
+/*====================*/
+    trx_t* trx) /* in: trx to check for lock priority */
+{
+       if (wsrep_on(trx->mysql_thd) &&
+           wsrep_thd_is_BF(trx->mysql_thd, FALSE)) {
+               fprintf(stderr, "WSREP: BF lock wait long\n");
+                srv_print_innodb_monitor       = TRUE;
+                srv_print_innodb_lock_monitor  = TRUE;
+                os_event_set(srv_monitor_event);
+                return TRUE;
+       }
+       return FALSE;
+ }
+#endif /* WITH_WSREP */
+
 /***************************************************************//**
 Puts a user OS thread to wait for a lock to be released. If an error
 occurs during the wait trx->error_state associated with thr is
@@ -375,9 +397,15 @@ lock_wait_suspend_thread(
 
 	if (lock_wait_timeout < 100000000
 	    && wait_time > (double) lock_wait_timeout) {
+#ifdef WITH_WSREP
+                if (!wsrep_is_BF_lock_timeout(trx)) {
+#endif /* WITH_WSREP */
 
 		trx->error_state = DB_LOCK_WAIT_TIMEOUT;
 
+#ifdef WITH_WSREP
+                }
+#endif /* WITH_WSREP */
 		MONITOR_INC(MONITOR_TIMEOUT);
 	}
 
@@ -461,8 +489,13 @@ lock_wait_check_and_cancel(
 		if (trx->lock.wait_lock) {
 
 			ut_a(trx->lock.que_state == TRX_QUE_LOCK_WAIT);
-
+#ifdef WITH_WSREP
+                        if (!wsrep_is_BF_lock_timeout(trx)) {
+#endif /* WITH_WSREP */
 			lock_cancel_waiting_and_release(trx->lock.wait_lock);
+#ifdef WITH_WSREP
+                        }
+#endif /* WITH_WSREP */
 		}
 
 		lock_mutex_exit();
diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc
index 394c3199f76..edcb6662fe9 100644
--- a/storage/xtradb/os/os0file.cc
+++ b/storage/xtradb/os/os0file.cc
@@ -110,6 +110,12 @@ UNIV_INTERN os_ib_mutex_t	os_file_seek_mutexes[OS_FILE_N_SEEK_MUTEXES];
 /* In simulated aio, merge at most this many consecutive i/os */
 #define OS_AIO_MERGE_N_CONSECUTIVE	64
 
+#ifdef WITH_INNODB_DISALLOW_WRITES
+#define WAIT_ALLOW_WRITES() os_event_wait(srv_allow_writes_event)
+#else
+#define WAIT_ALLOW_WRITES() do { } while (0)
+#endif /* WITH_INNODB_DISALLOW_WRITES */
+
 /**********************************************************************
 
 InnoDB AIO Implementation:
@@ -999,7 +1005,9 @@ os_file_create_tmpfile(void)
 /*========================*/
 {
 	FILE*	file	= NULL;
-	int	fd	= innobase_mysql_tmpfile();
+	int	fd;
+	WAIT_ALLOW_WRITES();
+	fd	= innobase_mysql_tmpfile();
 
 	ut_ad(!srv_read_only_mode);
 
@@ -1325,6 +1333,7 @@ os_file_create_directory(
 	return(TRUE);
 #else
 	int	rcode;
+	WAIT_ALLOW_WRITES();
 
 	rcode = mkdir(pathname, 0770);
 
@@ -1451,6 +1460,8 @@ os_file_create_simple_func(
 
 #else /* __WIN__ */
 	int		create_flag;
+	if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW)
+		WAIT_ALLOW_WRITES();
 
 	ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT));
 	ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT));
@@ -1638,6 +1649,8 @@ os_file_create_simple_no_error_handling_func(
 	int		create_flag;
 
 	ut_a(name);
+	if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW)
+		WAIT_ALLOW_WRITES();
 
 	ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT));
 	ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT));
@@ -2013,6 +2026,8 @@ os_file_create_func(
 #else /* __WIN__ */
 	int		create_flag;
 	const char*	mode_str	= NULL;
+	if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW)
+		WAIT_ALLOW_WRITES();
 
 	on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT
 		? TRUE : FALSE;
@@ -2212,6 +2227,7 @@ loop:
 	goto loop;
 #else
 	int	ret;
+	WAIT_ALLOW_WRITES();
 
 	ret = unlink(name);
 
@@ -2276,6 +2292,7 @@ loop:
 	goto loop;
 #else
 	int	ret;
+	WAIT_ALLOW_WRITES();
 
 	ret = unlink(name);
 
@@ -2329,6 +2346,7 @@ os_file_rename_func(
 	return(FALSE);
 #else
 	int	ret;
+	WAIT_ALLOW_WRITES();
 
 	ret = rename(oldpath, newpath);
 
@@ -2562,6 +2580,7 @@ os_file_set_eof(
 	HANDLE h = (HANDLE) _get_osfhandle(fileno(file));
 	return(SetEndOfFile(h));
 #else /* __WIN__ */
+	WAIT_ALLOW_WRITES();
 	return(!ftruncate(fileno(file), ftell(file)));
 #endif /* __WIN__ */
 }
@@ -2581,6 +2600,7 @@ os_file_set_eof_at(
 	return(SetFilePointerEx(file, li, &li2,FILE_BEGIN)
 	       && SetEndOfFile(file));
 #else
+	WAIT_ALLOW_WRITES();
 	/* TODO: works only with -D_FILE_OFFSET_BITS=64 ? */
 	return(!ftruncate(file, new_len));
 #endif
@@ -2680,6 +2700,7 @@ os_file_flush_func(
 	return(FALSE);
 #else
 	int	ret;
+	WAIT_ALLOW_WRITES();
 
 #if defined(HAVE_DARWIN_THREADS)
 # ifndef F_FULLFSYNC
@@ -3382,6 +3403,7 @@ retry:
 	return(FALSE);
 #else
 	ssize_t	ret;
+	WAIT_ALLOW_WRITES();
 
 	ret = os_file_pwrite(file, buf, n, offset);
 
diff --git a/storage/xtradb/rem/rem0rec.cc b/storage/xtradb/rem/rem0rec.cc
index 0d7b7c16785..ca9b0a6b841 100644
--- a/storage/xtradb/rem/rem0rec.cc
+++ b/storage/xtradb/rem/rem0rec.cc
@@ -33,6 +33,9 @@ Created 5/30/1994 Heikki Tuuri
 #include "mtr0mtr.h"
 #include "mtr0log.h"
 #include "fts0fts.h"
+#ifdef WITH_WSREP
+#include <ha_prototypes.h>
+#endif /* WITH_WSREP */
 
 /*			PHYSICAL RECORD (OLD STYLE)
 			===========================
@@ -1917,6 +1920,138 @@ rec_print(
 		}
 	}
 }
+#endif /* !UNIV_HOTBACKUP */
+
+#ifdef WITH_WSREP
+int
+wsrep_rec_get_foreign_key(
+	byte 		*buf,     /* out: extracted key */
+	ulint 		*buf_len, /* in/out: length of buf */
+	const rec_t*	rec,	  /* in: physical record */
+	dict_index_t*	index_for,  /* in: index in foreign table */
+	dict_index_t*	index_ref,  /* in: index in referenced table */
+	ibool		new_protocol) /* in: protocol > 1 */
+{
+	const byte*	data;
+	ulint		len;
+	ulint		key_len = 0;
+	ulint		i;
+	uint            key_parts;
+	mem_heap_t*	heap	= NULL;
+	ulint		offsets_[REC_OFFS_NORMAL_SIZE];
+        const ulint*    offsets;
+
+	ut_ad(index_for);
+	ut_ad(index_ref);
+
+        rec_offs_init(offsets_);
+	offsets = rec_get_offsets(rec, index_for, offsets_,
+				  ULINT_UNDEFINED, &heap);
+
+	ut_ad(rec_offs_validate(rec, NULL, offsets));
+
+	ut_ad(rec);
+
+	key_parts = dict_index_get_n_unique_in_tree(index_for);
+	for (i = 0; 
+	     i < key_parts && 
+	       (index_for->type & DICT_CLUSTERED || i < key_parts - 1); 
+	     i++) {
+		dict_field_t*	  field_f = 
+			dict_index_get_nth_field(index_for, i);
+		const dict_col_t* col_f = dict_field_get_col(field_f);
+                dict_field_t*	  field_r = 
+			dict_index_get_nth_field(index_ref, i);
+		const dict_col_t* col_r = dict_field_get_col(field_r);
+
+		data = rec_get_nth_field(rec, offsets, i, &len);
+		if (key_len + ((len != UNIV_SQL_NULL) ? len + 1 : 1) > 
+		    *buf_len) {
+			fprintf (stderr, 
+				 "WSREP: FK key len exceeded %lu %lu %lu\n", 
+				 key_len, len, *buf_len);
+			goto err_out;
+		}
+
+		if (len == UNIV_SQL_NULL) {
+			ut_a(!(col_f->prtype & DATA_NOT_NULL));
+			*buf++ = 1;
+			key_len++;
+		} else if (!new_protocol) {
+			if (!(col_r->prtype & DATA_NOT_NULL)) {
+				*buf++ = 0;
+				key_len++;
+			}
+			memcpy(buf, data, len);
+			*buf_len = wsrep_innobase_mysql_sort(
+				(int)(col_f->prtype & DATA_MYSQL_TYPE_MASK),
+				(uint)dtype_get_charset_coll(col_f->prtype),
+				buf, len, *buf_len);
+		} else { /* new protocol */
+			if (!(col_r->prtype & DATA_NOT_NULL)) {
+				*buf++ = 0;
+				key_len++;
+			}
+			switch (col_f->mtype) {
+			case DATA_INT: {
+				byte* ptr = buf+len;
+				for (;;) {
+					ptr--;
+					*ptr = *data;
+					if (ptr == buf) {
+						break;
+					}
+					data++;
+				}
+		
+				if (!(col_f->prtype & DATA_UNSIGNED)) {
+					buf[len-1] = (byte) (buf[len-1] ^ 128);
+				}
+
+				break;
+			}
+			case DATA_VARCHAR:
+			case DATA_VARMYSQL:
+			case DATA_CHAR:
+			case DATA_MYSQL:
+				/* Copy the actual data */
+				ut_memcpy(buf, data, len);
+				len = wsrep_innobase_mysql_sort(
+					(int)
+					(col_f->prtype & DATA_MYSQL_TYPE_MASK),
+					(uint)
+					dtype_get_charset_coll(col_f->prtype),
+					buf, len, *buf_len);
+				break;
+			case DATA_BLOB:
+			case DATA_BINARY:
+				memcpy(buf, data, len);
+				break;
+			default: 
+				break;
+			}
+
+			key_len += len;
+			buf 	+= len;
+		}
+	}
+
+	rec_validate(rec, offsets);
+
+	if (UNIV_LIKELY_NULL(heap)) {
+		mem_heap_free(heap);
+	}
+
+	*buf_len = key_len;
+	return DB_SUCCESS;
+
+ err_out:
+	if (UNIV_LIKELY_NULL(heap)) {
+		mem_heap_free(heap);
+	}
+	return DB_ERROR;
+}
+#endif /* WITH_WSREP */
 
 # ifdef UNIV_DEBUG
 /************************************************************//**
@@ -1959,5 +2094,5 @@ rec_get_trx_id(
 
 	return(trx_read_trx_id(trx_id));
 }
-# endif /* UNIV_DEBUG */
-#endif /* !UNIV_HOTBACKUP */
+#endif /* UNIV_DEBUG */
+
diff --git a/storage/xtradb/row/row0ins.cc b/storage/xtradb/row/row0ins.cc
index f8ca40fac12..444fac87842 100644
--- a/storage/xtradb/row/row0ins.cc
+++ b/storage/xtradb/row/row0ins.cc
@@ -918,6 +918,14 @@ row_ins_invalidate_query_cache(
 	innobase_invalidate_query_cache(thr_get_trx(thr), buf, len);
 	mem_free(buf);
 }
+#ifdef WITH_WSREP
+dberr_t wsrep_append_foreign_key(trx_t *trx,  
+			       dict_foreign_t*	foreign,
+			       const rec_t*	clust_rec,
+			       dict_index_t*	clust_index,
+			       ibool		referenced,
+			       ibool            shared);
+#endif /* WITH_WSREP */
 
 /*********************************************************************//**
 Perform referential actions or checks when a parent row is deleted or updated
@@ -1269,7 +1277,19 @@ row_ins_foreign_check_on_constraint(
 
 	cascade->state = UPD_NODE_UPDATE_CLUSTERED;
 
-	err = row_update_cascade_for_mysql(thr, cascade,
+#ifdef WITH_WSREP
+	err = wsrep_append_foreign_key(
+				       thr_get_trx(thr),
+				       foreign,
+				       clust_rec,
+				       clust_index,
+				       FALSE, FALSE);
+	if (err != DB_SUCCESS) {
+		fprintf(stderr,
+			"WSREP: foreign key append failed: %d\n", err);
+	} else
+#endif /* WITH_WSREP */
+		err = row_update_cascade_for_mysql(thr, cascade,
 					   foreign->foreign_table);
 
 	if (foreign->foreign_table->n_foreign_key_checks_running == 0) {
@@ -1607,7 +1627,14 @@ run_again:
 
 				if (check_ref) {
 					err = DB_SUCCESS;
-
+#ifdef WITH_WSREP
+					err = wsrep_append_foreign_key(
+						thr_get_trx(thr),
+						foreign,
+						rec, 
+						check_index, 
+						check_ref, TRUE);
+#endif /* WITH_WSREP */
 					goto end_scan;
 				} else if (foreign->type != 0) {
 					/* There is an ON UPDATE or ON DELETE
diff --git a/storage/xtradb/row/row0upd.cc b/storage/xtradb/row/row0upd.cc
index 3ead385c2cd..bdd2a691368 100644
--- a/storage/xtradb/row/row0upd.cc
+++ b/storage/xtradb/row/row0upd.cc
@@ -53,6 +53,9 @@ Created 12/27/1996 Heikki Tuuri
 #include "pars0sym.h"
 #include "eval0eval.h"
 #include "buf0lru.h"
+#ifdef WITH_WSREP
+extern my_bool wsrep_debug;
+#endif
 
 
 /* What kind of latch and lock can we assume when the control comes to
@@ -172,6 +175,50 @@ func_exit:
 	return(is_referenced);
 }
 
+#ifdef WITH_WSREP
+static
+ibool
+wsrep_row_upd_index_is_foreign(
+/*========================*/
+	dict_index_t*	index,	/*!< in: index */
+	trx_t*		trx)	/*!< in: transaction */
+{
+	dict_table_t*	table		= index->table;
+	dict_foreign_t*	foreign;
+	ibool		froze_data_dict	= FALSE;
+	ibool		is_referenced	= FALSE;
+
+	if (!UT_LIST_GET_FIRST(table->foreign_list)) {
+
+		return(FALSE);
+	}
+
+	if (trx->dict_operation_lock_mode == 0) {
+		row_mysql_freeze_data_dictionary(trx);
+		froze_data_dict = TRUE;
+	}
+
+	foreign = UT_LIST_GET_FIRST(table->foreign_list);
+
+	while (foreign) {
+		if (foreign->foreign_index == index) {
+
+			is_referenced = TRUE;
+			goto func_exit;
+		}
+
+		foreign = UT_LIST_GET_NEXT(foreign_list, foreign);
+	}
+
+func_exit:
+	if (froze_data_dict) {
+		row_mysql_unfreeze_data_dictionary(trx);
+	}
+
+	return(is_referenced);
+}
+#endif /* WITH_WSREP */
+
 /*********************************************************************//**
 Checks if possible foreign key constraints hold after a delete of the record
 under pcur.
@@ -289,7 +336,123 @@ run_again:
 	}
 
 	err = DB_SUCCESS;
+func_exit:
+	if (got_s_lock) {
+		row_mysql_unfreeze_data_dictionary(trx);
+	}
+
+	mem_heap_free(heap);
 
+	return(err);
+}
+#ifdef WITH_WSREP
+static
+dberr_t
+wsrep_row_upd_check_foreign_constraints(
+/*=================================*/
+	upd_node_t*	node,	/*!< in: row update node */
+	btr_pcur_t*	pcur,	/*!< in: cursor positioned on a record; NOTE: the
+				cursor position is lost in this function! */
+	dict_table_t*	table,	/*!< in: table in question */
+	dict_index_t*	index,	/*!< in: index of the cursor */
+	ulint*		offsets,/*!< in/out: rec_get_offsets(pcur.rec, index) */
+	que_thr_t*	thr,	/*!< in: query thread */
+	mtr_t*		mtr)	/*!< in: mtr */
+{
+	dict_foreign_t*	foreign;
+	mem_heap_t*	heap;
+	dtuple_t*	entry;
+	trx_t*		trx;
+	const rec_t*	rec;
+	ulint		n_ext;
+	dberr_t		err;
+	ibool		got_s_lock	= FALSE;
+	ibool		opened     	= FALSE;
+
+	if (UT_LIST_GET_FIRST(table->foreign_list) == NULL) {
+
+		return(DB_SUCCESS);
+	}
+
+	trx = thr_get_trx(thr);
+
+        /* TODO: make native slave thread bail out here */
+
+	rec = btr_pcur_get_rec(pcur);
+	ut_ad(rec_offs_validate(rec, index, offsets));
+
+	heap = mem_heap_create(500);
+
+	entry = row_rec_to_index_entry(rec, index, offsets,
+				       &n_ext, heap);
+
+	mtr_commit(mtr);
+
+	mtr_start(mtr);
+
+	if (trx->dict_operation_lock_mode == 0) {
+		got_s_lock = TRUE;
+
+		row_mysql_freeze_data_dictionary(trx);
+	}
+
+	foreign = UT_LIST_GET_FIRST(table->foreign_list);
+
+	while (foreign) {
+		/* Note that we may have an update which updates the index
+		record, but does NOT update the first fields which are
+		referenced in a foreign key constraint. Then the update does
+		NOT break the constraint. */
+
+		if (foreign->foreign_index == index
+		    && (node->is_delete
+			|| row_upd_changes_first_fields_binary(
+				entry, index, node->update,
+				foreign->n_fields))) {
+
+			if (foreign->referenced_table == NULL) {
+				foreign->referenced_table =
+					dict_table_open_on_name(
+					  foreign->referenced_table_name_lookup,
+					  FALSE, FALSE, DICT_ERR_IGNORE_NONE);
+				opened = TRUE;
+			}
+
+			if (foreign->referenced_table) {
+				os_inc_counter(dict_sys->mutex,
+					       foreign->referenced_table
+					       ->n_foreign_key_checks_running);
+			}
+
+			/* NOTE that if the thread ends up waiting for a lock
+			we will release dict_operation_lock temporarily!
+			But the counter on the table protects 'foreign' from
+			being dropped while the check is running. */
+
+			err = row_ins_check_foreign_constraint(
+				TRUE, foreign, table, entry, thr);
+
+			if (foreign->referenced_table) {
+				os_dec_counter(dict_sys->mutex,
+					       foreign->referenced_table
+					       ->n_foreign_key_checks_running);
+
+				if (opened == TRUE) {
+					dict_table_close(foreign->referenced_table, TRUE, FALSE);
+					opened = FALSE;
+				}
+			}
+
+			if (err != DB_SUCCESS) {
+
+				goto func_exit;
+			}
+		}
+
+		foreign = UT_LIST_GET_NEXT(foreign_list, foreign);
+	}
+
+	err = DB_SUCCESS;
 func_exit:
 	if (got_s_lock) {
 		row_mysql_unfreeze_data_dictionary(trx);
@@ -301,6 +464,7 @@ func_exit:
 
 	return(err);
 }
+#endif /* WITH_WSREP */
 
 /*********************************************************************//**
 Creates an update node for a query graph.
@@ -1675,6 +1839,9 @@ row_upd_sec_index_entry(
 	index = node->index;
 
 	referenced = row_upd_index_is_referenced(index, trx);
+#ifdef WITH_WSREP
+	ibool foreign = wsrep_row_upd_index_is_foreign(index, trx);
+#endif /* WITH_WSREP */
 
 	heap = mem_heap_create(1024);
 
@@ -1805,6 +1972,9 @@ row_upd_sec_index_entry(
 		row_ins_sec_index_entry() below */
 		if (!rec_get_deleted_flag(
 			    rec, dict_table_is_comp(index->table))) {
+#ifdef WITH_WSREP
+			que_node_t *parent = que_node_get_parent(node);
+#endif /* WITH_WSREP */
 			err = btr_cur_del_mark_set_sec_rec(
 				0, btr_cur, TRUE, thr, &mtr);
 
@@ -1822,6 +1992,37 @@ row_upd_sec_index_entry(
 					node, &pcur, index->table,
 					index, offsets, thr, &mtr);
 			}
+#ifdef WITH_WSREP
+			if (err == DB_SUCCESS && !referenced                  &&
+			    !(parent && que_node_get_type(parent) ==
+				QUE_NODE_UPDATE                               &&
+			      ((upd_node_t*)parent)->cascade_node == node)    &&
+			    foreign
+			) {
+				ulint*	offsets =
+					rec_get_offsets(
+						rec, index, NULL, ULINT_UNDEFINED,
+						&heap);
+				err = wsrep_row_upd_check_foreign_constraints(
+					node, &pcur, index->table,
+					index, offsets, thr, &mtr);
+				switch (err) {
+				case DB_SUCCESS:
+				case DB_NO_REFERENCED_ROW:
+					err = DB_SUCCESS;
+					break;
+				case DB_DEADLOCK:
+					if (wsrep_debug) fprintf (stderr, 
+						"WSREP: sec index FK check fail for deadlock");
+					break;
+				default:
+					fprintf (stderr, 
+						 "WSREP: referenced FK check fail: %d", 
+						 (int)err);
+					break;
+				}
+			}
+#endif /* WITH_WSREP */
 		}
 		break;
 	}
@@ -1976,6 +2177,9 @@ row_upd_clust_rec_by_insert(
 	que_thr_t*	thr,	/*!< in: query thread */
 	ibool		referenced,/*!< in: TRUE if index may be referenced in
 				a foreign key constraint */
+#ifdef WITH_WSREP
+	ibool		foreign, /*!< in: TRUE if index is foreign key index */
+#endif /* WITH_WSREP */
 	mtr_t*		mtr)	/*!< in/out: mtr; gets committed here */
 {
 	mem_heap_t*	heap;
@@ -1989,6 +2193,9 @@ row_upd_clust_rec_by_insert(
 	rec_t*		rec;
 	ulint*		offsets			= NULL;
 
+#ifdef WITH_WSREP
+	que_node_t *parent = que_node_get_parent(node);
+#endif /* WITH_WSREP */
 	ut_ad(node);
 	ut_ad(dict_index_is_clust(index));
 
@@ -2074,6 +2281,34 @@ err_exit:
 				goto err_exit;
 			}
 		}
+#ifdef WITH_WSREP
+		if (!referenced                                              &&
+		    !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE &&
+		      ((upd_node_t*)parent)->cascade_node == node)           &&
+		    foreign
+		) {
+			err = wsrep_row_upd_check_foreign_constraints(
+				node, pcur, table, index, offsets, thr, mtr);
+			switch (err) {
+			case DB_SUCCESS:
+			case DB_NO_REFERENCED_ROW:
+				err = DB_SUCCESS;
+				break;
+			case DB_DEADLOCK:
+				if (wsrep_debug) fprintf (stderr, 
+					"WSREP: insert FK check fail for deadlock");
+				break;
+			default:
+				fprintf (stderr, 
+					"WSREP: referenced FK check fail: %d", 
+					 (int)err);
+				break;
+			}
+			if (err != DB_SUCCESS) {
+				goto err_exit;
+			}
+		}
+#endif /* WITH_WSREP */
 	}
 
 	mtr_commit(mtr);
@@ -2269,11 +2504,18 @@ row_upd_del_mark_clust_rec(
 	ibool		referenced,
 				/*!< in: TRUE if index may be referenced in
 				a foreign key constraint */
+#ifdef WITH_WSREP
+	ibool		foreign,/*!< in: TRUE if index is foreign key index */
+#endif /* WITH_WSREP */
 	mtr_t*		mtr)	/*!< in: mtr; gets committed here */
 {
 	btr_pcur_t*	pcur;
 	btr_cur_t*	btr_cur;
 	dberr_t		err;
+#ifdef WITH_WSREP
+	rec_t*		rec;
+	que_node_t *parent = que_node_get_parent(node);
+#endif /* WITH_WSREP */
 
 	ut_ad(node);
 	ut_ad(dict_index_is_clust(index));
@@ -2290,8 +2532,16 @@ row_upd_del_mark_clust_rec(
 	/* Mark the clustered index record deleted; we do not have to check
 	locks, because we assume that we have an x-lock on the record */
 
+#ifdef WITH_WSREP
+	rec = btr_cur_get_rec(btr_cur);
+#endif /* WITH_WSREP */
+
 	err = btr_cur_del_mark_set_clust_rec(
+#ifdef WITH_WSREP
+		btr_cur_get_block(btr_cur), rec,
+#else
 		btr_cur_get_block(btr_cur), btr_cur_get_rec(btr_cur),
+#endif /* WITH_WSREP */
 		index, offsets, thr, mtr);
 	if (err == DB_SUCCESS && referenced) {
 		/* NOTE that the following call loses the position of pcur ! */
@@ -2299,6 +2549,32 @@ row_upd_del_mark_clust_rec(
 		err = row_upd_check_references_constraints(
 			node, pcur, index->table, index, offsets, thr, mtr);
 	}
+#ifdef WITH_WSREP
+	if (err == DB_SUCCESS && !referenced                         &&
+	    !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE &&
+	      ((upd_node_t*)parent)->cascade_node == node)           &&
+	    thr_get_trx(thr)                                         &&
+	    foreign
+	) {
+		err = wsrep_row_upd_check_foreign_constraints(
+			node, pcur, index->table, index, offsets, thr, mtr);
+		switch (err) {
+		case DB_SUCCESS:
+		case DB_NO_REFERENCED_ROW:
+			err = DB_SUCCESS;
+			break;
+		case DB_DEADLOCK:
+			if (wsrep_debug) fprintf (stderr, 
+				"WSREP: clust rec FK check fail for deadlock");
+			break;
+		default:
+			fprintf (stderr, 
+				"WSREP: clust rec referenced FK check fail: %d", 
+				 (int)err);
+			break;
+		}
+	}
+#endif /* WITH_WSREP */
 
 	mtr_commit(mtr);
 
@@ -2331,6 +2607,10 @@ row_upd_clust_step(
 	index = dict_table_get_first_index(node->table);
 
 	referenced = row_upd_index_is_referenced(index, thr_get_trx(thr));
+#ifdef WITH_WSREP
+	ibool foreign = wsrep_row_upd_index_is_foreign(
+		index, thr_get_trx(thr));
+#endif /* WITH_WSREP */
 
 	pcur = node->pcur;
 
@@ -2427,7 +2707,11 @@ row_upd_clust_step(
 
 	if (node->is_delete) {
 		err = row_upd_del_mark_clust_rec(
+#ifdef WITH_WSREP
+			node, index, offsets, thr, referenced, foreign, &mtr);
+#else
 			node, index, offsets, thr, referenced, &mtr);
+#endif /* WITH_WSREP */
 
 		if (err == DB_SUCCESS) {
 			node->state = UPD_NODE_UPDATE_ALL_SEC;
@@ -2472,7 +2756,11 @@ row_upd_clust_step(
 		externally! */
 
 		err = row_upd_clust_rec_by_insert(
+#ifdef WITH_WSREP
+			node, index, thr, referenced, foreign, &mtr);
+#else
 			node, index, thr, referenced, &mtr);
+#endif /* WITH_WSREP */
 
 		if (err != DB_SUCCESS) {
 
diff --git a/storage/xtradb/srv/srv0conc.cc b/storage/xtradb/srv/srv0conc.cc
index 6c15753246a..5868e4e012b 100644
--- a/storage/xtradb/srv/srv0conc.cc
+++ b/storage/xtradb/srv/srv0conc.cc
@@ -43,6 +43,10 @@ Created 2011/04/18 Sunny Bains
 #include "trx0trx.h"
 
 #include "mysql/plugin.h"
+#ifdef WITH_WSREP
+extern "C" int wsrep_trx_is_aborting(void *thd_ptr);
+extern my_bool wsrep_debug;
+#endif
 
 /** Number of times a thread is allowed to enter InnoDB within the same
 SQL query after it has once got the ticket. */
@@ -87,6 +91,9 @@ struct srv_conc_slot_t{
 					reserved may still be TRUE at that
 					point */
 	srv_conc_node_t	srv_conc_queue;	/*!< queue node */
+#ifdef WITH_WSREP
+	void				*thd;		/*!< to see priority */
+#endif
 };
 
 /** Queue of threads waiting to get in */
@@ -146,6 +153,9 @@ srv_conc_init(void)
 
 		conc_slot->event = os_event_create();
 		ut_a(conc_slot->event);
+#ifdef WITH_WSREP
+		conc_slot->thd = NULL;
+#endif /* WITH_WSREP */
 	}
 #endif /* !HAVE_ATOMIC_BUILTINS */
 }
@@ -203,6 +213,16 @@ srv_conc_enter_innodb_with_atomics(
 
 	for (;;) {
 		ulint	sleep_in_us;
+#ifdef WITH_WSREP
+		if (wsrep_on(trx->mysql_thd) && 
+		    wsrep_trx_is_aborting(trx->mysql_thd)) {
+			if (wsrep_debug)
+		  		fprintf(stderr,	
+					"srv_conc_enter due to MUST_ABORT");
+			srv_conc_force_enter_innodb(trx);
+			return;
+		}
+#endif /* WITH_WSREP */
 
 		if (srv_conc.n_active < (lint) srv_thread_concurrency) {
 			ulint	n_active;
@@ -321,6 +341,9 @@ srv_conc_exit_innodb_without_atomics(
 	slot = NULL;
 
 	if (srv_conc.n_active < (lint) srv_thread_concurrency) {
+#ifdef WITH_WSREP
+		srv_conc_slot_t*  wsrep_slot;
+#endif
 		/* Look for a slot where a thread is waiting and no other
 		thread has yet released the thread */
 
@@ -331,6 +354,19 @@ srv_conc_exit_innodb_without_atomics(
 			/* No op */
 		}
 
+#ifdef WITH_WSREP
+		/* look for aborting trx, they must be released asap */
+		wsrep_slot= slot;
+		while (wsrep_slot && (wsrep_slot->wait_ended == TRUE || 
+		    !wsrep_trx_is_aborting(wsrep_slot->thd))) {
+			wsrep_slot = UT_LIST_GET_NEXT(srv_conc_queue, wsrep_slot);
+		}
+		if (wsrep_slot) {
+			slot = wsrep_slot;
+			if (wsrep_debug)
+			    fprintf(stderr, "WSREP: releasing aborting thd\n");
+		}
+#endif
 		if (slot != NULL) {
 			slot->wait_ended = TRUE;
 
@@ -390,6 +426,13 @@ retry:
 
 		return;
 	}
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd) && 
+	    wsrep_thd_is_brute_force(trx->mysql_thd)) {
+		srv_conc_force_enter_innodb(trx);
+		return;
+	}
+#endif
 
 	/* If the transaction is not holding resources, let it sleep
 	for srv_thread_sleep_delay microseconds, and try again then */
@@ -457,6 +500,9 @@ retry:
 	/* Add to the queue */
 	slot->reserved = TRUE;
 	slot->wait_ended = FALSE;
+#ifdef WITH_WSREP
+	slot->thd = trx->mysql_thd;
+#endif
 
 	UT_LIST_ADD_LAST(srv_conc_queue, srv_conc_queue, slot);
 
@@ -464,6 +510,18 @@ retry:
 
 	srv_conc.n_waiting++;
 
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd) && 
+	    wsrep_trx_is_aborting(trx->mysql_thd)) {
+		os_fast_mutex_unlock(&srv_conc_mutex);
+		if (wsrep_debug)
+			fprintf(stderr, "srv_conc_enter due to MUST_ABORT");
+		trx->declared_to_be_inside_innodb = TRUE;
+		trx->n_tickets_to_enter_innodb = srv_n_free_tickets_to_enter;
+		return;
+	}
+	trx->wsrep_event = slot->event;
+#endif /* WITH_WSREP */
 	os_fast_mutex_unlock(&srv_conc_mutex);
 
 	/* Go to wait for the event; when a thread leaves InnoDB it will
@@ -487,6 +545,9 @@ retry:
 
 	os_event_wait(slot->event);
 	thd_wait_end(trx->mysql_thd);
+#ifdef WITH_WSREP
+	trx->wsrep_event = NULL;
+#endif /* WITH_WSREP */
 
 	trx->op_info = "";
 
@@ -504,6 +565,9 @@ retry:
 	incremented the thread counter on behalf of this thread */
 
 	slot->reserved = FALSE;
+#ifdef WITH_WSREP
+	slot->thd = NULL;
+#endif
 
 	UT_LIST_REMOVE(srv_conc_queue, srv_conc_queue, slot);
 
@@ -614,5 +678,32 @@ srv_conc_get_active_threads(void)
 /*==============================*/
 {
 	return(srv_conc.n_active);
- }
+}
+
+#ifdef WITH_WSREP
+UNIV_INTERN
+void
+wsrep_srv_conc_cancel_wait(
+/*==================*/
+	trx_t*	trx)	/*!< in: transaction object associated with the
+			thread */
+{
+#ifdef HAVE_ATOMIC_BUILTINS
+	/* aborting transactions will enter innodb by force in 
+	   srv_conc_enter_innodb_with_atomics(). No need to cancel here,
+	   thr will wake up after os_sleep and let to enter innodb
+	*/
+	if (wsrep_debug)
+		fprintf(stderr, "WSREP: conc slot cancel, no atomics\n");
+#else
+	os_fast_mutex_lock(&srv_conc_mutex);
+	if (trx->wsrep_event) {
+		if (wsrep_debug) 
+			fprintf(stderr, "WSREP: conc slot cancel\n");
+		os_event_set(trx->wsrep_event);
+	}
+	os_fast_mutex_unlock(&srv_conc_mutex);
+#endif
+}
+#endif /* WITH_WSREP */
 
diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc
index 8e01ea7402e..d421980d41a 100644
--- a/storage/xtradb/srv/srv0srv.cc
+++ b/storage/xtradb/srv/srv0srv.cc
@@ -84,6 +84,14 @@ ulong		innobase_thd_get_thread_id(const void* thd);
 /* prototypes for new functions added to ha_innodb.cc */
 ibool	innobase_get_slow_log();
 
+#ifdef WITH_WSREP
+extern int wsrep_debug;
+extern int wsrep_trx_is_aborting(void *thd_ptr);
+#endif
+/* The following counter is incremented whenever there is some user activity
+in the server */
+UNIV_INTERN ulint	srv_activity_count	= 0;
+
 /* The following is the maximum allowed duration of a lock wait. */
 UNIV_INTERN ulint	srv_fatal_semaphore_wait_threshold = 600;
 
@@ -254,6 +262,10 @@ srv_printf_innodb_monitor() will request mutex acquisition
 with mutex_enter(), which will wait until it gets the mutex. */
 #define MUTEX_NOWAIT(mutex_skipped)	((mutex_skipped) < MAX_MUTEX_NOWAIT)
 
+#ifdef WITH_INNODB_DISALLOW_WRITES
+UNIV_INTERN os_event_t	srv_allow_writes_event;
+#endif /* WITH_INNODB_DISALLOW_WRITES */
+
 /** The sort order table of the MySQL latin1_swedish_ci character set
 collation */
 UNIV_INTERN const byte*	srv_latin1_ordering;
@@ -1150,6 +1162,14 @@ srv_init(void)
 	dict_ind_init();
 
 	srv_conc_init();
+#ifdef WITH_INNODB_DISALLOW_WRITES
+	/* Writes have to be enabled on init or else we hang. Thus, we
+	always set the event here regardless of innobase_disallow_writes.
+	That flag will always be 0 at this point because it isn't settable
+	via my.cnf or command line arg. */
+	srv_allow_writes_event = os_event_create();
+	os_event_set(srv_allow_writes_event);
+#endif /* WITH_INNODB_DISALLOW_WRITES */
 
 	/* Initialize some INFORMATION SCHEMA internal structures */
 	trx_i_s_cache_init(trx_i_s_cache);
@@ -2158,7 +2178,20 @@ loop:
 
 	if (sync_array_print_long_waits(&waiter, &sema)
 	    && sema == old_sema && os_thread_eq(waiter, old_waiter)) {
+#if defined(WITH_WSREP) && defined(WITH_INNODB_DISALLOW_WRITES)
+	  if (srv_allow_writes_event->is_set) {
+#endif /* WITH_WSREP */
 		fatal_cnt++;
+#if defined(WITH_WSREP) && defined(WITH_INNODB_DISALLOW_WRITES)
+	  } else {
+		fprintf(stderr,
+			"WSREP: avoiding InnoDB self crash due to long "
+			"semaphore wait of  > %lu seconds\n"
+			"Server is processing SST donor operation, "
+			"fatal_cnt now: %lu",
+			(ulong) srv_fatal_semaphore_wait_threshold, fatal_cnt);
+	  }
+#endif /* WITH_WSREP */
 		if (fatal_cnt > 10) {
 
 			fprintf(stderr,
diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc
index 1089607c6d1..eb2af877a6d 100644
--- a/storage/xtradb/trx/trx0roll.cc
+++ b/storage/xtradb/trx/trx0roll.cc
@@ -45,6 +45,9 @@ Created 3/26/1996 Heikki Tuuri
 #include "pars0pars.h"
 #include "srv0mon.h"
 #include "trx0sys.h"
+#ifdef WITH_WSREP
+#include "ha_prototypes.h"
+#endif /* WITH_WSREP */
 
 /** This many pages must be undone before a truncate is tried within
 rollback */
@@ -382,6 +385,13 @@ trx_rollback_to_savepoint_for_mysql_low(
 
 	trx->op_info = "";
 
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd) &&
+	    trx->lock.was_chosen_as_deadlock_victim) {
+		trx->lock.was_chosen_as_deadlock_victim = FALSE;
+	}
+#endif
+
 	return(err);
 }
 
@@ -1020,6 +1030,12 @@ trx_roll_try_truncate(
 	if (trx->update_undo) {
 		trx_undo_truncate_end(trx, trx->update_undo, limit);
 	}
+
+#ifdef WITH_WSREP_OUT
+	if (wsrep_on(trx->mysql_thd)) {
+		trx->lock.was_chosen_as_deadlock_victim = FALSE;
+	}
+#endif /* WITH_WSREP */
 }
 
 /***********************************************************************//**
diff --git a/storage/xtradb/trx/trx0sys.cc b/storage/xtradb/trx/trx0sys.cc
index aab99a793f6..19d14bf6f38 100644
--- a/storage/xtradb/trx/trx0sys.cc
+++ b/storage/xtradb/trx/trx0sys.cc
@@ -44,6 +44,10 @@ Created 3/26/1996 Heikki Tuuri
 #include "os0file.h"
 #include "read0read.h"
 
+#ifdef WITH_WSREP
+#include "ha_prototypes.h" /* wsrep_is_wsrep_xid() */
+#endif /* */
+
 /** The file format tag structure with id and name. */
 struct file_format_t {
 	ulint		id;		/*!< id of the file format */
@@ -174,7 +178,12 @@ trx_sys_flush_max_trx_id(void)
 	mtr_t		mtr;
 	trx_sysf_t*	sys_header;
 
+#ifndef WITH_WSREP
+       /* wsrep_fake_trx_id  violates this assert
+        * Copied from trx_sys_get_new_trx_id
+        */
 	ut_ad(mutex_own(&trx_sys->mutex));
+#endif /* WITH_WSREP */
 
 	if (!srv_read_only_mode) {
 		mtr_start(&mtr);
@@ -202,9 +211,14 @@ trx_sys_update_mysql_binlog_offset(
 	ib_int64_t	offset,	/*!< in: position in that log file */
 	ulint		field,	/*!< in: offset of the MySQL log info field in
 				the trx sys header */
+#ifdef WITH_WSREP
+        trx_sysf_t*     sys_header, /*!< in: trx sys header */
+#endif /* WITH_WSREP */
 	mtr_t*		mtr)	/*!< in: mtr */
 {
+#ifndef WITH_WSREP
 	trx_sysf_t*	sys_header;
+#endif /* !WITH_WSREP */
 
 	if (ut_strlen(file_name) >= TRX_SYS_MYSQL_LOG_NAME_LEN) {
 
@@ -213,7 +227,9 @@ trx_sys_update_mysql_binlog_offset(
 		return;
 	}
 
+#ifndef WITH_WSREP
 	sys_header = trx_sysf_get(mtr);
+#endif /* !WITH_WSREP */
 
 	if (mach_read_from_4(sys_header + field
 			     + TRX_SYS_MYSQL_LOG_MAGIC_N_FLD)
@@ -300,6 +316,124 @@ trx_sys_print_mysql_binlog_offset(void)
 	mtr_commit(&mtr);
 }
 
+#ifdef WITH_WSREP
+
+#ifdef UNIV_DEBUG
+static long long trx_sys_cur_xid_seqno = -1;
+static unsigned char trx_sys_cur_xid_uuid[16];
+
+long long read_wsrep_xid_seqno(const XID* xid)
+{
+    long long seqno;
+    memcpy(&seqno, xid->data + 24, sizeof(long long));
+    return seqno;
+}
+
+void read_wsrep_xid_uuid(const XID* xid, unsigned char* buf)
+{
+    memcpy(buf, xid->data + 8, 16);
+}
+
+#endif /* UNIV_DEBUG */
+
+void
+trx_sys_update_wsrep_checkpoint(
+        const XID*      xid,        /*!< in: transaction XID */
+        trx_sysf_t*     sys_header, /*!< in: sys_header */
+        mtr_t*          mtr)        /*!< in: mtr */
+{
+#ifdef UNIV_DEBUG
+        {
+            /* Check that seqno is monotonically increasing */
+            unsigned char xid_uuid[16];
+            long long xid_seqno = read_wsrep_xid_seqno(xid);
+            read_wsrep_xid_uuid(xid, xid_uuid);
+            if (!memcmp(xid_uuid, trx_sys_cur_xid_uuid, 8))
+            {
+                ut_ad(xid_seqno > trx_sys_cur_xid_seqno);
+                trx_sys_cur_xid_seqno = xid_seqno;
+            }
+            else
+            {
+                memcpy(trx_sys_cur_xid_uuid, xid_uuid, 16);
+            }
+            trx_sys_cur_xid_seqno = xid_seqno;
+        }
+#endif /* UNIV_DEBUG */
+
+        ut_ad(xid && mtr);
+        ut_a(xid->formatID == -1 || wsrep_is_wsrep_xid((const void *)xid));
+
+        if (mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
+                             + TRX_SYS_WSREP_XID_MAGIC_N_FLD)
+            != TRX_SYS_WSREP_XID_MAGIC_N) {
+                mlog_write_ulint(sys_header + TRX_SYS_WSREP_XID_INFO
+                                 + TRX_SYS_WSREP_XID_MAGIC_N_FLD,
+                                 TRX_SYS_WSREP_XID_MAGIC_N,
+                                 MLOG_4BYTES, mtr);
+        }
+
+        mlog_write_ulint(sys_header + TRX_SYS_WSREP_XID_INFO
+                         + TRX_SYS_WSREP_XID_FORMAT,
+                         (int)xid->formatID,
+                         MLOG_4BYTES, mtr);
+        mlog_write_ulint(sys_header + TRX_SYS_WSREP_XID_INFO
+                         + TRX_SYS_WSREP_XID_GTRID_LEN,
+                         (int)xid->gtrid_length,
+                         MLOG_4BYTES, mtr);
+        mlog_write_ulint(sys_header + TRX_SYS_WSREP_XID_INFO
+                         + TRX_SYS_WSREP_XID_BQUAL_LEN,
+                         (int)xid->bqual_length,
+                         MLOG_4BYTES, mtr);
+        mlog_write_string(sys_header + TRX_SYS_WSREP_XID_INFO
+                          + TRX_SYS_WSREP_XID_DATA,
+                          (const unsigned char*) xid->data,
+                          XIDDATASIZE, mtr);
+
+}
+
+void
+trx_sys_read_wsrep_checkpoint(XID* xid)
+/*===================================*/
+{
+        trx_sysf_t* sys_header;
+	mtr_t	    mtr;
+        ulint       magic;
+
+        ut_ad(xid);
+
+	mtr_start(&mtr);
+
+	sys_header = trx_sysf_get(&mtr);
+
+        if ((magic = mach_read_from_4(sys_header + TRX_SYS_WSREP_XID_INFO
+                                      + TRX_SYS_WSREP_XID_MAGIC_N_FLD))
+            != TRX_SYS_WSREP_XID_MAGIC_N) {
+                memset(xid, 0, sizeof(*xid));
+                xid->formatID = -1;
+                trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr);
+                mtr_commit(&mtr);
+                return;
+        }
+
+        xid->formatID     = (int)mach_read_from_4(
+                sys_header
+                + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_FORMAT);
+        xid->gtrid_length = (int)mach_read_from_4(
+                sys_header
+                + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_GTRID_LEN);
+        xid->bqual_length = (int)mach_read_from_4(
+                sys_header
+                + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_BQUAL_LEN);
+        ut_memcpy(xid->data,
+                  sys_header + TRX_SYS_WSREP_XID_INFO + TRX_SYS_WSREP_XID_DATA,
+                  XIDDATASIZE);
+
+	mtr_commit(&mtr);
+}
+
+#endif /* WITH_WSREP */
+
 /*****************************************************************//**
 Prints to stderr the MySQL master log offset info in the trx system header if
 the magic number shows it valid. */
diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc
index f2c78bafd86..d26517637cd 100644
--- a/storage/xtradb/trx/trx0trx.cc
+++ b/storage/xtradb/trx/trx0trx.cc
@@ -294,6 +294,9 @@ trx_create(void)
 
 	trx->lock.table_locks = ib_vector_create(
 		heap_alloc, sizeof(void**), 32);
+#ifdef WITH_WSREP
+	trx->wsrep_event = NULL;
+#endif /* WITH_WSREP */
 
 	return(trx);
 }
@@ -1041,6 +1044,11 @@ trx_start_low(
 			srv_undo_logs, srv_undo_tablespaces);
 	}
 
+#ifdef WITH_WSREP
+        memset(&trx->xid, 0, sizeof(trx->xid));
+        trx->xid.formatID = -1;
+#endif /* WITH_WSREP */
+
 	/* The initial value for trx->no: TRX_ID_MAX is used in
 	read_view_open_now: */
 
@@ -1166,6 +1174,9 @@ trx_write_serialisation_history(
 	trx_t*		trx,	/*!< in/out: transaction */
 	mtr_t*		mtr)	/*!< in/out: mini-transaction */
 {
+#ifdef WITH_WSREP
+        trx_sysf_t* sys_header;
+#endif /* WITH_WSREP */
 	trx_rseg_t*	rseg;
 
 	rseg = trx->rseg;
@@ -1212,6 +1223,15 @@ trx_write_serialisation_history(
 
 	MONITOR_INC(MONITOR_TRX_COMMIT_UNDO);
 
+#ifdef WITH_WSREP
+	sys_header = trx_sysf_get(mtr);
+	/* Update latest MySQL wsrep XID in trx sys header. */
+	if (wsrep_is_wsrep_xid((const void *)&trx->xid))
+	{
+		trx_sys_update_wsrep_checkpoint(&trx->xid, sys_header, mtr);
+	}
+#endif /* WITH_WSREP */
+
 	/* Update the latest MySQL binlog name and offset info
 	in trx sys header if MySQL binlogging is on or the database
 	server is a MySQL replication slave */
@@ -1222,7 +1242,11 @@ trx_write_serialisation_history(
 		trx_sys_update_mysql_binlog_offset(
 			trx->mysql_log_file_name,
 			trx->mysql_log_offset,
-			TRX_SYS_MYSQL_LOG_INFO, mtr);
+			TRX_SYS_MYSQL_LOG_INFO, 
+#ifdef WITH_WSREP
+                        sys_header,
+#endif /* WITH_WSREP */
+			mtr);
 
 		trx->mysql_log_file_name = NULL;
 	}
@@ -1527,6 +1551,11 @@ trx_commit_in_memory(
 	ut_ad(!trx->in_ro_trx_list);
 	ut_ad(!trx->in_rw_trx_list);
 
+#ifdef WITH_WSREP
+	if (wsrep_on(trx->mysql_thd)) {
+		trx->lock.was_chosen_as_deadlock_victim = FALSE;
+	}
+#endif
 	trx->dict_operation = TRX_DICT_OP_NONE;
 
 	trx->error_state = DB_SUCCESS;
@@ -1709,6 +1738,10 @@ trx_commit_or_rollback_prepare(
 
 	switch (trx->state) {
 	case TRX_STATE_NOT_STARTED:
+#ifdef WITH_WSREP
+		ut_d(trx->start_file = __FILE__);
+		ut_d(trx->start_line = __LINE__);
+#endif /* WITH_WSREP */
 		trx_start_low(trx);
 		/* fall through */
 	case TRX_STATE_ACTIVE:
@@ -2480,6 +2513,10 @@ trx_start_if_not_started_low(
 {
 	switch (trx->state) {
 	case TRX_STATE_NOT_STARTED:
+#ifdef WITH_WSREP
+		ut_d(trx->start_file = __FILE__);
+		ut_d(trx->start_line = __LINE__);
+#endif /* WITH_WSREP */
 		trx_start_low(trx);
 		/* fall through */
 	case TRX_STATE_ACTIVE:
@@ -2514,6 +2551,10 @@ trx_start_for_ddl_low(
 
 		trx->ddl = true;
 
+#ifdef WITH_WSREP
+		ut_d(trx->start_file = __FILE__);
+		ut_d(trx->start_line = __LINE__);
+#endif /* WITH_WSREP */
 		trx_start_low(trx);
 		return;
 
diff --git a/storage/xtradb/wsrep/md5.cc b/storage/xtradb/wsrep/md5.cc
new file mode 100644
index 00000000000..30be6ab7dd3
--- /dev/null
+++ b/storage/xtradb/wsrep/md5.cc
@@ -0,0 +1,74 @@
+/*
+   Copyright (c) 2014 SkySQL AB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifdef WITH_WSREP
+
+#if defined(HAVE_YASSL)
+#include "my_config.h"
+#include "md5.hpp"
+#elif defined(HAVE_OPENSSL)
+#include <openssl/md5.h>
+#endif /* HAVE_YASSL */
+
+/* Initialize md5 object. */
+void *wsrep_md5_init()
+{
+#if defined(HAVE_YASSL)
+  TaoCrypt::MD5 *hasher= new TaoCrypt::MD5;
+  return (void*)hasher;
+#elif defined(HAVE_OPENSSL)
+  MD5_CTX *ctx = new MD5_CTX();
+  MD5_Init (ctx);
+  return (void *)ctx;
+#endif /* HAVE_YASSL */
+}
+
+/**
+  Supply message to be hashed.
+
+  @param ctx [IN]    Pointer to MD5 context
+  @param buf [IN]    Message to be computed.
+  @param len [IN]    Length of the message.
+*/
+void wsrep_md5_update(void *ctx, char* buf, int len)
+{
+#if defined(HAVE_YASSL)
+  ((TaoCrypt::MD5 *)ctx)->Update((TaoCrypt::byte *) buf, len);
+#elif defined(HAVE_OPENSSL)
+  MD5_Update((MD5_CTX*)(ctx), buf, len);
+#endif /* HAVE_YASSL */
+}
+
+/**
+  Place computed MD5 digest into the given buffer.
+
+  @param digest [OUT] Computed MD5 digest
+  @param ctx    [IN]  Pointer to MD5 context
+*/
+void wsrep_compute_md5_hash(char *digest, void *ctx)
+{
+#if defined(HAVE_YASSL)
+  ((TaoCrypt::MD5*)ctx)->Final((TaoCrypt::byte *) digest);
+  delete (TaoCrypt::MD5*)ctx;
+#elif defined(HAVE_OPENSSL)
+  MD5_Final ((unsigned char*)digest, (MD5_CTX*)ctx);
+  delete (MD5_CTX*)ctx;
+#endif /* HAVE_YASSL */
+}
+
+#endif /* WITH_WSREP */
+
diff --git a/storage/xtradb/wsrep/wsrep_md5.h b/storage/xtradb/wsrep/wsrep_md5.h
new file mode 100644
index 00000000000..1339f7f4b86
--- /dev/null
+++ b/storage/xtradb/wsrep/wsrep_md5.h
@@ -0,0 +1,26 @@
+#ifndef WSREP_MD5_INCLUDED
+#define WSREP_MD5_INCLUDED
+
+/* Copyright (c) 2014 SkySQL AB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+*/
+
+#ifdef WITH_WSREP
+void *wsrep_md5_init();
+void wsrep_md5_update(void *ctx, char* buf, int len);
+void wsrep_compute_md5_hash(char *digest, void *ctx);
+#endif /* WITH_WSREP */
+
+#endif /* WSREP_MD5_INCLUDED */
diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt
index 698d06d1889..19f6bc14663 100644
--- a/support-files/CMakeLists.txt
+++ b/support-files/CMakeLists.txt
@@ -41,7 +41,7 @@ ELSE()
   SET(inst_location ${INSTALL_SUPPORTFILESDIR})
 ENDIF()
 
-FOREACH(inifile my-huge my-innodb-heavy-4G my-large my-medium my-small)
+FOREACH(inifile my-huge my-innodb-heavy-4G my-large my-medium my-small wsrep)
   CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${inifile}.cnf.sh 
                  ${CMAKE_CURRENT_BINARY_DIR}/${inifile}.${ini_file_extension} @ONLY)
   INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${inifile}.${ini_file_extension}
@@ -65,6 +65,7 @@ IF(UNIX)
   IF(INSTALL_SUPPORTFILESDIR)
     INSTALL(FILES magic DESTINATION ${inst_location} COMPONENT SupportFiles)
     INSTALL(DIRECTORY RHEL4-SElinux/ DESTINATION ${inst_location}/SELinux/RHEL4 COMPONENT SupportFiles)
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/wsrep_notify DESTINATION ${inst_location} COMPONENT SupportFiles)
   ENDIF()
 
   INSTALL(FILES mysql.m4 DESTINATION ${INSTALL_SHAREDIR}/aclocal COMPONENT Development)
diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh
index a1ee5c4c653..c98c8284a97 100644
--- a/support-files/mysql.server.sh
+++ b/support-files/mysql.server.sh
@@ -253,6 +253,8 @@ wait_for_gone () {
 
 wait_for_ready () {
 
+  sst_progress_file=$datadir/sst_in_progress
+
   i=0
   while test $i -ne $service_startup_timeout ; do
 
@@ -261,6 +263,11 @@ wait_for_ready () {
       return 0
     fi
 
+    if test -e $sst_progress_file && [ $startup_sleep -ne 10 ];then
+      echo $echo_n "SST in progress, setting sleep higher"
+      startup_sleep=10
+    fi
+
     echo $echo_n ".$echo_c"
     i=`expr $i + 1`
     sleep 1
@@ -349,7 +356,10 @@ case "$mode" in
     # Stop the service and regardless of whether it was
     # running or not, start it again.
     if $0 stop  $other_args; then
-      $0 start $other_args
+      if ! $0 start $other_args; then
+        log_failure_msg "Failed to restart server."
+        exit 1
+      fi
     else
       log_failure_msg "Failed to stop running server, so refusing to try to start."
       exit 1
@@ -426,10 +436,16 @@ case "$mode" in
     fi
     exit $r
     ;;
+  'bootstrap')
+      # Bootstrap the cluster, start the first node
+      # that initiate the cluster
+      echo $echo_n "Bootstrapping the cluster"
+      $0 start $other_args --wsrep-new-cluster
+      ;;
   *)
       # usage
       basename=`basename "$0"`
-      echo "Usage: $basename  {start|stop|restart|reload|force-reload|status|configtest}  [ MySQL server options ]"
+      echo "Usage: $basename  {start|stop|restart|reload|force-reload|status|configtest|bootstrap}  [ MySQL server options ]"
       exit 1
     ;;
 esac
diff --git a/support-files/mysql.spec.sh b/support-files/mysql.spec.sh
index d8de3a41e7f..0a158b4fe49 100644
--- a/support-files/mysql.spec.sh
+++ b/support-files/mysql.spec.sh
@@ -69,6 +69,14 @@
 #
 
 # ----------------------------------------------------------------------------
+# wsrep builds
+# ----------------------------------------------------------------------------
+%if %{defined with_wsrep}
+%define mysql_version @VERSION@_wsrep_@WSREP_API_VERSION@.@WSREP_PATCH_VERSION@
+%define wsrep_version @WSREP_VERSION@
+%endif
+
+# ----------------------------------------------------------------------------
 # Commercial builds
 # ----------------------------------------------------------------------------
 %if %{undefined commercial}
@@ -283,9 +291,16 @@ documentation and the manual for more information.
 ##############################################################################
 
 %package -n MySQL-server%{product_suffix}
+%if %{defined with_wsrep}
+Version:        %{mysql_version}
+%endif
 Summary:        MySQL: a very fast and reliable SQL database server
 Group:          Applications/Databases
+%if %{defined with_wsrep}
+Requires:       %{distro_requires} rsync lsof
+%else
 Requires:       %{distro_requires}
+%endif
 %if 0%{?commercial}
 Obsoletes:      MySQL-server
 %else
@@ -319,6 +334,9 @@ and the manual for more information.
 This package includes the MySQL server binary as well as related utilities
 to run and administer a MySQL server.
 
+%if %{defined with_wsrep}
+Built with wsrep patch %{wsrep_version}.
+%endif
 If you want to access and work with the database, you have to install
 package "MySQL-client%{product_suffix}" as well!
 
@@ -410,6 +428,7 @@ This package contains the shared libraries (*.so*) which certain languages
 and applications need to dynamically load and use MySQL.
 
 # ----------------------------------------------------------------------------
+%if %{undefined with_wsrep}
 %package -n MySQL-embedded%{product_suffix}
 Summary:        MySQL - Embedded library
 Group:          Applications/Databases
@@ -439,6 +458,7 @@ The API is identical for the embedded MySQL version and the
 client/server version.
 
 For a description of MySQL see the base MySQL RPM or http://www.mysql.com/
+%endif
 
 ##############################################################################
 %prep
@@ -514,6 +534,9 @@ mkdir debug
            -DMYSQL_UNIX_ADDR="%{mysqldatadir}/mysql.sock" \
            -DFEATURE_SET="%{feature_set}" \
            -DCOMPILATION_COMMENT="%{compilation_comment_debug}" \
+%if %{defined with_wsrep}
+           -DWITH_WSREP=1 \
+%endif
            -DMYSQL_SERVER_SUFFIX="%{server_suffix}"
   echo BEGIN_DEBUG_CONFIG ; egrep '^#define' include/config.h ; echo END_DEBUG_CONFIG
   make ${MAKE_JFLAG} VERBOSE=1
@@ -529,6 +552,9 @@ mkdir release
            -DMYSQL_UNIX_ADDR="%{mysqldatadir}/mysql.sock" \
            -DFEATURE_SET="%{feature_set}" \
            -DCOMPILATION_COMMENT="%{compilation_comment_release}" \
+%if %{defined with_wsrep}
+           -DWITH_WSREP=1 \
+%endif
            -DMYSQL_SERVER_SUFFIX="%{server_suffix}"
   echo BEGIN_NORMAL_CONFIG ; egrep '^#define' include/config.h ; echo END_NORMAL_CONFIG
   make ${MAKE_JFLAG} VERBOSE=1
@@ -591,11 +617,20 @@ install -m 755 $MBD/release/support-files/mysql.server $RBR%{_sysconfdir}/init.d
 
 # Create a symlink "rcmysql", pointing to the init.script. SuSE users
 # will appreciate that, as all services usually offer this.
-ln -s %{_sysconfdir}/init.d/mysql $RBR%{_sbindir}/rcmysql
+ln -sf %{_sysconfdir}/init.d/mysql $RBR%{_sbindir}/rcmysql
+
+%if %{defined with_wsrep}
+# Create a wsrep_sst_rsync_wan symlink.
+install -d $RBR%{_bindir}
+ln -sf wsrep_sst_rsync $RBR%{_bindir}/wsrep_sst_rsync_wan
+%endif
 
 # Touch the place where the my.cnf config file might be located
 # Just to make sure it's in the file list and marked as a config file
 touch $RBR%{_sysconfdir}/my.cnf
+%if %{defined with_wsrep}
+touch $RBR%{_sysconfdir}/wsrep.cnf
+%endif
 
 # Install SELinux files in datadir
 install -m 600 $MBD/%{src_dir}/support-files/RHEL4-SElinux/mysql.{fc,te} \
@@ -1057,6 +1092,11 @@ echo "====="                                     >> $STATUS_HISTORY
 %doc %{src_dir}/Docs/INFO_SRC*
 %doc release/Docs/INFO_BIN*
 %doc release/support-files/my-*.cnf
+%if %{defined with_wsrep}
+%doc %{src_dir}/Docs/README-wsrep
+%doc release/support-files/wsrep.cnf
+%doc release/support-files/wsrep_notify
+%endif
 
 %if 0%{?commercial}
 %doc %attr(644, root, root) %{_infodir}/mysql.info*
@@ -1092,6 +1132,9 @@ echo "====="                                     >> $STATUS_HISTORY
 %doc %attr(644, root, man) %{_mandir}/man1/resolveip.1*
 
 %ghost %config(noreplace,missingok) %{_sysconfdir}/my.cnf
+%if %{defined with_wsrep}
+%ghost %config(noreplace,missingok) %{_sysconfdir}/wsrep.cnf
+%endif
 
 %attr(755, root, root) %{_bindir}/innochecksum
 %attr(755, root, root) %{_bindir}/my_print_defaults
@@ -1118,6 +1161,14 @@ echo "====="                                     >> $STATUS_HISTORY
 %attr(755, root, root) %{_bindir}/replace
 %attr(755, root, root) %{_bindir}/resolve_stack_dump
 %attr(755, root, root) %{_bindir}/resolveip
+%if %{defined with_wsrep}
+%attr(755, root, root) %{_bindir}/wsrep_sst_common
+%attr(755, root, root) %{_bindir}/wsrep_sst_mysqldump
+%attr(755, root, root) %{_bindir}/wsrep_sst_rsync
+%attr(755, root, root) %{_bindir}/wsrep_sst_rsync_wan
+%attr(755, root, root) %{_bindir}/wsrep_sst_xtrabackup
+%attr(755, root, root) %{_bindir}/wsrep_sst_xtrabackup-v2
+%endif
 
 %attr(755, root, root) %{_sbindir}/mysqld
 %attr(755, root, root) %{_sbindir}/mysqld-debug
@@ -1196,8 +1247,10 @@ echo "====="                                     >> $STATUS_HISTORY
 %defattr(-, root, root, 0755)
 %attr(-, root, root) %{_datadir}/mysql-test
 %attr(755, root, root) %{_bindir}/mysql_client_test
+%if %{undefined with_wsrep}
 %attr(755, root, root) %{_bindir}/mysql_client_test_embedded
 %attr(755, root, root) %{_bindir}/mysqltest_embedded
+%endif
 %doc %attr(644, root, man) %{_mandir}/man1/mysql_client_test.1*
 %doc %attr(644, root, man) %{_mandir}/man1/mysql-stress-test.pl.1*
 %doc %attr(644, root, man) %{_mandir}/man1/mysql-test-run.pl.1*
@@ -1205,11 +1258,13 @@ echo "====="                                     >> $STATUS_HISTORY
 %doc %attr(644, root, man) %{_mandir}/man1/mysqltest_embedded.1*
 
 # ----------------------------------------------------------------------------
+%if %{undefined with_wsrep}
 %files -n MySQL-embedded%{product_suffix}
 %defattr(-, root, root, 0755)
 %attr(755, root, root) %{_bindir}/mysql_embedded
 %attr(644, root, root) %{_libdir}/mysql/libmysqld.a
 %attr(644, root, root) %{_libdir}/mysql/libmysqld-debug.a
+%endif
 
 ##############################################################################
 # The spec file changelog only includes changes made to the spec file
@@ -1241,6 +1296,10 @@ echo "====="                                     >> $STATUS_HISTORY
 
 - Make sure newly added "SPECIFIC-ULN/" directory does not disturb packaging.
   
+* Wed Dec 07 2011 Alexey Yurchenko <alexey.yurchenko@codership.com>
+
+- wsrep-related cleanups.
+
 * Wed Sep 28 2011 Joerg Bruehe <joerg.bruehe@oracle.com>
 
 - Fix duplicate mentioning of "mysql_plugin" and its manual page,
diff --git a/support-files/rpm/server.cnf b/support-files/rpm/server.cnf
index fb52635d4d8..6170a2e4c49 100644
--- a/support-files/rpm/server.cnf
+++ b/support-files/rpm/server.cnf
@@ -11,6 +11,22 @@
 # this is only for the mysqld standalone daemon
 [mysqld]
 
+#
+# * Galera-related settings
+#
+[galera]
+# Mandatory settings
+#wsrep_provider=
+#wsrep_cluster_address=
+#wsrep_slave_threads=1
+#binlog_format=row
+#default_storage_engine=InnoDB
+#innodb_autoinc_lock_mode=2
+#query_cache_size=0
+#
+# Optional setting
+#innodb_flush_log_at_trx_commit=0
+
 # this is only for embedded server
 [embedded]
 
diff --git a/support-files/wsrep.cnf b/support-files/wsrep.cnf
new file mode 100644
index 00000000000..756d4f6783b
--- /dev/null
+++ b/support-files/wsrep.cnf
@@ -0,0 +1,129 @@
+# This file contains wsrep-related mysqld options. It should be included
+# in the main MySQL configuration file.
+#
+# Options that need to be customized:
+#  - wsrep_provider
+#  - wsrep_cluster_address
+#  - wsrep_sst_auth
+# The rest of defaults should work out of the box.
+
+##
+## mysqld options _MANDATORY_ for correct opration of the cluster
+##
+[mysqld]
+
+# (This must be substituted by wsrep_format)
+binlog_format=ROW
+
+# Currently only InnoDB storage engine is supported
+default-storage-engine=innodb
+
+# to avoid issues with 'bulk mode inserts' using autoinc
+innodb_autoinc_lock_mode=2
+
+# This is a must for paralell applying
+innodb_locks_unsafe_for_binlog=1
+
+# Query Cache is not supported with wsrep
+query_cache_size=0
+query_cache_type=0
+
+# Override bind-address
+# In some systems bind-address defaults to 127.0.0.1, and with mysqldump SST
+# it will have (most likely) disastrous consequences on donor node
+bind-address=0.0.0.0
+
+##
+## WSREP options
+##
+
+# Full path to wsrep provider library or 'none'
+wsrep_provider=none
+
+# Provider specific configuration options
+#wsrep_provider_options=
+
+# Logical cluster name. Should be the same for all nodes.
+wsrep_cluster_name="my_wsrep_cluster"
+
+# Group communication system handle
+#wsrep_cluster_address="dummy://"
+
+# Human-readable node name (non-unique). Hostname by default.
+#wsrep_node_name=
+
+# Base replication <address|hostname>[:port] of the node.
+# The values supplied will be used as defaults for state transfer receiving,
+# listening ports and so on. Default: address of the first network interface.
+#wsrep_node_address=
+
+# Address for incoming client connections. Autodetect by default.
+#wsrep_node_incoming_address=
+
+# How many threads will process writesets from other nodes
+wsrep_slave_threads=1
+
+# DBUG options for wsrep provider
+#wsrep_dbug_option
+
+# Generate fake primary keys for non-PK tables (required for multi-master
+# and parallel applying operation)
+wsrep_certify_nonPK=1
+
+# Maximum number of rows in write set
+wsrep_max_ws_rows=131072
+
+# Maximum size of write set
+wsrep_max_ws_size=1073741824
+
+# to enable debug level logging, set this to 1
+wsrep_debug=0
+
+# convert locking sessions into transactions
+wsrep_convert_LOCK_to_trx=0
+
+# how many times to retry deadlocked autocommits
+wsrep_retry_autocommit=1
+
+# change auto_increment_increment and auto_increment_offset automatically
+wsrep_auto_increment_control=1
+
+# retry autoinc insert, which failed for duplicate key error
+wsrep_drupal_282555_workaround=0
+
+# enable "strictly synchronous" semantics for read operations
+wsrep_causal_reads=0
+
+# Command to call when node status or cluster membership changes.
+# Will be passed all or some of the following options:
+# --status  - new status of this node
+# --uuid    - UUID of the cluster
+# --primary - whether the component is primary or not ("yes"/"no")
+# --members - comma-separated list of members
+# --index   - index of this node in the list
+wsrep_notify_cmd=
+
+##
+## WSREP State Transfer options
+##
+
+# State Snapshot Transfer method
+wsrep_sst_method=rsync
+
+# Address which donor should send State Snapshot to.
+# Should be the address of THIS node. DON'T SET IT TO DONOR ADDRESS!!!
+# (SST method dependent. Defaults to the first IP of the first interface)
+#wsrep_sst_receive_address=
+
+# SST authentication string. This will be used to send SST to joining nodes.
+# Depends on SST method. For mysqldump method it is root:<root password>
+wsrep_sst_auth=root:
+
+# Desired SST donor name.
+#wsrep_sst_donor=
+
+# Reject client queries when donating SST (false)
+#wsrep_sst_donor_rejects_queries=0
+
+# Protocol version to use
+# wsrep_protocol_version=
diff --git a/support-files/wsrep.cnf.sh b/support-files/wsrep.cnf.sh
new file mode 100644
index 00000000000..756d4f6783b
--- /dev/null
+++ b/support-files/wsrep.cnf.sh
@@ -0,0 +1,129 @@
+# This file contains wsrep-related mysqld options. It should be included
+# in the main MySQL configuration file.
+#
+# Options that need to be customized:
+#  - wsrep_provider
+#  - wsrep_cluster_address
+#  - wsrep_sst_auth
+# The rest of defaults should work out of the box.
+
+##
+## mysqld options _MANDATORY_ for correct opration of the cluster
+##
+[mysqld]
+
+# (This must be substituted by wsrep_format)
+binlog_format=ROW
+
+# Currently only InnoDB storage engine is supported
+default-storage-engine=innodb
+
+# to avoid issues with 'bulk mode inserts' using autoinc
+innodb_autoinc_lock_mode=2
+
+# This is a must for paralell applying
+innodb_locks_unsafe_for_binlog=1
+
+# Query Cache is not supported with wsrep
+query_cache_size=0
+query_cache_type=0
+
+# Override bind-address
+# In some systems bind-address defaults to 127.0.0.1, and with mysqldump SST
+# it will have (most likely) disastrous consequences on donor node
+bind-address=0.0.0.0
+
+##
+## WSREP options
+##
+
+# Full path to wsrep provider library or 'none'
+wsrep_provider=none
+
+# Provider specific configuration options
+#wsrep_provider_options=
+
+# Logical cluster name. Should be the same for all nodes.
+wsrep_cluster_name="my_wsrep_cluster"
+
+# Group communication system handle
+#wsrep_cluster_address="dummy://"
+
+# Human-readable node name (non-unique). Hostname by default.
+#wsrep_node_name=
+
+# Base replication <address|hostname>[:port] of the node.
+# The values supplied will be used as defaults for state transfer receiving,
+# listening ports and so on. Default: address of the first network interface.
+#wsrep_node_address=
+
+# Address for incoming client connections. Autodetect by default.
+#wsrep_node_incoming_address=
+
+# How many threads will process writesets from other nodes
+wsrep_slave_threads=1
+
+# DBUG options for wsrep provider
+#wsrep_dbug_option
+
+# Generate fake primary keys for non-PK tables (required for multi-master
+# and parallel applying operation)
+wsrep_certify_nonPK=1
+
+# Maximum number of rows in write set
+wsrep_max_ws_rows=131072
+
+# Maximum size of write set
+wsrep_max_ws_size=1073741824
+
+# to enable debug level logging, set this to 1
+wsrep_debug=0
+
+# convert locking sessions into transactions
+wsrep_convert_LOCK_to_trx=0
+
+# how many times to retry deadlocked autocommits
+wsrep_retry_autocommit=1
+
+# change auto_increment_increment and auto_increment_offset automatically
+wsrep_auto_increment_control=1
+
+# retry autoinc insert, which failed for duplicate key error
+wsrep_drupal_282555_workaround=0
+
+# enable "strictly synchronous" semantics for read operations
+wsrep_causal_reads=0
+
+# Command to call when node status or cluster membership changes.
+# Will be passed all or some of the following options:
+# --status  - new status of this node
+# --uuid    - UUID of the cluster
+# --primary - whether the component is primary or not ("yes"/"no")
+# --members - comma-separated list of members
+# --index   - index of this node in the list
+wsrep_notify_cmd=
+
+##
+## WSREP State Transfer options
+##
+
+# State Snapshot Transfer method
+wsrep_sst_method=rsync
+
+# Address which donor should send State Snapshot to.
+# Should be the address of THIS node. DON'T SET IT TO DONOR ADDRESS!!!
+# (SST method dependent. Defaults to the first IP of the first interface)
+#wsrep_sst_receive_address=
+
+# SST authentication string. This will be used to send SST to joining nodes.
+# Depends on SST method. For mysqldump method it is root:<root password>
+wsrep_sst_auth=root:
+
+# Desired SST donor name.
+#wsrep_sst_donor=
+
+# Reject client queries when donating SST (false)
+#wsrep_sst_donor_rejects_queries=0
+
+# Protocol version to use
+# wsrep_protocol_version=
diff --git a/support-files/wsrep.cnf.sh.moved b/support-files/wsrep.cnf.sh.moved
new file mode 100644
index 00000000000..756d4f6783b
--- /dev/null
+++ b/support-files/wsrep.cnf.sh.moved
@@ -0,0 +1,129 @@
+# This file contains wsrep-related mysqld options. It should be included
+# in the main MySQL configuration file.
+#
+# Options that need to be customized:
+#  - wsrep_provider
+#  - wsrep_cluster_address
+#  - wsrep_sst_auth
+# The rest of defaults should work out of the box.
+
+##
+## mysqld options _MANDATORY_ for correct opration of the cluster
+##
+[mysqld]
+
+# (This must be substituted by wsrep_format)
+binlog_format=ROW
+
+# Currently only InnoDB storage engine is supported
+default-storage-engine=innodb
+
+# to avoid issues with 'bulk mode inserts' using autoinc
+innodb_autoinc_lock_mode=2
+
+# This is a must for paralell applying
+innodb_locks_unsafe_for_binlog=1
+
+# Query Cache is not supported with wsrep
+query_cache_size=0
+query_cache_type=0
+
+# Override bind-address
+# In some systems bind-address defaults to 127.0.0.1, and with mysqldump SST
+# it will have (most likely) disastrous consequences on donor node
+bind-address=0.0.0.0
+
+##
+## WSREP options
+##
+
+# Full path to wsrep provider library or 'none'
+wsrep_provider=none
+
+# Provider specific configuration options
+#wsrep_provider_options=
+
+# Logical cluster name. Should be the same for all nodes.
+wsrep_cluster_name="my_wsrep_cluster"
+
+# Group communication system handle
+#wsrep_cluster_address="dummy://"
+
+# Human-readable node name (non-unique). Hostname by default.
+#wsrep_node_name=
+
+# Base replication <address|hostname>[:port] of the node.
+# The values supplied will be used as defaults for state transfer receiving,
+# listening ports and so on. Default: address of the first network interface.
+#wsrep_node_address=
+
+# Address for incoming client connections. Autodetect by default.
+#wsrep_node_incoming_address=
+
+# How many threads will process writesets from other nodes
+wsrep_slave_threads=1
+
+# DBUG options for wsrep provider
+#wsrep_dbug_option
+
+# Generate fake primary keys for non-PK tables (required for multi-master
+# and parallel applying operation)
+wsrep_certify_nonPK=1
+
+# Maximum number of rows in write set
+wsrep_max_ws_rows=131072
+
+# Maximum size of write set
+wsrep_max_ws_size=1073741824
+
+# to enable debug level logging, set this to 1
+wsrep_debug=0
+
+# convert locking sessions into transactions
+wsrep_convert_LOCK_to_trx=0
+
+# how many times to retry deadlocked autocommits
+wsrep_retry_autocommit=1
+
+# change auto_increment_increment and auto_increment_offset automatically
+wsrep_auto_increment_control=1
+
+# retry autoinc insert, which failed for duplicate key error
+wsrep_drupal_282555_workaround=0
+
+# enable "strictly synchronous" semantics for read operations
+wsrep_causal_reads=0
+
+# Command to call when node status or cluster membership changes.
+# Will be passed all or some of the following options:
+# --status  - new status of this node
+# --uuid    - UUID of the cluster
+# --primary - whether the component is primary or not ("yes"/"no")
+# --members - comma-separated list of members
+# --index   - index of this node in the list
+wsrep_notify_cmd=
+
+##
+## WSREP State Transfer options
+##
+
+# State Snapshot Transfer method
+wsrep_sst_method=rsync
+
+# Address which donor should send State Snapshot to.
+# Should be the address of THIS node. DON'T SET IT TO DONOR ADDRESS!!!
+# (SST method dependent. Defaults to the first IP of the first interface)
+#wsrep_sst_receive_address=
+
+# SST authentication string. This will be used to send SST to joining nodes.
+# Depends on SST method. For mysqldump method it is root:<root password>
+wsrep_sst_auth=root:
+
+# Desired SST donor name.
+#wsrep_sst_donor=
+
+# Reject client queries when donating SST (false)
+#wsrep_sst_donor_rejects_queries=0
+
+# Protocol version to use
+# wsrep_protocol_version=
diff --git a/support-files/wsrep_notify b/support-files/wsrep_notify
new file mode 100644
index 00000000000..bdbe3d12a39
--- /dev/null
+++ b/support-files/wsrep_notify
@@ -0,0 +1,102 @@
+#!/bin/sh -eu
+
+# This is a simple example of wsrep notification script (wsrep_notify_cmd).
+# It will create 'wsrep' schema and two tables in it: 'membeship' and 'status'
+# and fill them on every membership or node status change.
+#
+# Edit parameters below to specify the address and login to server.
+
+USER=root
+PSWD=rootpass
+HOST=127.0.0.1
+PORT=3306
+
+SCHEMA="wsrep"
+MEMB_TABLE="$SCHEMA.membership"
+STATUS_TABLE="$SCHEMA.status"
+
+BEGIN="
+SET wsrep_on=0;
+DROP SCHEMA IF EXISTS $SCHEMA; CREATE SCHEMA $SCHEMA;
+CREATE TABLE $MEMB_TABLE (
+    idx  INT UNIQUE PRIMARY KEY,
+    uuid CHAR(40) UNIQUE, /* node UUID */
+    name VARCHAR(32),     /* node name */
+    addr VARCHAR(256)     /* node address */
+) ENGINE=MEMORY;
+CREATE TABLE $STATUS_TABLE (
+    size   INT,      /* component size   */
+    idx    INT,      /* this node index  */
+    status CHAR(16), /* this node status */
+    uuid   CHAR(40), /* cluster UUID */
+    prim   BOOLEAN   /* if component is primary */
+) ENGINE=MEMORY;
+BEGIN;
+DELETE FROM $MEMB_TABLE;
+DELETE FROM $STATUS_TABLE;
+"
+END="COMMIT;"
+
+configuration_change()
+{
+    echo "$BEGIN;"
+
+    local idx=0
+
+    for NODE in $(echo $MEMBERS | sed s/,/\ /g)
+    do
+        echo "INSERT INTO $MEMB_TABLE VALUES ( $idx, "
+        # Don't forget to properly quote string values
+        echo "'$NODE'" | sed  s/\\//\',\'/g
+        echo ");"
+        idx=$(( $idx + 1 ))
+    done
+
+    echo "INSERT INTO $STATUS_TABLE VALUES($idx, $INDEX, '$STATUS', '$CLUSTER_UUID', $PRIMARY);"
+
+    echo "$END"
+}
+
+status_update()
+{
+    echo "SET wsrep_on=0; BEGIN; UPDATE $STATUS_TABLE SET status='$STATUS'; COMMIT;"
+}
+
+COM=status_update # not a configuration change by default
+
+while [ $# -gt 0 ]
+do
+    case $1 in
+    --status)
+        STATUS=$2
+        shift
+        ;;
+    --uuid)
+        CLUSTER_UUID=$2
+        shift
+        ;;
+    --primary)
+        [ "$2" = "yes" ] && PRIMARY="1" || PRIMARY="0"
+        COM=configuration_change
+        shift
+        ;;
+    --index)
+        INDEX=$2
+        shift
+        ;;
+    --members)
+        MEMBERS=$2
+        shift
+        ;;
+    esac
+    shift
+done
+
+# Undefined means node is shutting down
+if [ "$STATUS" != "Undefined" ]
+then
+    $COM | mysql -B -u$USER -p$PSWD -h$HOST -P$PORT
+fi
+
+exit 0
+#
diff --git a/support-files/wsrep_notify.sh b/support-files/wsrep_notify.sh
new file mode 100644
index 00000000000..bdbe3d12a39
--- /dev/null
+++ b/support-files/wsrep_notify.sh
@@ -0,0 +1,102 @@
+#!/bin/sh -eu
+
+# This is a simple example of wsrep notification script (wsrep_notify_cmd).
+# It will create 'wsrep' schema and two tables in it: 'membeship' and 'status'
+# and fill them on every membership or node status change.
+#
+# Edit parameters below to specify the address and login to server.
+
+USER=root
+PSWD=rootpass
+HOST=127.0.0.1
+PORT=3306
+
+SCHEMA="wsrep"
+MEMB_TABLE="$SCHEMA.membership"
+STATUS_TABLE="$SCHEMA.status"
+
+BEGIN="
+SET wsrep_on=0;
+DROP SCHEMA IF EXISTS $SCHEMA; CREATE SCHEMA $SCHEMA;
+CREATE TABLE $MEMB_TABLE (
+    idx  INT UNIQUE PRIMARY KEY,
+    uuid CHAR(40) UNIQUE, /* node UUID */
+    name VARCHAR(32),     /* node name */
+    addr VARCHAR(256)     /* node address */
+) ENGINE=MEMORY;
+CREATE TABLE $STATUS_TABLE (
+    size   INT,      /* component size   */
+    idx    INT,      /* this node index  */
+    status CHAR(16), /* this node status */
+    uuid   CHAR(40), /* cluster UUID */
+    prim   BOOLEAN   /* if component is primary */
+) ENGINE=MEMORY;
+BEGIN;
+DELETE FROM $MEMB_TABLE;
+DELETE FROM $STATUS_TABLE;
+"
+END="COMMIT;"
+
+configuration_change()
+{
+    echo "$BEGIN;"
+
+    local idx=0
+
+    for NODE in $(echo $MEMBERS | sed s/,/\ /g)
+    do
+        echo "INSERT INTO $MEMB_TABLE VALUES ( $idx, "
+        # Don't forget to properly quote string values
+        echo "'$NODE'" | sed  s/\\//\',\'/g
+        echo ");"
+        idx=$(( $idx + 1 ))
+    done
+
+    echo "INSERT INTO $STATUS_TABLE VALUES($idx, $INDEX, '$STATUS', '$CLUSTER_UUID', $PRIMARY);"
+
+    echo "$END"
+}
+
+status_update()
+{
+    echo "SET wsrep_on=0; BEGIN; UPDATE $STATUS_TABLE SET status='$STATUS'; COMMIT;"
+}
+
+COM=status_update # not a configuration change by default
+
+while [ $# -gt 0 ]
+do
+    case $1 in
+    --status)
+        STATUS=$2
+        shift
+        ;;
+    --uuid)
+        CLUSTER_UUID=$2
+        shift
+        ;;
+    --primary)
+        [ "$2" = "yes" ] && PRIMARY="1" || PRIMARY="0"
+        COM=configuration_change
+        shift
+        ;;
+    --index)
+        INDEX=$2
+        shift
+        ;;
+    --members)
+        MEMBERS=$2
+        shift
+        ;;
+    esac
+    shift
+done
+
+# Undefined means node is shutting down
+if [ "$STATUS" != "Undefined" ]
+then
+    $COM | mysql -B -u$USER -p$PSWD -h$HOST -P$PORT
+fi
+
+exit 0
+#
diff --git a/wsrep/CMakeLists.txt b/wsrep/CMakeLists.txt
new file mode 100644
index 00000000000..d9e66739fcc
--- /dev/null
+++ b/wsrep/CMakeLists.txt
@@ -0,0 +1,25 @@
+# Copyright (c) 2012, Codership Oy. All rights reserved.
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+
+INCLUDE_DIRECTORIES( "." )
+BUILD_WITH_WSREP()
+
+SET(WSREP_SOURCES wsrep_gtid.c wsrep_uuid.c wsrep_loader.c wsrep_dummy.c)
+
+ADD_CONVENIENCE_LIBRARY(wsrep ${WSREP_SOURCES})
+DTRACE_INSTRUMENT(wsrep)
+
+#ADD_EXECUTABLE(listener wsrep_listener.c ${WSREP_SOURCES})
+#TARGET_LINK_LIBRARIES(listener ${LIBDL})
diff --git a/wsrep/wsrep_api.h b/wsrep/wsrep_api.h
new file mode 100644
index 00000000000..c3304d7ed7c
--- /dev/null
+++ b/wsrep/wsrep_api.h
@@ -0,0 +1,1118 @@
+/* Copyright (C) 2009-2013 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with this program; if not, write to the Free Software Foundation, Inc.,
+   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/*!
+  @file wsrep API declaration.
+
+  HOW TO READ THIS FILE.
+
+  Due to C language rules this header layout doesn't lend itself to intuitive
+  reading. So here's the scoop: in the end this header declares two main types:
+
+  * struct wsrep_init_args
+
+  and
+
+  * struct wsrep
+
+  wsrep_init_args contains initialization parameters for wsrep provider like
+  names, addresses, etc. and pointers to callbacks. The callbacks will be called
+  by provider when it needs to do something application-specific, like log a
+  message or apply a writeset. It should be passed to init() call from
+  wsrep API. It is an application part of wsrep API contract.
+
+  struct wsrep is the interface to wsrep provider. It contains all wsrep API
+  calls. It is a provider part of wsrep API contract.
+
+  Finally, wsrep_load() method loads (dlopens) wsrep provider library. It is
+  defined in wsrep_loader.c unit and is part of libwsrep.a (which is not a
+  wsrep provider, but a convenience library).
+
+  wsrep_unload() does the reverse.
+
+*/
+#ifndef WSREP_H
+#define WSREP_H
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**************************************************************************
+ *                                                                        *
+ *                       wsrep replication API                            *
+ *                                                                        *
+ **************************************************************************/
+
+#define WSREP_INTERFACE_VERSION "25"
+
+/*! Empty backend spec */
+#define WSREP_NONE "none"
+
+
+/*!
+ * @brief log severity levels, passed as first argument to log handler
+ */
+typedef enum wsrep_log_level
+{
+    WSREP_LOG_FATAL, //!< Unrecoverable error, application must quit.
+    WSREP_LOG_ERROR, //!< Operation failed, must be repeated.
+    WSREP_LOG_WARN,  //!< Unexpected condition, but no operational failure.
+    WSREP_LOG_INFO,  //!< Informational message.
+    WSREP_LOG_DEBUG  //!< Debug message. Shows only of compiled with debug.
+} wsrep_log_level_t;
+
+/*!
+ * @brief error log handler
+ *
+ *        All messages from wsrep provider are directed to this
+ *        handler, if present.
+ *
+ * @param level   log level
+ * @param message log message
+ */
+typedef void (*wsrep_log_cb_t)(wsrep_log_level_t, const char *);
+
+
+/*!
+ *  Certain provider capabilities application may want to know about
+ */
+#define WSREP_CAP_MULTI_MASTER          ( 1ULL << 0 )
+#define WSREP_CAP_CERTIFICATION         ( 1ULL << 1 )
+#define WSREP_CAP_PARALLEL_APPLYING     ( 1ULL << 2 )
+#define WSREP_CAP_TRX_REPLAY            ( 1ULL << 3 )
+#define WSREP_CAP_ISOLATION             ( 1ULL << 4 )
+#define WSREP_CAP_PAUSE                 ( 1ULL << 5 )
+#define WSREP_CAP_CAUSAL_READS          ( 1ULL << 6 )
+#define WSREP_CAP_CAUSAL_TRX            ( 1ULL << 7 )
+#define WSREP_CAP_INCREMENTAL_WRITESET  ( 1ULL << 8 )
+#define WSREP_CAP_SESSION_LOCKS         ( 1ULL << 9 )
+#define WSREP_CAP_DISTRIBUTED_LOCKS     ( 1ULL << 10 )
+#define WSREP_CAP_CONSISTENCY_CHECK     ( 1ULL << 11 )
+#define WSREP_CAP_UNORDERED             ( 1ULL << 12 )
+#define WSREP_CAP_ANNOTATION            ( 1ULL << 13 )
+#define WSREP_CAP_PREORDERED            ( 1ULL << 14 )
+
+
+/*!
+ *  Writeset flags
+ *
+ * COMMIT       the writeset and all preceding writesets must be committed
+ * ROLLBACK     all preceding writesets in a transaction must be rolled back
+ * ISOLATION    the writeset must be applied AND committed in isolation
+ * PA_UNSAFE    the writeset cannot be applied in parallel
+ * COMMUTATIVE  the order in which the writeset is applied does not matter
+ * NATIVE       the writeset contains another writeset in this provider format
+ *
+ * Note that some of the flags are mutually exclusive (e.g. COMMIT and
+ * ROLLBACK).
+ */
+#define WSREP_FLAG_COMMIT               ( 1ULL << 0 )
+#define WSREP_FLAG_ROLLBACK             ( 1ULL << 1 )
+#define WSREP_FLAG_ISOLATION            ( 1ULL << 2 )
+#define WSREP_FLAG_PA_UNSAFE            ( 1ULL << 3 )
+#define WSREP_FLAG_COMMUTATIVE          ( 1ULL << 4 )
+#define WSREP_FLAG_NATIVE               ( 1ULL << 5 )
+
+
+typedef uint64_t wsrep_trx_id_t;  //!< application transaction ID
+typedef uint64_t wsrep_conn_id_t; //!< application connection ID
+typedef int64_t  wsrep_seqno_t;   //!< sequence number of a writeset, etc.
+#ifdef __cplusplus
+typedef bool     wsrep_bool_t;
+#else
+typedef _Bool    wsrep_bool_t;    //!< should be the same as standard (C99) bool
+#endif /* __cplusplus */
+
+/*! undefined seqno */
+#define WSREP_SEQNO_UNDEFINED (-1)
+
+
+/*! wsrep provider status codes */
+typedef enum wsrep_status
+{
+    WSREP_OK = 0,          //!< success
+    WSREP_WARNING,         //!< minor warning, error logged
+    WSREP_TRX_MISSING,     //!< transaction is not known by wsrep
+    WSREP_TRX_FAIL,        //!< transaction aborted, server can continue
+    WSREP_BF_ABORT,        //!< trx was victim of brute force abort
+    WSREP_SIZE_EXCEEDED,   //!< data exceeded maximum supported size
+    WSREP_CONN_FAIL,       //!< error in client connection, must abort
+    WSREP_NODE_FAIL,       //!< error in node state, wsrep must reinit
+    WSREP_FATAL,           //!< fatal error, server must abort
+    WSREP_NOT_IMPLEMENTED  //!< feature not implemented
+} wsrep_status_t;
+
+
+/*! wsrep callbacks status codes */
+typedef enum wsrep_cb_status
+{
+    WSREP_CB_SUCCESS =  0, //!< success (as in "not critical failure")
+    WSREP_CB_FAILURE       //!< critical failure (consistency violation)
+    /* Technically, wsrep provider has no use for specific failure codes since
+     * there is nothing it can do about it but abort execution. Therefore any
+     * positive number shall indicate a critical failure. Optionally that value
+     * may be used by provider to come to a consensus about state consistency
+     * in a group of nodes. */
+} wsrep_cb_status_t;
+
+
+/*!
+ * UUID type - for all unique IDs
+ */
+typedef struct wsrep_uuid {
+    uint8_t data[16];
+} wsrep_uuid_t;
+
+/*! Undefined UUID */
+static const wsrep_uuid_t WSREP_UUID_UNDEFINED = {{0,}};
+
+/*! UUID string representation length, terminating '\0' not included */
+#define WSREP_UUID_STR_LEN 36
+
+/*!
+ * Scan UUID from string
+ * @return length of UUID string representation or negative error code
+ */
+extern int
+wsrep_uuid_scan (const char* str, size_t str_len, wsrep_uuid_t* uuid);
+
+/*!
+ * Print UUID to string
+ * @return length of UUID string representation or negative error code
+ */
+extern int
+wsrep_uuid_print (const wsrep_uuid_t* uuid, char* str, size_t str_len);
+
+#define WSREP_MEMBER_NAME_LEN 32  //!< maximum logical member name length
+#define WSREP_INCOMING_LEN    256 //!< max Domain Name length + 0x00
+
+
+/*!
+ * Global transaction identifier
+ */
+typedef struct wsrep_gtid
+{
+    wsrep_uuid_t  uuid;  /*!< History UUID */
+    wsrep_seqno_t seqno; /*!< Sequence number */
+} wsrep_gtid_t;
+
+/*! Undefined GTID */
+static const wsrep_gtid_t WSREP_GTID_UNDEFINED = {{{0, }}, -1};
+
+/*! Minimum number of bytes guaranteed to store GTID string representation,
+ * terminating '\0' not included (36 + 1 + 20) */
+#define WSREP_GTID_STR_LEN 57
+
+
+/*!
+ * Scan GTID from string
+ * @return length of GTID string representation or negative error code
+ */
+extern int
+wsrep_gtid_scan(const char* str, size_t str_len, wsrep_gtid_t* gtid);
+
+/*!
+ * Print GTID to string
+ * @return length of GTID string representation or negative error code
+ */
+extern int
+wsrep_gtid_print(const wsrep_gtid_t* gtid, char* str, size_t str_len);
+
+
+/*!
+ * Transaction meta data
+ */
+typedef struct wsrep_trx_meta
+{
+    wsrep_gtid_t  gtid;       /*!< Global transaction identifier */
+    wsrep_seqno_t depends_on; /*!< Sequence number part of the last transaction
+                                   this transaction depends on */
+} wsrep_trx_meta_t;
+
+
+/*!
+ * member status
+ */
+typedef enum wsrep_member_status {
+    WSREP_MEMBER_UNDEFINED, //!< undefined state
+    WSREP_MEMBER_JOINER,    //!< incomplete state, requested state transfer
+    WSREP_MEMBER_DONOR,     //!< complete state, donates state transfer
+    WSREP_MEMBER_JOINED,    //!< complete state
+    WSREP_MEMBER_SYNCED,    //!< complete state, synchronized with group
+    WSREP_MEMBER_ERROR,     //!< this and above is provider-specific error code
+    WSREP_MEMBER_MAX
+} wsrep_member_status_t;
+
+/*!
+ * static information about a group member (some fields are tentative yet)
+ */
+typedef struct wsrep_member_info {
+    wsrep_uuid_t id;                           //!< group-wide unique member ID
+    char         name[WSREP_MEMBER_NAME_LEN];  //!< human-readable name
+    char         incoming[WSREP_INCOMING_LEN]; //!< address for client requests
+} wsrep_member_info_t;
+
+/*!
+ * group status
+ */
+typedef enum wsrep_view_status {
+    WSREP_VIEW_PRIMARY,      //!< primary group configuration (quorum present)
+    WSREP_VIEW_NON_PRIMARY,  //!< non-primary group configuration (quorum lost)
+    WSREP_VIEW_DISCONNECTED, //!< not connected to group, retrying.
+    WSREP_VIEW_MAX
+} wsrep_view_status_t;
+
+/*!
+ * view of the group
+ */
+typedef struct wsrep_view_info {
+    wsrep_gtid_t        state_id;  //!< global state ID
+    wsrep_seqno_t       view;      //!< global view number
+    wsrep_view_status_t status;    //!< view status
+    wsrep_bool_t        state_gap; //!< gap between global and local states
+    int                 my_idx;    //!< index of this member in the view
+    int                 memb_num;  //!< number of members in the view
+    int                 proto_ver; //!< application protocol agreed on the view
+    wsrep_member_info_t members[1];//!< array of member information
+} wsrep_view_info_t;
+
+/*!
+ * Magic string to tell provider to engage into trivial (empty) state transfer.
+ * No data will be passed, but the node shall be considered JOINED.
+ * Should be passed in sst_req parameter of wsrep_view_cb_t.
+ */
+#define WSREP_STATE_TRANSFER_TRIVIAL "trivial"
+
+/*!
+ * Magic string to tell provider not to engage in state transfer at all.
+ * The member will stay in WSREP_MEMBER_UNDEFINED state but will keep on
+ * receiving all writesets.
+ * Should be passed in sst_req parameter of wsrep_view_cb_t.
+ */
+#define WSREP_STATE_TRANSFER_NONE "none"
+
+/*!
+ * @brief group view handler
+ *
+ * This handler is called in total order corresponding to the group
+ * configuration change. It is to provide a vital information about
+ * new group view. If view info indicates existence of discontinuity
+ * between group and member states, state transfer request message
+ * should be filled in by the callback implementation.
+ *
+ * @note Currently it is assumed that sst_req is allocated using
+ *       malloc()/calloc()/realloc() and it will be freed by
+ *       wsrep implementation.
+ *
+ * @param app_ctx     application context
+ * @param recv_ctx    receiver context
+ * @param view        new view on the group
+ * @param state       current state
+ * @param state_len   lenght of current state
+ * @param sst_req     location to store SST request
+ * @param sst_req_len location to store SST request length or error code,
+ *                    value of 0 means no SST.
+ */
+typedef enum wsrep_cb_status (*wsrep_view_cb_t) (
+    void*                    app_ctx,
+    void*                    recv_ctx,
+    const wsrep_view_info_t* view,
+    const char*              state,
+    size_t                   state_len,
+    void**                   sst_req,
+    size_t*                  sst_req_len
+);
+
+
+/*!
+ * @brief apply callback
+ *
+ * This handler is called from wsrep library to apply replicated writeset
+ * Must support brute force applying for multi-master operation
+ *
+ * @param recv_ctx receiver context pointer provided by the application
+ * @param data     data buffer containing the writeset
+ * @param size     data buffer size
+ * @param flags    WSREP_FLAG_... flags
+ * @param meta     transaction meta data of the writeset to be applied
+ *
+ * @return success code:
+ * @retval WSREP_OK
+ * @retval WSREP_NOT_IMPLEMENTED appl. does not support the writeset format
+ * @retval WSREP_ERROR failed to apply the writeset
+ */
+typedef enum wsrep_cb_status (*wsrep_apply_cb_t) (
+    void*                   recv_ctx,
+    const void*             data,
+    size_t                  size,
+    uint32_t                flags,
+    const wsrep_trx_meta_t* meta
+);
+
+
+/*!
+ * @brief commit callback
+ *
+ * This handler is called to commit the changes made by apply callback.
+ *
+ * @param recv_ctx receiver context pointer provided by the application
+ * @param flags    WSREP_FLAG_... flags
+ * @param meta     transaction meta data of the writeset to be committed
+ * @param exit     set to true to exit recv loop
+ * @param commit   true - commit writeset, false - rollback writeset
+ *
+ * @return success code:
+ * @retval WSREP_OK
+ * @retval WSREP_ERROR call failed
+ */
+typedef enum wsrep_cb_status (*wsrep_commit_cb_t) (
+    void*                   recv_ctx,
+    uint32_t                flags,
+    const wsrep_trx_meta_t* meta,
+    wsrep_bool_t*           exit,
+    wsrep_bool_t            commit
+);
+
+
+/*!
+ * @brief unordered callback
+ *
+ * This handler is called to execute unordered actions (actions that need not
+ * to be executed in any particular order) attached to writeset.
+ *
+ * @param recv_ctx receiver context pointer provided by the application
+ * @param data     data buffer containing the writeset
+ * @param size     data buffer size
+ */
+typedef enum wsrep_cb_status (*wsrep_unordered_cb_t) (
+    void*       recv_ctx,
+    const void* data,
+    size_t      size
+);
+
+
+/*!
+ * @brief a callback to donate state snapshot
+ *
+ * This handler is called from wsrep library when it needs this node
+ * to deliver state to a new cluster member.
+ * No state changes will be committed for the duration of this call.
+ * Wsrep implementation may provide internal state to be transmitted
+ * to new cluster member for initial state.
+ *
+ * @param app_ctx   application context
+ * @param recv_ctx  receiver context
+ * @param msg       state transfer request message
+ * @param msg_len   state transfer request message length
+ * @param gtid      current state ID on this node
+ * @param state     current wsrep internal state buffer
+ * @param state_len current wsrep internal state buffer len
+ * @param bypass    bypass snapshot transfer, only transfer uuid:seqno pair
+ */
+typedef enum wsrep_cb_status (*wsrep_sst_donate_cb_t) (
+    void*               app_ctx,
+    void*               recv_ctx,
+    const void*         msg,
+    size_t              msg_len,
+    const wsrep_gtid_t* state_id,
+    const char*         state,
+    size_t              state_len,
+    wsrep_bool_t        bypass
+);
+
+
+/*!
+ * @brief a callback to signal application that wsrep state is synced
+ *        with cluster
+ *
+ * This callback is called after wsrep library has got in sync with
+ * rest of the cluster.
+ *
+ * @param app_ctx application context
+ */
+typedef void (*wsrep_synced_cb_t) (void* app_ctx);
+
+
+/*!
+ * Initialization parameters for wsrep provider.
+ */
+struct wsrep_init_args
+{
+    void* app_ctx;             //!< Application context for callbacks
+
+    /* Configuration parameters */
+    const char* node_name;     //!< Symbolic name of this node (e.g. hostname)
+    const char* node_address;  //!< Address to be used by wsrep provider
+    const char* node_incoming; //!< Address for incoming client connections
+    const char* data_dir;      //!< Directory where wsrep files are kept if any
+    const char* options;       //!< Provider-specific configuration string
+    int         proto_ver;     //!< Max supported application protocol version
+
+    /* Application initial state information. */
+    const wsrep_gtid_t* state_id;    //!< Application state GTID
+    const char*         state;       //!< Initial state for wsrep provider
+    size_t              state_len;   //!< Length of state buffer
+
+    /* Application callbacks */
+    wsrep_log_cb_t        logger_cb;       //!< logging handler
+    wsrep_view_cb_t       view_handler_cb; //!< group view change handler
+
+    /* Applier callbacks */
+    wsrep_apply_cb_t      apply_cb;        //!< apply  callback
+    wsrep_commit_cb_t     commit_cb;       //!< commit callback
+    wsrep_unordered_cb_t  unordered_cb;    //!< callback for unordered actions
+
+    /* State Snapshot Transfer callbacks */
+    wsrep_sst_donate_cb_t sst_donate_cb;   //!< starting to donate
+    wsrep_synced_cb_t     synced_cb;       //!< synced with group
+};
+
+
+/*! Type of the stats variable value in struct wsrep_status_var */
+typedef enum wsrep_var_type
+{
+    WSREP_VAR_STRING, //!< pointer to null-terminated string
+    WSREP_VAR_INT64,  //!< int64_t
+    WSREP_VAR_DOUBLE  //!< double
+}
+wsrep_var_type_t;
+
+/*! Generalized stats variable representation */
+struct wsrep_stats_var
+{
+    const char*      name;     //!< variable name
+    wsrep_var_type_t type;     //!< variable value type
+    union {
+        int64_t     _int64;
+        double      _double;
+        const char* _string;
+    } value;                   //!< variable value
+};
+
+
+/*! Abstract data buffer structure */
+typedef struct wsrep_buf
+{
+    const void* ptr; /*!< Pointer to data buffer */
+    size_t      len; /*!< Length of buffer */
+} wsrep_buf_t;
+
+/*! Key struct used to pass certification keys for transaction handling calls.
+ *  A key consists of zero or more key parts. */
+typedef struct wsrep_key
+{
+    const wsrep_buf_t* key_parts;     /*!< Array of key parts  */
+    size_t             key_parts_num; /*!< Number of key parts */
+} wsrep_key_t;
+
+/*! Key type:
+ *  EXCLUSIVE conflicts with any key type
+ *  SEMI      reserved. If not supported, should be interpeted as EXCLUSIVE
+ *  SHARED    conflicts only with EXCLUSIVE keys */
+typedef enum wsrep_key_type
+{
+    WSREP_KEY_SHARED = 0,
+    WSREP_KEY_SEMI,
+    WSREP_KEY_EXCLUSIVE
+} wsrep_key_type_t;
+
+/*! Data type:
+ *  ORDERED    state modification event that should be applied and committed
+ *             in order.
+ *  UNORDERED  some action that does not modify state and execution of which is
+ *             optional and does not need to happen in order.
+ *  ANNOTATION (human readable) writeset annotation. */
+typedef enum wsrep_data_type
+{
+    WSREP_DATA_ORDERED = 0,
+    WSREP_DATA_UNORDERED,
+    WSREP_DATA_ANNOTATION
+} wsrep_data_type_t;
+
+
+/*! Transaction handle struct passed for wsrep transaction handling calls */
+typedef struct wsrep_ws_handle
+{
+    wsrep_trx_id_t trx_id; //!< transaction ID
+    void*          opaque; //!< opaque provider transaction context data
+} wsrep_ws_handle_t;
+
+/*!
+ * @brief Helper method to reset trx writeset handle state when trx id changes
+ *
+ * Instead of passing wsrep_ws_handle_t directly to wsrep calls,
+ * wrapping handle with this call offloads bookkeeping from
+ * application.
+ */
+static inline wsrep_ws_handle_t* wsrep_ws_handle_for_trx(
+    wsrep_ws_handle_t* ws_handle,
+    wsrep_trx_id_t     trx_id)
+{
+    if (ws_handle->trx_id != trx_id)
+    {
+        ws_handle->trx_id = trx_id;
+        ws_handle->opaque = NULL;
+    }
+    return ws_handle;
+}
+
+
+/*!
+ *  A handle for processing preordered actions.
+ *  Must be initialized to WSREP_PO_INITIALIZER before use.
+ */
+typedef struct wsrep_po_handle { void* opaque; } wsrep_po_handle_t;
+
+static const wsrep_po_handle_t WSREP_PO_INITIALIZER = { NULL };
+
+
+typedef struct wsrep wsrep_t;
+/*!
+ * wsrep interface for dynamically loadable libraries
+ */
+struct wsrep {
+
+    const char *version; //!< interface version string
+
+  /*!
+   * @brief Initializes wsrep provider
+   *
+   * @param wsrep provider handle
+   * @param args  wsrep initialization parameters
+   */
+    wsrep_status_t (*init)   (wsrep_t*                      wsrep,
+                              const struct wsrep_init_args* args);
+
+  /*!
+   * @brief Returns provider capabilities flag bitmap
+   *
+   * @param wsrep provider handle
+   */
+    uint64_t (*capabilities) (wsrep_t* wsrep);
+
+  /*!
+   * @brief Passes provider-specific configuration string to provider.
+   *
+   * @param wsrep provider handle
+   * @param conf  configuration string
+   *
+   * @retval WSREP_OK      configuration string was parsed successfully
+   * @retval WSREP_WARNING could't not parse conf string, no action taken
+   */
+    wsrep_status_t (*options_set) (wsrep_t* wsrep, const char* conf);
+
+  /*!
+   * @brief Returns provider-specific string with current configuration values.
+   *
+   * @param wsrep provider handle
+   *
+   * @return a dynamically allocated string with current configuration
+   *         parameter values
+   */
+    char*          (*options_get) (wsrep_t* wsrep);
+
+  /*!
+   * @brief Opens connection to cluster
+   *
+   * Returns when either node is ready to operate as a part of the clsuter
+   * or fails to reach operating status.
+   *
+   * @param wsrep        provider handle
+   * @param cluster_name unique symbolic cluster name
+   * @param cluster_url  URL-like cluster address (backend://address)
+   * @param state_donor  name of the node to be asked for state transfer.
+   * @param bootstrap    a flag to request initialization of a new wsrep
+   *                     service rather then a connection to the existing one.
+   *                     clister_url may still carry important initialization
+   *                     parameters, like backend spec and/or listen address.
+   */
+    wsrep_status_t (*connect) (wsrep_t*     wsrep,
+                               const char*  cluster_name,
+                               const char*  cluster_url,
+                               const char*  state_donor,
+                               wsrep_bool_t bootstrap);
+
+  /*!
+   * @brief Closes connection to cluster.
+   *
+   * If state_uuid and/or state_seqno is not NULL, will store final state
+   * in there.
+   *
+   * @param wsrep this  wsrep handler
+   */
+    wsrep_status_t (*disconnect)(wsrep_t* wsrep);
+
+  /*!
+   * @brief start receiving replication events
+   *
+   * This function never returns
+   *
+   * @param wsrep provider handle
+   * @param recv_ctx receiver context
+   */
+    wsrep_status_t (*recv)(wsrep_t* wsrep, void* recv_ctx);
+
+  /*!
+   * @brief Replicates/logs result of transaction to other nodes and allocates
+   * required resources.
+   *
+   * Must be called before transaction commit. Returns success code, which
+   * caller must check.
+   * In case of WSREP_OK, starts commit critical section, transaction can
+   * commit. Otherwise transaction must rollback.
+   *
+   * @param wsrep      provider handle
+   * @param ws_handle  writeset of committing transaction
+   * @param conn_id    connection ID
+   * @param flags      fine tuning the replication WSREP_FLAG_*
+   * @param meta       transaction meta data
+   *
+   * @retval WSREP_OK         cluster-wide commit succeeded
+   * @retval WSREP_TRX_FAIL   must rollback transaction
+   * @retval WSREP_CONN_FAIL  must close client connection
+   * @retval WSREP_NODE_FAIL  must close all connections and reinit
+   */
+    wsrep_status_t (*pre_commit)(wsrep_t*                wsrep,
+                                 wsrep_conn_id_t         conn_id,
+                                 wsrep_ws_handle_t*      ws_handle,
+                                 uint32_t                flags,
+                                 wsrep_trx_meta_t*       meta);
+
+  /*!
+   * @brief Releases resources after transaction commit.
+   *
+   * Ends commit critical section.
+   *
+   * @param wsrep      provider handle
+   * @param ws_handle  writeset of committing transaction
+   * @retval WSREP_OK  post_commit succeeded
+   */
+    wsrep_status_t (*post_commit) (wsrep_t*            wsrep,
+                                   wsrep_ws_handle_t*  ws_handle);
+
+  /*!
+   * @brief Releases resources after transaction rollback.
+   *
+   * @param wsrep      provider handle
+   * @param ws_handle  writeset of committing transaction
+   * @retval WSREP_OK  post_rollback succeeded
+   */
+    wsrep_status_t (*post_rollback)(wsrep_t*            wsrep,
+                                    wsrep_ws_handle_t*  ws_handle);
+
+  /*!
+   * @brief Replay trx as a slave writeset
+   *
+   * If local trx has been aborted by brute force, and it has already
+   * replicated before this abort, we must try if we can apply it as
+   * slave trx. Note that slave nodes see only trx writesets and certification
+   * test based on write set content can be different to DBMS lock conflicts.
+   *
+   * @param wsrep      provider handle
+   * @param ws_handle  writeset of committing transaction
+   * @param trx_ctx    transaction context
+   *
+   * @retval WSREP_OK         cluster commit succeeded
+   * @retval WSREP_TRX_FAIL   must rollback transaction
+   * @retval WSREP_BF_ABORT   brute force abort happened after trx replicated
+   *                          must rollback transaction and try to replay
+   * @retval WSREP_CONN_FAIL  must close client connection
+   * @retval WSREP_NODE_FAIL  must close all connections and reinit
+   */
+    wsrep_status_t (*replay_trx)(wsrep_t*            wsrep,
+                                 wsrep_ws_handle_t*  ws_handle,
+                                 void*               trx_ctx);
+
+  /*!
+   * @brief Abort pre_commit() call of another thread.
+   *
+   * It is possible, that some high-priority transaction needs to abort
+   * another transaction which is in pre_commit() call waiting for resources.
+   *
+   * The kill routine checks that abort is not attmpted against a transaction
+   * which is front of the caller (in total order).
+   *
+   * @param wsrep      provider handle
+   * @param bf_seqno   seqno of brute force trx, running this cancel
+   * @param victim_trx transaction to be aborted, and which is committing
+   *
+   * @retval WSREP_OK       abort secceded
+   * @retval WSREP_WARNING  abort failed
+   */
+    wsrep_status_t (*abort_pre_commit)(wsrep_t*       wsrep,
+                                       wsrep_seqno_t  bf_seqno,
+                                       wsrep_trx_id_t victim_trx);
+
+  /*!
+   * @brief Appends a row reference to transaction writeset
+   *
+   * Both copy flag and key_type can be ignored by provider (key type
+   * interpreted as WSREP_KEY_EXCLUSIVE).
+   *
+   * @param wsrep      provider handle
+   * @param ws_handle  writeset handle
+   * @param keys       array of keys
+   * @param count      length of the array of keys
+   * @param type       type ot the key
+   * @param copy       can be set to FALSE if keys persist through commit.
+   */
+    wsrep_status_t (*append_key)(wsrep_t*            wsrep,
+                                 wsrep_ws_handle_t*  ws_handle,
+                                 const wsrep_key_t*  keys,
+                                 size_t              count,
+                                 enum wsrep_key_type type,
+                                 wsrep_bool_t        copy);
+
+  /*!
+   * @brief Appends data to transaction writeset
+   *
+   * This method can be called any time before commit and it
+   * appends a number of data buffers to transaction writeset.
+   *
+   * Both copy and unordered flags can be ignored by provider.
+   *
+   * @param wsrep      provider handle
+   * @param ws_handle  writeset handle
+   * @param data       array of data buffers
+   * @param count      buffer count
+   * @param type       type of data
+   * @param copy       can be set to FALSE if data persists through commit.
+   */
+    wsrep_status_t (*append_data)(wsrep_t*                wsrep,
+                                  wsrep_ws_handle_t*      ws_handle,
+                                  const struct wsrep_buf* data,
+                                  size_t                  count,
+                                  enum wsrep_data_type    type,
+                                  wsrep_bool_t            copy);
+
+  /*!
+   * @brief Get causal ordering for read operation
+   *
+   * This call will block until causal ordering with all possible
+   * preceding writes in the cluster is guaranteed. If pointer to
+   * gtid is non-null, the call stores the global transaction ID
+   * of the last transaction which is guaranteed to be ordered
+   * causally before this call.
+   *
+   * @param wsrep provider handle
+   * @param gtid  location to store GTID
+   */
+    wsrep_status_t (*causal_read)(wsrep_t* wsrep, wsrep_gtid_t* gtid);
+
+  /*!
+   * @brief Clears allocated connection context.
+   *
+   * Whenever a new connection ID is passed to wsrep provider through
+   * any of the API calls, a connection context is allocated for this
+   * connection. This call is to explicitly notify provider fo connection
+   * closing.
+   *
+   * @param wsrep       provider handle
+   * @param conn_id     connection ID
+   * @param query       the 'set database' query
+   * @param query_len   length of query (does not end with 0)
+   */
+    wsrep_status_t (*free_connection)(wsrep_t*        wsrep,
+                                      wsrep_conn_id_t conn_id);
+
+  /*!
+   * @brief Replicates a query and starts "total order isolation" section.
+   *
+   * Replicates the action spec and returns success code, which caller must
+   * check. Total order isolation continues until to_execute_end() is called.
+   *
+   * @param wsrep       provider handle
+   * @param conn_id     connection ID
+   * @param keys        array of keys
+   * @param keys_num    lenght of the array of keys
+   * @param action      action buffer array to be executed
+   * @param count       action buffer count
+   * @param meta        transaction meta data
+   *
+   * @retval WSREP_OK         cluster commit succeeded
+   * @retval WSREP_CONN_FAIL  must close client connection
+   * @retval WSREP_NODE_FAIL  must close all connections and reinit
+   */
+    wsrep_status_t (*to_execute_start)(wsrep_t*                wsrep,
+                                       wsrep_conn_id_t         conn_id,
+                                       const wsrep_key_t*      keys,
+                                       size_t                  keys_num,
+                                       const struct wsrep_buf* action,
+                                       size_t                  count,
+                                       wsrep_trx_meta_t*       meta);
+
+  /*!
+   * @brief Ends the total order isolation section.
+   *
+   * Marks the end of total order isolation. TO locks are freed
+   * and other transactions are free to commit from this point on.
+   *
+   * @param wsrep provider handle
+   * @param conn_id connection ID
+   *
+   * @retval WSREP_OK         cluster commit succeeded
+   * @retval WSREP_CONN_FAIL  must close client connection
+   * @retval WSREP_NODE_FAIL  must close all connections and reinit
+   */
+    wsrep_status_t (*to_execute_end)(wsrep_t* wsrep, wsrep_conn_id_t conn_id);
+
+  /*!
+   * @brief Collects preordered replication events into a writeset.
+   *
+   * @param wsrep   wsrep provider handle
+   * @param handle  a handle associated with a given writeset
+   * @param data    an array of data buffers.
+   * @param count   length of data buffer array.
+   * @param copy    whether provider needs to make a copy of events.
+   *
+   * @retval WSREP_OK         cluster-wide commit succeeded
+   * @retval WSREP_TRX_FAIL   operation failed (e.g. trx size exceeded limit)
+   * @retval WSREP_NODE_FAIL  must close all connections and reinit
+   */
+    wsrep_status_t (*preordered_collect) (wsrep_t*                 wsrep,
+                                          wsrep_po_handle_t*       handle,
+                                          const struct wsrep_buf*  data,
+                                          size_t                   count,
+                                          wsrep_bool_t             copy);
+
+  /*!
+   * @brief "Commits" preordered writeset to cluster.
+   *
+   * The contract is that the writeset will be committed in the same (partial)
+   * order this method was called. Frees resources associated with the writeset
+   * handle and reinitializes the handle.
+   *
+   * @param wsrep     wsrep provider handle
+   * @param po_handle a handle associated with a given writeset
+   * @param source_id ID of the event producer, also serves as the partial order
+   *                  or stream ID - events with different source_ids won't be
+   *                  ordered with respect to each other.
+   * @param flags     WSREP_FLAG_... flags
+   * @param pa_range  the number of preceding events this event can be processed
+   *                  in parallel with. A value of 0 means strict serial
+   *                  processing. Note: commits always happen in wsrep order.
+   * @param commit    'true'  to commit writeset to cluster (replicate) or
+   *                  'false' to rollback (cancel) the writeset.
+   *
+   * @retval WSREP_OK         cluster-wide commit succeeded
+   * @retval WSREP_TRX_FAIL   operation failed (e.g. NON-PRIMARY component)
+   * @retval WSREP_NODE_FAIL  must close all connections and reinit
+   */
+    wsrep_status_t (*preordered_commit)  (wsrep_t*             wsrep,
+                                          wsrep_po_handle_t*   handle,
+                                          const wsrep_uuid_t*  source_id,
+                                          uint32_t             flags,
+                                          int                  pa_range,
+                                          wsrep_bool_t         commit);
+
+  /*!
+   * @brief Signals to wsrep provider that state snapshot has been sent to
+   *        joiner.
+   *
+   * @param wsrep    provider handle
+   * @param state_id state ID
+   * @param rcode    0 or negative error code of the operation.
+   */
+    wsrep_status_t (*sst_sent)(wsrep_t*            wsrep,
+                               const wsrep_gtid_t* state_id,
+                               int                 rcode);
+
+  /*!
+   * @brief Signals to wsrep provider that new state snapshot has been received.
+   *        May deadlock if called from sst_prepare_cb.
+   *
+   * @param wsrep     provider handle
+   * @param state_id  state ID
+   * @param state     initial state provided by SST donor
+   * @param state_len length of state buffer
+   * @param rcode     0 or negative error code of the operation.
+   */
+    wsrep_status_t (*sst_received)(wsrep_t*            wsrep,
+                                   const wsrep_gtid_t* state_id,
+                                   const void*         state,
+                                   size_t              state_len,
+                                   int                 rcode);
+
+
+  /*!
+   * @brief Generate request for consistent snapshot.
+   *
+   * If successfull, this call will generate internally SST request
+   * which in turn triggers calling SST donate callback on the nodes
+   * specified in donor_spec. If donor_spec is null, callback is
+   * called only locally. This call will block until sst_sent is called
+   * from callback.
+   *
+   * @param wsrep      provider handle
+   * @param msg        context message for SST donate callback
+   * @param msg_len    length of context message
+   * @param donor_spec list of snapshot donors
+   */
+    wsrep_status_t (*snapshot)(wsrep_t*    wsrep,
+                               const void* msg,
+                               size_t      msg_len,
+                               const char* donor_spec);
+
+  /*!
+   * @brief Returns an array fo status variables.
+   *        Array is terminated by Null variable name.
+   *
+   * @param wsrep provider handle
+   * @return array of struct wsrep_status_var.
+   */
+    struct wsrep_stats_var* (*stats_get) (wsrep_t* wsrep);
+
+  /*!
+   * @brief Release resources that might be associated with the array.
+   *
+   * @param wsrep     provider handle.
+   * @param var_array array returned by stats_get().
+   */
+    void (*stats_free) (wsrep_t* wsrep, struct wsrep_stats_var* var_array);
+
+  /*!
+   * @brief Reset some stats variables to inital value, provider-dependent.
+   *
+   * @param wsrep provider handle.
+   */
+    void (*stats_reset) (wsrep_t* wsrep);
+
+  /*!
+   * @brief Pauses writeset applying/committing.
+   *
+   * @return global sequence number of the paused state or negative error code.
+   */
+    wsrep_seqno_t (*pause) (wsrep_t* wsrep);
+
+  /*!
+   * @brief Resumes writeset applying/committing.
+   */
+    wsrep_status_t (*resume) (wsrep_t* wsrep);
+
+  /*!
+   * @brief Desynchronize from cluster
+   *
+   * Effectively turns off flow control for this node, allowing it
+   * to fall behind the cluster.
+   */
+    wsrep_status_t (*desync) (wsrep_t* wsrep);
+
+  /*!
+   * @brief Request to resynchronize with cluster.
+   *
+   * Effectively turns on flow control. Asynchronous - actual synchronization
+   * event to be deliverred via sync_cb.
+   */
+    wsrep_status_t (*resync) (wsrep_t* wsrep);
+
+  /*!
+   * @brief Acquire global named lock
+   *
+   * @param wsrep  wsrep provider handle
+   * @param name   lock name
+   * @param shared shared or exclusive lock
+   * @param owner  64-bit owner ID
+   * @param tout   timeout in nanoseconds.
+   *               0 - return immediately, -1 wait forever.
+   * @return          wsrep status or negative error code
+   * @retval -EDEADLK lock was already acquired by this thread
+   * @retval -EBUSY   lock was busy
+   */
+    wsrep_status_t (*lock) (wsrep_t* wsrep,
+                            const char* name, wsrep_bool_t shared,
+                            uint64_t owner, int64_t tout);
+
+  /*!
+   * @brief Release global named lock
+   *
+   * @param wsrep   wsrep provider handle
+   * @param name    lock name
+   * @param owner   64-bit owner ID
+   * @return        wsrep status or negative error code
+   * @retval -EPERM lock does not belong to this owner
+   */
+    wsrep_status_t (*unlock) (wsrep_t* wsrep, const char* name, uint64_t owner);
+
+  /*!
+   * @brief Check if global named lock is locked
+   *
+   * @param wsrep wsrep provider handle
+   * @param name  lock name
+   * @param owner if not NULL will contain 64-bit owner ID
+   * @param node  if not NULL will contain owner's node UUID
+   * @return true if lock is locked
+   */
+    wsrep_bool_t (*is_locked) (wsrep_t* wsrep, const char* name, uint64_t* conn,
+                               wsrep_uuid_t* node);
+
+  /*!
+   * wsrep provider name
+   */
+    const char* provider_name;
+
+  /*!
+   * wsrep provider version
+   */
+    const char* provider_version;
+
+  /*!
+   * wsrep provider vendor name
+   */
+    const char* provider_vendor;
+
+  /*!
+   * @brief Frees allocated resources before unloading the library.
+   * @param wsrep provider handle
+   */
+    void (*free)(wsrep_t* wsrep);
+
+    void *dlh;    //!< reserved for future use
+    void *ctx;    //!< reserved for implemetation private context
+};
+
+
+/*!
+ *
+ * @brief Loads wsrep library
+ *
+ * @param spec   path to wsrep library. If NULL or WSREP_NONE initialises dummy
+ *               pass-through implementation.
+ * @param hptr   wsrep handle
+ * @param log_cb callback to handle loader messages. Otherwise writes to stderr.
+ *
+ * @return zero on success, errno on failure
+ */
+int wsrep_load(const char* spec, wsrep_t** hptr, wsrep_log_cb_t log_cb);
+
+/*!
+ * @brief Unloads wsrep library and frees associated resources
+ *
+ * @param hptr wsrep handler pointer
+ */
+void wsrep_unload(wsrep_t* hptr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* WSREP_H */
diff --git a/wsrep/wsrep_dummy.c b/wsrep/wsrep_dummy.c
new file mode 100644
index 00000000000..bab5329dc02
--- /dev/null
+++ b/wsrep/wsrep_dummy.c
@@ -0,0 +1,407 @@
+/* Copyright (C) 2009-2010 Codership Oy <info@codersihp.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+/*! @file Dummy wsrep API implementation. */
+
+#include "wsrep_api.h"
+
+#include <errno.h>
+#include <stdbool.h>
+#include <string.h>
+
+/*! Dummy backend context. */
+typedef struct wsrep_dummy
+{
+    wsrep_log_cb_t log_fn;
+    char* options;
+} wsrep_dummy_t;
+
+/* Get pointer to wsrep_dummy context from wsrep_t pointer */
+#define WSREP_DUMMY(_p) ((wsrep_dummy_t *) (_p)->ctx)
+
+/* Trace function usage a-la DBUG */
+#define WSREP_DBUG_ENTER(_w) do {                                       \
+        if (WSREP_DUMMY(_w)) {                                          \
+            if (WSREP_DUMMY(_w)->log_fn)                                \
+                WSREP_DUMMY(_w)->log_fn(WSREP_LOG_DEBUG, __FUNCTION__); \
+        }                                                               \
+    } while (0)
+
+
+static void dummy_free(wsrep_t *w)
+{
+    WSREP_DBUG_ENTER(w);
+    if (WSREP_DUMMY(w)->options) {
+        free(WSREP_DUMMY(w)->options);
+        WSREP_DUMMY(w)->options = NULL;
+    }
+    free(w->ctx);
+    w->ctx = NULL;
+}
+
+static wsrep_status_t dummy_init (wsrep_t* w,
+                                  const struct wsrep_init_args* args)
+{
+    WSREP_DUMMY(w)->log_fn = args->logger_cb;
+    WSREP_DBUG_ENTER(w);
+    if (args->options) {
+        WSREP_DUMMY(w)->options = strdup(args->options);
+    }
+    return WSREP_OK;
+}
+
+static uint64_t dummy_capabilities (wsrep_t* w __attribute__((unused)))
+{
+    return 0;
+}
+
+static wsrep_status_t dummy_options_set(
+    wsrep_t* w,
+    const char* conf)
+{
+    WSREP_DBUG_ENTER(w);
+    if (WSREP_DUMMY(w)->options) {
+        free(WSREP_DUMMY(w)->options);
+        WSREP_DUMMY(w)->options = NULL;
+    }
+    if (conf) {
+        WSREP_DUMMY(w)->options = strdup(conf);
+    }
+    return WSREP_OK;
+}
+
+static char* dummy_options_get (wsrep_t* w)
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_DUMMY(w)->options;
+}
+
+static wsrep_status_t dummy_connect(
+    wsrep_t* w,
+    const char*  name      __attribute__((unused)),
+    const char*  url       __attribute__((unused)),
+    const char*  donor     __attribute__((unused)),
+    wsrep_bool_t bootstrap __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_disconnect(wsrep_t* w)
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_recv(wsrep_t* w,
+                                 void*    recv_ctx __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_pre_commit(
+    wsrep_t* w,
+    const wsrep_conn_id_t   conn_id    __attribute__((unused)),
+    wsrep_ws_handle_t*      ws_handle  __attribute__((unused)),
+    uint32_t                flags      __attribute__((unused)),
+    wsrep_trx_meta_t*       meta       __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_post_commit(
+    wsrep_t* w,
+    wsrep_ws_handle_t*  ws_handle  __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_post_rollback(
+    wsrep_t* w,
+    wsrep_ws_handle_t*  ws_handle  __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_replay_trx(
+    wsrep_t* w,
+    wsrep_ws_handle_t*  ws_handle  __attribute__((unused)),
+    void*               trx_ctx    __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_abort_pre_commit(
+    wsrep_t* w,
+    const wsrep_seqno_t  bf_seqno __attribute__((unused)),
+    const wsrep_trx_id_t trx_id   __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_append_key(
+    wsrep_t* w,
+    wsrep_ws_handle_t*     ws_handle  __attribute__((unused)),
+    const wsrep_key_t*     key        __attribute__((unused)),
+    const size_t           key_num    __attribute__((unused)),
+    const wsrep_key_type_t key_type   __attribute__((unused)),
+    const bool             copy       __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_append_data(
+    wsrep_t* w,
+    wsrep_ws_handle_t*      ws_handle  __attribute__((unused)),
+    const struct wsrep_buf* data       __attribute__((unused)),
+    const size_t            count      __attribute__((unused)),
+    const wsrep_data_type_t type       __attribute__((unused)),
+    const bool              copy       __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_causal_read(
+    wsrep_t* w,
+    wsrep_gtid_t* gtid __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_free_connection(
+    wsrep_t* w,
+    const wsrep_conn_id_t  conn_id   __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_to_execute_start(
+    wsrep_t* w,
+    const wsrep_conn_id_t   conn_id __attribute__((unused)),
+    const wsrep_key_t*      key     __attribute__((unused)),
+    const size_t            key_num __attribute__((unused)),
+    const struct wsrep_buf* data    __attribute__((unused)),
+    const size_t            count   __attribute__((unused)),
+    wsrep_trx_meta_t*       meta    __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_to_execute_end(
+    wsrep_t* w,
+    const wsrep_conn_id_t  conn_id   __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_preordered_collect(
+    wsrep_t*                 w,
+    wsrep_po_handle_t*       handle    __attribute__((unused)),
+    const struct wsrep_buf*  data      __attribute__((unused)),
+    size_t                   count     __attribute__((unused)),
+    wsrep_bool_t             copy      __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_preordered_commit(
+    wsrep_t*                 w,
+    wsrep_po_handle_t*       handle    __attribute__((unused)),
+    const wsrep_uuid_t*      source_id __attribute__((unused)),
+    uint32_t                 flags     __attribute__((unused)),
+    int                      pa_range  __attribute__((unused)),
+    wsrep_bool_t             commit    __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_sst_sent(
+    wsrep_t* w,
+    const wsrep_gtid_t* state_id  __attribute__((unused)),
+    const int           rcode     __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_sst_received(
+    wsrep_t* w,
+    const wsrep_gtid_t* state_id  __attribute__((unused)),
+    const void*         state     __attribute__((unused)),
+    const size_t        state_len __attribute__((unused)),
+    const int           rcode     __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_snapshot(
+    wsrep_t* w,
+    const void*  msg        __attribute__((unused)),
+    const size_t msg_len    __attribute__((unused)),
+    const char*  donor_spec __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static struct wsrep_stats_var dummy_stats[] = {
+    { NULL, WSREP_VAR_STRING, { 0 } }
+};
+
+static struct wsrep_stats_var* dummy_stats_get (wsrep_t* w)
+{
+    WSREP_DBUG_ENTER(w);
+    return dummy_stats;
+}
+
+static void dummy_stats_free (
+    wsrep_t* w,
+    struct wsrep_stats_var* stats __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+}
+
+static void dummy_stats_reset (wsrep_t* w)
+{
+    WSREP_DBUG_ENTER(w);
+}
+
+static wsrep_seqno_t dummy_pause (wsrep_t* w)
+{
+    WSREP_DBUG_ENTER(w);
+    return -ENOSYS;
+}
+
+static wsrep_status_t dummy_resume (wsrep_t* w)
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_desync (wsrep_t* w)
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_NOT_IMPLEMENTED;
+}
+
+static wsrep_status_t dummy_resync (wsrep_t* w)
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static wsrep_status_t dummy_lock (wsrep_t* w,
+                                  const char* s __attribute__((unused)),
+                                  bool        r __attribute__((unused)),
+                                  uint64_t    o __attribute__((unused)),
+                                  int64_t     t __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_NOT_IMPLEMENTED;
+}
+
+static wsrep_status_t dummy_unlock (wsrep_t* w,
+                                    const char* s __attribute__((unused)),
+                                    uint64_t    o __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return WSREP_OK;
+}
+
+static bool dummy_is_locked (wsrep_t* w,
+                             const char*   s __attribute__((unused)),
+                             uint64_t*     o __attribute__((unused)),
+                             wsrep_uuid_t* t __attribute__((unused)))
+{
+    WSREP_DBUG_ENTER(w);
+    return false;
+}
+
+static wsrep_t dummy_iface = {
+    WSREP_INTERFACE_VERSION,
+    &dummy_init,
+    &dummy_capabilities,
+    &dummy_options_set,
+    &dummy_options_get,
+    &dummy_connect,
+    &dummy_disconnect,
+    &dummy_recv,
+    &dummy_pre_commit,
+    &dummy_post_commit,
+    &dummy_post_rollback,
+    &dummy_replay_trx,
+    &dummy_abort_pre_commit,
+    &dummy_append_key,
+    &dummy_append_data,
+    &dummy_causal_read,
+    &dummy_free_connection,
+    &dummy_to_execute_start,
+    &dummy_to_execute_end,
+    &dummy_preordered_collect,
+    &dummy_preordered_commit,
+    &dummy_sst_sent,
+    &dummy_sst_received,
+    &dummy_snapshot,
+    &dummy_stats_get,
+    &dummy_stats_free,
+    &dummy_stats_reset,
+    &dummy_pause,
+    &dummy_resume,
+    &dummy_desync,
+    &dummy_resync,
+    &dummy_lock,
+    &dummy_unlock,
+    &dummy_is_locked,
+    WSREP_NONE,
+    WSREP_INTERFACE_VERSION,
+    "Codership Oy <info@codership.com>",
+    &dummy_free,
+    NULL,
+    NULL
+};
+
+int wsrep_dummy_loader(wsrep_t* w)
+{
+    if (!w)
+        return EINVAL;
+
+    *w = dummy_iface;
+
+    // allocate private context
+    if (!(w->ctx = malloc(sizeof(wsrep_dummy_t))))
+        return ENOMEM;
+
+    // initialize private context
+    WSREP_DUMMY(w)->log_fn = NULL;
+    WSREP_DUMMY(w)->options = NULL;
+
+    return 0;
+}
diff --git a/wsrep/wsrep_gtid.c b/wsrep/wsrep_gtid.c
new file mode 100644
index 00000000000..e618c5ab7d0
--- /dev/null
+++ b/wsrep/wsrep_gtid.c
@@ -0,0 +1,74 @@
+/* Copyright (C) 2013 Codership Oy <info@codersihp.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+/*! @file Helper functions to deal with GTID string representations */
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "wsrep_api.h"
+
+/*!
+ * Read GTID from string
+ * @return length of GTID string representation or -EINVAL in case of error
+ */
+int
+wsrep_gtid_scan(const char* str, size_t str_len, wsrep_gtid_t* gtid)
+{
+    unsigned int offset;
+    char* endptr;
+
+    if ((offset = wsrep_uuid_scan(str, str_len, &gtid->uuid)) > 0 &&
+        offset < str_len && str[offset] == ':') {
+        ++offset;
+        if (offset < str_len)
+        {
+            errno = 0;
+            gtid->seqno = strtoll(str + offset, &endptr, 0);
+
+            if (errno == 0) {
+                offset = endptr - str;
+                return offset;
+            }
+        }
+    }
+    *gtid = WSREP_GTID_UNDEFINED;
+    return -EINVAL;
+}
+
+/*!
+ * Write GTID to string
+ * @return length of GTID stirng representation of -EMSGSIZE if string is too
+ *         short
+ */
+int
+wsrep_gtid_print(const wsrep_gtid_t* gtid, char* str, size_t str_len)
+{
+    unsigned int offset, ret;
+    if ((offset = wsrep_uuid_print(&gtid->uuid, str, str_len)) > 0)
+    {
+        ret = snprintf(str + offset, str_len - offset,
+                       ":%" PRId64, gtid->seqno);
+        if (ret <= str_len - offset) {
+            return (offset + ret);
+        }
+
+    }
+
+    return -EMSGSIZE;
+}
diff --git a/wsrep/wsrep_loader.c b/wsrep/wsrep_loader.c
new file mode 100644
index 00000000000..8ae6ea962ec
--- /dev/null
+++ b/wsrep/wsrep_loader.c
@@ -0,0 +1,203 @@
+/* Copyright (C) 2009-2011 Codership Oy <info@codersihp.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+/*! @file wsrep implementation loader */
+
+#include <dlfcn.h>
+#include <errno.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "wsrep_api.h"
+
+// Logging stuff for the loader
+static const char* log_levels[] = {"FATAL", "ERROR", "WARN", "INFO", "DEBUG"};
+
+static void default_logger (wsrep_log_level_t lvl, const char* msg)
+{
+    fprintf (stderr, "wsrep loader: [%s] %s\n", log_levels[lvl], msg);
+}
+
+static wsrep_log_cb_t logger = default_logger;
+
+/**************************************************************************
+ * Library loader
+ **************************************************************************/
+
+static int verify(const wsrep_t *wh, const char *iface_ver)
+{
+    const size_t msg_len = 128;
+    char msg[msg_len];
+
+#define VERIFY(_p) if (!(_p)) {                                       \
+        snprintf(msg, msg_len, "wsrep_load(): verify(): %s\n", # _p); \
+        logger (WSREP_LOG_ERROR, msg);                                \
+        return EINVAL;                                                \
+    }
+
+    VERIFY(wh);
+    VERIFY(wh->version);
+
+    if (strcmp(wh->version, iface_ver)) {
+        snprintf (msg, msg_len,
+                  "provider interface version mismatch: need '%s', found '%s'",
+                  iface_ver, wh->version);
+        logger (WSREP_LOG_ERROR, msg);
+        return EINVAL;
+    }
+
+    VERIFY(wh->init);
+    VERIFY(wh->options_set);
+    VERIFY(wh->options_get);
+    VERIFY(wh->connect);
+    VERIFY(wh->disconnect);
+    VERIFY(wh->recv);
+    VERIFY(wh->pre_commit);
+    VERIFY(wh->post_commit);
+    VERIFY(wh->post_rollback);
+    VERIFY(wh->replay_trx);
+    VERIFY(wh->abort_pre_commit);
+    VERIFY(wh->append_key);
+    VERIFY(wh->append_data);
+    VERIFY(wh->free_connection);
+    VERIFY(wh->to_execute_start);
+    VERIFY(wh->to_execute_end);
+    VERIFY(wh->preordered_collect);
+    VERIFY(wh->preordered_commit);
+    VERIFY(wh->sst_sent);
+    VERIFY(wh->sst_received);
+    VERIFY(wh->stats_get);
+    VERIFY(wh->stats_free);
+    VERIFY(wh->stats_reset);
+    VERIFY(wh->pause);
+    VERIFY(wh->resume);
+    VERIFY(wh->desync);
+    VERIFY(wh->resync);
+    VERIFY(wh->lock);
+    VERIFY(wh->unlock);
+    VERIFY(wh->is_locked);
+    VERIFY(wh->provider_name);
+    VERIFY(wh->provider_version);
+    VERIFY(wh->provider_vendor);
+    VERIFY(wh->free);
+    return 0;
+}
+
+typedef int (*wsrep_loader_fun)(wsrep_t*);
+
+static wsrep_loader_fun wsrep_dlf(void *dlh, const char *sym)
+{
+    union {
+        wsrep_loader_fun dlfun;
+        void *obj;
+    } alias;
+    alias.obj = dlsym(dlh, sym);
+    return alias.dlfun;
+}
+
+extern int wsrep_dummy_loader(wsrep_t *w);
+
+int wsrep_load(const char *spec, wsrep_t **hptr, wsrep_log_cb_t log_cb)
+{
+    int ret = 0;
+    void *dlh = NULL;
+    wsrep_loader_fun dlfun;
+    const size_t msg_len = 1024;
+    char msg[msg_len + 1];
+    msg[msg_len] = 0;
+
+    if (NULL != log_cb)
+        logger = log_cb;
+
+    if (!(spec && hptr))
+        return EINVAL;
+
+    snprintf (msg, msg_len,
+              "wsrep_load(): loading provider library '%s'", spec);
+    logger (WSREP_LOG_INFO, msg);
+
+    if (!(*hptr = malloc(sizeof(wsrep_t)))) {
+        logger (WSREP_LOG_FATAL, "wsrep_load(): out of memory");
+        return ENOMEM;
+    }
+
+    if (!spec || strcmp(spec, WSREP_NONE) == 0) {
+        if ((ret = wsrep_dummy_loader(*hptr)) != 0) {
+            free (*hptr);
+            *hptr = NULL;
+        }
+        return ret;
+    }
+
+    if (!(dlh = dlopen(spec, RTLD_NOW | RTLD_LOCAL))) {
+        snprintf(msg, msg_len, "wsrep_load(): dlopen(): %s", dlerror());
+        logger (WSREP_LOG_ERROR, msg);
+        ret = EINVAL;
+        goto out;
+    }
+
+    if (!(dlfun = wsrep_dlf(dlh, "wsrep_loader"))) {
+        ret = EINVAL;
+        goto out;
+    }
+
+    if ((ret = (*dlfun)(*hptr)) != 0) {
+        snprintf(msg, msg_len, "wsrep_load(): loader failed: %s",
+                 strerror(ret));
+        logger (WSREP_LOG_ERROR, msg);
+        goto out;
+    }
+
+    if ((ret = verify(*hptr, WSREP_INTERFACE_VERSION)) != 0) {
+        snprintf (msg, msg_len,
+                  "wsrep_load(): interface version mismatch: my version %s, "
+                  "provider version %s", WSREP_INTERFACE_VERSION,
+                  (*hptr)->version);
+        logger (WSREP_LOG_ERROR, msg);
+        goto out;
+    }
+
+    (*hptr)->dlh = dlh;
+
+out:
+    if (ret != 0) {
+        if (dlh) dlclose(dlh);
+        free(*hptr);
+        *hptr = NULL;
+    } else {
+        snprintf (msg, msg_len,
+                  "wsrep_load(): %s %s by %s loaded successfully.",
+                  (*hptr)->provider_name, (*hptr)->provider_version,
+                  (*hptr)->provider_vendor);
+        logger (WSREP_LOG_INFO, msg);
+    }
+
+    return ret;
+}
+
+void wsrep_unload(wsrep_t *hptr)
+{
+    if (!hptr) {
+        logger (WSREP_LOG_WARN, "wsrep_unload(): null pointer.");
+    } else {
+        if (hptr->free)
+            hptr->free(hptr);
+        if (hptr->dlh)
+            dlclose(hptr->dlh);
+        free(hptr);
+    }
+}
+
diff --git a/wsrep/wsrep_uuid.c b/wsrep/wsrep_uuid.c
new file mode 100644
index 00000000000..baa95b2578a
--- /dev/null
+++ b/wsrep/wsrep_uuid.c
@@ -0,0 +1,83 @@
+/* Copyright (C) 2009 Codership Oy <info@codersihp.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+/*! @file Helper functions to deal with history UUID string representations */
+
+#include <errno.h>
+#include <ctype.h>
+#include <stdio.h>
+
+#include "wsrep_api.h"
+
+/*!
+ * Read UUID from string
+ * @return length of UUID string representation or -EINVAL in case of error
+ */
+int
+wsrep_uuid_scan (const char* str, size_t str_len, wsrep_uuid_t* uuid)
+{
+    unsigned int uuid_len  = 0;
+    unsigned int uuid_offt = 0;
+
+    while (uuid_len + 1 < str_len) {
+        /* We are skipping potential '-' after uuid_offt == 4, 6, 8, 10
+         * which means
+         *     (uuid_offt >> 1) == 2, 3, 4, 5,
+         * which in turn means
+         *     (uuid_offt >> 1) - 2 <= 3
+         * since it is always >= 0, because uuid_offt is unsigned */
+        if (((uuid_offt >> 1) - 2) <= 3 && str[uuid_len] == '-') {
+            // skip dashes after 4th, 6th, 8th and 10th positions
+            uuid_len += 1;
+            continue;
+        }
+
+        if (isxdigit(str[uuid_len]) && isxdigit(str[uuid_len + 1])) {
+            // got hex digit, scan another byte to uuid, increment uuid_offt
+            sscanf (str + uuid_len, "%2hhx", uuid->data + uuid_offt);
+            uuid_len  += 2;
+            uuid_offt += 1;
+            if (sizeof (uuid->data) == uuid_offt)
+                return uuid_len;
+        }
+        else {
+            break;
+        }
+    }
+
+    *uuid = WSREP_UUID_UNDEFINED;
+    return -EINVAL;
+}
+
+/*!
+ * Write UUID to string
+ * @return length of UUID string representation or -EMSGSIZE if string is too
+ *         short
+ */
+int
+wsrep_uuid_print (const wsrep_uuid_t* uuid, char* str, size_t str_len)
+{
+    if (str_len > 36) {
+        const unsigned char* u = uuid->data;
+        return snprintf(str, str_len, "%02x%02x%02x%02x-%02x%02x-%02x%02x-"
+                        "%02x%02x-%02x%02x%02x%02x%02x%02x",
+                        u[ 0], u[ 1], u[ 2], u[ 3], u[ 4], u[ 5], u[ 6], u[ 7],
+                        u[ 8], u[ 9], u[10], u[11], u[12], u[13], u[14], u[15]);
+    }
+    else {
+        return -EMSGSIZE;
+    }
+}