MENT-1626: JSON file interface to wsrep node state.bb-10.8-MDEV-26971

Integration with status reporter in wsrep-lib.

Status reporter reports changes in wsrep state and logged errors/
warnings to a json file which then can be read and interpreted by
an external monitoring tool.

Rationale: until the server is fully initialized it is unaccessible
by client and the only source of information is an error log which
is not machine-friendly. Since wsrep node can spend a very long time
in initialization phase (state transfer), it may be a very long time
that automatic tools can't easily monitor its liveness and progression.

New variable: wsrep_status_file specifies the output file name.
If not set, no file is created and no reporting is done.

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

17 files changed, 180 insertions, 11 deletions

diff --git a/mysql-test/include/galera_variables_ok.inc b/mysql-test/include/galera_variables_ok.inc
index 82c5174bc14..c9a54724c17 100644
--- a/mysql-test/include/galera_variables_ok.inc
+++ b/mysql-test/include/galera_variables_ok.inc
@@ -1,6 +1,6 @@
 --disable_query_log
 
---let $galera_variables_ok = `SELECT COUNT(*) = 49 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep%'`
+--let $galera_variables_ok = `SELECT COUNT(*) = 50 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep%'`
 
 --if (!$galera_variables_ok) {
 	--skip Galera number of variables has changed!
diff --git a/mysql-test/include/galera_variables_ok_debug.inc b/mysql-test/include/galera_variables_ok_debug.inc
index c9a54724c17..e420b3af6c3 100644
--- a/mysql-test/include/galera_variables_ok_debug.inc
+++ b/mysql-test/include/galera_variables_ok_debug.inc
@@ -1,6 +1,6 @@
 --disable_query_log
 
---let $galera_variables_ok = `SELECT COUNT(*) = 50 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep%'`
+--let $galera_variables_ok = `SELECT COUNT(*) = 51 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep%'`
 
 --if (!$galera_variables_ok) {
 	--skip Galera number of variables has changed!
diff --git a/mysql-test/suite/galera/r/galera_defaults.result b/mysql-test/suite/galera/r/galera_defaults.result
index d6bbcbae680..04f45a7c770 100644
--- a/mysql-test/suite/galera/r/galera_defaults.result
+++ b/mysql-test/suite/galera/r/galera_defaults.result
@@ -3,7 +3,7 @@ connection node_1;
 # Correct Galera library found
 SELECT COUNT(*) `expect 50` FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep_%';
 expect 50
-49
+50
 SELECT VARIABLE_NAME, VARIABLE_VALUE
 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
 WHERE VARIABLE_NAME LIKE 'wsrep_%'
@@ -57,6 +57,7 @@ WSREP_SST_AUTH
 WSREP_SST_DONOR	
 WSREP_SST_DONOR_REJECTS_QUERIES	OFF
 WSREP_SST_METHOD	rsync
+WSREP_STATUS_FILE	
 WSREP_SYNC_WAIT	15
 WSREP_TRX_FRAGMENT_SIZE	0
 WSREP_TRX_FRAGMENT_UNIT	bytes
diff --git a/mysql-test/suite/sys_vars/r/sysvars_wsrep.result b/mysql-test/suite/sys_vars/r/sysvars_wsrep.result
index bc1f49195b5..693af55ed66 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_wsrep.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_wsrep.result
@@ -706,6 +706,21 @@ ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 GLOBAL_VALUE_PATH	NULL
+VARIABLE_NAME	WSREP_STATUS_FILE
+SESSION_VALUE	NULL
+GLOBAL_VALUE	
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	VARCHAR
+VARIABLE_COMMENT	wsrep status output filename
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	NULL
+READ_ONLY	YES
+COMMAND_LINE_ARGUMENT	REQUIRED
+GLOBAL_VALUE_PATH	NULL
 VARIABLE_NAME	WSREP_SYNC_WAIT
 SESSION_VALUE	0
 GLOBAL_VALUE	0
diff --git a/mysql-test/suite/wsrep/r/variables.result b/mysql-test/suite/wsrep/r/variables.result
index 8e1af35fd62..8df0210b2d1 100644
--- a/mysql-test/suite/wsrep/r/variables.result
+++ b/mysql-test/suite/wsrep/r/variables.result
@@ -135,6 +135,7 @@ WSREP_SST_DONOR_REJECTS_QUERIES
 WSREP_SST_METHOD
 WSREP_SST_RECEIVE_ADDRESS
 WSREP_START_POSITION
+WSREP_STATUS_FILE
 WSREP_SYNC_WAIT
 WSREP_TRX_FRAGMENT_SIZE
 WSREP_TRX_FRAGMENT_UNIT
diff --git a/mysql-test/suite/wsrep/r/variables_debug.result b/mysql-test/suite/wsrep/r/variables_debug.result
index 0690f540a37..109b25cd898 100644
--- a/mysql-test/suite/wsrep/r/variables_debug.result
+++ b/mysql-test/suite/wsrep/r/variables_debug.result
@@ -136,6 +136,7 @@ WSREP_SST_DONOR_REJECTS_QUERIES
 WSREP_SST_METHOD
 WSREP_SST_RECEIVE_ADDRESS
 WSREP_START_POSITION
+WSREP_STATUS_FILE
 WSREP_SYNC_WAIT
 WSREP_TRX_FRAGMENT_SIZE
 WSREP_TRX_FRAGMENT_UNIT
diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
index 972cc0b736c..997f6c2dc14 100644
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@ -22,6 +22,7 @@ IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY)
     wsrep_server_service.cc
     wsrep_storage_service.cc
     wsrep_server_state.cc
+    wsrep_status.cc
     wsrep_utils.cc
     wsrep_xid.cc
     wsrep_check_opts.cc
diff --git a/sql/log.cc b/sql/log.cc
index 969e2820975..aa4fe5f5888 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -61,6 +61,7 @@
 #include "wsrep_mysqld.h"
 #ifdef WITH_WSREP
 #include "wsrep_trans_observer.h"
+#include "wsrep_status.h"
 #endif /* WITH_WSREP */
 
 #ifdef HAVE_REPLICATION
@@ -9141,6 +9142,16 @@ static void print_buffer_to_file(enum loglevel level, const char *buffer,
 
   fflush(stderr);
 
+#ifdef WITH_WSREP
+  if (level <= WARNING_LEVEL)
+  {
+    wsrep::reporter::log_level const lvl = (level <= ERROR_LEVEL ?
+                                            wsrep::reporter::error :
+                                            wsrep::reporter::warning);
+    Wsrep_status::report_log_msg(lvl, tag, tag_length, buffer, length, skr);
+  }
+#endif /* WITH_WSREP */
+
   mysql_mutex_unlock(&LOCK_error_log);
   DBUG_VOID_RETURN;
 }
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index e18ca6392d8..fe9559ec843 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -5961,6 +5961,11 @@ static Sys_var_charptr Sys_wsrep_notify_cmd(
        READ_ONLY GLOBAL_VAR(wsrep_notify_cmd), CMD_LINE(REQUIRED_ARG),
        DEFAULT(""));
 
+static Sys_var_charptr_fscs Sys_wsrep_status_file(
+       "wsrep_status_file", "wsrep status output filename",
+       READ_ONLY GLOBAL_VAR(wsrep_status_file), CMD_LINE(REQUIRED_ARG),
+       DEFAULT(""));
+
 static Sys_var_mybool Sys_wsrep_certify_nonPK(
        "wsrep_certify_nonPK", "Certify tables with no primary key",
        GLOBAL_VAR(wsrep_certify_nonPK), 
diff --git a/sql/wsrep_mutex.h b/sql/wsrep_mutex.h
index 3454b44e0ec..9ceb263618d 100644
--- a/sql/wsrep_mutex.h
+++ b/sql/wsrep_mutex.h
@@ -20,6 +20,7 @@
 #include "wsrep/mutex.hpp"
 
 /* implementation */
+#include "my_global.h"
 #include "my_pthread.h"
 
 class Wsrep_mutex : public wsrep::mutex
diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index d61494e31be..b1af1d0d1a8 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -17,6 +17,7 @@
 #include "sql_plugin.h"                         /* wsrep_plugins_pre_init() */
 #include "my_global.h"
 #include "wsrep_server_state.h"
+#include "wsrep_status.h"
 
 #include "mariadb.h"
 #include <mysqld.h>
@@ -82,6 +83,7 @@ const char *wsrep_start_position;
 const char *wsrep_data_home_dir;
 const char *wsrep_dbug_option;
 const char *wsrep_notify_cmd;
+const char *wsrep_status_file;
 
 ulong   wsrep_debug;                            // Debug level logging
 my_bool wsrep_convert_LOCK_to_trx;              // Convert locking sessions to trx
@@ -764,6 +766,8 @@ int wsrep_init_server()
   wsrep::log::logger_fn(wsrep_log_cb);
   try
   {
+    Wsrep_status::init_once(wsrep_status_file);
+
     std::string server_name;
     std::string server_id;
     std::string node_address;
@@ -826,6 +830,7 @@ void wsrep_deinit_server()
 {
   wsrep_deinit_schema();
   Wsrep_server_state::destroy();
+  Wsrep_status::destroy();
 }
 
 int wsrep_init()
diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h
index 73eff87e42e..89c2ffba94b 100644
--- a/sql/wsrep_mysqld.h
+++ b/sql/wsrep_mysqld.h
@@ -79,6 +79,7 @@ extern const char* wsrep_start_position;
 extern ulong       wsrep_max_ws_size;
 extern ulong       wsrep_max_ws_rows;
 extern const char* wsrep_notify_cmd;
+extern const char* wsrep_status_file;
 extern my_bool     wsrep_certify_nonPK;
 extern long int    wsrep_protocol_version;
 extern ulong       wsrep_forced_binlog_format;
diff --git a/sql/wsrep_status.cc b/sql/wsrep_status.cc
new file mode 100644
index 00000000000..055982aa341
--- /dev/null
+++ b/sql/wsrep_status.cc
@@ -0,0 +1,60 @@
+/* Copyright 2021 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#include "wsrep_status.h"
+
+mysql_mutex_t LOCK_wsrep_status;
+
+#ifdef HAVE_PSI_INTERFACE
+PSI_mutex_key key_LOCK_wsrep_status;
+#endif
+
+Wsrep_mutex*     Wsrep_status::m_mutex    = 0;
+wsrep::reporter* Wsrep_status::m_instance = 0;
+
+void Wsrep_status::report_log_msg(wsrep::reporter::log_level const level,
+                                  const char* const tag, size_t const tag_len,
+                                  const char* const buf, size_t const buf_len,
+                                  double const tstamp)
+{
+  if (!Wsrep_status::m_instance) return;
+
+  Wsrep_status::m_instance->report_log_msg(level,
+    std::string(tag, tag_len) + std::string(buf, buf_len),
+    tstamp);
+}
+
+void Wsrep_status::init_once(const std::string& file_name)
+{
+  if (file_name.length() > 0 && m_instance == 0)
+  {
+    mysql_mutex_init(key_LOCK_wsrep_status, &LOCK_wsrep_status,
+                     MY_MUTEX_INIT_FAST);
+    m_mutex    = new Wsrep_mutex(LOCK_wsrep_status);
+    m_instance = new wsrep::reporter(*m_mutex, file_name, 4);
+  }
+}
+
+void Wsrep_status::destroy()
+{
+  if (m_instance)
+  {
+    delete m_instance;
+    m_instance= 0;
+    delete m_mutex;
+    m_mutex= 0;
+    mysql_mutex_destroy(&LOCK_wsrep_status);
+  }
+}
diff --git a/sql/wsrep_status.h b/sql/wsrep_status.h
new file mode 100644
index 00000000000..1c99a4cfe88
--- /dev/null
+++ b/sql/wsrep_status.h
@@ -0,0 +1,61 @@
+/* Copyright 2021 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef WSREP_STATUS_H
+#define WSREP_STATUS_H
+
+/* wsrep-lib */
+#include "wsrep/reporter.hpp"
+
+/* implementation */
+#include "wsrep_mutex.h"
+
+class Wsrep_status
+{
+public:
+  static void init_once(const std::string& file_name);
+  static void destroy();
+
+  static void report_state(enum wsrep::server_state::state const state,
+                           float const progress = wsrep::reporter::indefinite)
+  {
+    if (!Wsrep_status::m_instance) return;
+
+    Wsrep_status::m_instance->report_state(state, progress);
+  }
+
+  static void report_log_msg(wsrep::reporter::log_level level,
+                             const char* tag, size_t tag_len,
+                             const char* buf, size_t buf_len,
+                             double const tstamp = wsrep::reporter::undefined);
+//  {
+//    if (!Wsrep_status::m_instance) return;
+//
+//    Wsrep_status::m_instance->report_log_msg(level, msg, tstamp);
+//  }
+
+  static bool is_instance_initialized()
+  {
+      return m_instance;
+  }
+
+private:
+  Wsrep_status(const std::string& file_name);
+
+  static Wsrep_mutex*     m_mutex;
+  static wsrep::reporter* m_instance;
+};
+
+#endif /* WSREP_STATUS_H */
diff --git a/sql/wsrep_utils.cc b/sql/wsrep_utils.cc
index a679304c40a..a85533e52bb 100644
--- a/sql/wsrep_utils.cc
+++ b/sql/wsrep_utils.cc
@@ -26,6 +26,7 @@
 #include "wsrep_utils.h"
 #include "wsrep_mysqld.h"
 #include "wsrep_thd.h"
+#include "wsrep_status.h"
 
 #include <sql_class.h>
 
@@ -93,6 +94,18 @@ wsrep_prepend_PATH (const char* path)
 namespace wsp
 {
 
+void
+node_status::set(enum wsrep::server_state::state new_status,
+                 const wsrep::view* view)
+{
+    if (status != new_status || 0 != view)
+    {
+        wsrep_notify_status(new_status, view);
+        status= new_status;
+        Wsrep_status::report_state(status, 0);
+    }
+}
+
 bool
 env::ctor_common(char** e)
 {
diff --git a/sql/wsrep_utils.h b/sql/wsrep_utils.h
index 974c623521e..39c0e56dac1 100644
--- a/sql/wsrep_utils.h
+++ b/sql/wsrep_utils.h
@@ -27,14 +27,7 @@ class node_status
 public:
   node_status() : status(wsrep::server_state::s_disconnected) {}
   void set(enum wsrep::server_state::state new_status,
-           const wsrep::view* view= 0)
-  {
-    if (status != new_status || 0 != view)
-    {
-      wsrep_notify_status(new_status, view);
-      status= new_status;
-    }
-  }
+           const wsrep::view* view= 0);
   enum wsrep::server_state::state get() const { return status; }
 private:
   enum wsrep::server_state::state status;
diff --git a/wsrep-lib b/wsrep-lib
-Subproject 22921e7082ddfb45222f21a585aa8b877e62aa8
+Subproject 7a28f49506ab45ce308f4bc44303d06793c9dd5