MDEV-26971: JSON file interface to wsrep node state.

Integration with status reporter in wsrep-lib.

Status reporter reports changes in wsrep state and logged errors/
warnings to a json file which then can be read and interpreted by
an external monitoring tool.

Rationale: until the server is fully initialized it is unaccessible
by client and the only source of information is an error log which
is not machine-friendly. Since wsrep node can spend a very long time
in initialization phase (state transfer), it may be a very long time
that automatic tools can't easily monitor its liveness and progression.

New variable: wsrep_status_file specifies the output file name.
If not set, no file is created and no reporting is done.

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

10 files changed, 172 insertions, 0 deletions

diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
index 8aadc22046e..a96ab1dfced 100644
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@ -22,6 +22,7 @@ IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY)
     wsrep_server_service.cc
     wsrep_storage_service.cc
     wsrep_server_state.cc
+    wsrep_status.cc
     wsrep_utils.cc
     wsrep_xid.cc
     wsrep_check_opts.cc
diff --git a/sql/log.cc b/sql/log.cc
index 5b5f42ffee2..a4c0da2ab70 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -62,6 +62,7 @@
 #include "wsrep_mysqld.h"
 #ifdef WITH_WSREP
 #include "wsrep_trans_observer.h"
+#include "wsrep_status.h"
 #endif /* WITH_WSREP */
 
 #ifdef HAVE_REPLICATION
@@ -9284,6 +9285,16 @@ static void print_buffer_to_file(enum loglevel level, const char *buffer,
 
   fflush(stderr);
 
+#ifdef WITH_WSREP
+  if (level <= WARNING_LEVEL)
+  {
+    wsrep::reporter::log_level const lvl = (level <= ERROR_LEVEL ?
+                                            wsrep::reporter::error :
+                                            wsrep::reporter::warning);
+    Wsrep_status::report_log_msg(lvl, tag, tag_length, buffer, length, skr);
+  }
+#endif /* WITH_WSREP */
+
   mysql_mutex_unlock(&LOCK_error_log);
   DBUG_VOID_RETURN;
 }
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index 2174f8be2e7..4a10e5e2c7f 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -5990,6 +5990,11 @@ static Sys_var_charptr Sys_wsrep_notify_cmd(
        READ_ONLY GLOBAL_VAR(wsrep_notify_cmd), CMD_LINE(REQUIRED_ARG),
        DEFAULT(""));
 
+static Sys_var_charptr_fscs Sys_wsrep_status_file(
+       "wsrep_status_file", "wsrep status output filename",
+       READ_ONLY GLOBAL_VAR(wsrep_status_file), CMD_LINE(REQUIRED_ARG),
+       DEFAULT(""));
+
 static Sys_var_mybool Sys_wsrep_certify_nonPK(
        "wsrep_certify_nonPK", "Certify tables with no primary key",
        GLOBAL_VAR(wsrep_certify_nonPK), 
diff --git a/sql/wsrep_mutex.h b/sql/wsrep_mutex.h
index f396c1be331..50f129c96cd 100644
--- a/sql/wsrep_mutex.h
+++ b/sql/wsrep_mutex.h
@@ -20,6 +20,7 @@
 #include "wsrep/mutex.hpp"
 
 /* implementation */
+#include "my_global.h"
 #include "my_pthread.h"
 
 class Wsrep_mutex : public wsrep::mutex
diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index a2a9ef68e96..df0d715650f 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -17,6 +17,7 @@
 #include "sql_plugin.h"                         /* wsrep_plugins_pre_init() */
 #include "my_global.h"
 #include "wsrep_server_state.h"
+#include "wsrep_status.h"
 
 #include "mariadb.h"
 #include <mysqld.h>
@@ -82,6 +83,7 @@ const char *wsrep_start_position;
 const char *wsrep_data_home_dir;
 const char *wsrep_dbug_option;
 const char *wsrep_notify_cmd;
+const char *wsrep_status_file;
 
 ulong   wsrep_debug;                            // Debug level logging
 my_bool wsrep_convert_LOCK_to_trx;              // Convert locking sessions to trx
@@ -763,6 +765,8 @@ int wsrep_init_server()
   wsrep::log::logger_fn(wsrep_log_cb);
   try
   {
+    Wsrep_status::init_once(wsrep_status_file);
+
     std::string server_name;
     std::string server_id;
     std::string node_address;
@@ -831,6 +835,7 @@ void wsrep_deinit_server()
 {
   wsrep_deinit_schema();
   Wsrep_server_state::destroy();
+  Wsrep_status::destroy();
 }
 
 int wsrep_init()
diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h
index 9cbe186aedf..e5aacadf2b6 100644
--- a/sql/wsrep_mysqld.h
+++ b/sql/wsrep_mysqld.h
@@ -79,6 +79,7 @@ extern const char* wsrep_start_position;
 extern ulong       wsrep_max_ws_size;
 extern ulong       wsrep_max_ws_rows;
 extern const char* wsrep_notify_cmd;
+extern const char* wsrep_status_file;
 extern my_bool     wsrep_certify_nonPK;
 extern long int    wsrep_protocol_version;
 extern ulong       wsrep_forced_binlog_format;
diff --git a/sql/wsrep_status.cc b/sql/wsrep_status.cc
new file mode 100644
index 00000000000..055982aa341
--- /dev/null
+++ b/sql/wsrep_status.cc
@@ -0,0 +1,60 @@
+/* Copyright 2021 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#include "wsrep_status.h"
+
+mysql_mutex_t LOCK_wsrep_status;
+
+#ifdef HAVE_PSI_INTERFACE
+PSI_mutex_key key_LOCK_wsrep_status;
+#endif
+
+Wsrep_mutex*     Wsrep_status::m_mutex    = 0;
+wsrep::reporter* Wsrep_status::m_instance = 0;
+
+void Wsrep_status::report_log_msg(wsrep::reporter::log_level const level,
+                                  const char* const tag, size_t const tag_len,
+                                  const char* const buf, size_t const buf_len,
+                                  double const tstamp)
+{
+  if (!Wsrep_status::m_instance) return;
+
+  Wsrep_status::m_instance->report_log_msg(level,
+    std::string(tag, tag_len) + std::string(buf, buf_len),
+    tstamp);
+}
+
+void Wsrep_status::init_once(const std::string& file_name)
+{
+  if (file_name.length() > 0 && m_instance == 0)
+  {
+    mysql_mutex_init(key_LOCK_wsrep_status, &LOCK_wsrep_status,
+                     MY_MUTEX_INIT_FAST);
+    m_mutex    = new Wsrep_mutex(LOCK_wsrep_status);
+    m_instance = new wsrep::reporter(*m_mutex, file_name, 4);
+  }
+}
+
+void Wsrep_status::destroy()
+{
+  if (m_instance)
+  {
+    delete m_instance;
+    m_instance= 0;
+    delete m_mutex;
+    m_mutex= 0;
+    mysql_mutex_destroy(&LOCK_wsrep_status);
+  }
+}
diff --git a/sql/wsrep_status.h b/sql/wsrep_status.h
new file mode 100644
index 00000000000..1c99a4cfe88
--- /dev/null
+++ b/sql/wsrep_status.h
@@ -0,0 +1,61 @@
+/* Copyright 2021 Codership Oy <info@codership.com>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef WSREP_STATUS_H
+#define WSREP_STATUS_H
+
+/* wsrep-lib */
+#include "wsrep/reporter.hpp"
+
+/* implementation */
+#include "wsrep_mutex.h"
+
+class Wsrep_status
+{
+public:
+  static void init_once(const std::string& file_name);
+  static void destroy();
+
+  static void report_state(enum wsrep::server_state::state const state,
+                           float const progress = wsrep::reporter::indefinite)
+  {
+    if (!Wsrep_status::m_instance) return;
+
+    Wsrep_status::m_instance->report_state(state, progress);
+  }
+
+  static void report_log_msg(wsrep::reporter::log_level level,
+                             const char* tag, size_t tag_len,
+                             const char* buf, size_t buf_len,
+                             double const tstamp = wsrep::reporter::undefined);
+//  {
+//    if (!Wsrep_status::m_instance) return;
+//
+//    Wsrep_status::m_instance->report_log_msg(level, msg, tstamp);
+//  }
+
+  static bool is_instance_initialized()
+  {
+      return m_instance;
+  }
+
+private:
+  Wsrep_status(const std::string& file_name);
+
+  static Wsrep_mutex*     m_mutex;
+  static wsrep::reporter* m_instance;
+};
+
+#endif /* WSREP_STATUS_H */
diff --git a/sql/wsrep_utils.cc b/sql/wsrep_utils.cc
index a679304c40a..a85533e52bb 100644
--- a/sql/wsrep_utils.cc
+++ b/sql/wsrep_utils.cc
@@ -26,6 +26,7 @@
 #include "wsrep_utils.h"
 #include "wsrep_mysqld.h"
 #include "wsrep_thd.h"
+#include "wsrep_status.h"
 
 #include <sql_class.h>
 
@@ -93,6 +94,18 @@ wsrep_prepend_PATH (const char* path)
 namespace wsp
 {
 
+void
+node_status::set(enum wsrep::server_state::state new_status,
+                 const wsrep::view* view)
+{
+    if (status != new_status || 0 != view)
+    {
+        wsrep_notify_status(new_status, view);
+        status= new_status;
+        Wsrep_status::report_state(status, 0);
+    }
+}
+
 bool
 env::ctor_common(char** e)
 {
diff --git a/sql/wsrep_utils.h b/sql/wsrep_utils.h
index 88c5770d3fe..39c0e56dac1 100644
--- a/sql/wsrep_utils.h
+++ b/sql/wsrep_utils.h
@@ -21,6 +21,20 @@
 
 unsigned int wsrep_check_ip (const char* const addr, bool *is_ipv6);
 size_t wsrep_guess_ip (char* buf, size_t buf_len);
+namespace wsp {
+class node_status
+{
+public:
+  node_status() : status(wsrep::server_state::s_disconnected) {}
+  void set(enum wsrep::server_state::state new_status,
+           const wsrep::view* view= 0);
+  enum wsrep::server_state::state get() const { return status; }
+private:
+  enum wsrep::server_state::state status;
+};
+} /* namespace wsp */
+
+extern wsp::node_status local_status;
 
 /* returns the length of the host part of the address string */
 size_t wsrep_host_len(const char* addr, size_t addr_len);