WL#2575 - Fulltext: Parser plugin for FTS

Manual merge.


Makefile.am:
  Added new 'plugin' subdir.
configure.in:
  Added plugin related makefiles.
include/my_base.h:
  Added HA_OPEN_FROM_SQL_LAYER flag - indicates that a table was openned from the sql layer.
  Added HA_OPTION_RELIES_ON_SQL_LAYER flag - indicates that a table relies on the sql layer.
  Added HA_CREATE_RELIES_ON_SQL_LAYER flag - indicates that a table must be created with
  HA_OPTION_RELIES_ON_SQL_LAYER flag.
include/myisam.h:
  Distinct fulltext parser number added.
include/plugin.h:
  Revise comment.
sql/ha_myisam.cc:
  Pass HA_OPEN_FROM_SQL_LAYER flag to mi_open().
  Pass HA_CREATE_RELIES_ON_SQL_LAYER flag to mi_create().
sql/sql_plugin.cc:
  Reuse "unused" dynamic array elements.
  A check for plugin info interface version.
sql/sql_plugin.h:
  Added plugin_type_names[] - string plugin type names.
sql/sql_show.cc:
  Use plugin_type_names array instead of switch to find literal parser name representation.
sql/sql_table.cc:
  Fixed that ALTER TABLE ... ADD INDEX loses WITH PARSER info.
storage/myisam/ft_boolean_search.c:
  Call fulltext parser init() function, pass MYSQL_FTPARSER_PARAM, returned by
  ftparser_call_initializer(), to parser->parse().
storage/myisam/ft_nlq_search.c:
  Call fulltext parser init() function, pass MYSQL_FTPARSER_PARAM, returned by
  ftparser_call_initializer(), to parser->parse().
storage/myisam/ft_parser.c:
  Added two functions:
  ftparser_call_initializer() - calls parser->init() function if specified and parser is not yet
  initialized. Returns MYSQL_FTPARSER_PARAM *.
  ftparser_call_deinitializer() - calls parser->deinit() function if specified and parser was
  initialized. Deinitializes all parsers.
  ft_parse() accepts additional param now - MYSQL_FTPARSER_PARM and passes it to parser->parse().
storage/myisam/ft_update.c:
  Call fulltext parser init() function, pass MYSQL_FTPARSER_PARAM, returned by
  ftparser_call_initializer(), to _mi_ft_parse().
  _mi_ft_parse() accepts additional param now - MYSQL_FTPARSER_PARAM and passes
  it to parser->parse().
storage/myisam/ftdefs.h:
  Prototypes for new functions were added. MYSQL_FTPARSER_PARAM was added
  to ft_parse and _mi_ft_parse().
storage/myisam/mi_close.c:
  Free ftparser_param allocated by ftparser_call_initializer().
storage/myisam/mi_create.c:
  If a table relies on the sql layer, set HA_OPTION_RELIES_ON_SQL_LAYER.
storage/myisam/mi_locking.c:
  Call deinitializer for each initialized parser.
storage/myisam/mi_open.c:
  Set default values for share->ftparser and keydef->ftparser_nr.
  If a table is openned from the non-sql layer and HA_OPTION_RELIES_ON_SQL_LAYER is set, raise
  HA_ERR_UNSUPPORTED error.
storage/myisam/myisamdef.h:
  Added number of distinct parsers to MYISAM_SHARE.
  Added ftparser_param to MI_INFO.
plugin/Makefile.am:
  New BitKeeper file ``plugin/Makefile.am''
plugin/fulltext/Makefile.am:
  New BitKeeper file ``plugin/fulltext/Makefile.am''
plugin/fulltext/plugin_example.c:
  New BitKeeper file ``plugin/fulltext/plugin_example.c''

2 files changed, 220 insertions, 0 deletions

diff --git a/plugin/fulltext/Makefile.am b/plugin/fulltext/Makefile.am
new file mode 100644
index 00000000000..70c207a992f
--- /dev/null
+++ b/plugin/fulltext/Makefile.am
@@ -0,0 +1,4 @@
+INCLUDES= -I$(top_builddir)/include
+noinst_LTLIBRARIES= libmypluglib.la
+libmypluglib_la_SOURCES= plugin_example.c
+libmypluglib_la_LDFLAGS= -module
diff --git a/plugin/fulltext/plugin_example.c b/plugin/fulltext/plugin_example.c
new file mode 100644
index 00000000000..c222d3d85c3
--- /dev/null
+++ b/plugin/fulltext/plugin_example.c
@@ -0,0 +1,216 @@
+/*
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include <my_global.h>
+#include <m_string.h>
+#include <m_ctype.h>
+#include <plugin.h>
+
+/*
+  Simple full-text parser plugin that acts as a replacement for the
+  built-in full-text parser:
+  - All non-whitespace characters are significant and are interpreted as
+   "word characters."
+  - Whitespace characters are space, tab, CR, LF.
+  - There is no minimum word length.  Non-whitespace sequences of one
+    character or longer are words.
+  - Stopwords are used in non-boolean mode, not used in boolean mode.
+*/
+
+/*
+  simple_parser interface functions:
+
+  Plugin declaration functions:
+  - simple_parser_plugin_init()
+  - simple_parser_plugin_deinit()
+
+  Parser descriptor functions:
+  - simple_parser_parse()
+  - simple_parser_init()
+  - simple_parser_deinit()
+*/
+
+
+/*
+  Initialize the parser plugin at server start or plugin installation.
+
+  SYNOPSIS
+    simple_parser_plugin_init()
+
+  DESCRIPTION
+    Does nothing.
+
+  RETURN VALUE
+    0                    success
+    1                    failure (cannot happen)
+*/
+
+static int simple_parser_plugin_init(void)
+{
+  return(0);
+}
+
+
+/*
+  Terminate the parser plugin at server shutdown or plugin deinstallation.
+
+  SYNOPSIS
+    simple_parser_plugin_deinit()
+    Does nothing.
+
+  RETURN VALUE
+    0                    success
+    1                    failure (cannot happen)
+
+*/
+
+static int simple_parser_plugin_deinit(void)
+{
+  return(0);
+}
+
+
+/*
+  Initialize the parser at ... [WHEN]
+
+  SYNOPSIS
+    simple_parser_init()
+
+  DESCRIPTION
+    Does nothing.
+
+  RETURN VALUE
+    0                    success
+    1                    failure (cannot happen)
+*/
+
+static int simple_parser_init(MYSQL_FTPARSER_PARAM *param)
+{
+  return(0);
+}
+
+
+/*
+  Terminate the parser at ... [WHEN]
+
+  SYNOPSIS
+    simple_parser_deinit()
+
+  DESCRIPTION
+    Does nothing.
+
+  RETURN VALUE
+    0                    success
+    1                    failure (cannot happen)
+*/
+
+static int simple_parser_deinit(MYSQL_FTPARSER_PARAM *param)
+{
+  return(0);
+}
+
+
+/*
+  Pass a word back to the server.
+
+  SYNOPSIS
+    add_word()
+      param              parsing context of the plugin
+      word               a word
+      len                word length
+
+  DESCRIPTION
+    Fill in boolean metadata for the word (if parsing in boolean mode)
+    and pass the word to the server.  The server adds the word to
+    a full-text index when parsing for indexing, or adds the word to
+    the list of search terms when parsing a search string.
+*/
+
+static void add_word(MYSQL_FTPARSER_PARAM *param, char *word, size_t len)
+{
+  MYSQL_FTPARSER_BOOLEAN_INFO bool_info=
+    { FT_TOKEN_WORD, 0, 0, 0, 0, ' ', 0 };
+
+  if (param->mode == MYSQL_FTPARSER_FULL_BOOLEAN_INFO)
+    param->mysql_add_word(param->mysql_ftparam, word, len, &bool_info);
+  else
+    param->mysql_add_word(param->mysql_ftparam, word, len, 0);
+}
+
+/*
+  Parse a document or a search query.
+
+  SYNOPSIS
+    simple_parser_parse()
+      param              parsing context
+
+  DESCRIPTION
+    This is the main plugin function which is called to parse
+    a document or a search query. The call mode is set in
+    param->mode.  This function simply splits the text into words
+    and passes every word to the MySQL full-text indexing engine.
+*/
+
+int simple_parser_parse(MYSQL_FTPARSER_PARAM *param)
+{
+  char *end, *start, *docend= param->doc + param->length;
+
+  for (end= start= param->doc;; end++)
+  {
+    if (end == docend)
+    {
+      if (end > start)
+        add_word(param, start, end - start);
+      break;
+    }
+    else if (isspace(*end))
+    {
+      if (end > start)
+        add_word(param, start, end - start);
+      start= end + 1;
+    }
+  }
+  return(0);
+}
+
+
+/*
+  Plugin type-specific descriptor
+*/
+
+static struct st_mysql_ftparser simple_parser_descriptor=
+{
+  MYSQL_FTPARSER_INTERFACE_VERSION, /* interface version      */
+  simple_parser_parse,              /* parsing function       */
+  simple_parser_init,               /* parser init function   */
+  simple_parser_deinit              /* parser deinit function */
+};
+
+
+/*
+  Plugin library descriptor
+*/
+
+mysql_declare_plugin
+{
+  MYSQL_FTPARSER_PLUGIN,      /* type                            */
+  &simple_parser_descriptor,  /* descriptor                      */
+  "simple_parser",            /* name                            */
+  "MySQL AB",                 /* author                          */
+  "Simple Full-Text Parser",  /* description                     */
+  simple_parser_plugin_init,  /* init function (when loaded)     */
+  simple_parser_plugin_deinit /* deinit function (when unloaded) */
+}
+mysql_declare_plugin_end;