Adding Full Text Search support to partitions

Contains Spiral patches:
007_mariadb-10.2.0.partition_fulltext.diff  MDEV-7705
038_mariadb-10.2.0.partition_fulltext2.diff MDEV-7734

This commit has the following differences compared to the original
patches:

- Added necessary full text search cleanup at the storage engine layer
  that was omitted in the original patch.
- Added test case.
- A lot of code cleanups to make the code notable smaller.
- Changed SQL code to use ha_ft_end() instead of ft_end()

Original author: Kentoku SHIBA
First reviewer:  Jacob Mathew
Second reviewer: Michael Widenius

1 files changed, 465 insertions, 9 deletions

diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc
index f1b484f7c03..49e99b1d609 100644
--- a/sql/ha_partition.cc
+++ b/sql/ha_partition.cc
@@ -74,7 +74,6 @@
                                        HA_REC_NOT_IN_SEQ | \
                                        HA_CAN_REPAIR)
 #define PARTITION_DISABLED_TABLE_FLAGS (HA_CAN_GEOMETRY | \
-                                        HA_CAN_FULLTEXT | \
                                         HA_DUPLICATE_POS | \
                                         HA_CAN_INSERT_DELAYED | \
                                         HA_READ_BEFORE_WRITE_REMOVAL |\
@@ -126,7 +125,6 @@ static void init_partition_psi_keys(void)
 
 static int partition_initialize(void *p)
 {
-
   handlerton *partition_hton;
   partition_hton= (handlerton *)p;
 
@@ -245,12 +243,19 @@ ha_partition::ha_partition(handlerton *hton, TABLE_SHARE *share)
   :handler(hton, share)
 {
   DBUG_ENTER("ha_partition::ha_partition(table)");
-  init_alloc_root(&m_mem_root, 512, 512, MYF(0));
-  init_handler_variables();
+  ha_partition_init();
   DBUG_VOID_RETURN;
 }
 
 
+/* Initialize all partition variables */
+
+void ha_partition::ha_partition_init()
+{
+  init_alloc_root(&m_mem_root, 512, 512, MYF(0));
+  init_handler_variables();
+}
+
 /*
   Constructor method
 
@@ -267,8 +272,7 @@ ha_partition::ha_partition(handlerton *hton, partition_info *part_info)
 {
   DBUG_ENTER("ha_partition::ha_partition(part_info)");
   DBUG_ASSERT(part_info);
-  init_alloc_root(&m_mem_root, 512, 512, MYF(0));
-  init_handler_variables();
+  ha_partition_init();
   m_part_info= part_info;
   m_create_handler= TRUE;
   m_is_sub_partitioned= m_part_info->is_sub_partitioned();
@@ -294,8 +298,7 @@ ha_partition::ha_partition(handlerton *hton, TABLE_SHARE *share,
   :handler(hton, share)
 {
   DBUG_ENTER("ha_partition::ha_partition(clone)");
-  init_alloc_root(&m_mem_root, 512, 512, MYF(0));
-  init_handler_variables();
+  ha_partition_init();
   m_part_info= part_info_arg;
   m_create_handler= TRUE;
   m_is_sub_partitioned= m_part_info->is_sub_partitioned();
@@ -383,6 +386,9 @@ void ha_partition::init_handler_variables()
   m_pre_calling= FALSE;
   m_pre_call_use_parallel= FALSE;
 
+  ft_first= ft_current=  NULL;
+  bulk_access_executing= FALSE;                 // For future
+
   /*
     Clear bitmaps to allow on one to call my_bitmap_free() on them at any time
   */
@@ -3759,14 +3765,20 @@ int ha_partition::close(void)
   bool first= TRUE;
   handler **file;
   uint i;
+  st_partition_ft_info *tmp_ft_info;
   DBUG_ENTER("ha_partition::close");
-
   DBUG_ASSERT(table->s == table_share);
   DBUG_ASSERT(m_part_info);
 
   destroy_record_priority_queue();
   free_partition_bitmaps();
 
+  for (; ft_first ; ft_first= tmp_ft_info)
+  {
+    tmp_ft_info= ft_first->next;
+    my_free(ft_first);
+  }
+
   /* Free active mrr_ranges */
   for (i= 0; i < m_tot_parts; i++)
   {
@@ -6460,6 +6472,450 @@ int ha_partition::multi_range_read_explain_info(uint mrr_mode, char *str,
 }
 
 
+/**
+  Find and retrieve the Full Text Search relevance ranking for a search string
+  in a full text index.
+
+  @param  handler           Full Text Search handler
+  @param  record            Search string
+  @param  length            Length of the search string
+
+  @retval                   Relevance value
+*/
+
+float partition_ft_find_relevance(FT_INFO *handler,
+                                  uchar *record, uint length)
+{
+  st_partition_ft_info *info= (st_partition_ft_info *)handler;
+  uint m_last_part= ((ha_partition*) info->file)->last_part();
+  FT_INFO *m_handler= info->part_ft_info[m_last_part];
+  DBUG_ENTER("partition_ft_find_relevance");
+  if (!m_handler)
+    DBUG_RETURN((float)-1.0);
+  DBUG_RETURN(m_handler->please->find_relevance(m_handler, record, length));
+}
+
+
+/**
+  Retrieve the Full Text Search relevance ranking for the current
+  full text search.
+
+  @param  handler           Full Text Search handler
+
+  @retval                   Relevance value
+*/
+
+float partition_ft_get_relevance(FT_INFO *handler)
+{
+  st_partition_ft_info *info= (st_partition_ft_info *)handler;
+  uint m_last_part= ((ha_partition*) info->file)->last_part();
+  FT_INFO *m_handler= info->part_ft_info[m_last_part];
+  DBUG_ENTER("partition_ft_get_relevance");
+  if (!m_handler)
+    DBUG_RETURN((float)-1.0);
+  DBUG_RETURN(m_handler->please->get_relevance(m_handler));
+}
+
+
+/**
+  Free the memory for a full text search handler.
+
+  @param  handler           Full Text Search handler
+*/
+
+void partition_ft_close_search(FT_INFO *handler)
+{
+  st_partition_ft_info *info= (st_partition_ft_info *)handler;
+  info->file->ft_close_search(handler);
+}
+
+
+/**
+  Free the memory for a full text search handler.
+
+  @param  handler           Full Text Search handler
+*/
+
+void ha_partition::ft_close_search(FT_INFO *handler)
+{
+  uint i;
+  st_partition_ft_info *info= (st_partition_ft_info *)handler;
+  DBUG_ENTER("ha_partition::ft_close_search");
+
+  for (i= 0; i < m_tot_parts; i++)
+  {
+    FT_INFO *m_handler= info->part_ft_info[i];
+    DBUG_ASSERT(!m_handler ||
+                (m_handler->please && m_handler->please->close_search));
+    if (m_handler &&
+        m_handler->please &&
+        m_handler->please->close_search)
+      m_handler->please->close_search(m_handler);
+  }
+  DBUG_VOID_RETURN;
+}
+
+
+/* Partition Full Text search function table */
+_ft_vft partition_ft_vft =
+{
+  NULL, // partition_ft_read_next
+  partition_ft_find_relevance,
+  partition_ft_close_search,
+  partition_ft_get_relevance,
+  NULL  // partition_ft_reinit_search
+};
+
+
+/**
+  Initialize a full text search.
+*/
+
+int ha_partition::ft_init()
+{
+  int error;
+  uint i= 0;
+  uint32 part_id;
+  DBUG_ENTER("ha_partition::ft_init");
+  DBUG_PRINT("info", ("partition this: %p", this));
+
+  /*
+    For operations that may need to change data, we may need to extend
+    read_set.
+  */
+  if (get_lock_type() == F_WRLCK)
+  {
+    /*
+      If write_set contains any of the fields used in partition and
+      subpartition expression, we need to set all bits in read_set because
+      the row may need to be inserted in a different [sub]partition. In
+      other words update_row() can be converted into write_row(), which
+      requires a complete record.
+    */
+    if (bitmap_is_overlapping(&m_part_info->full_part_field_set,
+                              table->write_set))
+      bitmap_set_all(table->read_set);
+    else
+    {
+      /*
+        Some handlers only read fields as specified by the bitmap for the
+        read set. For partitioned handlers we always require that the
+        fields of the partition functions are read such that we can
+        calculate the partition id to place updated and deleted records.
+      */
+      bitmap_union(table->read_set, &m_part_info->full_part_field_set);
+    }
+  }
+
+  /* Now we see what the index of our first important partition is */
+  DBUG_PRINT("info", ("m_part_info->read_partitions: %p",
+             (void *) m_part_info->read_partitions.bitmap));
+  part_id= bitmap_get_first_set(&(m_part_info->read_partitions));
+  DBUG_PRINT("info", ("m_part_spec.start_part %d", part_id));
+
+  if (part_id == MY_BIT_NONE)
+  {
+    error= 0;
+    goto err1;
+  }
+
+  DBUG_PRINT("info", ("ft_init on partition %d", part_id));
+  /*
+    ft_end() is needed for partitioning to reset internal data if scan
+    is already in use
+  */
+  if (m_pre_calling)
+  {
+    if ((error= pre_ft_end()))
+      goto err1;
+  }
+  else
+    ft_end();
+  m_index_scan_type= partition_ft_read;
+  for (i= part_id; i < m_tot_parts; i++)
+  {
+    if (bitmap_is_set(&(m_part_info->read_partitions), i))
+    {
+      error= m_pre_calling ? m_file[i]->pre_ft_init() : m_file[i]->ft_init();
+      if (error)
+        goto err2;
+    }
+  }
+  m_scan_value= 1;
+  m_part_spec.start_part= part_id;
+  m_part_spec.end_part= m_tot_parts - 1;
+  m_ft_init_and_first= TRUE;
+  DBUG_PRINT("info", ("m_scan_value: %d", m_scan_value));
+  DBUG_RETURN(0);
+
+err2:
+  late_extra_no_cache(part_id);
+  while ((int)--i >= (int)part_id)
+  {
+    if (bitmap_is_set(&(m_part_info->read_partitions), i))
+    {
+      if (m_pre_calling)
+        m_file[i]->pre_ft_end();
+      else
+        m_file[i]->ft_end();
+    }
+  }
+err1:
+  m_scan_value= 2;
+  m_part_spec.start_part= NO_CURRENT_PART_ID;
+  DBUG_RETURN(error);
+}
+
+
+/**
+  Initialize a full text search during a bulk access request.
+*/
+
+int ha_partition::pre_ft_init()
+{
+  bool save_m_pre_calling;
+  int error;
+  DBUG_ENTER("ha_partition::pre_ft_init");
+  save_m_pre_calling= m_pre_calling;
+  m_pre_calling= TRUE;
+  error= ft_init();
+  m_pre_calling= save_m_pre_calling;
+  DBUG_RETURN(error);
+}
+
+
+/**
+  Terminate a full text search.
+*/
+
+void ha_partition::ft_end()
+{
+  handler **file;
+  DBUG_ENTER("ha_partition::ft_end");
+  DBUG_PRINT("info", ("partition this: %p", this));
+
+  switch (m_scan_value) {
+  case 2:                                       // Error
+    break;
+  case 1:                                       // Table scan
+    if (NO_CURRENT_PART_ID != m_part_spec.start_part)
+      late_extra_no_cache(m_part_spec.start_part);
+    file= m_file;
+    do
+    {
+      if (bitmap_is_set(&(m_part_info->read_partitions), (uint)(file - m_file)))
+      {
+        if (m_pre_calling)
+          (*file)->pre_ft_end();
+        else
+          (*file)->ft_end();
+      }
+    } while (*(++file));
+    break;
+  }
+  m_scan_value= 2;
+  m_part_spec.start_part= NO_CURRENT_PART_ID;
+  ft_current= 0;
+  DBUG_VOID_RETURN;
+}
+
+
+/**
+  Terminate a full text search during a bulk access request.
+*/
+
+int ha_partition::pre_ft_end()
+{
+  bool save_m_pre_calling;
+  DBUG_ENTER("ha_partition::pre_ft_end");
+  save_m_pre_calling= m_pre_calling;
+  m_pre_calling= TRUE;
+  ft_end();
+  m_pre_calling= save_m_pre_calling;
+  DBUG_RETURN(0);
+}
+
+
+/**
+  Initialize a full text search using the extended API.
+
+  @param  flags             Search flags
+  @param  inx               Key number
+  @param  key               Key value
+
+  @return FT_INFO structure if successful
+          NULL              otherwise
+*/
+
+FT_INFO *ha_partition::ft_init_ext(uint flags, uint inx, String *key)
+{
+  FT_INFO *ft_handler;
+  handler **file;
+  st_partition_ft_info *ft_target, **parent;
+  DBUG_ENTER("ha_partition::ft_init_ext");
+
+  if (ft_current)
+    parent= &ft_current->next;
+  else
+    parent= &ft_first;
+
+  if (!(ft_target= *parent))
+  {
+    FT_INFO **tmp_ft_info;
+    if (!(ft_target= (st_partition_ft_info *)
+          my_multi_malloc(MYF(MY_WME | MY_ZEROFILL),
+                          &ft_target,
+                          sizeof(st_partition_ft_info),
+                          &tmp_ft_info,
+                          sizeof(FT_INFO *) * m_tot_parts,
+                          NullS)))
+    {
+      my_error(ER_OUT_OF_RESOURCES, MYF(ME_FATALERROR));
+      DBUG_RETURN(NULL);
+    }
+    ft_target->part_ft_info= tmp_ft_info;
+    (*parent)= ft_target;
+  }
+
+  ft_current= ft_target;
+  file= m_file;
+  do
+  {
+    if (bitmap_is_set(&(m_part_info->read_partitions), (uint)(file - m_file)))
+    {
+      if ((ft_handler= (*file)->ft_init_ext(flags, inx, key)))
+        (*file)->ft_handler= ft_handler;
+      else
+        (*file)->ft_handler= NULL;
+      ft_target->part_ft_info[file - m_file]= ft_handler;
+    }
+    else
+    {
+      (*file)->ft_handler= NULL;
+      ft_target->part_ft_info[file - m_file]= NULL;
+    }
+  } while (*(++file));
+
+  ft_target->please= &partition_ft_vft;
+  ft_target->file= this;
+  DBUG_RETURN((FT_INFO*)ft_target);
+}
+
+/**
+  Return the next record from the FT result set during an ordered index
+  pre-scan
+
+  @param  use_parallel      Is it a parallel search
+
+  @return >0                Error code
+          0                 Success
+*/
+
+int ha_partition::pre_ft_read(bool use_parallel)
+{
+  bool save_m_pre_calling;
+  int error;
+  DBUG_ENTER("ha_partition::pre_ft_read");
+  DBUG_PRINT("info", ("partition this: %p", this));
+  save_m_pre_calling= m_pre_calling;
+  m_pre_calling= TRUE;
+  m_pre_call_use_parallel= use_parallel;
+  error= ft_read(table->record[0]);
+  m_pre_calling= save_m_pre_calling;
+  DBUG_RETURN(error);
+}
+
+
+/**
+  Return the first or next record in a full text search.
+
+  @param  buf               Buffer where the record should be returned
+
+  @return >0                Error code
+          0                 Success
+*/
+
+int ha_partition::ft_read(uchar *buf)
+{
+  handler *file;
+  int result= HA_ERR_END_OF_FILE, error;
+  uint part_id= m_part_spec.start_part;
+  DBUG_ENTER("ha_partition::ft_read");
+  DBUG_PRINT("info", ("partition this: %p", this));
+  DBUG_PRINT("info", ("part_id: %u", part_id));
+
+  if (part_id == NO_CURRENT_PART_ID)
+  {
+    /*
+      The original set of partitions to scan was empty and thus we report
+      the result here.
+    */
+    DBUG_PRINT("info", ("NO_CURRENT_PART_ID"));
+    goto end;
+  }
+
+  DBUG_ASSERT(m_scan_value == 1);
+
+  if (m_ft_init_and_first)                      // First call to ft_read()
+  {
+    m_ft_init_and_first= FALSE;
+    if (!bulk_access_executing)
+    {
+      error= handle_pre_scan(FALSE, check_parallel_search());
+      if (m_pre_calling || error)
+        DBUG_RETURN(error);
+    }
+    late_extra_cache(part_id);
+  }
+
+  file= m_file[part_id];
+
+  while (TRUE)
+  {
+    if (!(result= file->ft_read(buf)))
+    {
+      /* Found row: remember position and return it. */
+      m_part_spec.start_part= m_last_part= part_id;
+      table->status= 0;
+      DBUG_RETURN(0);
+    }
+
+    /*
+      if we get here, then the current partition ft_next returned failure
+    */
+    if (result == HA_ERR_RECORD_DELETED)
+      continue;                               // Probably MyISAM
+
+    if (result != HA_ERR_END_OF_FILE)
+      goto end_dont_reset_start_part;         // Return error
+
+    /* End current partition */
+    late_extra_no_cache(part_id);
+    DBUG_PRINT("info", ("stopping using partition %d", part_id));
+
+    /* Shift to next partition */
+    while (++part_id < m_tot_parts &&
+           !bitmap_is_set(&(m_part_info->read_partitions), part_id))
+      ;
+    if (part_id >= m_tot_parts)
+    {
+      result= HA_ERR_END_OF_FILE;
+      break;
+    }
+    m_part_spec.start_part= m_last_part= part_id;
+    file= m_file[part_id];
+    DBUG_PRINT("info", ("now using partition %d", part_id));
+    late_extra_cache(part_id);
+  }
+
+end:
+  m_part_spec.start_part= NO_CURRENT_PART_ID;
+end_dont_reset_start_part:
+  table->status= STATUS_NOT_FOUND;
+  DBUG_RETURN(result);
+}
+
+
 /*
   Common routine to set up index scans