New modules 'unistr/u8-strtok', 'unistr/u16-strtok', 'unistr/u32-strtok'.

7 files changed, 219 insertions, 0 deletions

diff --git a/lib/unistr/u-strtok.h b/lib/unistr/u-strtok.h
new file mode 100644
index 0000000000..c76c0e13c6
--- /dev/null
+++ b/lib/unistr/u-strtok.h
@@ -0,0 +1,54 @@
+/* Tokenize UTF-8/UTF-16/UTF-32 string.
+   Copyright (C) 1999, 2002, 2006 Free Software Foundation, Inc.
+   Written by Bruno Haible <bruno@clisp.org>, 2002.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU Library General Public License as published
+   by the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Library General Public License for more details.
+
+   You should have received a copy of the GNU Library General Public
+   License along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+   USA.  */
+
+UNIT *
+FUNC (UNIT *str, const UNIT *delim, UNIT **ptr)
+{
+  if (str == NULL)
+    {
+      str = *ptr;
+      if (str == NULL)
+	return NULL; /* reminder that end of token sequence has been reached */
+    }
+
+  /* Skip leading delimiters.  */
+  str += U_STRSPN (str, delim);
+
+  /* Found a token?  */
+  if (*str == 0)
+    {
+      *ptr = NULL;
+      return NULL;
+    }
+
+  /* Move past the token. */
+  {
+    UNIT *token_end = U_STRPBRK (str, delim);
+    if (token_end)
+      {
+	/* NUL-terminate the token.  */
+	*token_end = 0;
+	*ptr = token_end + 1;
+      }
+    else
+      *ptr = NULL;
+  }
+
+  return str;
+}
diff --git a/lib/unistr/u16-strtok.c b/lib/unistr/u16-strtok.c
new file mode 100644
index 0000000000..dfb908b83a
--- /dev/null
+++ b/lib/unistr/u16-strtok.c
@@ -0,0 +1,29 @@
+/* Tokenize UTF-16 string.
+   Copyright (C) 1999, 2002, 2006 Free Software Foundation, Inc.
+   Written by Bruno Haible <bruno@clisp.org>, 2002.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU Library General Public License as published
+   by the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Library General Public License for more details.
+
+   You should have received a copy of the GNU Library General Public
+   License along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+   USA.  */
+
+#include <config.h>
+
+/* Specification.  */
+#include "unistr.h"
+
+#define FUNC u16_strtok
+#define UNIT uint16_t
+#define U_STRSPN u16_strspn
+#define U_STRPBRK u16_strpbrk
+#include "u-strtok.h"
diff --git a/lib/unistr/u32-strtok.c b/lib/unistr/u32-strtok.c
new file mode 100644
index 0000000000..ed272dd89c
--- /dev/null
+++ b/lib/unistr/u32-strtok.c
@@ -0,0 +1,29 @@
+/* Tokenize UTF-32 string.
+   Copyright (C) 1999, 2002, 2006 Free Software Foundation, Inc.
+   Written by Bruno Haible <bruno@clisp.org>, 2002.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU Library General Public License as published
+   by the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Library General Public License for more details.
+
+   You should have received a copy of the GNU Library General Public
+   License along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+   USA.  */
+
+#include <config.h>
+
+/* Specification.  */
+#include "unistr.h"
+
+#define FUNC u32_strtok
+#define UNIT uint32_t
+#define U_STRSPN u32_strspn
+#define U_STRPBRK u32_strpbrk
+#include "u-strtok.h"
diff --git a/lib/unistr/u8-strtok.c b/lib/unistr/u8-strtok.c
new file mode 100644
index 0000000000..77702c20c4
--- /dev/null
+++ b/lib/unistr/u8-strtok.c
@@ -0,0 +1,29 @@
+/* Tokenize UTF-8 string.
+   Copyright (C) 1999, 2002, 2006 Free Software Foundation, Inc.
+   Written by Bruno Haible <bruno@clisp.org>, 2002.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU Library General Public License as published
+   by the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Library General Public License for more details.
+
+   You should have received a copy of the GNU Library General Public
+   License along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+   USA.  */
+
+#include <config.h>
+
+/* Specification.  */
+#include "unistr.h"
+
+#define FUNC u8_strtok
+#define UNIT uint8_t
+#define U_STRSPN u8_strspn
+#define U_STRPBRK u8_strpbrk
+#include "u-strtok.h"
diff --git a/modules/unistr/u16-strtok b/modules/unistr/u16-strtok
new file mode 100644
index 0000000000..2eb33b09c0
--- /dev/null
+++ b/modules/unistr/u16-strtok
@@ -0,0 +1,26 @@
+Description:
+Tokenize UTF-16 string.
+
+Files:
+lib/unistr/u16-strtok.c
+lib/unistr/u-strtok.h
+
+Depends-on:
+unistr/base
+unistr/u16-strspn
+unistr/u16-strpbrk
+
+configure.ac:
+
+Makefile.am:
+lib_SOURCES += unistr/u16-strtok.c
+
+Include:
+"unistr.h"
+
+License:
+LGPL
+
+Maintainer:
+Bruno Haible
+
diff --git a/modules/unistr/u32-strtok b/modules/unistr/u32-strtok
new file mode 100644
index 0000000000..e338fc2273
--- /dev/null
+++ b/modules/unistr/u32-strtok
@@ -0,0 +1,26 @@
+Description:
+Tokenize UTF-32 string.
+
+Files:
+lib/unistr/u32-strtok.c
+lib/unistr/u-strtok.h
+
+Depends-on:
+unistr/base
+unistr/u32-strspn
+unistr/u32-strpbrk
+
+configure.ac:
+
+Makefile.am:
+lib_SOURCES += unistr/u32-strtok.c
+
+Include:
+"unistr.h"
+
+License:
+LGPL
+
+Maintainer:
+Bruno Haible
+
diff --git a/modules/unistr/u8-strtok b/modules/unistr/u8-strtok
new file mode 100644
index 0000000000..77c30bc64a
--- /dev/null
+++ b/modules/unistr/u8-strtok
@@ -0,0 +1,26 @@
+Description:
+Tokenize UTF-8 string.
+
+Files:
+lib/unistr/u8-strtok.c
+lib/unistr/u-strtok.h
+
+Depends-on:
+unistr/base
+unistr/u8-strspn
+unistr/u8-strpbrk
+
+configure.ac:
+
+Makefile.am:
+lib_SOURCES += unistr/u8-strtok.c
+
+Include:
+"unistr.h"
+
+License:
+LGPL
+
+Maintainer:
+Bruno Haible
+