1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
/**
* Copyright (C) 2015 MongoDB Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
* linked combinations including the program with the OpenSSL library. You
* must comply with the GNU Affero General Public License in all respects for
* all of the code used other than as permitted herein. If you modify file(s)
* with this exception, you may extend this exception to your version of the
* file(s), but you are not obligated to do so. If you do not wish to do so,
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*/
#pragma once
#include <cstdint>
#include <memory>
#include <string>
#include "mongo/base/string_data.h"
#include "mongo/bson/util/builder.h"
#include "mongo/db/fts/unicode/codepoints.h"
namespace mongo {
namespace unicode {
/**
* A string class that support basic Unicode functionality such as removing diacritic marks, and
* lowercasing. The String is constructed with UTF-8 source data, and is converted under the hood to
* a u32string (UTF-32) so operations can be easily done with individual Unicode code points.
*/
class String {
public:
String() = default;
/**
* Construct a String with UTF-8 source data (supports standard C++ string literals, and
* std::strings).
*/
explicit String(StringData utf8_src);
/**
* Reset the String with the new UTF-8 source data, reusing the underlying buffer when possible.
*/
void resetData(const StringData utf8_src);
/**
* Takes a substring of the current String and puts it in another String.
* Overwrites buffer's previous contents rather than appending.
*/
StringData substrToBuf(StackBufBuilder* buffer, size_t pos, size_t len) const;
/**
* Lowercases a substring of the current String and stores the UTF8 result in buffer.
* Overwrites buffer's previous contents rather than appending.
*/
StringData toLowerToBuf(StackBufBuilder* buffer,
CaseFoldMode mode,
size_t offset = 0,
size_t len = std::string::npos) const;
/**
* Returns a UTF-8 encoded std::string version of the String instance. Uses the conversion
* stored in the output buffer when possible.
*/
std::string toString();
/**
* Returns the number Unicode codepoints in the String.
*/
size_t size() const {
return _data.size();
}
/**
* Returns the Unicode codepoint at index i of the String.
*/
const char32_t& operator[](int i) const {
return _data[i];
}
/**
* Options for the substrMatch method.
*/
using SubstrMatchOptions = uint8_t;
/**
* No options (case insensitive and diacritic insensitive).
*/
static const SubstrMatchOptions kNone = 0;
/**
* Perform case sensitive substring match.
*/
static const SubstrMatchOptions kCaseSensitive = 1 << 0;
/**
* Perform diacritic sensitive substring match.
*/
static const SubstrMatchOptions kDiacriticSensitive = 1 << 1;
/**
* Search the string 'str' for the string 'find'. If 'find' exists in 'str', return true, else
* return false. Optionally searches can be made case sensitive and diacritic insensitive. If
* the search is case insensitive, non-Turkish case folding is used unless the
* CaseFoldMode::Turkish is passed to mode.
*/
static bool substrMatch(const std::string& str,
const std::string& find,
SubstrMatchOptions options,
CaseFoldMode mode = CaseFoldMode::kNormal);
/**
* Strips diacritics and case-folds the utf8 input string, as needed to support options.
*
* The options field specifies what operations to *skip*, so kCaseSensitive means to skip case
* folding and kDiacriticSensitive means to skip diacritic striping. If both flags are
* specified, the input utf8 StringData is returned directly without any processing or copying.
*
* If processing is performed, the returned StringData will be placed in buffer. buffer's
* contents (if any) will be replaced. Since we may return the input unmodified the returned
* StringData's lifetime is the shorter of the input utf8 and the next modification to buffer.
* The input utf8 must not point into buffer.
*/
static StringData caseFoldAndStripDiacritics(StackBufBuilder* buffer,
StringData utf8,
SubstrMatchOptions options,
CaseFoldMode mode);
private:
/**
* Helper method for converting a UTF-8 string to a UTF-32 string.
*/
void setData(const StringData utf8_src);
/**
* Unified implementation of substrToBuf and toLowerToBuf.
*/
template <typename Func>
StringData substrToBufWithTransform(StackBufBuilder* buffer,
size_t pos,
size_t len,
Func transform) const;
/**
* The underlying UTF-32 data.
*/
std::u32string _data;
/**
* A buffer for storing the result of the UTF-32 to UTF-8 conversion.
*/
std::string _outputBuf;
/**
* A bool flag that is set to true when toString() will require that the UTF-32 to UTF-8
* conversion be applied again.
*/
bool _needsOutputConversion;
};
} // namespace unicode
} // namespace mongo
|
