diff options
| author | Kentoku SHIBA <kentokushiba@gmail.com> | 2015-04-30 04:44:30 +0900 |
|---|---|---|
| committer | Kentoku SHIBA <kentokushiba@gmail.com> | 2015-04-30 04:44:30 +0900 |
| commit | a0fdb258a435290980993ae025a7fc19c09d2cdb (patch) | |
| tree | 67023cf51c542ed78443d68e9c61b28b39583245 | |
| parent | 060ec5b6b9384456695b6fc903ccfeb8c7ccd298 (diff) | |
| download | mariadb-git-a0fdb258a435290980993ae025a7fc19c09d2cdb.tar.gz | |
Update Mroonga to the latest version on 2015-04-30T04:44:30+0900
413 files changed, 25886 insertions, 5251 deletions
diff --git a/storage/mroonga/CMakeLists.txt b/storage/mroonga/CMakeLists.txt index 4c5156ba96d..00aec7b874d 100644 --- a/storage/mroonga/CMakeLists.txt +++ b/storage/mroonga/CMakeLists.txt @@ -20,24 +20,35 @@ cmake_minimum_required(VERSION 2.6) project(mroonga) +if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") + set(MRN_BUNDLED FALSE) +else() + set(MRN_BUNDLED TRUE) +endif() + include(TestBigEndian) test_big_endian(BIG_ENDIAN) if(BIG_ENDIAN) - message(STATUS "Mroonga is not supported on big-endian") - return() -endif() - -if(MSVC) - if(MSVC_VERSION LESS 1600) - message(STATUS "Mroonga supports only MSVC 2010 or later") + set(MRN_BIG_ENDIAN_NOT_SUPPORTED_MESSAGE + "Mroonga doesn't support on big-endian") + if(MRN_BUNDLED) + message(STATUS ${MRN_BIG_ENDIAN_NOT_SUPPORTED_MESSAGE}) return() + else() + message(FATAL_ERROR ${MRN_BIG_ENDIAN_NOT_SUPPORTED_MESSAGE}) endif() endif() -if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") - set(MRN_BUNDLED FALSE) -else() - set(MRN_BUNDLED TRUE) +if(MSVC) + if(MSVC_VERSION LESS 1800) + set(MRN_OLD_MSVC_MESSAGE "Mroonga supports only MSVC 2013 or later") + if(MRN_BUNDLED) + message(STATUS ${MRN_OLD_MSVC_MESSAGE}) + return() + else() + message(FATAL_ERROR ${MRN_OLD_MSVC_MESSAGE}) + endif() + endif() endif() if(MRN_BUNDLED) @@ -46,11 +57,6 @@ if(MRN_BUNDLED) endif() endif() -IF(HAVE_WVLA) - SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-vla") - SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Wno-vla") -ENDIF() - set(MRN_BUNDLED_GROONGA_RELATIVE_DIR "vendor/groonga") set(MRN_BUNDLED_GROONGA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/${MRN_BUNDLED_GROONGA_RELATIVE_DIR}") @@ -184,6 +190,12 @@ else() endif() find_path(MYSQL_CONFIG "${MYSQL_CONFIG}") +if(EXISTS "${MYSQL_SOURCE_DIR}/storage/maria") + set(MYSQL_VARIANT "MariaDB") +else() + set(MYSQL_VARIANT "MySQL") +endif() + if(EXISTS "${MYSQL_SOURCE_DIR}/pcre") set(MYSQL_REGEX_INCLUDE_DIR "${MYSQL_SOURCE_DIR}/pcre") else() @@ -196,12 +208,24 @@ else() set(MYSQL_RAPIDJSON_INCLUDE_DIR) endif() +if(EXISTS "${MYSQL_SOURCE_DIR}/libbinlogevents") + set(MYSQL_LIBBINLOGEVENTS_EXPORT_DIR + "${MYSQL_SOURCE_DIR}/libbinlogevents/export") + set(MYSQL_LIBBINLOGEVENTS_INCLUDE_DIR + "${MYSQL_SOURCE_DIR}/libbinlogevents/include") +else() + set(MYSQL_LIBBINLOGEVENTS_EXPORT_DIR) + set(MYSQL_LIBBINLOGEVENTS_INCLUDE_DIR) +endif() + set(MYSQL_INCLUDE_DIRS "${MYSQL_BUILD_DIR}/include" "${MYSQL_SOURCE_DIR}/sql" "${MYSQL_SOURCE_DIR}/include" "${MYSQL_REGEX_INCLUDE_DIR}" "${MYSQL_RAPIDJSON_INCLUDE_DIR}" + "${MYSQL_LIBBINLOGEVENTS_EXPORT_DIR}" + "${MYSQL_LIBBINLOGEVENTS_INCLUDE_DIR}" "${MYSQL_SOURCE_DIR}") if(MRN_BUNDLED) @@ -261,6 +285,9 @@ else() pkg_check_modules(GROONGA REQUIRED "groonga >= ${REQUIRED_GROONGA_VERSION}") pkg_check_modules(GROONGA_NORMALIZER_MYSQL "groonga-normalizer-mysql >= ${REQUIRED_GROONGA_NORMALIZER_MYSQL_VERSION}") + set(MRN_LIBRARY_DIRS + ${MRN_LIBRARY_DIRS} + ${GROONGA_LIBRARY_DIRS}) endif() include_directories( @@ -337,8 +364,12 @@ else() mrn_build_flag("-Wno-strict-aliasing") mrn_build_flag("-Wno-deprecated") mrn_check_cxxflag("-fno-implicit-templates") - mrn_check_cxxflag("-fno-exceptions") - mrn_check_cxxflag("-fno-rtti") + if(("${MYSQL_VARIANT}" STREQUAL "MariaDB") OR + ("${MYSQL_VARIANT}" STREQUAL "MySQL" AND + ${MYSQL_VERSION} VERSION_LESS "5.7.0")) + mrn_check_cxxflag("-fno-exceptions") + mrn_check_cxxflag("-fno-rtti") + endif() mrn_check_cxxflag("-felide-constructors") endif() set_source_files_properties(${MRN_SOURCES} PROPERTIES diff --git a/storage/mroonga/COPYING b/storage/mroonga/COPYING new file mode 100644 index 00000000000..be8dfebc1c0 --- /dev/null +++ b/storage/mroonga/COPYING @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + <one line to give the library's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + <signature of Ty Coon>, 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/storage/mroonga/Makefile.am b/storage/mroonga/Makefile.am index 733347eb2c2..d783ec883f9 100644 --- a/storage/mroonga/Makefile.am +++ b/storage/mroonga/Makefile.am @@ -40,7 +40,6 @@ SUBDIRS = \ EXTRA_DIST = \ AUTHORS \ gpg_uid \ - plug.in \ CMakeLists.txt installcheck-local: install diff --git a/storage/mroonga/appveyor.yml b/storage/mroonga/appveyor.yml index a048d7d02e1..de368b85660 100644 --- a/storage/mroonga/appveyor.yml +++ b/storage/mroonga/appveyor.yml @@ -2,14 +2,17 @@ version: "{build}" clone_depth: 10 install: - cd .. - - choco install curl 7zip.commandline - - curl -O http://mirror.jmu.edu/pub/mariadb/mariadb-10.0.16/source/mariadb-10.0.16.tar.gz - - 7z x mariadb-10.0.16.tar.gz - - 7z x mariadb-10.0.16.tar > nul - - cd mariadb-10.0.16 + - choco install -y curl 7zip.commandline + - curl -O http://mirror.jmu.edu/pub/mariadb/mariadb-10.0.17/source/mariadb-10.0.17.tar.gz + - 7z x mariadb-10.0.17.tar.gz + - 7z x mariadb-10.0.17.tar > nul + - cd mariadb-10.0.17 - rmdir /S /Q storage\mroonga\ - move ..\mroonga storage\mroonga - git clone --quiet --depth 1 https://github.com/groonga/groonga.git ..\groonga + - cd ..\groonga + - git submodule update --init + - cd ..\mariadb-10.0.17 - rmdir /S /Q ..\groonga\test\ - mkdir storage\mroonga\vendor - move ..\groonga storage\mroonga\vendor\groonga diff --git a/storage/mroonga/configure.ac b/storage/mroonga/configure.ac index 6d8fa0fe31d..dc127c17838 100644 --- a/storage/mroonga/configure.ac +++ b/storage/mroonga/configure.ac @@ -184,6 +184,10 @@ AC_DEFUN([CONFIG_OPTION_MYSQL],[ mysql_regex_include_dir="$ac_mysql_source_dir/regex" fi MYSQL_INCLUDES="$MYSQL_INCLUDES -I$mysql_regex_include_dir" + if test -d "$ac_mysql_source_dir/libbinlogevents"; then + MYSQL_INCLUDES="$MYSQL_INCLUDES -I$ac_mysql_source_dir/libbinlogevents/export" + MYSQL_INCLUDES="$MYSQL_INCLUDES -I$ac_mysql_source_dir/libbinlogevents/include" + fi MYSQL_INCLUDES="$MYSQL_INCLUDES -I$ac_mysql_source_dir" MYSQL_INCLUDES="$MYSQL_INCLUDES $($ac_mysql_config --include)" AC_SUBST(MYSQL_INCLUDES) @@ -191,8 +195,19 @@ AC_DEFUN([CONFIG_OPTION_MYSQL],[ MYSQL_VERSION="$($ac_mysql_config --version)" AC_SUBST(MYSQL_VERSION) + MYSQL_MAJOR_MINOR_VERSION=["$(echo $MYSQL_VERSION | sed -e 's/\([0-9]*\)\.\([0-9]*\)\.\([0-9]*[a-z]*\)[.0-9a-z-]*$/\1.\2/')"] + + MYSQL_CXXFLAGS="-fno-implicit-templates -felide-constructors" + case "$MYSQL_MAJOR_MINOR_VERSION" in + 5.7) + : + ;; + *) + MYSQL_CXXFLAGS="$MYSQL_CXXFLAGS -fno-exceptions -fno-rtti" + ;; + esac + if test "${with_libmysqlservices_compat}" = "no"; then - MYSQL_MAJOR_MINOR_VERSION=["$(echo $MYSQL_VERSION | sed -e 's/\([0-9]*\)\.\([0-9]*\)\.\([0-9]*[a-z]*\)$/\1.\2/')"] case "$MYSQL_MAJOR_MINOR_VERSION" in 5.1) MYSQL_LIBS="" @@ -451,7 +466,7 @@ AC_SUBST(DOCUMENT_VERSION) AC_SUBST(DOCUMENT_VERSION_FULL) CFLAGS="$CFLAGS" -CXXFLAGS="$CXXFLAGS -fno-implicit-templates -fno-exceptions -fno-rtti -felide-constructors" +CXXFLAGS="$CXXFLAGS $MYSQL_CXXFLAGS" AC_CONFIG_FILES([ Makefile @@ -489,6 +504,7 @@ AC_OUTPUT([ packages/rpm/centos/mysql55-mroonga.spec packages/rpm/centos/mysql56-community-mroonga.spec packages/rpm/centos/mariadb-mroonga.spec + packages/rpm/centos/percona-server-56-mroonga.spec packages/yum/env.sh data/install.sql ]) diff --git a/storage/mroonga/ha_mroonga.cpp b/storage/mroonga/ha_mroonga.cpp index dbc2f0ca6e3..83f03fcb272 100644 --- a/storage/mroonga/ha_mroonga.cpp +++ b/storage/mroonga/ha_mroonga.cpp @@ -21,6 +21,7 @@ */ #include "mrn_mysql.h" +#include "mrn_mysql_compat.h" #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation @@ -35,6 +36,11 @@ #endif #include <sql_select.h> +#include <item_sum.h> + +#ifdef MRN_HAVE_BINLOG_H +# include <binlog.h> +#endif #ifdef MRN_HAVE_SQL_OPTIMIZER_H # include <sql_optimizer.h> @@ -49,9 +55,6 @@ #ifdef WIN32 # include <math.h> # include <direct.h> -# define MRN_ALLOCATE_VARIABLE_LENGTH_ARRAYS(type, variable_name, variable_size) \ - type *variable_name = (type *)_malloca(sizeof(type) * (variable_size)) -# define MRN_FREE_VARIABLE_LENGTH_ARRAYS(variable_name) _freea(variable_name) # define MRN_TABLE_SHARE_LOCK_SHARE_PROC "?key_TABLE_SHARE_LOCK_share@@3IA" # define MRN_TABLE_SHARE_LOCK_HA_DATA_PROC "?key_TABLE_SHARE_LOCK_ha_data@@3IA" # ifdef _WIN64 @@ -64,11 +67,14 @@ #else # include <dirent.h> # include <unistd.h> -# define MRN_ALLOCATE_VARIABLE_LENGTH_ARRAYS(type, variable_name, variable_size) \ - type variable_name[variable_size] -# define MRN_FREE_VARIABLE_LENGTH_ARRAYS(variable_name) #endif +#define MRN_ALLOCATE_VARIABLE_LENGTH_ARRAYS(type, variable_name, variable_size) \ + type *variable_name = \ + (type *)mrn_my_malloc(sizeof(type) * (variable_size), MYF(MY_WME)) +#define MRN_FREE_VARIABLE_LENGTH_ARRAYS(variable_name) \ + my_free(variable_name) + #include "mrn_err.h" #include "mrn_table.hpp" #include "ha_mroonga.hpp" @@ -89,6 +95,8 @@ #include <mrn_smart_grn_obj.hpp> #include <mrn_database_manager.hpp> #include <mrn_grn.hpp> +#include <mrn_value_decoder.hpp> +#include <mrn_database_repairer.hpp> #ifdef MRN_SUPPORT_FOREIGN_KEYS # include <sql_table.h> @@ -148,7 +156,11 @@ static mysql_mutex_t *mrn_LOCK_open; # define mrn_declare_plugin(NAME) maria_declare_plugin(NAME) # define mrn_declare_plugin_end maria_declare_plugin_end # define MRN_PLUGIN_LAST_VALUES MRN_VERSION, MariaDB_PLUGIN_MATURITY_STABLE -# define MRN_ABORT_ON_WARNING(thd) thd_kill_level(thd) +# if MYSQL_VERSION_ID >= 100000 +# define MRN_ABORT_ON_WARNING(thd) thd_kill_level(thd) +# else +# define MRN_ABORT_ON_WARNING(thd) thd->abort_on_warning +# endif #else # define mrn_declare_plugin(NAME) mysql_declare_plugin(NAME) # define mrn_declare_plugin_end mysql_declare_plugin_end @@ -157,7 +169,11 @@ static mysql_mutex_t *mrn_LOCK_open; # else # define MRN_PLUGIN_LAST_VALUES NULL # endif -# define MRN_ABORT_ON_WARNING(thd) thd->abort_on_warning +# if MYSQL_VERSION_ID >= 50706 +# define MRN_ABORT_ON_WARNING(thd) false +# else +# define MRN_ABORT_ON_WARNING(thd) thd->abort_on_warning +# endif #endif #if MYSQL_VERSION_ID >= 100007 && defined(MRN_MARIADB_P) @@ -172,12 +188,69 @@ static mysql_mutex_t *mrn_LOCK_open; # define MRN_GET_ERR_MSG(code) ER(code) #endif +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_LEX_GET_TABLE_LIST(lex) (lex)->select_lex->table_list.first +#else +# define MRN_LEX_GET_TABLE_LIST(lex) (lex)->select_lex.table_list.first +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_KEYTYPE_FOREIGN KEYTYPE_FOREIGN +#else +# define MRN_KEYTYPE_FOREIGN Key::FOREIGN_KEY +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define mrn_calculate_key_len(table, key_index, buffer, keypart_map) \ + calculate_key_len(table, key_index, keypart_map) +#else +# define mrn_calculate_key_len(table, key_index, buffer, keypart_map) \ + calculate_key_len(table, key_index, buffer, keypart_map) +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_SELECT_LEX_GET_WHERE_COND(select_lex) \ + ((select_lex)->where_cond()) +# define MRN_SELECT_LEX_GET_HAVING_COND(select_lex) \ + ((select_lex)->having_cond()) +# define MRN_SELECT_LEX_GET_ACTIVE_OPTIONS(select_lex) \ + ((select_lex)->active_options()) +#else +# define MRN_SELECT_LEX_GET_WHERE_COND(select_lex) \ + ((select_lex)->where) +# define MRN_SELECT_LEX_GET_HAVING_COND(select_lex) \ + ((select_lex)->having) +# define MRN_SELECT_LEX_GET_ACTIVE_OPTIONS(select_lex) \ + ((select_lex)->options) +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_TABLE_LIST_GET_DERIVED(table_list) NULL +#else +# define MRN_TABLE_LIST_GET_DERIVED(table_list) (table_list)->derived +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_GEOMETRY_FREE(geometry) +#else +# define MRN_GEOMETRY_FREE(geometry) delete (geometry) +#endif + Rpl_filter *mrn_binlog_filter; Time_zone *mrn_my_tz_UTC; #ifdef MRN_HAVE_TABLE_DEF_CACHE HASH *mrn_table_def_cache; #endif +#ifdef MRN_HAVE_PSI_MEMORY_KEY +PSI_memory_key mrn_memory_key; + +static PSI_memory_info mrn_all_memory_keys[]= +{ + {&mrn_memory_key, "Mroonga", 0} +}; +#endif + static const char *INDEX_COLUMN_NAME = "index"; static const char *MRN_PLUGIN_AUTHOR = "The Mroonga project"; @@ -189,31 +262,22 @@ extern "C" { int grn_atoi(const char *nptr, const char *end, const char **rest); uint grn_atoui(const char *nptr, const char *end, const char **rest); -/* global variables */ -handlerton *mrn_hton_ptr; -HASH mrn_open_tables; -mysql_mutex_t mrn_open_tables_mutex; -HASH mrn_long_term_share; -mysql_mutex_t mrn_long_term_share_mutex; - -HASH mrn_allocated_thds; -mysql_mutex_t mrn_allocated_thds_mutex; -/* internal variables */ -static grn_ctx mrn_ctx; -static mysql_mutex_t mrn_log_mutex; -static grn_obj *mrn_db; -static grn_ctx mrn_db_manager_ctx; -static mysql_mutex_t mrn_db_manager_mutex; -mrn::DatabaseManager *mrn_db_manager = NULL; - #ifdef HAVE_PSI_INTERFACE -PSI_mutex_key mrn_allocated_thds_mutex_key; +# ifdef WIN32 +# ifdef MRN_TABLE_SHARE_HAVE_LOCK_SHARE +PSI_mutex_key *mrn_table_share_lock_share; +# endif +# ifdef MRN_TABLE_SHARE_HAVE_LOCK_HA_DATA +PSI_mutex_key *mrn_table_share_lock_ha_data; +# endif +# endif static PSI_mutex_key mrn_open_tables_mutex_key; static PSI_mutex_key mrn_long_term_share_mutex_key; +static PSI_mutex_key mrn_allocated_thds_mutex_key; PSI_mutex_key mrn_share_mutex_key; PSI_mutex_key mrn_long_term_share_auto_inc_mutex_key; -static PSI_mutex_key mrn_db_manager_mutex_key; static PSI_mutex_key mrn_log_mutex_key; +static PSI_mutex_key mrn_db_manager_mutex_key; static PSI_mutex_info mrn_mutexes[] = { @@ -226,20 +290,25 @@ static PSI_mutex_info mrn_mutexes[] = {&mrn_log_mutex_key, "log", PSI_FLAG_GLOBAL}, {&mrn_db_manager_mutex_key, "DatabaseManager", PSI_FLAG_GLOBAL} }; - -#ifdef WIN32 -# ifdef MRN_TABLE_SHARE_HAVE_LOCK_SHARE - PSI_mutex_key *mrn_table_share_lock_share; -# endif -# ifdef MRN_TABLE_SHARE_HAVE_LOCK_HA_DATA - PSI_mutex_key *mrn_table_share_lock_ha_data; -# endif #endif -#else -#undef MRN_TABLE_SHARE_HAVE_LOCK_SHARE -#undef MRN_TABLE_SHARE_HAVE_LOCK_HA_DATA -#endif +/* global variables */ +handlerton *mrn_hton_ptr; +HASH mrn_open_tables; +mysql_mutex_t mrn_open_tables_mutex; +HASH mrn_long_term_share; +mysql_mutex_t mrn_long_term_share_mutex; + +HASH mrn_allocated_thds; +mysql_mutex_t mrn_allocated_thds_mutex; + +/* internal variables */ +static grn_ctx mrn_ctx; +static mysql_mutex_t mrn_log_mutex; +static grn_obj *mrn_db; +static grn_ctx mrn_db_manager_ctx; +static mysql_mutex_t mrn_db_manager_mutex; +mrn::DatabaseManager *mrn_db_manager = NULL; #ifdef WIN32 @@ -293,12 +362,19 @@ static const char *mrn_inspect_thr_lock_type(enum thr_lock_type lock_type) inspected = "TL_WRITE_ALLOW_READ"; break; #endif +#ifdef MRN_HAVE_TL_WRITE_CONCURRENT_DEFAULT + case TL_WRITE_CONCURRENT_DEFAULT: + inspected = "TL_WRITE_CONCURRENT_DEFAULT"; + break; +#endif case TL_WRITE_CONCURRENT_INSERT: inspected = "TL_WRITE_CONCURRENT_INSERT"; break; +#ifdef MRN_HAVE_TL_WRITE_DELAYED case TL_WRITE_DELAYED: inspected = "TL_WRITE_DELAYED"; break; +#endif case TL_WRITE_DEFAULT: inspected = "TL_WRITE_DEFAULT"; break; @@ -537,6 +613,7 @@ typedef enum { MRN_BOOLEAN_MODE_SYNTAX_FLAG_ALLOW_UPDATE = (1 << 4), MRN_BOOLEAN_MODE_SYNTAX_FLAG_ALLOW_LEADING_NOT = (1 << 5) } mrn_boolean_mode_syntax_flag; +#ifdef MRN_SUPPORT_THDVAR_SET static const char *mrn_boolean_mode_sytnax_flag_names[] = { "DEFAULT", "SYNTAX_QUERY", @@ -552,6 +629,7 @@ static TYPELIB mrn_boolean_mode_syntax_flags_typelib = { mrn_boolean_mode_sytnax_flag_names, NULL }; +#endif typedef enum { MRN_ACTION_ON_ERROR_ERROR, @@ -612,13 +690,25 @@ static uchar *mrn_allocated_thds_get_key(const uchar *record, static struct st_mysql_storage_engine storage_engine_structure = { MYSQL_HANDLERTON_INTERFACE_VERSION }; +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_STATUS_VARIABLE_ENTRY(name, value, type, scope) \ + {name, value, type, scope} +#else +# define MRN_STATUS_VARIABLE_ENTRY(name, value, type, scope) \ + {name, value, type} +#endif + static struct st_mysql_show_var mrn_status_variables[] = { - {MRN_STATUS_VARIABLE_NAME_PREFIX_STRING "_count_skip", - (char *)&mrn_count_skip, SHOW_LONG}, - {MRN_STATUS_VARIABLE_NAME_PREFIX_STRING "_fast_order_limit", - (char *)&mrn_fast_order_limit, SHOW_LONG}, - {NullS, NullS, SHOW_LONG} + MRN_STATUS_VARIABLE_ENTRY(MRN_STATUS_VARIABLE_NAME_PREFIX_STRING "_count_skip", + (char *)&mrn_count_skip, + SHOW_LONG, + SHOW_SCOPE_GLOBAL), + MRN_STATUS_VARIABLE_ENTRY(MRN_STATUS_VARIABLE_NAME_PREFIX_STRING "_fast_order_limit", + (char *)&mrn_fast_order_limit, + SHOW_LONG, + SHOW_SCOPE_GLOBAL), + MRN_STATUS_VARIABLE_ENTRY(NullS, NullS, SHOW_LONG, SHOW_SCOPE_GLOBAL) }; static const char *mrn_log_level_type_names[] = { @@ -728,10 +818,10 @@ static void mrn_log_file_update(THD *thd, struct st_mysql_sys_var *var, #ifdef MRN_NEED_FREE_STRING_MEMALLOC_PLUGIN_VAR char *old_log_file_name = *old_value_ptr; - *old_value_ptr = my_strdup(new_log_file_name, MYF(MY_WME)); + *old_value_ptr = mrn_my_strdup(new_log_file_name, MYF(MY_WME)); my_free(old_log_file_name); #else - *old_value_ptr = my_strdup(new_log_file_name, MYF(MY_WME)); + *old_value_ptr = mrn_my_strdup(new_log_file_name, MYF(MY_WME)); #endif grn_ctx_fin(&ctx); @@ -769,7 +859,7 @@ static void mrn_default_parser_update(THD *thd, struct st_mysql_sys_var *var, #ifdef MRN_NEED_FREE_STRING_MEMALLOC_PLUGIN_VAR my_free(*old_value_ptr); - *old_value_ptr = my_strdup(new_value, MYF(MY_WME)); + *old_value_ptr = mrn_my_strdup(new_value, MYF(MY_WME)); #else *old_value_ptr = (char *)new_value; #endif @@ -811,7 +901,7 @@ static MYSQL_THDVAR_LONGLONG(match_escalation_threshold, NULL, grn_get_default_match_escalation_threshold(), -1, - LONGLONG_MAX, + INT_MAX64, 0); static void mrn_vector_column_delimiter_update(THD *thd, struct st_mysql_sys_var *var, @@ -823,7 +913,7 @@ static void mrn_vector_column_delimiter_update(THD *thd, struct st_mysql_sys_var #ifdef MRN_NEED_FREE_STRING_MEMALLOC_PLUGIN_VAR my_free(*old_value_ptr); - *old_value_ptr = my_strdup(new_value, MYF(MY_WME)); + *old_value_ptr = mrn_my_strdup(new_value, MYF(MY_WME)); #else *old_value_ptr = (char *)new_value; #endif @@ -849,7 +939,7 @@ static void mrn_database_path_prefix_update(THD *thd, if (*old_value_ptr) my_free(*old_value_ptr); if (new_value) - *old_value_ptr = my_strdup(new_value, MYF(MY_WME)); + *old_value_ptr = mrn_my_strdup(new_value, MYF(MY_WME)); else *old_value_ptr = NULL; #else @@ -967,6 +1057,7 @@ static MYSQL_SYSVAR_BOOL(libgroonga_support_lz4, mrn_libgroonga_support_lz4, NULL, grn_check_lz4_support()); +#ifdef MRN_SUPPORT_THDVAR_SET static MYSQL_THDVAR_SET(boolean_mode_syntax_flags, PLUGIN_VAR_RQCMDARG, "The flags to custom syntax in BOOLEAN MODE. " @@ -978,6 +1069,20 @@ static MYSQL_THDVAR_SET(boolean_mode_syntax_flags, NULL, MRN_BOOLEAN_MODE_SYNTAX_FLAG_DEFAULT, &mrn_boolean_mode_syntax_flags_typelib); +#endif + +static const int MRN_MAX_N_RECORDS_FOR_ESTIMATE_DEFAULT = 1000; + +static MYSQL_THDVAR_INT(max_n_records_for_estimate, + PLUGIN_VAR_RQCMDARG, + "The max number of records to " + "estimate the number of matched records", + NULL, + NULL, + MRN_MAX_N_RECORDS_FOR_ESTIMATE_DEFAULT, + -1, + INT_MAX, + 0); static struct st_mysql_sys_var *mrn_system_variables[] = { @@ -996,7 +1101,10 @@ static struct st_mysql_sys_var *mrn_system_variables[] = MYSQL_SYSVAR(vector_column_delimiter), MYSQL_SYSVAR(libgroonga_support_zlib), MYSQL_SYSVAR(libgroonga_support_lz4), +#ifdef MRN_SUPPORT_THDVAR_SET MYSQL_SYSVAR(boolean_mode_syntax_flags), +#endif + MYSQL_SYSVAR(max_n_records_for_estimate), NULL }; @@ -1116,7 +1224,11 @@ static int mrn_close_connection(handlerton *hton, THD *thd) DBUG_RETURN(0); } +#ifdef MRN_FLUSH_LOGS_HAVE_BINLOG_GROUP_FLUSH +static bool mrn_flush_logs(handlerton *hton, bool binlog_group_flush) +#else static bool mrn_flush_logs(handlerton *hton) +#endif { MRN_DBUG_ENTER_FUNCTION(); bool result = 0; @@ -1322,7 +1434,7 @@ grn_obj_flags mrn_parse_grn_column_create_flags(THD *thd, if (mrn_libgroonga_support_zlib) { flags |= GRN_OBJ_COMPRESS_ZLIB; } else { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + push_warning_printf(thd, MRN_SEVERITY_WARNING, ER_MRN_UNSUPPORTED_COLUMN_FLAG_NUM, ER_MRN_UNSUPPORTED_COLUMN_FLAG_STR, "COMPRESS_ZLIB"); @@ -1332,7 +1444,7 @@ grn_obj_flags mrn_parse_grn_column_create_flags(THD *thd, if (mrn_libgroonga_support_lz4) { flags |= GRN_OBJ_COMPRESS_LZ4; } else { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + push_warning_printf(thd, MRN_SEVERITY_WARNING, ER_MRN_UNSUPPORTED_COLUMN_FLAG_NUM, ER_MRN_UNSUPPORTED_COLUMN_FLAG_STR, "COMPRESS_LZ4"); @@ -1344,7 +1456,7 @@ grn_obj_flags mrn_parse_grn_column_create_flags(THD *thd, "%.*s", static_cast<int>(rest_length), flag_names); - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + push_warning_printf(thd, MRN_SEVERITY_WARNING, ER_MRN_INVALID_COLUMN_FLAG_NUM, ER_MRN_INVALID_COLUMN_FLAG_STR, invalid_flag_name, @@ -1356,7 +1468,53 @@ grn_obj_flags mrn_parse_grn_column_create_flags(THD *thd, return flags; } -#ifdef HAVE_SPATIAL +bool mrn_parse_grn_index_column_flags(THD *thd, + grn_ctx *ctx, + const char *flag_names, + uint flag_names_length, + grn_obj_flags *index_column_flags) +{ + const char *flag_names_end = flag_names + flag_names_length; + bool found = false; + + while (flag_names < flag_names_end) { + uint rest_length = flag_names_end - flag_names; + + if (*flag_names == '|' || *flag_names == ' ') { + flag_names += 1; + continue; + } + if (rest_length >= 4 && !memcmp(flag_names, "NONE", 4)) { + flag_names += 4; + found = true; + } else if (rest_length >= 13 && !memcmp(flag_names, "WITH_POSITION", 13)) { + *index_column_flags |= GRN_OBJ_WITH_POSITION; + flag_names += 13; + found = true; + } else if (rest_length >= 12 && !memcmp(flag_names, "WITH_SECTION", 12)) { + *index_column_flags |= GRN_OBJ_WITH_SECTION; + flag_names += 12; + found = true; + } else if (rest_length >= 11 && !memcmp(flag_names, "WITH_WEIGHT", 11)) { + *index_column_flags |= GRN_OBJ_WITH_WEIGHT; + flag_names += 11; + found = true; + } else { + char invalid_flag_name[MRN_MESSAGE_BUFFER_SIZE]; + snprintf(invalid_flag_name, MRN_MESSAGE_BUFFER_SIZE, + "%.*s", + static_cast<int>(rest_length), + flag_names); + push_warning_printf(thd, MRN_SEVERITY_WARNING, + ER_MRN_INVALID_INDEX_FLAG_NUM, + ER_MRN_INVALID_INDEX_FLAG_STR, + invalid_flag_name); + } + } + return found; +} + +#ifdef MRN_HAVE_SPATIAL static int mrn_set_geometry(grn_ctx *ctx, grn_obj *buf, const char *wkb, uint wkb_size) { @@ -1393,12 +1551,13 @@ static int mrn_set_geometry(grn_ctx *ctx, grn_obj *buf, error = ER_MRN_GEOMETRY_NOT_SUPPORT_NUM; break; } - delete geometry; + MRN_GEOMETRY_FREE(geometry); return error; } #endif +#ifdef MRN_HAVE_HTON_ALTER_TABLE_FLAGS static uint mrn_alter_table_flags(uint flags) { uint alter_flags = 0; @@ -1419,18 +1578,15 @@ static uint mrn_alter_table_flags(uint flags) HA_INPLACE_DROP_INDEX_NO_READ_WRITE | HA_INPLACE_ADD_UNIQUE_INDEX_NO_READ_WRITE | HA_INPLACE_DROP_UNIQUE_INDEX_NO_READ_WRITE | - HA_INPLACE_ADD_PK_INDEX_NO_READ_WRITE | - HA_INPLACE_DROP_PK_INDEX_NO_READ_WRITE | HA_INPLACE_ADD_INDEX_NO_WRITE | HA_INPLACE_DROP_INDEX_NO_WRITE | HA_INPLACE_ADD_UNIQUE_INDEX_NO_WRITE | - HA_INPLACE_DROP_UNIQUE_INDEX_NO_WRITE | - HA_INPLACE_ADD_PK_INDEX_NO_WRITE | - HA_INPLACE_DROP_PK_INDEX_NO_WRITE; + HA_INPLACE_DROP_UNIQUE_INDEX_NO_WRITE; } #endif return alter_flags; } +#endif static int mrn_init(void *p) { @@ -1440,11 +1596,16 @@ static int mrn_init(void *p) hton = (handlerton *)p; hton->state = SHOW_OPTION_YES; hton->create = mrn_handler_create; - hton->flags = 0; + hton->flags = HTON_NO_FLAGS; +#ifndef MRN_SUPPORT_PARTITION + hton->flags |= HTON_NO_PARTITION; +#endif hton->drop_database = mrn_drop_database; hton->close_connection = mrn_close_connection; hton->flush_logs = mrn_flush_logs; +#ifdef MRN_HAVE_HTON_ALTER_TABLE_FLAGS hton->alter_table_flags = mrn_alter_table_flags; +#endif mrn_hton_ptr = hton; #ifdef _WIN32 @@ -1462,13 +1623,17 @@ static int mrn_init(void *p) (mysql_mutex_t *)GetProcAddress(current_module, "?LOCK_open@@3Ust_mysql_mutex@@A"); # endif -# ifdef MRN_TABLE_SHARE_HAVE_LOCK_SHARE - mrn_table_share_lock_share = - (PSI_mutex_key *)GetProcAddress(current_module, MRN_TABLE_SHARE_LOCK_SHARE_PROC); -# endif -# ifdef MRN_TABLE_SHARE_HAVE_LOCK_HA_DATA - mrn_table_share_lock_ha_data = - (PSI_mutex_key *)GetProcAddress(current_module, MRN_TABLE_SHARE_LOCK_HA_DATA_PROC); +# ifdef HAVE_PSI_INTERFACE +# ifdef MRN_TABLE_SHARE_HAVE_LOCK_SHARE + mrn_table_share_lock_share = + (PSI_mutex_key *)GetProcAddress(current_module, + MRN_TABLE_SHARE_LOCK_SHARE_PROC); +# endif +# ifdef MRN_TABLE_SHARE_HAVE_LOCK_HA_DATA + mrn_table_share_lock_ha_data = + (PSI_mutex_key *)GetProcAddress(current_module, + MRN_TABLE_SHARE_LOCK_HA_DATA_PROC); +# endif # endif #else mrn_binlog_filter = binlog_filter; @@ -1502,6 +1667,14 @@ static int mrn_init(void *p) if (mrn_change_encoding(ctx, system_charset_info)) goto err_mrn_change_encoding; +#ifdef MRN_HAVE_PSI_MEMORY_KEY + { + const char *category = "ha_mroonga"; + int n_keys = array_elements(mrn_all_memory_keys); + mysql_memory_register(category, mrn_all_memory_keys, n_keys); + } +#endif + if (mysql_mutex_init(mrn_log_mutex_key, &mrn_log_mutex, MY_MUTEX_INIT_FAST) != 0) { @@ -2164,6 +2337,9 @@ ha_mroonga::~ha_mroonga() if (share_for_create.wrapper_mode) { plugin_unlock(NULL, share_for_create.plugin); } + if (share_for_create.table_name) { + my_free(share_for_create.table_name); + } mrn_free_share_alloc(&share_for_create); free_root(&mem_root_for_create, MYF(0)); } @@ -2545,7 +2721,7 @@ int ha_mroonga::create_share_for_create() const THD *thd = ha_thd(); LEX *lex = thd->lex; HA_CREATE_INFO *create_info = &lex->create_info; - TABLE_LIST *table_list = lex->select_lex.table_list.first; + TABLE_LIST *table_list = MRN_LEX_GET_TABLE_LIST(lex); MRN_DBUG_ENTER_METHOD(); wrap_handler_for_create = NULL; memset(&table_for_create, 0, sizeof(TABLE)); @@ -2589,10 +2765,12 @@ int ha_mroonga::create_share_for_create() const } } } - init_alloc_root(&mem_root_for_create, 1024, 0, MYF(0)); + mrn_init_alloc_root(&mem_root_for_create, 1024, 0, MYF(0)); analyzed_for_create = true; if (table_list) { - share_for_create.table_name = table_list->table_name; + share_for_create.table_name = mrn_my_strndup(table_list->table_name, + table_list->table_name_length, + MYF(MY_WME)); share_for_create.table_name_length = table_list->table_name_length; } share_for_create.table_share = &table_share_for_create; @@ -2741,10 +2919,13 @@ int ha_mroonga::wrapper_create_index_fulltext(const char *grn_table_name, GRN_OBJ_PERSISTENT; grn_obj *index_table; - grn_obj_flags index_column_flags = - GRN_OBJ_COLUMN_INDEX | GRN_OBJ_WITH_POSITION | GRN_OBJ_PERSISTENT; - if (KEY_N_KEY_PARTS(key_info) > 1) { - index_column_flags |= GRN_OBJ_WITH_SECTION; + grn_obj_flags index_column_flags = GRN_OBJ_COLUMN_INDEX | GRN_OBJ_PERSISTENT; + + if (!find_index_column_flags(key_info, &index_column_flags)) { + index_column_flags |= GRN_OBJ_WITH_POSITION; + if (KEY_N_KEY_PARTS(key_info) > 1) { + index_column_flags |= GRN_OBJ_WITH_SECTION; + } } mrn::SmartGrnObj lexicon_key_type(ctx, GRN_DB_SHORT_TEXT); @@ -3029,7 +3210,8 @@ int ha_mroonga::storage_create(const char *name, TABLE *table, DBUG_RETURN(error); } - if (table_flags == (GRN_OBJ_PERSISTENT | GRN_OBJ_TABLE_PAT_KEY)) { + if (table_flags == (GRN_OBJ_PERSISTENT | GRN_OBJ_TABLE_PAT_KEY) || + table_flags == (GRN_OBJ_PERSISTENT | GRN_OBJ_TABLE_HASH_KEY)) { KEY key_info = table->s->key_info[pkey_nr]; int key_parts = KEY_N_KEY_PARTS(&key_info); if (key_parts == 1) { @@ -3182,7 +3364,7 @@ bool ha_mroonga::storage_create_foreign_key(TABLE *table, char ref_db_buff[NAME_LEN + 1], ref_table_buff[NAME_LEN + 1]; while ((key = key_iterator++)) { - if (key->type != Key::FOREIGN_KEY) + if (key->type != MRN_KEYTYPE_FOREIGN) { continue; } @@ -3207,7 +3389,11 @@ bool ha_mroonga::storage_create_foreign_key(TABLE *table, Key_part_spec *key_part_ref_col = key_part_ref_col_iterator++; LEX_STRING ref_field_name = key_part_ref_col->field_name; DBUG_PRINT("info", ("mroonga: ref_field_name=%s", ref_field_name.str)); +#ifdef MRN_FOREIGN_KEY_USE_CONST_STRING + LEX_CSTRING ref_db_name = fk->ref_db; +#else LEX_STRING ref_db_name = fk->ref_db; +#endif DBUG_PRINT("info", ("mroonga: ref_db_name=%s", ref_db_name.str)); if (ref_db_name.str && lower_case_table_names) { strmake(ref_db_buff, ref_db_name.str, sizeof(ref_db_buff) - 1); @@ -3215,7 +3401,11 @@ bool ha_mroonga::storage_create_foreign_key(TABLE *table, ref_db_name.str = ref_db_buff; DBUG_PRINT("info", ("mroonga: casedn ref_db_name=%s", ref_db_name.str)); } +#ifdef MRN_FOREIGN_KEY_USE_CONST_STRING + LEX_CSTRING ref_table_name = fk->ref_table; +#else LEX_STRING ref_table_name = fk->ref_table; +#endif DBUG_PRINT("info", ("mroonga: ref_table_name=%s", ref_table_name.str)); if (ref_table_name.str && lower_case_table_names) { strmake(ref_table_buff, ref_table_name.str, sizeof(ref_table_buff) - 1); @@ -3545,10 +3735,13 @@ int ha_mroonga::storage_create_index(TABLE *table, const char *grn_table_name, if (error) DBUG_RETURN(error); - grn_obj_flags index_column_flags = - GRN_OBJ_COLUMN_INDEX | GRN_OBJ_WITH_POSITION | GRN_OBJ_PERSISTENT; - if (is_multiple_column_index) { - index_column_flags |= GRN_OBJ_WITH_SECTION; + grn_obj_flags index_column_flags = GRN_OBJ_COLUMN_INDEX | GRN_OBJ_PERSISTENT; + + if (!find_index_column_flags(key_info, &index_column_flags)) { + index_column_flags |= GRN_OBJ_WITH_POSITION; + if (is_multiple_column_index) { + index_column_flags |= GRN_OBJ_WITH_SECTION; + } } index_table = index_tables[i]; @@ -3753,7 +3946,7 @@ int ha_mroonga::wrapper_open(const char *name, int mode, uint test_if_locked) } } - init_alloc_root(&mem_root, 1024, 0, MYF(0)); + mrn_init_alloc_root(&mem_root, 1024, 0, MYF(0)); wrap_key_info = mrn_create_key_info_for_table(share, table, &error); if (error) DBUG_RETURN(error); @@ -3988,15 +4181,17 @@ int ha_mroonga::storage_open(const char *name, int mode, uint test_if_locked) DBUG_RETURN(error); } - error = storage_open_indexes(name); - if (error) { - // TODO: free grn_columns and set NULL; - grn_obj_unlink(ctx, grn_table); - grn_table = NULL; - DBUG_RETURN(error); - } + if (!(ha_thd()->open_options & HA_OPEN_FOR_REPAIR)) { + error = storage_open_indexes(name); + if (error) { + // TODO: free grn_columns and set NULL; + grn_obj_unlink(ctx, grn_table); + grn_table = NULL; + DBUG_RETURN(error); + } - storage_set_keys_in_use(); + storage_set_keys_in_use(); + } ref_length = sizeof(grn_id); DBUG_RETURN(0); @@ -4043,6 +4238,11 @@ int ha_mroonga::storage_open_columns(void) int n_columns = table->s->fields; grn_columns = (grn_obj **)malloc(sizeof(grn_obj *) * n_columns); grn_column_ranges = (grn_obj **)malloc(sizeof(grn_obj *) * n_columns); + for (int i = 0; i < n_columns; i++) { + grn_columns[i] = NULL; + grn_column_ranges[i] = NULL; + } + if (table_share->blob_fields) { if (blob_buffers) @@ -4055,8 +4255,7 @@ int ha_mroonga::storage_open_columns(void) } } - int i; - for (i = 0; i < n_columns; i++) { + for (int i = 0; i < n_columns; i++) { Field *field = table->field[i]; const char *column_name = field->field_name; int column_name_size = strlen(column_name); @@ -4065,24 +4264,46 @@ int ha_mroonga::storage_open_columns(void) blob_buffers[i].set_charset(field->charset()); } if (strcmp(MRN_COLUMN_NAME_ID, column_name) == 0) { - grn_columns[i] = NULL; - grn_column_ranges[i] = NULL; continue; } grn_columns[i] = grn_obj_column(ctx, grn_table, column_name, column_name_size); + if (!grn_columns[i]) { + error = ER_CANT_OPEN_FILE; + my_message(error, ctx->errbuf, MYF(0)); + break; + } + grn_id range_id = grn_obj_get_range(ctx, grn_columns[i]); grn_column_ranges[i] = grn_ctx_at(ctx, range_id); - if (ctx->rc) { - // TODO: free grn_columns and set NULL; - int error = ER_CANT_OPEN_FILE; + if (!grn_column_ranges[i]) { + error = ER_CANT_OPEN_FILE; my_message(error, ctx->errbuf, MYF(0)); - DBUG_RETURN(error); + break; } } - DBUG_RETURN(0); + if (error != 0) { + for (int i = 0; i < n_columns; i++) { + grn_obj *column = grn_columns[i]; + if (column) { + grn_obj_unlink(ctx, column); + } + + grn_obj *range = grn_column_ranges[i]; + if (range) { + grn_obj_unlink(ctx, range); + } + } + + free(grn_columns); + grn_columns = NULL; + free(grn_column_ranges); + grn_column_ranges = NULL; + } + + DBUG_RETURN(error); } int ha_mroonga::storage_open_indexes(const char *name) @@ -4147,7 +4368,7 @@ int ha_mroonga::storage_open_indexes(const char *name) grn_index_tables[i], INDEX_COLUMN_NAME, strlen(INDEX_COLUMN_NAME)); - if (!grn_index_columns[i]) { + if (!grn_index_columns[i] && ctx->rc == GRN_SUCCESS) { /* just for backward compatibility before 1.0. */ Field *field = key_info.key_part[0].field; grn_index_columns[i] = grn_obj_column(ctx, grn_index_tables[i], @@ -5176,7 +5397,7 @@ int ha_mroonga::wrapper_write_row_index(uchar *buf) (int)GRN_TEXT_LEN(&key_buffer), GRN_TEXT_VALUE(&key_buffer)); error = ER_ERROR_ON_WRITE; - push_warning(ha_thd(), Sql_condition::WARN_LEVEL_WARN, error, + push_warning(ha_thd(), MRN_SEVERITY_WARNING, error, error_message); DBUG_RETURN(0); } @@ -5259,7 +5480,7 @@ int ha_mroonga::storage_write_row(uchar *buf) if (field->is_null()) continue; if (strcmp(MRN_COLUMN_NAME_ID, column_name) == 0) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + push_warning_printf(thd, MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, MRN_GET_ERR_MSG(WARN_DATA_TRUNCATED), MRN_COLUMN_NAME_ID, MRN_GET_CURRENT_ROW_FOR_WARNING(thd)); @@ -5339,7 +5560,7 @@ int ha_mroonga::storage_write_row(uchar *buf) if (field->is_null()) continue; -#ifdef HAVE_SPATIAL +#ifdef MRN_HAVE_SPATIAL bool is_null_geometry_value = field->real_type() == MYSQL_TYPE_GEOMETRY && static_cast<Field_geom *>(field)->get_length() == 0; @@ -5687,7 +5908,7 @@ int ha_mroonga::wrapper_get_record_id(uchar *data, grn_id *record_id, "%s: key=<%.*s>", context, (int)GRN_TEXT_LEN(&key), GRN_TEXT_VALUE(&key)); error = ER_ERROR_ON_WRITE; - push_warning(ha_thd(), Sql_condition::WARN_LEVEL_WARN, error, + push_warning(ha_thd(), MRN_SEVERITY_WARNING, error, error_message); } grn_obj_unlink(ctx, &key); @@ -5851,7 +6072,7 @@ int ha_mroonga::storage_update_row(const uchar *old_data, uchar *new_data) if (bitmap_is_set(table->write_set, field->field_index)) { if (field->is_null()) continue; if (strcmp(MRN_COLUMN_NAME_ID, column_name) == 0) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + push_warning_printf(thd, MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, MRN_GET_ERR_MSG(WARN_DATA_TRUNCATED), MRN_COLUMN_NAME_ID, MRN_GET_CURRENT_ROW_FOR_WARNING(thd)); @@ -5909,7 +6130,7 @@ int ha_mroonga::storage_update_row(const uchar *old_data, uchar *new_data) snprintf(message, MRN_BUFFER_SIZE, "data truncated for primary key column: <%s>", column_name); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + push_warning(thd, MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, message); } have_pkey = true; @@ -6489,6 +6710,7 @@ uint ha_mroonga::storage_max_supported_key_parts() const uint ha_mroonga::max_supported_key_parts() const { MRN_DBUG_ENTER_METHOD(); + uint parts; if (!share && !analyzed_for_create && ( @@ -6506,6 +6728,7 @@ uint ha_mroonga::max_supported_key_parts() const } else { parts = storage_max_supported_key_parts(); } + DBUG_RETURN(parts); } @@ -6601,6 +6824,7 @@ ha_rows ha_mroonga::storage_records_in_range(uint key_nr, key_range *range_min, } } + int cursor_limit = THDVAR(ha_thd(), max_n_records_for_estimate); uint pkey_nr = table->s->primary_key; if (key_nr == pkey_nr) { DBUG_PRINT("info", ("mroonga: use primary key")); @@ -6608,7 +6832,7 @@ ha_rows ha_mroonga::storage_records_in_range(uint key_nr, key_range *range_min, cursor = grn_table_cursor_open(ctx, grn_table, key_min, size_min, key_max, size_max, - 0, -1, flags); + 0, cursor_limit, flags); while (grn_table_cursor_next(ctx, cursor) != GRN_ID_NIL) { row_count++; } @@ -6621,18 +6845,13 @@ ha_rows ha_mroonga::storage_records_in_range(uint key_nr, key_range *range_min, } grn_table_cursor *cursor; - grn_table_cursor *index_cursor; cursor = grn_table_cursor_open(ctx, grn_index_tables[key_nr], key_min, size_min, key_max, size_max, - 0, -1, flags); - index_cursor = grn_index_cursor_open(ctx, cursor, - grn_index_columns[key_nr], - 0, GRN_ID_MAX, 0); - while (grn_table_cursor_next(ctx, index_cursor) != GRN_ID_NIL) { - row_count++; - } - grn_obj_unlink(ctx, index_cursor); + 0, cursor_limit, flags); + grn_obj *index_column = grn_index_columns[key_nr]; + grn_ii *ii = reinterpret_cast<grn_ii *>(index_column); + row_count = grn_ii_estimate_size_for_lexicon_cursor(ctx, ii, cursor); grn_table_cursor_close(ctx, cursor); } DBUG_RETURN(row_count); @@ -6814,7 +7033,8 @@ int ha_mroonga::storage_index_read_map(uchar *buf, const uchar *key, bool is_multiple_column_index = KEY_N_KEY_PARTS(&key_info) > 1; if (is_multiple_column_index) { mrn_change_encoding(ctx, NULL); - uint key_length = calculate_key_len(table, active_index, key, keypart_map); + uint key_length = + mrn_calculate_key_len(table, active_index, key, keypart_map); DBUG_PRINT("info", ("mroonga: multiple column index: " "search key length=<%u>, " @@ -6832,6 +7052,10 @@ int ha_mroonga::storage_index_read_map(uchar *buf, const uchar *key, storage_encode_multiple_column_key(&key_info, key, key_length, key_min, &size_min); + if (find_flag == HA_READ_KEY_EXACT) { + key_max = key_min; + size_max = size_min; + } } } else { flags |= GRN_CURSOR_PREFIX; @@ -7018,7 +7242,8 @@ int ha_mroonga::storage_index_read_last_map(uchar *buf, const uchar *key, if (is_multiple_column_index) { mrn_change_encoding(ctx, NULL); flags |= GRN_CURSOR_PREFIX; - uint key_length = calculate_key_len(table, active_index, key, keypart_map); + uint key_length = + mrn_calculate_key_len(table, active_index, key, keypart_map); key_min = key_min_entity; storage_encode_multiple_column_key(&key_info, key, key_length, @@ -7630,7 +7855,8 @@ void ha_mroonga::generic_ft_init_ext_add_conditions_fast_order_limit( { MRN_DBUG_ENTER_METHOD(); - Item *where = table->pos_in_table_list->select_lex->where; + Item *where = + MRN_SELECT_LEX_GET_WHERE_COND(table->pos_in_table_list->select_lex); bool is_storage_mode = !(share->wrapper_mode); mrn::ConditionConverter converter(info->ctx, grn_table, is_storage_mode); @@ -7806,7 +8032,10 @@ grn_expr_flags ha_mroonga::expr_flags_in_boolean_mode() { MRN_DBUG_ENTER_METHOD(); - ulonglong syntax_flags = THDVAR(ha_thd(), boolean_mode_syntax_flags); + ulonglong syntax_flags = MRN_BOOLEAN_MODE_SYNTAX_FLAG_DEFAULT; +#ifdef MRN_SUPPORT_THDVAR_SET + syntax_flags = THDVAR(ha_thd(), boolean_mode_syntax_flags); +#endif grn_expr_flags expression_flags = 0; if (syntax_flags == MRN_BOOLEAN_MODE_SYNTAX_FLAG_DEFAULT) { expression_flags = GRN_EXPR_SYNTAX_QUERY | GRN_EXPR_ALLOW_LEADING_NOT; @@ -8093,7 +8322,7 @@ FT_INFO *ha_mroonga::generic_ft_init_ext(uint flags, uint key_nr, String *key) for (int i = 0; i < n_sort_keys; i++) { grn_obj_unlink(info->ctx, sort_keys[i].key); } - free(sort_keys); + my_free(sort_keys); } DBUG_RETURN((FT_INFO *)info); @@ -8372,7 +8601,7 @@ void ha_mroonga::push_warning_unsupported_spatial_index_search(enum ha_rkey_func sprintf(search_name, "unknown: %d", flag); } push_warning_printf(ha_thd(), - Sql_condition::WARN_LEVEL_WARN, + MRN_SEVERITY_WARNING, ER_UNSUPPORTED_EXTENSION, "spatial index search " "except MBRContains aren't supported: <%s>", @@ -8638,7 +8867,7 @@ grn_obj *ha_mroonga::find_tokenizer(const char *name, int name_length) name_length, name, MRN_PARSER_DEFAULT); push_warning(ha_thd(), - Sql_condition::WARN_LEVEL_WARN, ER_UNSUPPORTED_EXTENSION, + MRN_SEVERITY_WARNING, ER_UNSUPPORTED_EXTENSION, message); tokenizer = grn_ctx_get(ctx, MRN_PARSER_DEFAULT, @@ -8646,7 +8875,7 @@ grn_obj *ha_mroonga::find_tokenizer(const char *name, int name_length) } if (!tokenizer) { push_warning(ha_thd(), - Sql_condition::WARN_LEVEL_WARN, ER_UNSUPPORTED_EXTENSION, + MRN_SEVERITY_WARNING, ER_UNSUPPORTED_EXTENSION, "couldn't find fulltext parser. " "Bigram fulltext parser is used instead."); tokenizer = grn_ctx_at(ctx, GRN_DB_BIGRAM); @@ -8682,6 +8911,28 @@ grn_obj *ha_mroonga::find_normalizer(KEY *key_info) DBUG_RETURN(normalizer); } +bool ha_mroonga::find_index_column_flags(KEY *key_info, grn_obj_flags *index_column_flags) +{ + MRN_DBUG_ENTER_METHOD(); + bool found = false; +#if MYSQL_VERSION_ID >= 50500 + if (key_info->comment.length > 0) { + mrn::ParametersParser parser(key_info->comment.str, + key_info->comment.length); + parser.parse(); + const char *names = parser["index_flags"]; + if (names) { + found = mrn_parse_grn_index_column_flags(ha_thd(), + ctx, + names, + strlen(names), + index_column_flags); + } + } +#endif + DBUG_RETURN(found); +} + bool ha_mroonga::find_token_filters(KEY *key_info, grn_obj *token_filters) { MRN_DBUG_ENTER_METHOD(); @@ -8718,7 +8969,7 @@ bool ha_mroonga::find_token_filters_put(grn_obj *token_filters, "nonexistent token filter: <%.*s>", token_filter_name_length, token_filter_name); push_warning(ha_thd(), - Sql_condition::WARN_LEVEL_WARN, ER_UNSUPPORTED_EXTENSION, + MRN_SEVERITY_WARNING, ER_UNSUPPORTED_EXTENSION, message); return false; } @@ -8778,7 +9029,7 @@ break_loop: (int)(current - last_name_end), last_name_end, (int)(end - current), current); push_warning(ha_thd(), - Sql_condition::WARN_LEVEL_WARN, ER_UNSUPPORTED_EXTENSION, + MRN_SEVERITY_WARNING, ER_UNSUPPORTED_EXTENSION, message); return false; } @@ -9050,9 +9301,9 @@ void ha_mroonga::check_count_skip(key_part_map start_key_part_map, if ( thd_sql_command(ha_thd()) == SQLCOM_SELECT && - !select_lex->non_agg_fields.elements && + select_lex->item_list.elements == 1 && !select_lex->group_list.elements && - !select_lex->having && + !MRN_SELECT_LEX_GET_HAVING_COND(select_lex) && select_lex->table_list.elements == 1 ) { Item *info = (Item *) select_lex->item_list.first_node()->info; @@ -9073,7 +9324,7 @@ void ha_mroonga::check_count_skip(key_part_map start_key_part_map, Item *where; if (fulltext) { DBUG_PRINT("info", ("mroonga: count skip: fulltext")); - where = select_lex->where; + where = MRN_SELECT_LEX_GET_WHERE_COND(select_lex); if (!where || where->type() != Item::FUNC_ITEM || ((Item_func *)where)->functype() != Item_func::FT_FUNC) { @@ -9081,20 +9332,10 @@ void ha_mroonga::check_count_skip(key_part_map start_key_part_map, count_skip = false; DBUG_VOID_RETURN; } - where = where->next; - if (!where || - where->type() != Item::STRING_ITEM) { - DBUG_PRINT("info", ("mroonga: count skip: string item is not match")); - count_skip = false; - DBUG_VOID_RETURN; - } - for (where = where->next; where; where = where->next) { - if (where->type() != Item::FIELD_ITEM) - break; - DBUG_PRINT("info", ("mroonga: count skip: FIELD_ITEM=%p", where)); - } - if (where != info) { - DBUG_PRINT("info", ("mroonga: count skip: where clause is not match")); + if (select_lex->select_n_where_fields != 1) { + DBUG_PRINT("info", + ("mroonga: count skip: " + "where clause is not fulltext search only")); count_skip = false; DBUG_VOID_RETURN; } @@ -9117,10 +9358,29 @@ void ha_mroonga::check_count_skip(key_part_map start_key_part_map, uint key_nr = active_index; KEY key_info = table->key_info[key_nr]; KEY_PART_INFO *key_part = key_info.key_part; - for (where = select_lex->where; where; where = where->next) { - if (where->type() == Item::FIELD_ITEM) + for (where = MRN_SELECT_LEX_GET_WHERE_COND(select_lex); + where; + where = where->next) { + Item *target = where; + + if (where->type() == Item::FUNC_ITEM) { + Item_func *func_item = static_cast<Item_func *>(where); + if (func_item->arg_count == 0) { + break; + } + target = func_item->key_item(); + where = where->next; + if (func_item->arguments()[0] == where) { + uint n_args = func_item->arg_count; + for (; n_args > 0; --n_args) { + where = where->next; + } + } + } + + if (target->type() == Item::FIELD_ITEM) { - Field *field = ((Item_field *)where)->field; + Field *field = ((Item_field *)target)->field; if (!field) break; if (field->table != table) @@ -9174,6 +9434,27 @@ bool ha_mroonga::is_grn_zero_column_value(grn_obj *column, grn_obj *value) DBUG_RETURN(true); } +bool ha_mroonga::is_primary_key_field(Field *field) const +{ + MRN_DBUG_ENTER_METHOD(); + + if (table->s->primary_key == MAX_INDEXES) { + DBUG_RETURN(false); + } + + KEY *key_info = &(table->s->key_info[table->s->primary_key]); + if (KEY_N_KEY_PARTS(key_info) != 1) { + DBUG_RETURN(false); + } + + if (strcmp(field->field_name, + key_info->key_part[0].field->field_name) == 0) { + DBUG_RETURN(true); + } else { + DBUG_RETURN(false); + } +} + void ha_mroonga::check_fast_order_limit(grn_table_sort_key **sort_keys, int *n_sort_keys, longlong *limit) @@ -9188,7 +9469,7 @@ void ha_mroonga::check_fast_order_limit(grn_table_sort_key **sort_keys, TABLE_LIST *table_list = table->pos_in_table_list; st_select_lex *select_lex = table_list->select_lex; - SELECT_LEX_UNIT *unit = table_list->derived; + SELECT_LEX_UNIT *unit = MRN_TABLE_LIST_GET_DERIVED(table_list); st_select_lex *first_select_lex; if (unit) { @@ -9198,13 +9479,13 @@ void ha_mroonga::check_fast_order_limit(grn_table_sort_key **sort_keys, } DBUG_PRINT("info", ("mroonga: first_select_lex->options=%llu", - first_select_lex ? first_select_lex->options : 0)); + first_select_lex ? MRN_SELECT_LEX_GET_ACTIVE_OPTIONS(first_select_lex) : 0)); if ( thd_sql_command(ha_thd()) == SQLCOM_SELECT && !select_lex->with_sum_func && !select_lex->group_list.elements && - !select_lex->having && + !MRN_SELECT_LEX_GET_HAVING_COND(select_lex) && select_lex->table_list.elements == 1 && select_lex->order_list.elements && select_lex->explicit_limit && @@ -9225,17 +9506,18 @@ void ha_mroonga::check_fast_order_limit(grn_table_sort_key **sort_keys, fast_order_limit = false; DBUG_VOID_RETURN; } - if (first_select_lex && (first_select_lex->options & OPTION_FOUND_ROWS)) { + if (first_select_lex && + (MRN_SELECT_LEX_GET_ACTIVE_OPTIONS(first_select_lex) & OPTION_FOUND_ROWS)) { DBUG_PRINT("info", ("mroonga: fast_order_limit = false: " "SQL_CALC_FOUND_ROWS is specified")); fast_order_limit = false; DBUG_VOID_RETURN; } - Item *where = select_lex->where; + bool is_storage_mode = !(share->wrapper_mode); + Item *where = MRN_SELECT_LEX_GET_WHERE_COND(select_lex); const Item_func *match_against = NULL; if (where) { - bool is_storage_mode = !(share->wrapper_mode); mrn::ConditionConverter converter(ctx, grn_table, is_storage_mode); if (!converter.is_convertable(where)) { DBUG_PRINT("info", @@ -9253,18 +9535,22 @@ void ha_mroonga::check_fast_order_limit(grn_table_sort_key **sort_keys, DBUG_VOID_RETURN; } } - *n_sort_keys = select_lex->order_list.elements; - *sort_keys = (grn_table_sort_key *)malloc(sizeof(grn_table_sort_key) * - *n_sort_keys); + int n_max_sort_keys = select_lex->order_list.elements; + *n_sort_keys = 0; + size_t sort_keys_size = sizeof(grn_table_sort_key) * n_max_sort_keys; + *sort_keys = (grn_table_sort_key *)mrn_my_malloc(sort_keys_size, + MYF(MY_WME)); + memset(*sort_keys, 0, sort_keys_size); ORDER *order; int i; mrn_change_encoding(ctx, system_charset_info); - for (order = (ORDER *) select_lex->order_list.first, i = 0; order; + for (order = (ORDER *) select_lex->order_list.first, i = 0; + order; order = order->next, i++) { Item *item = *order->item; - if (grn_columns && item->type() == Item::FIELD_ITEM) + if (item->type() == Item::FIELD_ITEM) { - Field *field = ((Item_field *) (*order->item))->field; + Field *field = static_cast<Item_field *>(item)->field; const char *column_name = field->field_name; int column_name_size = strlen(column_name); @@ -9273,14 +9559,31 @@ void ha_mroonga::check_fast_order_limit(grn_table_sort_key **sort_keys, DBUG_PRINT("info", ("mroonga: fast_order_limit = false: " "sort by collated value isn't supported yet.")); fast_order_limit = false; - free(*sort_keys); + my_free(*sort_keys); *sort_keys = NULL; *n_sort_keys = 0; DBUG_VOID_RETURN; } - (*sort_keys)[i].key = grn_obj_column(ctx, matched_record_keys, - column_name, column_name_size); + if (is_storage_mode) { + (*sort_keys)[i].key = grn_obj_column(ctx, matched_record_keys, + column_name, column_name_size); + } else { + if (is_primary_key_field(field)) { + (*sort_keys)[i].key = grn_obj_column(ctx, matched_record_keys, + MRN_COLUMN_NAME_KEY, + strlen(MRN_COLUMN_NAME_KEY)); + } else { + DBUG_PRINT("info", ("mroonga: fast_order_limit = false: " + "sort by not primary key value " + "isn't supported in wrapper mode.")); + fast_order_limit = false; + my_free(*sort_keys); + *sort_keys = NULL; + *n_sort_keys = 0; + DBUG_VOID_RETURN; + } + } } else if (!match_against || match_against->eq(item, true)) { (*sort_keys)[i].key = grn_obj_column(ctx, matched_record_keys, MRN_COLUMN_NAME_SCORE, @@ -9289,7 +9592,7 @@ void ha_mroonga::check_fast_order_limit(grn_table_sort_key **sort_keys, DBUG_PRINT("info", ("mroonga: fast_order_limit = false: " "sort by computed value isn't supported.")); fast_order_limit = false; - free(*sort_keys); + my_free(*sort_keys); *sort_keys = NULL; *n_sort_keys = 0; DBUG_VOID_RETURN; @@ -9301,6 +9604,7 @@ void ha_mroonga::check_fast_order_limit(grn_table_sort_key **sort_keys, } else { (*sort_keys)[i].flags = GRN_TABLE_SORT_DESC; } + (*n_sort_keys)++; } DBUG_PRINT("info", ("mroonga: fast_order_limit = true")); fast_order_limit = true; @@ -9329,7 +9633,8 @@ int ha_mroonga::generic_store_bulk_variable_size_string(Field *field, String value; field->val_str(NULL, &value); grn_obj_reinit(ctx, buf, GRN_DB_SHORT_TEXT, 0); - DBUG_PRINT("info", ("mroonga: length=%u", value.length())); + DBUG_PRINT("info", ("mroonga: length=%" MRN_FORMAT_STRING_LENGTH, + value.length())); DBUG_PRINT("info", ("mroonga: value=%s", value.c_ptr_safe())); GRN_TEXT_SET(ctx, buf, value.ptr(), value.length()); DBUG_RETURN(error); @@ -9392,7 +9697,7 @@ int ha_mroonga::generic_store_bulk_integer(Field *field, grn_obj *buf) "unknown integer value size: <%u>: " "available sizes: [1, 2, 3, 4, 8]", size); - push_warning(ha_thd(), Sql_condition::WARN_LEVEL_WARN, + push_warning(ha_thd(), MRN_SEVERITY_WARNING, error, error_message); break; } @@ -9432,7 +9737,7 @@ int ha_mroonga::generic_store_bulk_unsigned_integer(Field *field, grn_obj *buf) "unknown unsigned integer value size: <%u>: " "available sizes: [1, 2, 3, 4, 8]", size); - push_warning(ha_thd(), Sql_condition::WARN_LEVEL_WARN, + push_warning(ha_thd(), MRN_SEVERITY_WARNING, error, error_message); break; } @@ -9459,7 +9764,7 @@ int ha_mroonga::generic_store_bulk_float(Field *field, grn_obj *buf) "unknown float value size: <%u>: " "available sizes: [4, 8]", size); - push_warning(ha_thd(), Sql_condition::WARN_LEVEL_WARN, + push_warning(ha_thd(), MRN_SEVERITY_WARNING, error, error_message); break; } @@ -9519,7 +9824,7 @@ int ha_mroonga::generic_store_bulk_date(Field *field, grn_obj *buf) mrn::TimeConverter time_converter; long long int time = time_converter.tm_to_grn_time(&date, usec, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } grn_obj_reinit(ctx, buf, GRN_DB_TIME, 0); @@ -9539,7 +9844,7 @@ int ha_mroonga::generic_store_bulk_time(Field *field, grn_obj *buf) long long int time = time_converter.mysql_time_to_grn_time(&mysql_time, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } grn_obj_reinit(ctx, buf, GRN_DB_TIME, 0); @@ -9559,7 +9864,7 @@ int ha_mroonga::generic_store_bulk_datetime(Field *field, grn_obj *buf) long long int time = time_converter.mysql_time_to_grn_time(&mysql_time, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } grn_obj_reinit(ctx, buf, GRN_DB_TIME, 0); @@ -9591,7 +9896,7 @@ int ha_mroonga::generic_store_bulk_year(Field *field, grn_obj *buf) mrn::TimeConverter time_converter; long long int time = time_converter.tm_to_grn_time(&date, usec, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } grn_obj_reinit(ctx, buf, GRN_DB_TIME, 0); @@ -9612,7 +9917,7 @@ int ha_mroonga::generic_store_bulk_datetime2(Field *field, grn_obj *buf) long long int time = time_converter.mysql_time_to_grn_time(&mysql_time, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } grn_obj_reinit(ctx, buf, GRN_DB_TIME, 0); @@ -9633,7 +9938,7 @@ int ha_mroonga::generic_store_bulk_time2(Field *field, grn_obj *buf) long long int time = time_converter.mysql_time_to_grn_time(&mysql_time, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } grn_obj_reinit(ctx, buf, GRN_DB_TIME, 0); @@ -9654,7 +9959,7 @@ int ha_mroonga::generic_store_bulk_new_date(Field *field, grn_obj *buf) long long int time = time_converter.mysql_time_to_grn_time(&mysql_date, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } grn_obj_reinit(ctx, buf, GRN_DB_TIME, 0); @@ -9690,7 +9995,7 @@ int ha_mroonga::generic_store_bulk_geometry(Field *field, grn_obj *buf) { MRN_DBUG_ENTER_METHOD(); int error = 0; -#ifdef HAVE_SPATIAL +#ifdef MRN_HAVE_SPATIAL String buffer; Field_geom *geometry = (Field_geom *)field; String *value = geometry->val_str(0, &buffer); @@ -9876,7 +10181,7 @@ void ha_mroonga::storage_store_field_integer(Field *field, "unknown integer value size: <%d>: " "available sizes: [1, 2, 4, 8]", value_length); - push_warning(ha_thd(), Sql_condition::WARN_LEVEL_WARN, + push_warning(ha_thd(), MRN_SEVERITY_WARNING, HA_ERR_UNSUPPORTED, error_message); storage_store_field_string(field, value, value_length); break; @@ -9928,7 +10233,7 @@ void ha_mroonga::storage_store_field_unsigned_integer(Field *field, "unknown integer value size: <%d>: " "available sizes: [1, 2, 4, 8]", value_length); - push_warning(ha_thd(), Sql_condition::WARN_LEVEL_WARN, + push_warning(ha_thd(), MRN_SEVERITY_WARNING, HA_ERR_UNSUPPORTED, error_message); storage_store_field_string(field, value, value_length); break; @@ -10121,7 +10426,7 @@ void ha_mroonga::storage_store_field_geometry(Field *field, uint value_length) { MRN_DBUG_ENTER_METHOD(); -#ifdef HAVE_SPATIAL +#ifdef MRN_HAVE_SPATIAL uchar wkb[SRID_SIZE + WKB_HEADER_SIZE + POINT_DATA_SIZE]; grn_geo_point *field_value = (grn_geo_point *)value; int latitude, longitude; @@ -10526,7 +10831,7 @@ int ha_mroonga::storage_encode_key_timestamp(Field *field, const uchar *key, mrn::TimeConverter time_converter; time = time_converter.mysql_time_to_grn_time(&mysql_time, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } memcpy(buf, &time, 8); @@ -10573,7 +10878,7 @@ int ha_mroonga::storage_encode_key_time(Field *field, const uchar *key, mrn::TimeConverter time_converter; time = time_converter.mysql_time_to_grn_time(&mysql_time, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } #else @@ -10608,7 +10913,7 @@ int ha_mroonga::storage_encode_key_year(Field *field, const uchar *key, long long int time = time_converter.tm_to_grn_time(&datetime, usec, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } memcpy(buf, &time, 8); @@ -10657,7 +10962,7 @@ int ha_mroonga::storage_encode_key_datetime(Field *field, const uchar *key, time = time_converter.tm_to_grn_time(&date, usec, &truncated); } if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } memcpy(buf, &time, 8); @@ -10683,7 +10988,7 @@ int ha_mroonga::storage_encode_key_timestamp2(Field *field, const uchar *key, long long int grn_time = time_converter.mysql_time_to_grn_time(&mysql_time, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } memcpy(buf, &grn_time, 8); @@ -10710,7 +11015,7 @@ int ha_mroonga::storage_encode_key_datetime2(Field *field, const uchar *key, long long int grn_time = time_converter.mysql_time_to_grn_time(&mysql_time, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } memcpy(buf, &grn_time, 8); @@ -10737,7 +11042,7 @@ int ha_mroonga::storage_encode_key_time2(Field *field, const uchar *key, long long int grn_time = time_converter.mysql_time_to_grn_time(&mysql_time, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } memcpy(buf, &grn_time, 8); @@ -10759,7 +11064,7 @@ int ha_mroonga::storage_encode_key_enum(Field *field, const uchar *key, memcpy(buf, &value, *size); } else { uint16 value; - shortget(value, key); + mrn::value_decoder::decode(&value, key); *size = 2; memcpy(buf, &value, *size); } @@ -10871,7 +11176,7 @@ int ha_mroonga::storage_encode_key(Field *field, const uchar *key, { float float_value; double double_value; - float4get(float_value, ptr); + mrn::value_decoder::decode(&float_value, ptr); double_value = float_value; memcpy(buf, &double_value, 8); *size = 8; @@ -10880,7 +11185,7 @@ int ha_mroonga::storage_encode_key(Field *field, const uchar *key, case MYSQL_TYPE_DOUBLE: { double val; - float8get(val, ptr); + mrn::value_decoder::decode(&val, ptr); memcpy(buf, &val, 8); *size = 8; break; @@ -10907,7 +11212,7 @@ int ha_mroonga::storage_encode_key(Field *field, const uchar *key, long long int time = time_converter.tm_to_grn_time(&date, usec, &truncated); if (truncated) { - field->set_warning(Sql_condition::WARN_LEVEL_WARN, + field->set_warning(MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, 1); } memcpy(buf, &time, 8); @@ -10994,16 +11299,29 @@ int ha_mroonga::generic_reset() { MRN_DBUG_ENTER_METHOD(); int error = 0; - if (thd_sql_command(ha_thd()) == SQLCOM_SELECT) { - st_select_lex *select_lex = table->pos_in_table_list->select_lex; - List_iterator<Item_func_match> iterator(*(select_lex->ftfunc_list)); - Item_func_match *item; - while ((item = iterator++)) { - if (item->ft_handler) { - mrn_generic_ft_clear(item->ft_handler); - } + + if (thd_sql_command(ha_thd()) != SQLCOM_SELECT) { + DBUG_RETURN(error); + } + + TABLE_LIST *table_list = table->pos_in_table_list; + if (!table_list) { + DBUG_RETURN(error); + } + + st_select_lex *select_lex = table_list->select_lex; + if (!select_lex) { + DBUG_RETURN(error); + } + + List_iterator<Item_func_match> iterator(*(select_lex->ftfunc_list)); + Item_func_match *item; + while ((item = iterator++)) { + if (item->ft_handler) { + mrn_generic_ft_clear(item->ft_handler); } } + DBUG_RETURN(error); } @@ -12040,9 +12358,9 @@ void ha_mroonga::update_create_info(HA_CREATE_INFO* create_info) } if (create_info->connect_string.str) { slot_data->alter_connect_string = - my_strndup(create_info->connect_string.str, - create_info->connect_string.length, - MYF(MY_WME)); + mrn_my_strndup(create_info->connect_string.str, + create_info->connect_string.length, + MYF(MY_WME)); } if (slot_data->alter_comment) { my_free(slot_data->alter_comment); @@ -12050,9 +12368,9 @@ void ha_mroonga::update_create_info(HA_CREATE_INFO* create_info) } if (create_info->comment.str) { slot_data->alter_comment = - my_strndup(create_info->comment.str, - create_info->comment.length, - MYF(MY_WME)); + mrn_my_strndup(create_info->comment.str, + create_info->comment.length, + MYF(MY_WME)); } if (share && share->disable_keys) { slot_data->disable_keys_create_info = create_info; @@ -12365,14 +12683,15 @@ bool ha_mroonga::wrapper_is_crashed() const bool ha_mroonga::storage_is_crashed() const { MRN_DBUG_ENTER_METHOD(); - bool crashed = handler::is_crashed(); + mrn::DatabaseRepairer repairer(ctx, ha_thd()); + bool crashed = repairer.is_crashed(); DBUG_RETURN(crashed); } bool ha_mroonga::is_crashed() const { MRN_DBUG_ENTER_METHOD(); - int crashed; + bool crashed; if (share->wrapper_mode) { crashed = wrapper_is_crashed(); @@ -12384,52 +12703,52 @@ bool ha_mroonga::is_crashed() const bool ha_mroonga::wrapper_auto_repair(int error) const { - bool crashed; + bool repaired; MRN_DBUG_ENTER_METHOD(); MRN_SET_WRAP_SHARE_KEY(share, table->s); MRN_SET_WRAP_TABLE_KEY(this, table); #ifdef MRN_HANDLER_AUTO_REPAIR_HAVE_ERROR - crashed = wrap_handler->auto_repair(error); + repaired = wrap_handler->auto_repair(error); #else - crashed = wrap_handler->auto_repair(); + repaired = wrap_handler->auto_repair(); #endif MRN_SET_BASE_SHARE_KEY(share, table->s); MRN_SET_BASE_TABLE_KEY(this, table); - DBUG_RETURN(crashed); + DBUG_RETURN(repaired); } bool ha_mroonga::storage_auto_repair(int error) const { MRN_DBUG_ENTER_METHOD(); - bool crashed; + bool repaired; #ifdef MRN_HANDLER_AUTO_REPAIR_HAVE_ERROR - crashed = handler::auto_repair(error); + repaired = handler::auto_repair(error); #else - crashed = handler::auto_repair(); + repaired = handler::auto_repair(); #endif - DBUG_RETURN(crashed); + DBUG_RETURN(repaired); } bool ha_mroonga::auto_repair(int error) const { MRN_DBUG_ENTER_METHOD(); - bool crashed; + bool repaired; // TODO: We should consider about creating share for error = // ER_CANT_OPEN_FILE. The following code just ignores the error. if (share && share->wrapper_mode) { - crashed = wrapper_auto_repair(error); + repaired = wrapper_auto_repair(error); } else { - crashed = storage_auto_repair(error); + repaired = storage_auto_repair(error); } - DBUG_RETURN(crashed); + DBUG_RETURN(repaired); } bool ha_mroonga::auto_repair() const { MRN_DBUG_ENTER_METHOD(); - bool crashed = auto_repair(HA_ERR_CRASHED_ON_USAGE); - DBUG_RETURN(crashed); + bool repaired = auto_repair(HA_ERR_CRASHED_ON_USAGE); + DBUG_RETURN(repaired); } int ha_mroonga::generic_disable_index(int i, KEY *key_info) @@ -12760,7 +13079,12 @@ int ha_mroonga::wrapper_check(THD* thd, HA_CHECK_OPT* check_opt) int ha_mroonga::storage_check(THD* thd, HA_CHECK_OPT* check_opt) { MRN_DBUG_ENTER_METHOD(); - DBUG_RETURN(HA_ADMIN_NOT_IMPLEMENTED); + mrn::DatabaseRepairer repairer(ctx, thd); + if (repairer.repair()) { + DBUG_RETURN(HA_ADMIN_OK); + } else { + DBUG_RETURN(HA_ADMIN_CORRUPT); + } } int ha_mroonga::check(THD* thd, HA_CHECK_OPT* check_opt) @@ -12929,6 +13253,58 @@ int ha_mroonga::wrapper_recreate_indexes(THD *thd) DBUG_RETURN(error); } +int ha_mroonga::storage_recreate_indexes(THD *thd) +{ + MRN_DBUG_ENTER_METHOD(); + + if (share->disable_keys) + DBUG_RETURN(HA_ADMIN_OK); + + clear_indexes(); + + int n_columns = table->s->fields; + for (int i = 0; i < n_columns; i++) { + grn_obj *column = grn_columns[i]; + + if (!column) + continue; + + int n_hooks = grn_obj_get_nhooks(ctx, column, GRN_HOOK_SET); + for (int j = 0; j < n_hooks; j++) { + grn_obj_delete_hook(ctx, column, GRN_HOOK_SET, j); + } + } + + uint n_keys = table_share->keys; + mrn::PathMapper mapper(table_share->normalized_path.str); + for (uint i = 0; i < n_keys; i++) { + if (share->index_table && share->index_table[i]) + continue; + + if (i == table_share->primary_key) + continue; + + mrn::IndexTableName index_table_name(mapper.table_name(), + table_share->key_info[i].name); + char index_column_full_name[MRN_MAX_PATH_SIZE]; + snprintf(index_column_full_name, MRN_MAX_PATH_SIZE, + "%s.%s", index_table_name.c_str(), INDEX_COLUMN_NAME); + remove_grn_obj_force(index_column_full_name); + remove_grn_obj_force(index_table_name.c_str()); + } + + int error; + error = storage_create_indexes(table, mapper.table_name(), grn_table, share); + if (error) + DBUG_RETURN(HA_ADMIN_FAILED); + + error = storage_open_indexes(table_share->normalized_path.str); + if (error) + DBUG_RETURN(HA_ADMIN_FAILED); + + DBUG_RETURN(HA_ADMIN_OK); +} + int ha_mroonga::wrapper_repair(THD* thd, HA_CHECK_OPT* check_opt) { int error; @@ -12947,7 +13323,8 @@ int ha_mroonga::wrapper_repair(THD* thd, HA_CHECK_OPT* check_opt) int ha_mroonga::storage_repair(THD* thd, HA_CHECK_OPT* check_opt) { MRN_DBUG_ENTER_METHOD(); - DBUG_RETURN(HA_ADMIN_NOT_IMPLEMENTED); + int error = storage_recreate_indexes(thd); + DBUG_RETURN(error); } int ha_mroonga::repair(THD* thd, HA_CHECK_OPT* check_opt) @@ -12966,35 +13343,36 @@ int ha_mroonga::repair(THD* thd, HA_CHECK_OPT* check_opt) bool ha_mroonga::wrapper_check_and_repair(THD *thd) { - // XXX: success is valid variable name? - bool success; + bool is_error_or_not_supported; MRN_DBUG_ENTER_METHOD(); MRN_SET_WRAP_SHARE_KEY(share, table->s); MRN_SET_WRAP_TABLE_KEY(this, table); - success = wrap_handler->ha_check_and_repair(thd); + is_error_or_not_supported = wrap_handler->ha_check_and_repair(thd); MRN_SET_BASE_SHARE_KEY(share, table->s); MRN_SET_BASE_TABLE_KEY(this, table); - DBUG_RETURN(success); + DBUG_RETURN(is_error_or_not_supported); } bool ha_mroonga::storage_check_and_repair(THD *thd) { MRN_DBUG_ENTER_METHOD(); - DBUG_RETURN(true); + bool is_error = false; + mrn::DatabaseRepairer repairer(ctx, thd); + is_error = !repairer.repair(); + DBUG_RETURN(is_error); } bool ha_mroonga::check_and_repair(THD *thd) { MRN_DBUG_ENTER_METHOD(); - // XXX: success is valid variable name? - bool success; + bool is_error_or_not_supported; if (share->wrapper_mode) { - success = wrapper_check_and_repair(thd); + is_error_or_not_supported = wrapper_check_and_repair(thd); } else { - success = storage_check_and_repair(thd); + is_error_or_not_supported = storage_check_and_repair(thd); } - DBUG_RETURN(success); + DBUG_RETURN(is_error_or_not_supported); } int ha_mroonga::wrapper_analyze(THD* thd, HA_CHECK_OPT* check_opt) @@ -13059,7 +13437,11 @@ bool ha_mroonga::wrapper_is_fatal_error(int error_num, uint flags) MRN_DBUG_ENTER_METHOD(); MRN_SET_WRAP_SHARE_KEY(share, table->s); MRN_SET_WRAP_TABLE_KEY(this, table); +#ifdef MRN_HANDLER_IS_FATAL_ERROR_HAVE_FLAGS res = wrap_handler->is_fatal_error(error_num, flags); +#else + res = wrap_handler->is_fatal_error(error_num); +#endif MRN_SET_BASE_SHARE_KEY(share, table->s); MRN_SET_BASE_TABLE_KEY(this, table); DBUG_RETURN(res); @@ -13068,7 +13450,11 @@ bool ha_mroonga::wrapper_is_fatal_error(int error_num, uint flags) bool ha_mroonga::storage_is_fatal_error(int error_num, uint flags) { MRN_DBUG_ENTER_METHOD(); +#ifdef MRN_HANDLER_IS_FATAL_ERROR_HAVE_FLAGS bool is_fatal_error = handler::is_fatal_error(error_num, flags); +#else + bool is_fatal_error = handler::is_fatal_error(error_num); +#endif DBUG_RETURN(is_fatal_error); } @@ -13234,7 +13620,9 @@ enum_alter_inplace_result ha_mroonga::wrapper_check_if_supported_inplace_alter( uint n_keys; uint i; enum_alter_inplace_result result_mroonga = HA_ALTER_INPLACE_NO_LOCK; - DBUG_PRINT("info", ("mroonga: handler_flags=%lu", ha_alter_info->handler_flags)); + DBUG_PRINT("info", + ("mroonga: handler_flags=%lu", + static_cast<ulong>(ha_alter_info->handler_flags))); if (wrapper_is_comment_changed(table, altered_table)) { DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); @@ -13267,7 +13655,7 @@ enum_alter_inplace_result ha_mroonga::wrapper_check_if_supported_inplace_alter( alter_index_add_count = 0; alter_handler_flags = ha_alter_info->handler_flags; if (!(alter_key_info_buffer = (KEY *) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), + mrn_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &alter_key_info_buffer, sizeof(KEY) * ha_alter_info->key_count, &alter_index_drop_buffer, sizeof(KEY) * ha_alter_info->index_drop_count, &alter_index_add_buffer, sizeof(uint) * ha_alter_info->index_add_count, @@ -13362,8 +13750,6 @@ enum_alter_inplace_result ha_mroonga::storage_check_if_supported_inplace_alter( Alter_inplace_info::DROP_INDEX | Alter_inplace_info::ADD_UNIQUE_INDEX | Alter_inplace_info::DROP_UNIQUE_INDEX | - Alter_inplace_info::ADD_PK_INDEX | - Alter_inplace_info::DROP_PK_INDEX | Alter_inplace_info::ADD_COLUMN | Alter_inplace_info::DROP_COLUMN | Alter_inplace_info::ALTER_COLUMN_NAME; @@ -13480,7 +13866,7 @@ bool ha_mroonga::wrapper_inplace_alter_table( tmp_table_share.keys = ha_alter_info->key_count; tmp_table_share.fields = 0; if (!(tmp_share = (MRN_SHARE *) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), + mrn_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &tmp_share, sizeof(*tmp_share), &key_parser, sizeof(char *) * (tmp_table_share.keys), &key_parser_length, sizeof(uint) * (tmp_table_share.keys), @@ -13629,7 +14015,7 @@ bool ha_mroonga::storage_inplace_alter_table_index( tmp_table_share.keys = ha_alter_info->key_count; tmp_table_share.fields = 0; if (!(tmp_share = (MRN_SHARE *) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), + mrn_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &tmp_share, sizeof(*tmp_share), &index_table, sizeof(char *) * tmp_table_share.keys, &index_table_length, sizeof(uint) * tmp_table_share.keys, @@ -13763,7 +14149,7 @@ bool ha_mroonga::storage_inplace_alter_table_add_column( uint *index_table_length, *key_parser_length, *col_flags_length, *col_type_length; tmp_table_share.keys = 0; tmp_table_share.fields = altered_table->s->fields; - tmp_share = (MRN_SHARE *)my_multi_malloc( + tmp_share = (MRN_SHARE *)mrn_my_multi_malloc( MYF(MY_WME | MY_ZEROFILL), &tmp_share, sizeof(*tmp_share), &index_table, sizeof(char *) * tmp_table_share.keys, @@ -14155,8 +14541,8 @@ int ha_mroonga::wrapper_add_index(TABLE *table_arg, KEY *key_info, char **key_parser; uint *key_parser_length; MRN_DBUG_ENTER_METHOD(); - if (!(wrap_alter_key_info = (KEY *) my_malloc(sizeof(KEY) * num_of_keys, - MYF(MY_WME)))) { + if (!(wrap_alter_key_info = (KEY *) mrn_my_malloc(sizeof(KEY) * num_of_keys, + MYF(MY_WME)))) { MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_tables); MRN_FREE_VARIABLE_LENGTH_ARRAYS(index_columns); DBUG_RETURN(HA_ERR_OUT_OF_MEM); @@ -14165,7 +14551,7 @@ int ha_mroonga::wrapper_add_index(TABLE *table_arg, KEY *key_info, tmp_table_share.keys = n_keys + num_of_keys; tmp_table_share.fields = 0; if (!(tmp_share = (MRN_SHARE *) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), + mrn_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &tmp_share, sizeof(*tmp_share), &key_parser, sizeof(char *) * (n_keys + num_of_keys), &key_parser_length, sizeof(uint) * (n_keys + num_of_keys), @@ -14299,7 +14685,7 @@ int ha_mroonga::storage_add_index(TABLE *table_arg, KEY *key_info, tmp_table_share.keys = n_keys + num_of_keys; tmp_table_share.fields = 0; if (!(tmp_share = (MRN_SHARE *) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), + mrn_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &tmp_share, sizeof(*tmp_share), &index_table, sizeof(char*) * tmp_table_share.keys, &index_table_length, sizeof(uint) * tmp_table_share.keys, @@ -14663,7 +15049,7 @@ void ha_mroonga::storage_get_auto_increment(ulonglong offset, *first_value = long_term_share->auto_inc_value; DBUG_PRINT("info", ("mroonga: *first_value(auto_inc_value)=%llu", *first_value)); - *nb_reserved_values = ULONGLONG_MAX; + *nb_reserved_values = UINT_MAX64; } else { handler::get_auto_increment(offset, increment, nb_desired_values, first_value, nb_reserved_values); @@ -14811,6 +15197,7 @@ int ha_mroonga::check_for_upgrade(HA_CHECK_OPT *check_opt) DBUG_RETURN(error); } +#ifdef MRN_HANDLER_HAVE_RESET_AUTO_INCREMENT int ha_mroonga::wrapper_reset_auto_increment(ulonglong value) { int res; @@ -14847,6 +15234,7 @@ int ha_mroonga::reset_auto_increment(ulonglong value) } DBUG_RETURN(res); } +#endif void ha_mroonga::set_pk_bitmap() { @@ -15199,8 +15587,8 @@ char *ha_mroonga::storage_get_foreign_key_create_info() } create_info_str.q_append(") ON DELETE RESTRICT ON UPDATE RESTRICT", 39); } - if (!(create_info = (char *) my_malloc(create_info_str.length() + 1, - MYF(MY_WME)))) { + if (!(create_info = (char *) mrn_my_malloc(create_info_str.length() + 1, + MYF(MY_WME)))) { DBUG_RETURN(NULL); } memcpy(create_info, create_info_str.ptr(), create_info_str.length()); diff --git a/storage/mroonga/ha_mroonga.hpp b/storage/mroonga/ha_mroonga.hpp index a5379dbddbb..78d5c3d2465 100644 --- a/storage/mroonga/ha_mroonga.hpp +++ b/storage/mroonga/ha_mroonga.hpp @@ -98,7 +98,13 @@ extern "C" { # define MRN_TABLE_LIST_INIT_REQUIRE_ALIAS #endif -#ifdef BIG_TABLES +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_BIG_TABLES +#elif defined(BIG_TABLES) +# define MRN_BIG_TABLES +#endif + +#ifdef MRN_BIG_TABLES # define MRN_HA_ROWS_FORMAT "llu" #else # define MRN_HA_ROWS_FORMAT "lu" @@ -145,6 +151,14 @@ extern "C" { # define MRN_HAVE_TL_WRITE_ALLOW_READ #endif +#if MYSQL_VERSION_ID < 50706 || defined(MRN_MARIADB_P) +# define MRN_HAVE_TL_WRITE_DELAYED +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_HAVE_TL_WRITE_CONCURRENT_DEFAULT +#endif + #if (defined(MRN_MARIADB_P) && \ ((MYSQL_VERSION_ID >= 50306 && MYSQL_VERSION_ID < 50500) || \ MYSQL_VERSION_ID >= 50523)) @@ -179,6 +193,44 @@ extern "C" { # define MRN_HAVE_TDC_LOCK_TABLE_SHARE #endif +#ifdef MRN_MARIADB_P +# if MYSQL_VERSION_ID >= 50542 && MYSQL_VERSION_ID < 100000 +# define MRN_SUPPORT_THDVAR_SET +# elif MYSQL_VERSION_ID >= 100017 +# define MRN_SUPPORT_THDVAR_SET +# endif +#else +# define MRN_SUPPORT_THDVAR_SET +#endif + +#ifdef MRN_MARIADB_P +# if MYSQL_VERSION_ID < 100000 +# define MRN_SUPPORT_PARTITION +# endif +#else +# define MRN_SUPPORT_PARTITION +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_FLUSH_LOGS_HAVE_BINLOG_GROUP_FLUSH +#endif + +#if MYSQL_VERSION_ID < 50706 || defined(MRN_MARIADB_P) +# define MRN_HAVE_HTON_ALTER_TABLE_FLAGS +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_FOREIGN_KEY_USE_CONST_STRING +#endif + +#if MYSQL_VERSION_ID < 50706 || defined(MRN_MARIADB_P) +# define MRN_HANDLER_IS_FATAL_ERROR_HAVE_FLAGS +#endif + +#if MYSQL_VERSION_ID < 50706 || defined(MRN_MARIADB_P) +# define MRN_HANDLER_HAVE_RESET_AUTO_INCREMENT +#endif + class ha_mroonga; /* structs */ @@ -448,7 +500,7 @@ public: bool check_and_repair(THD *thd); int analyze(THD* thd, HA_CHECK_OPT* check_opt); int optimize(THD* thd, HA_CHECK_OPT* check_opt); - bool is_fatal_error(int error_num, uint flags); + bool is_fatal_error(int error_num, uint flags=0); bool check_if_incompatible_data(HA_CREATE_INFO *create_info, uint table_changes); #ifdef MRN_HANDLER_HAVE_CHECK_IF_SUPPORTED_INPLACE_ALTER @@ -474,7 +526,9 @@ public: void restore_auto_increment(ulonglong prev_insert_id); void release_auto_increment(); int check_for_upgrade(HA_CHECK_OPT *check_opt); +#ifdef MRN_HANDLER_HAVE_RESET_AUTO_INCREMENT int reset_auto_increment(ulonglong value); +#endif bool was_semi_consistent_read(); void try_semi_consistent_read(bool yes); void unlock_row(); @@ -556,6 +610,7 @@ private: int drop_index(MRN_SHARE *target_share, uint key_index); grn_obj *find_tokenizer(const char *name, int name_length); grn_obj *find_normalizer(KEY *key_info); + bool find_index_column_flags(KEY *key_info, grn_obj_flags *index_column_flags); bool find_token_filters(KEY *key_info, grn_obj *token_filters); bool find_token_filters_put(grn_obj *token_filters, const char *token_filter_name, @@ -579,6 +634,7 @@ private: void check_count_skip(key_part_map start_key_part_map, key_part_map end_key_part_map, bool fulltext); bool is_grn_zero_column_value(grn_obj *column, grn_obj *value); + bool is_primary_key_field(Field *field) const; void check_fast_order_limit(grn_table_sort_key **sort_keys, int *n_sort_keys, longlong *limit); @@ -1040,6 +1096,7 @@ private: int wrapper_fill_indexes(THD *thd, KEY *key_info, grn_obj **index_columns, uint n_keys); int wrapper_recreate_indexes(THD *thd); + int storage_recreate_indexes(THD *thd); int wrapper_repair(THD* thd, HA_CHECK_OPT* check_opt); int storage_repair(THD* thd, HA_CHECK_OPT* check_opt); bool wrapper_check_and_repair(THD *thd); @@ -1131,8 +1188,10 @@ private: void storage_release_auto_increment(); int wrapper_check_for_upgrade(HA_CHECK_OPT *check_opt); int storage_check_for_upgrade(HA_CHECK_OPT *check_opt); +#ifdef MRN_HANDLER_HAVE_RESET_AUTO_INCREMENT int wrapper_reset_auto_increment(ulonglong value); int storage_reset_auto_increment(ulonglong value); +#endif bool wrapper_was_semi_consistent_read(); bool storage_was_semi_consistent_read(); void wrapper_try_semi_consistent_read(bool yes); diff --git a/storage/mroonga/lib/libmrn_need_mysql_sources.am b/storage/mroonga/lib/libmrn_need_mysql_sources.am index bd852829467..575f38adbd1 100644 --- a/storage/mroonga/lib/libmrn_need_mysql_sources.am +++ b/storage/mroonga/lib/libmrn_need_mysql_sources.am @@ -24,4 +24,8 @@ libmrn_need_mysql_la_SOURCES = \ mrn_time_converter.cpp \ mrn_time_converter.hpp \ mrn_database_manager.cpp \ - mrn_database_manager.hpp + mrn_database_manager.hpp \ + mrn_value_decoder.cpp \ + mrn_value_decoder.hpp \ + mrn_database_repairer.cpp \ + mrn_database_repairer.hpp diff --git a/storage/mroonga/lib/mrn_condition_converter.hpp b/storage/mroonga/lib/mrn_condition_converter.hpp index 3cf97c62bbe..bb85f5cdef5 100644 --- a/storage/mroonga/lib/mrn_condition_converter.hpp +++ b/storage/mroonga/lib/mrn_condition_converter.hpp @@ -23,6 +23,8 @@ #include <groonga.h> #include <mrn_mysql_compat.h> +#include <item_cmpfunc.h> + namespace mrn { class ConditionConverter { public: diff --git a/storage/mroonga/lib/mrn_database_repairer.cpp b/storage/mroonga/lib/mrn_database_repairer.cpp new file mode 100644 index 00000000000..151330c9999 --- /dev/null +++ b/storage/mroonga/lib/mrn_database_repairer.cpp @@ -0,0 +1,246 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include <mrn_mysql.h> +#include <mrn_mysql_compat.h> +#include <mrn_constants.hpp> + +#include "mrn_database_repairer.hpp" +#include "mrn_path_mapper.hpp" + +// for debug +#define MRN_CLASS_NAME "mrn::DatabaseRepairer" + +#include <sys/stat.h> +#include <sys/types.h> +#include <errno.h> + +#ifndef WIN32 +# include <dirent.h> +#endif + +namespace mrn { + DatabaseRepairer::DatabaseRepairer(grn_ctx *ctx, THD *thd) + : ctx_(ctx), + thd_(thd), + base_directory_(NULL), + base_directory_buffer_(), + path_prefix_(NULL), + path_prefix_buffer_(), + path_prefix_length_(0), + mrn_db_file_suffix_length_(strlen(MRN_DB_FILE_SUFFIX)) { + } + + DatabaseRepairer::~DatabaseRepairer() { + } + + bool DatabaseRepairer::is_crashed(void) { + MRN_DBUG_ENTER_METHOD(); + + bool is_crashed = false; + each_database(&DatabaseRepairer::is_crashed_body, &is_crashed); + + DBUG_RETURN(is_crashed); + } + + bool DatabaseRepairer::repair(void) { + MRN_DBUG_ENTER_METHOD(); + + bool succeeded = true; + each_database(&DatabaseRepairer::repair_body, &succeeded); + + DBUG_RETURN(succeeded); + } + + void DatabaseRepairer::each_database(EachBodyFunc each_body_func, + void *user_data) { + MRN_DBUG_ENTER_METHOD(); + + detect_paths(); + +#ifdef WIN32 + WIN32_FIND_DATA data; + HANDLE finder = FindFirstFile(base_directory_, &data); + if (finder == INVALID_HANDLE_VALUE) { + DBUG_VOID_RETURN; + } + + do { + each_database_body(data.cFileName, each_body_func, user_data); + } while (FindNextFile(finder, &data) != 0); + FindClose(finder); +#else + DIR *dir = opendir(base_directory_); + if (!dir) { + DBUG_VOID_RETURN; + } + + while (struct dirent *entry = readdir(dir)) { + each_database_body(entry->d_name, each_body_func, user_data); + } + closedir(dir); +#endif + + DBUG_VOID_RETURN; + } + + void DatabaseRepairer::each_database_body(const char *base_path, + EachBodyFunc each_body_func, + void *user_data) { + MRN_DBUG_ENTER_METHOD(); + + if (path_prefix_length_ > 0 && + strncmp(base_path, path_prefix_, path_prefix_length_) != 0) { + DBUG_VOID_RETURN; + } + + size_t path_length = strlen(base_path); + if (path_length <= mrn_db_file_suffix_length_) { + DBUG_VOID_RETURN; + } + + if (strncmp(base_path + (path_length - mrn_db_file_suffix_length_), + MRN_DB_FILE_SUFFIX, mrn_db_file_suffix_length_) != 0) { + DBUG_VOID_RETURN; + } + + char db_path[MRN_MAX_PATH_SIZE]; + snprintf(db_path, MRN_MAX_PATH_SIZE, + "%s%c%s", base_directory_, FN_LIBCHAR, base_path); + grn_obj *db = grn_db_open(ctx_, db_path); + if (!db) { + DBUG_VOID_RETURN; + } + + (this->*each_body_func)(db, db_path, user_data); + + grn_obj_close(ctx_, db); + + DBUG_VOID_RETURN; + } + + void DatabaseRepairer::detect_paths(void) { + MRN_DBUG_ENTER_METHOD(); + + const char *raw_path_prefix = mrn::PathMapper::default_path_prefix; + + if (!raw_path_prefix) { + base_directory_ = "."; + path_prefix_ = NULL; + DBUG_VOID_RETURN; + } + + strcpy(base_directory_buffer_, raw_path_prefix); + size_t raw_path_prefix_length = strlen(raw_path_prefix); + size_t separator_position = raw_path_prefix_length; + for (; separator_position > 0; separator_position--) { + if (base_directory_buffer_[separator_position] == FN_LIBCHAR || + base_directory_buffer_[separator_position] == FN_LIBCHAR2) { + break; + } + } + if (separator_position == 0 || + separator_position == raw_path_prefix_length) { + base_directory_ = "."; + } else { + base_directory_buffer_[separator_position] = '\0'; + base_directory_ = base_directory_buffer_; + strcpy(path_prefix_buffer_, raw_path_prefix + separator_position + 1); + path_prefix_ = path_prefix_buffer_; + path_prefix_length_ = strlen(path_prefix_); + } + + DBUG_VOID_RETURN; + } + + void DatabaseRepairer::is_crashed_body(grn_obj *db, + const char *db_path, + void *user_data) { + MRN_DBUG_ENTER_METHOD(); + + bool *is_crashed = static_cast<bool *>(user_data); + + if (grn_obj_is_locked(ctx_, db)) { + *is_crashed = true; + DBUG_VOID_RETURN; + } + + grn_table_cursor *cursor; + cursor = grn_table_cursor_open(ctx_, db, + NULL, 0, + NULL, 0, + 0, -1, GRN_CURSOR_BY_ID); + if (!cursor) { + *is_crashed = true; + DBUG_VOID_RETURN; + } + + grn_id id; + while ((id = grn_table_cursor_next(ctx_, cursor)) != GRN_ID_NIL) { + grn_obj *object = grn_ctx_at(ctx_, id); + + if (!object) { + continue; + } + + switch (object->header.type) { + case GRN_TABLE_HASH_KEY : + case GRN_TABLE_PAT_KEY: + case GRN_TABLE_DAT_KEY: + case GRN_TABLE_NO_KEY: + case GRN_COLUMN_FIX_SIZE: + case GRN_COLUMN_VAR_SIZE: + case GRN_COLUMN_INDEX: + grn_obj_is_locked(ctx_, object); + *is_crashed = true; + break; + default: + break; + } + + grn_obj_unlink(ctx_, object); + + if (*is_crashed) { + break; + } + } + grn_table_cursor_close(ctx_, cursor); + + DBUG_VOID_RETURN; + } + + void DatabaseRepairer::repair_body(grn_obj *db, + const char *db_path, + void *user_data) { + MRN_DBUG_ENTER_METHOD(); + + bool *succeeded = static_cast<bool *>(user_data); + if (grn_db_recover(ctx_, db) != GRN_SUCCESS) { + push_warning_printf(thd_, + Sql_condition::WARN_LEVEL_WARN, + ER_NOT_KEYFILE, + "mroonga: repair: " + "Failed to recover database: <%s>: <%s>", + db_path, ctx_->errbuf); + *succeeded = false; + } + + DBUG_VOID_RETURN; + } +} diff --git a/storage/mroonga/lib/mrn_database_repairer.hpp b/storage/mroonga/lib/mrn_database_repairer.hpp new file mode 100644 index 00000000000..12e2bbc9c79 --- /dev/null +++ b/storage/mroonga/lib/mrn_database_repairer.hpp @@ -0,0 +1,58 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef MRN_DATABASE_REPAIRER_HPP_ +#define MRN_DATABASE_REPAIRER_HPP_ + +#include <groonga.h> + +namespace mrn { + class DatabaseRepairer { + public: + DatabaseRepairer(grn_ctx *ctx, THD *thd); + ~DatabaseRepairer(void); + bool is_crashed(void); + bool repair(void); + + private: + grn_ctx *ctx_; + THD *thd_; + const char *base_directory_; + char base_directory_buffer_[MRN_MAX_PATH_SIZE]; + const char *path_prefix_; + char path_prefix_buffer_[MRN_MAX_PATH_SIZE]; + size_t path_prefix_length_; + size_t mrn_db_file_suffix_length_; + + typedef void (DatabaseRepairer::*EachBodyFunc)(grn_obj *db, + const char *db_path, + void *user_data); + + void each_database(EachBodyFunc each_body_func, void *user_data); + void each_database_body(const char *base_path, + EachBodyFunc each_body_func, + void *user_data); + void detect_paths(void); + + void is_crashed_body(grn_obj *db, const char *db_path, void *user_data); + void repair_body(grn_obj *db, const char *db_path, void *user_data); + }; +} + +#endif /* MRN_DATABASE_REPAIRER_HPP_ */ diff --git a/storage/mroonga/lib/mrn_encoding.cpp b/storage/mroonga/lib/mrn_encoding.cpp index f6f66758b2f..35b8909fba2 100644 --- a/storage/mroonga/lib/mrn_encoding.cpp +++ b/storage/mroonga/lib/mrn_encoding.cpp @@ -18,6 +18,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include <mrn_mysql.h> #include <mrn_err.h> #include "mrn_encoding.hpp" diff --git a/storage/mroonga/lib/mrn_field_normalizer.cpp b/storage/mroonga/lib/mrn_field_normalizer.cpp index c34f9975e37..f0b9d921599 100644 --- a/storage/mroonga/lib/mrn_field_normalizer.cpp +++ b/storage/mroonga/lib/mrn_field_normalizer.cpp @@ -17,6 +17,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include <mrn_mysql.h> + #include "mrn_field_normalizer.hpp" #include "mrn_encoding.hpp" @@ -114,6 +116,9 @@ namespace mrn { } else if ((strcmp(charset_info->name, "utf8_unicode_ci") == 0) || (strcmp(charset_info->name, "utf8mb4_unicode_ci") == 0)) { normalizer_name = "NormalizerMySQLUnicodeCI"; + } else if ((strcmp(charset_info->name, "utf8_unicode_520_ci") == 0) || + (strcmp(charset_info->name, "utf8mb4_unicode_520_ci") == 0)) { + normalizer_name = "NormalizerMySQLUnicode520CI"; } grn_obj *normalizer = NULL; @@ -128,7 +133,7 @@ namespace mrn { normalizer_name, charset_info->name, default_normalizer_name); - push_warning(thread_, Sql_condition::WARN_LEVEL_WARN, + push_warning(thread_, MRN_SEVERITY_WARNING, HA_ERR_UNSUPPORTED, error_message); } } diff --git a/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp b/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp index 5ce736a49b8..1e55636f1bc 100644 --- a/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp +++ b/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2012-2014 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> Copyright(C) 2013 Kentoku SHIBA This library is free software; you can redistribute it and/or @@ -23,6 +23,7 @@ #include "mrn_multiple_column_key_codec.hpp" #include "mrn_field_normalizer.hpp" #include "mrn_smart_grn_obj.hpp" +#include "mrn_value_decoder.hpp" // for debug #define MRN_CLASS_NAME "mrn::MultipleColumnKeyCodec" @@ -121,14 +122,14 @@ namespace mrn { case TYPE_FLOAT: { float value; - float4get(value, current_mysql_key); + value_decoder::decode(&value, current_mysql_key); encode_float(value, data_size, current_grn_key); } break; case TYPE_DOUBLE: { double value; - float8get(value, current_mysql_key); + value_decoder::decode(&value, current_mysql_key); encode_double(value, data_size, current_grn_key); } break; @@ -523,7 +524,7 @@ namespace mrn { new_blob_data_length = normalized_length; } else { push_warning_printf(thread_, - Sql_condition::WARN_LEVEL_WARN, + MRN_SEVERITY_WARNING, WARN_DATA_TRUNCATED, "normalized data truncated " "for multiple column index: " diff --git a/storage/mroonga/lib/mrn_parameters_parser.cpp b/storage/mroonga/lib/mrn_parameters_parser.cpp index 3edf910ee91..bc075b07068 100644 --- a/storage/mroonga/lib/mrn_parameters_parser.cpp +++ b/storage/mroonga/lib/mrn_parameters_parser.cpp @@ -1,7 +1,7 @@ /* -*- c-basic-offset: 2 -*- */ /* Copyright(C) 2011-2013 Kentoku SHIBA - Copyright(C) 2011-2013 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2011-2015 Kouhei Sutou <kou@clear-code.com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -21,6 +21,7 @@ #include "mrn_parameters_parser.hpp" #include <mrn_mysql_compat.h> +#include <mrn_variables.hpp> namespace mrn { class Parameter { @@ -30,8 +31,8 @@ namespace mrn { Parameter(const char *key, unsigned int key_length, const char *value, unsigned int value_length) - : key_(my_strndup(key, key_length, MYF(0))), - value_(my_strndup(value, value_length, MYF(0))) { + : key_(mrn_my_strndup(key, key_length, MYF(0))), + value_(mrn_my_strndup(value, value_length, MYF(0))) { }; ~Parameter() { if (key_) { diff --git a/storage/mroonga/lib/mrn_path_mapper.cpp b/storage/mroonga/lib/mrn_path_mapper.cpp index 796101a10d1..7a595986f01 100644 --- a/storage/mroonga/lib/mrn_path_mapper.cpp +++ b/storage/mroonga/lib/mrn_path_mapper.cpp @@ -29,16 +29,17 @@ namespace mrn { char *PathMapper::default_path_prefix = NULL; char *PathMapper::default_mysql_data_home_path = NULL; - PathMapper::PathMapper(const char *mysql_path, + PathMapper::PathMapper(const char *original_mysql_path, const char *path_prefix, const char *mysql_data_home_path) - : mysql_path_(mysql_path), + : original_mysql_path_(original_mysql_path), path_prefix_(path_prefix), mysql_data_home_path_(mysql_data_home_path) { db_path_[0] = '\0'; db_name_[0] = '\0'; table_name_[0] = '\0'; mysql_table_name_[0] = '\0'; + mysql_path_[0] = '\0'; } /** @@ -52,22 +53,25 @@ namespace mrn { return db_path_; } - if (mysql_path_[0] == FN_CURLIB && mysql_path_[1] == FN_LIBCHAR) { + if (original_mysql_path_[0] == FN_CURLIB && + original_mysql_path_[1] == FN_LIBCHAR) { if (path_prefix_) { strcpy(db_path_, path_prefix_); } int i = 2, j = strlen(db_path_), len; - len = strlen(mysql_path_); - while (mysql_path_[i] != FN_LIBCHAR && i < len) { - db_path_[j++] = mysql_path_[i++]; + len = strlen(original_mysql_path_); + while (original_mysql_path_[i] != FN_LIBCHAR && i < len) { + db_path_[j++] = original_mysql_path_[i++]; } db_path_[j] = '\0'; } else if (mysql_data_home_path_) { - int len = strlen(mysql_path_); + int len = strlen(original_mysql_path_); int mysql_data_home_len = strlen(mysql_data_home_path_); if (len > mysql_data_home_len && - !strncmp(mysql_path_, mysql_data_home_path_, mysql_data_home_len)) { + !strncmp(original_mysql_path_, + mysql_data_home_path_, + mysql_data_home_len)) { int i = mysql_data_home_len, j; if (path_prefix_ && path_prefix_[0] == FN_LIBCHAR) { strcpy(db_path_, path_prefix_); @@ -87,19 +91,19 @@ namespace mrn { } } - while (mysql_path_[i] != FN_LIBCHAR && i < len) { - db_path_[j++] = mysql_path_[i++]; + while (original_mysql_path_[i] != FN_LIBCHAR && i < len) { + db_path_[j++] = original_mysql_path_[i++]; } if (i == len) { - memcpy(db_path_, mysql_path_, len); + memcpy(db_path_, original_mysql_path_, len); } else { db_path_[j] = '\0'; } } else { - strcpy(db_path_, mysql_path_); + strcpy(db_path_, original_mysql_path_); } } else { - strcpy(db_path_, mysql_path_); + strcpy(db_path_, original_mysql_path_); } strcat(db_path_, MRN_DB_FILE_SUFFIX); return db_path_; @@ -116,32 +120,35 @@ namespace mrn { return db_name_; } - if (mysql_path_[0] == FN_CURLIB && mysql_path_[1] == FN_LIBCHAR) { + if (original_mysql_path_[0] == FN_CURLIB && + original_mysql_path_[1] == FN_LIBCHAR) { int i = 2, j = 0, len; - len = strlen(mysql_path_); - while (mysql_path_[i] != FN_LIBCHAR && i < len) { - db_name_[j++] = mysql_path_[i++]; + len = strlen(original_mysql_path_); + while (original_mysql_path_[i] != FN_LIBCHAR && i < len) { + db_name_[j++] = original_mysql_path_[i++]; } db_name_[j] = '\0'; } else if (mysql_data_home_path_) { - int len = strlen(mysql_path_); + int len = strlen(original_mysql_path_); int mysql_data_home_len = strlen(mysql_data_home_path_); if (len > mysql_data_home_len && - !strncmp(mysql_path_, mysql_data_home_path_, mysql_data_home_len)) { + !strncmp(original_mysql_path_, + mysql_data_home_path_, + mysql_data_home_len)) { int i = mysql_data_home_len, j = 0; - while (mysql_path_[i] != FN_LIBCHAR && i < len) { - db_name_[j++] = mysql_path_[i++]; + while (original_mysql_path_[i] != FN_LIBCHAR && i < len) { + db_name_[j++] = original_mysql_path_[i++]; } if (i == len) { - memcpy(db_name_, mysql_path_, len); + memcpy(db_name_, original_mysql_path_, len); } else { db_name_[j] = '\0'; } } else { - strcpy(db_name_, mysql_path_); + strcpy(db_name_, original_mysql_path_); } } else { - strcpy(db_name_, mysql_path_); + strcpy(db_name_, original_mysql_path_); } return db_name_; } @@ -154,10 +161,10 @@ namespace mrn { return table_name_; } - int len = strlen(mysql_path_); + int len = strlen(original_mysql_path_); int i = len, j = 0; - for (; mysql_path_[--i] != FN_LIBCHAR ;) {} - if (mysql_path_[i + 1] == '_') { + for (; original_mysql_path_[--i] != FN_LIBCHAR ;) {} + if (original_mysql_path_[i + 1] == '_') { table_name_[j++] = '@'; table_name_[j++] = '0'; table_name_[j++] = '0'; @@ -166,7 +173,7 @@ namespace mrn { i++; } for (; i < len ;) { - table_name_[j++] = mysql_path_[++i]; + table_name_[j++] = original_mysql_path_[++i]; } table_name_[j] = '\0'; return table_name_; @@ -180,16 +187,39 @@ namespace mrn { return mysql_table_name_; } - int len = strlen(mysql_path_); + int len = strlen(original_mysql_path_); int i = len, j = 0; - for (; mysql_path_[--i] != FN_LIBCHAR ;) {} + for (; original_mysql_path_[--i] != FN_LIBCHAR ;) {} for (; i < len ;) { - if (len - i - 1 >= 3 && strncmp(mysql_path_ + i + 1, "#P#", 3) == 0) { + if (len - i - 1 >= 3 && + strncmp(original_mysql_path_ + i + 1, "#P#", 3) == 0) { break; } - mysql_table_name_[j++] = mysql_path_[++i]; + mysql_table_name_[j++] = original_mysql_path_[++i]; } mysql_table_name_[j] = '\0'; return mysql_table_name_; } + + /** + * "./${db}/${table}" ==> "./${db}/${table}" + * "./${db}/${table}#P#xxx" ==> "./${db}/${table}" + */ + const char *PathMapper::mysql_path() { + if (mysql_path_[0] != '\0') { + return mysql_path_; + } + + int i; + int len = strlen(original_mysql_path_); + for (i = 0; i < len; i++) { + if (len - i >= 3 && + strncmp(original_mysql_path_ + i, "#P#", 3) == 0) { + break; + } + mysql_path_[i] = original_mysql_path_[i]; + } + mysql_path_[i] = '\0'; + return mysql_path_; + } } diff --git a/storage/mroonga/lib/mrn_path_mapper.hpp b/storage/mroonga/lib/mrn_path_mapper.hpp index f70cd7b5587..607bfe4cdce 100644 --- a/storage/mroonga/lib/mrn_path_mapper.hpp +++ b/storage/mroonga/lib/mrn_path_mapper.hpp @@ -2,7 +2,7 @@ /* Copyright(C) 2010 Tetsuro IKEDA Copyright(C) 2010-2013 Kentoku SHIBA - Copyright(C) 2011-2012 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2011-2015 Kouhei Sutou <kou@clear-code.com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -30,21 +30,23 @@ namespace mrn { static char *default_path_prefix; static char *default_mysql_data_home_path; - PathMapper(const char *mysql_path, + PathMapper(const char *original_mysql_path, const char *path_prefix=default_path_prefix, const char *mysql_data_home_path=default_mysql_data_home_path); const char *db_path(); const char *db_name(); const char *table_name(); const char *mysql_table_name(); + const char *mysql_path(); private: - const char *mysql_path_; + const char *original_mysql_path_; const char *path_prefix_; const char *mysql_data_home_path_; char db_path_[MRN_MAX_PATH_SIZE]; char db_name_[MRN_MAX_PATH_SIZE]; char table_name_[MRN_MAX_PATH_SIZE]; char mysql_table_name_[MRN_MAX_PATH_SIZE]; + char mysql_path_[MRN_MAX_PATH_SIZE]; }; } diff --git a/storage/mroonga/lib/mrn_value_decoder.cpp b/storage/mroonga/lib/mrn_value_decoder.cpp new file mode 100644 index 00000000000..8356789915f --- /dev/null +++ b/storage/mroonga/lib/mrn_value_decoder.cpp @@ -0,0 +1,64 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "mrn_value_decoder.hpp" + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_DEST_IS_POINTER +#endif + +namespace mrn { + namespace value_decoder { + void decode(uint16 *dest, const uchar *source) { + MRN_DBUG_ENTER_FUNCTION(); +#ifdef MRN_DEST_IS_POINTER + ushortget(dest, source); +#else + uint16 value; + ushortget(value, source); + *dest = value; +#endif + DBUG_VOID_RETURN; + }; + + void decode(float *dest, const uchar *source) { + MRN_DBUG_ENTER_FUNCTION(); +#ifdef MRN_DEST_IS_POINTER + float4get(dest, source); +#else + float value; + float4get(value, source); + *dest = value; +#endif + DBUG_VOID_RETURN; + }; + + void decode(double *dest, const uchar *source) { + MRN_DBUG_ENTER_FUNCTION(); +#ifdef MRN_DEST_IS_POINTER + float8get(dest, source); +#else + double value; + float8get(value, source); + *dest = value; +#endif + DBUG_VOID_RETURN; + } + } +} diff --git a/storage/mroonga/lib/mrn_value_decoder.hpp b/storage/mroonga/lib/mrn_value_decoder.hpp new file mode 100644 index 00000000000..8a48de0b003 --- /dev/null +++ b/storage/mroonga/lib/mrn_value_decoder.hpp @@ -0,0 +1,33 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef MRN_VALUE_DECODER_HPP_ +#define MRN_VALUE_DECODER_HPP_ + +#include <mrn_mysql.h> + +namespace mrn { + namespace value_decoder { + void decode(uint16 *dest, const uchar *source); + void decode(float *dest, const uchar *source); + void decode(double *dest, const uchar *source); + } +} + +#endif // MRN_VALUE_DECODER_HPP_ diff --git a/storage/mroonga/mrn_constants.hpp b/storage/mroonga/mrn_constants.hpp index 494e08721bd..5bc6da6b9b6 100644 --- a/storage/mroonga/mrn_constants.hpp +++ b/storage/mroonga/mrn_constants.hpp @@ -22,6 +22,8 @@ #ifndef MRN_CONSTANTS_HPP_ #define MRN_CONSTANTS_HPP_ +#include <limits.h> + #include <groonga.h> #define MRN_BUFFER_SIZE 1024 diff --git a/storage/mroonga/mrn_err.h b/storage/mroonga/mrn_err.h index bfaf2d44fa8..cd4515a034e 100644 --- a/storage/mroonga/mrn_err.h +++ b/storage/mroonga/mrn_err.h @@ -34,5 +34,8 @@ #define ER_MRN_INVALID_COLUMN_FLAG_NUM 16507 #define ER_MRN_INVALID_COLUMN_FLAG_STR \ "The column flag '%-.64s' is invalid. '%-64s' is used instead" +#define ER_MRN_INVALID_INDEX_FLAG_NUM 16508 +#define ER_MRN_INVALID_INDEX_FLAG_STR \ + "The index flag '%-.64s' is invalid. It is ignored" #endif /* MRN_ERR_H_ */ diff --git a/storage/mroonga/mrn_mysql.h b/storage/mroonga/mrn_mysql.h index cf2a54621bb..644825dcf39 100644 --- a/storage/mroonga/mrn_mysql.h +++ b/storage/mroonga/mrn_mysql.h @@ -47,7 +47,7 @@ # include <mysql_priv.h> # include <mysql/plugin.h> #else -# include <sql_priv.h> +# include <sql_const.h> # include <sql_class.h> # include <probes_mysql.h> # include <sql_partition.h> @@ -58,12 +58,6 @@ # define MRN_MARIADB_P 1 #endif -#if MYSQL_VERSION_ID >= 50607 -# if !defined(MRN_MARIADB_P) -# define MRN_HAVE_SQL_OPTIMIZER_H -# endif -#endif - #define MRN_MESSAGE_BUFFER_SIZE 1024 #define MRN_DBUG_ENTER_FUNCTION() DBUG_ENTER(__FUNCTION__) diff --git a/storage/mroonga/mrn_mysql_compat.h b/storage/mroonga/mrn_mysql_compat.h index 61b4ddd67a2..a717220a35c 100644 --- a/storage/mroonga/mrn_mysql_compat.h +++ b/storage/mroonga/mrn_mysql_compat.h @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2011-2014 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2011-2015 Kouhei Sutou <kou@clear-code.com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -61,26 +61,34 @@ # define KEY_N_KEY_PARTS(key) (key)->key_parts #endif -#if MYSQL_VERSION_ID < 100000 || !defined(MRN_MARIADB_P) -# define init_alloc_root(PTR, SZ1, SZ2, FLAG) init_alloc_root(PTR, SZ1, SZ2) +#if defined(MRN_MARIADB_P) && MYSQL_VERSION_ID >= 100000 +# define mrn_init_alloc_root(PTR, SZ1, SZ2, FLAG) \ + init_alloc_root(PTR, SZ1, SZ2, FLAG) +#elif MYSQL_VERSION_ID >= 50706 +# define mrn_init_alloc_root(PTR, SZ1, SZ2, FLAG) \ + init_alloc_root(mrn_memory_key, PTR, SZ1, SZ2) +#else +# define mrn_init_alloc_root(PTR, SZ1, SZ2, FLAG) \ + init_alloc_root(PTR, SZ1, SZ2) #endif #if MYSQL_VERSION_ID < 100002 || !defined(MRN_MARIADB_P) # define GTS_TABLE 0 #endif -/* For MySQL 5.1. MySQL 5.1 doesn't have FN_LIBCHAR2. */ -#ifndef FN_LIBCHAR2 -# define FN_LIBCHAR2 FN_LIBCHAR -#endif - #if MYSQL_VERSION_ID >= 50607 # if MYSQL_VERSION_ID >= 100007 && defined(MRN_MARIADB_P) # define MRN_GET_ERROR_MESSAGE thd_get_error_message(current_thd) # define MRN_GET_CURRENT_ROW_FOR_WARNING(thd) thd_get_error_row(thd) # else # define MRN_GET_ERROR_MESSAGE current_thd->get_stmt_da()->message() -# define MRN_GET_CURRENT_ROW_FOR_WARNING(thd) thd->get_stmt_da()->current_row_for_warning() +# if MYSQL_VERSION_ID >= 50706 +# define MRN_GET_CURRENT_ROW_FOR_WARNING(thd) \ + thd->get_stmt_da()->current_row_for_condition() +# else +# define MRN_GET_CURRENT_ROW_FOR_WARNING(thd) \ + thd->get_stmt_da()->current_row_for_warning() +# endif # endif #else # if MYSQL_VERSION_ID >= 50500 @@ -96,7 +104,7 @@ # define MRN_ITEM_HAVE_ITEM_NAME #endif -#if MYSQL_VERSION_ID >= 50500 && MYSQL_VERSION_ID < 50700 +#if MYSQL_VERSION_ID >= 50500 && MYSQL_VERSION_ID < 100000 # define MRN_HAVE_TABLE_DEF_CACHE #endif @@ -131,4 +139,82 @@ # define MRN_USE_MYSQL_DATA_HOME #endif +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_SEVERITY_WARNING Sql_condition::SL_WARNING +#else +# define MRN_SEVERITY_WARNING Sql_condition::WARN_LEVEL_WARN +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_HAVE_PSI_MEMORY_KEY +#endif + +#ifdef MRN_HAVE_PSI_MEMORY_KEY +# define mrn_my_malloc(size, flags) \ + my_malloc(mrn_memory_key, size, flags) +# define mrn_my_strdup(string, flags) \ + my_strdup(mrn_memory_key, string, flags) +# define mrn_my_strndup(string, size, flags) \ + my_strndup(mrn_memory_key, string, size, flags) +# define mrn_my_multi_malloc(flags, ...) \ + my_multi_malloc(mrn_memory_key, flags, __VA_ARGS__) +#else +# define mrn_my_malloc(size, flags) my_malloc(size, flags) +# define mrn_my_strdup(string, flags) my_strdup(string, flags) +# define mrn_my_strndup(string, size, flags) \ + my_strndup(string, size, flags) +# define mrn_my_multi_malloc(flags, ...) \ + my_multi_malloc(flags, __VA_ARGS__) +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_STRING_FREE(string) string.mem_free(); +#else +# define MRN_STRING_FREE(string) string.free(); +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_THD_DB_PATH(thd) ((thd)->db().str) +#else +# define MRN_THD_DB_PATH(thd) ((thd)->db) +#endif + +#ifndef INT_MAX64 +# define INT_MAX64 LONGLONG_MAX +#endif + +#ifdef UINT_MAX +# define UINT_MAX64 UINT_MAX +#else +# define UINT_MAX64 LONGLONG_MAX +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define mrn_my_stpmov(dst, src) my_stpmov(dst, src) +#else +# define mrn_my_stpmov(dst, src) strmov(dst, src) +#endif + +#if MYSQL_VERSION_ID >= 50607 +# if !defined(MRN_MARIADB_P) +# define MRN_HAVE_SQL_OPTIMIZER_H +# endif +#endif + +#if MYSQL_VERSION_ID >= 50600 && !defined(MRN_MARIADB_P) +# define MRN_HAVE_BINLOG_H +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_HAVE_SPATIAL +#elif defined(HAVE_SPATIAL) +# define MRN_HAVE_SPATIAL +#endif + +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_FORMAT_STRING_LENGTH "zu" +#else +# define MRN_FORMAT_STRING_LENGTH "u" +#endif + #endif /* MRN_MYSQL_COMPAT_H_ */ diff --git a/storage/mroonga/mrn_table.cpp b/storage/mroonga/mrn_table.cpp index aa82c1845c1..ebd3e7e0528 100644 --- a/storage/mroonga/mrn_table.cpp +++ b/storage/mroonga/mrn_table.cpp @@ -1,7 +1,7 @@ /* -*- c-basic-offset: 2 -*- */ /* Copyright(C) 2011-2013 Kentoku SHIBA - Copyright(C) 2011-2014 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2011-2015 Kouhei Sutou <kou@clear-code.com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -24,9 +24,15 @@ # include <sql_servers.h> # include <sql_base.h> #endif +#ifdef WITH_PARTITION_STORAGE_ENGINE +# include <partition_info.h> +#endif +#include <sql_plugin.h> + #include "mrn_err.h" #include "mrn_table.hpp" #include "mrn_mysql_compat.h" +#include "mrn_variables.hpp" #include <mrn_lock.hpp> #if MYSQL_VERSION_ID >= 50603 && !defined(MRN_MARIADB_P) @@ -35,6 +41,12 @@ # define MRN_HA_RESOLVE_BY_NAME(name) ha_resolve_by_name(NULL, (name)) #endif +#if MYSQL_VERSION_ID >= 50706 && !defined(MRN_MARIADB_P) +# define MRN_PLUGIN_DATA(plugin, type) plugin_data<type>(plugin) +#else +# define MRN_PLUGIN_DATA(plugin, type) plugin_data(plugin, type) +#endif + #define LEX_STRING_IS_EMPTY(string) \ ((string).length == 0 || !(string).str || (string).str[0] == '\0') @@ -47,23 +59,23 @@ extern HASH *mrn_table_def_cache; #endif +#ifdef __cplusplus +extern "C" { +#endif + #ifdef HAVE_PSI_INTERFACE -#ifdef WIN32 -# ifdef MRN_TABLE_SHARE_HAVE_LOCK_SHARE +# ifdef WIN32 +# ifdef MRN_TABLE_SHARE_HAVE_LOCK_SHARE extern PSI_mutex_key *mrn_table_share_lock_share; -# endif -# ifdef MRN_TABLE_SHARE_HAVE_LOCK_HA_DATA +# endif +# ifdef MRN_TABLE_SHARE_HAVE_LOCK_HA_DATA extern PSI_mutex_key *mrn_table_share_lock_ha_data; +# endif # endif -#endif extern PSI_mutex_key mrn_share_mutex_key; extern PSI_mutex_key mrn_long_term_share_auto_inc_mutex_key; #endif -#ifdef __cplusplus -extern "C" { -#endif - extern HASH mrn_open_tables; extern mysql_mutex_t mrn_open_tables_mutex; extern HASH mrn_long_term_share; @@ -135,7 +147,7 @@ static char *mrn_get_string_between_quote(const char *ptr) DBUG_RETURN(NULL); size_t length = end_ptr - start_ptr; - char *extracted_string = (char *)my_malloc(length + 1, MYF(MY_WME)); + char *extracted_string = (char *)mrn_my_malloc(length + 1, MYF(MY_WME)); if (esc_flg) { size_t extracted_index = 0; const char *current_ptr = start_ptr; @@ -372,9 +384,9 @@ int mrn_parse_table_param(MRN_SHARE *share, TABLE *table) } { - params_string = my_strndup(params_string_value, - params_string_length, - MYF(MY_WME)); + params_string = mrn_my_strndup(params_string_value, + params_string_length, + MYF(MY_WME)); if (!params_string) { error = HA_ERR_OUT_OF_MEM; goto error; @@ -437,9 +449,9 @@ int mrn_parse_table_param(MRN_SHARE *share, TABLE *table) { share->engine_length = strlen(mrn_default_wrapper_engine); if ( - !(share->engine = my_strndup(mrn_default_wrapper_engine, - share->engine_length, - MYF(MY_WME))) + !(share->engine = mrn_my_strndup(mrn_default_wrapper_engine, + share->engine_length, + MYF(MY_WME))) ) { error = HA_ERR_OUT_OF_MEM; goto error; @@ -471,7 +483,7 @@ int mrn_parse_table_param(MRN_SHARE *share, TABLE *table) error = ER_UNKNOWN_STORAGE_ENGINE; goto error; } - share->hton = plugin_data(share->plugin, handlerton *); + share->hton = MRN_PLUGIN_DATA(share->plugin, handlerton *); share->wrapper_mode = TRUE; } } @@ -507,7 +519,7 @@ int mrn_add_index_param(MRN_SHARE *share, KEY *key_info, int i) my_free(share->key_parser[i]); } if ( - !(share->key_parser[i] = my_strdup(mrn_default_parser, MYF(MY_WME))) + !(share->key_parser[i] = mrn_my_strdup(mrn_default_parser, MYF(MY_WME))) ) { error = HA_ERR_OUT_OF_MEM; goto error; @@ -517,9 +529,9 @@ int mrn_add_index_param(MRN_SHARE *share, KEY *key_info, int i) } DBUG_PRINT("info", ("mroonga create comment string")); if ( - !(param_string = my_strndup(key_info->comment.str, - key_info->comment.length, - MYF(MY_WME))) + !(param_string = mrn_my_strndup(key_info->comment.str, + key_info->comment.length, + MYF(MY_WME))) ) { error = HA_ERR_OUT_OF_MEM; goto error_alloc_param_string; @@ -569,7 +581,7 @@ int mrn_add_index_param(MRN_SHARE *share, KEY *key_info, int i) #endif if (!share->key_parser[i]) { if ( - !(share->key_parser[i] = my_strdup(mrn_default_parser, MYF(MY_WME))) + !(share->key_parser[i] = mrn_my_strdup(mrn_default_parser, MYF(MY_WME))) ) { error = HA_ERR_OUT_OF_MEM; goto error; @@ -630,9 +642,9 @@ int mrn_add_column_param(MRN_SHARE *share, Field *field, int i) DBUG_PRINT("info", ("mroonga create comment string")); if ( - !(param_string = my_strndup(field->comment.str, - field->comment.length, - MYF(MY_WME))) + !(param_string = mrn_my_strndup(field->comment.str, + field->comment.length, + MYF(MY_WME))) ) { error = HA_ERR_OUT_OF_MEM; goto error_alloc_param_string; @@ -768,7 +780,7 @@ MRN_LONG_TERM_SHARE *mrn_get_long_term_share(const char *table_name, table_name_length))) { if (!(long_term_share = (MRN_LONG_TERM_SHARE *) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), + mrn_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &long_term_share, sizeof(*long_term_share), &tmp_name, table_name_length + 1, NullS)) @@ -817,7 +829,7 @@ MRN_SHARE *mrn_get_share(const char *table_name, TABLE *table, int *error) (uchar*) table_name, length))) { if (!(share = (MRN_SHARE *) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), + mrn_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &share, sizeof(*share), &tmp_name, length + 1, &index_table, sizeof(char *) * table->s->keys, @@ -847,7 +859,7 @@ MRN_SHARE *mrn_get_share(const char *table_name, TABLE *table, int *error) share->col_flags_length = col_flags_length; share->col_type = col_type; share->col_type_length = col_type_length; - strmov(share->table_name, table_name); + mrn_my_stpmov(share->table_name, table_name); share->table_share = table->s; if ( @@ -1033,7 +1045,7 @@ TABLE_SHARE *mrn_create_tmp_table_share(TABLE_LIST *table_list, const char *path share->tmp_table = INTERNAL_TMP_TABLE; // TODO: is this right? share->path.str = (char *) path; share->path.length = strlen(share->path.str); - share->normalized_path.str = my_strdup(path, MYF(MY_WME)); + share->normalized_path.str = mrn_my_strdup(path, MYF(MY_WME)); share->normalized_path.length = strlen(share->normalized_path.str); if (open_table_def(thd, share, GTS_TABLE)) { @@ -1060,7 +1072,7 @@ KEY *mrn_create_key_info_for_table(MRN_SHARE *share, TABLE *table, int *error) if (share->wrap_keys) { if (!(wrap_key_info = (KEY *) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), + mrn_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &wrap_key_info, sizeof(*wrap_key_info) * share->wrap_keys, NullS)) ) { diff --git a/storage/mroonga/mrn_variables.hpp b/storage/mroonga/mrn_variables.hpp new file mode 100644 index 00000000000..6bc948a1cac --- /dev/null +++ b/storage/mroonga/mrn_variables.hpp @@ -0,0 +1,29 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef MRN_VARIABLES_HPP_ +#define MRN_VARIABLES_HPP_ + +#include "mrn_mysql_compat.h" + +#ifdef MRN_HAVE_PSI_MEMORY_KEY +extern PSI_memory_key mrn_memory_key; +#endif + +#endif /* MRN_VARIABLES_HPP_ */ diff --git a/storage/mroonga/mysql-test/mroonga/include/mroonga/check_version.inc b/storage/mroonga/mysql-test/mroonga/include/mroonga/check_version.inc index b59a981d822..b78e79558d7 100644 --- a/storage/mroonga/mysql-test/mroonga/include/mroonga/check_version.inc +++ b/storage/mroonga/mysql-test/mroonga/include/mroonga/check_version.inc @@ -1,4 +1,4 @@ -# Copyright(C) 2012-2013 Kouhei Sutou <kou@clear-code.com> +# Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -20,9 +20,11 @@ let $version_major_minor = let $version_55 = `SELECT $version_major_minor = 5.5`; let $version_56 = `SELECT $version_major_minor = 5.6`; +let $version_57 = `SELECT $version_major_minor = 5.7`; let $version_100 = `SELECT $version_major_minor = 10.0`; let $version_55_or_later = `SELECT $version_major_minor >= 5.5`; let $version_56_or_later = `SELECT $version_major_minor >= 5.6`; +let $version_57_or_later = `SELECT $version_major_minor >= 5.7`; let $version_100_or_later = `SELECT $version_major_minor >= 10.0`; --enable_query_log diff --git a/storage/mroonga/mysql-test/mroonga/include/mroonga/have_version_57.inc b/storage/mroonga/mysql-test/mroonga/include/mroonga/have_version_57.inc new file mode 100644 index 00000000000..1ab664ed6a8 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/include/mroonga/have_version_57.inc @@ -0,0 +1,21 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/check_version.inc + +if (!$version_57) { + skip This test is for MySQL version 5.7.x; +} diff --git a/storage/mroonga/mysql-test/mroonga/include/mroonga/skip_mariadb_100_or_later.inc b/storage/mroonga/mysql-test/mroonga/include/mroonga/skip_mariadb_100_or_later.inc new file mode 100644 index 00000000000..8c9f76cd45f --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/include/mroonga/skip_mariadb_100_or_later.inc @@ -0,0 +1,24 @@ +# Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/check_version.inc +--source ../../include/mroonga/check_mariadb.inc + +if ($version_100_or_later) { + if ($mariadb) { + skip This test is not for MariaDB 10.x; + } +} diff --git a/storage/mroonga/mysql-test/mroonga/include/mroonga/skip_mysql_57.inc b/storage/mroonga/mysql-test/mroonga/include/mroonga/skip_mysql_57.inc new file mode 100644 index 00000000000..f1b1651a0b1 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/include/mroonga/skip_mysql_57.inc @@ -0,0 +1,24 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/check_version.inc +--source ../../include/mroonga/check_mariadb.inc + +if ($version_57) { + if (!$mariadb) { + skip This test is not for MySQL 5.7.x; + } +} diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_column_with_flags.result b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_column_with_flags.result index 1bc15c6efe2..e441df32c92 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_column_with_flags.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_column_with_flags.result @@ -2,8 +2,8 @@ CREATE TABLE tags ( id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY ) DEFAULT CHARSET=utf8; ALTER TABLE tags ADD COLUMN name VARCHAR(64) COMMENT 'flags "COLUMN_VECTOR"'; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create tags TABLE_PAT_KEY UInt32 column_create tags id COLUMN_SCALAR UInt32 column_create tags name COLUMN_VECTOR ShortText diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_column_with_type.result b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_column_with_type.result index 6ff92ec8e99..b4c3044c7d5 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_column_with_type.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_column_with_type.result @@ -5,12 +5,14 @@ CREATE TABLE bugs ( id INT UNSIGNED PRIMARY KEY ) DEFAULT CHARSET=utf8; ALTER TABLE bugs ADD COLUMN name VARCHAR(64) COMMENT 'type "tags"'; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create tags TABLE_PAT_KEY UInt32 column_create tags id COLUMN_SCALAR UInt32 + table_create bugs TABLE_PAT_KEY UInt32 column_create bugs id COLUMN_SCALAR UInt32 + column_create bugs name COLUMN_SCALAR tags DROP TABLE bugs; DROP TABLE tags; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_index_token_filters_one_token_filter.result b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_index_token_filters_one_token_filter.result index e0449e3e54b..21784decb88 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_index_token_filters_one_token_filter.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_index_token_filters_one_token_filter.result @@ -5,15 +5,13 @@ SET NAMES utf8; CREATE TABLE memos ( content VARCHAR(64) NOT NULL ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") -table_create memos TABLE_NO_KEY -column_create memos content COLUMN_SCALAR ShortText ALTER TABLE memos ADD FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord"'; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create memos TABLE_NO_KEY column_create memos content COLUMN_SCALAR ShortText + table_create memos-content TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI --token_filters TokenFilterStopWord + column_create memos-content index COLUMN_INDEX|WITH_POSITION memos content DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_primary_key.result b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_primary_key.result new file mode 100644 index 00000000000..439684ac5d6 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_add_primary_key.result @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS ids; +CREATE TABLE ids ( +id INT NOT NULL +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES (1); +INSERT INTO ids VALUES (2); +ALTER TABLE ids ADD PRIMARY KEY (id); +SHOW CREATE TABLE ids; +Table Create Table +ids CREATE TABLE `ids` ( + `id` int(11) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=Mroonga DEFAULT CHARSET=utf8 +SELECT * FROM ids WHERE id = 2; +id +2 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_change_token_filter.result b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_change_token_filter.result index 5046dc8e142..1c96236230b 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_change_token_filter.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_change_token_filter.result @@ -10,23 +10,39 @@ id INT NOT NULL PRIMARY KEY, content TEXT NOT NULL, FULLTEXT INDEX (content) COMMENT 'table "terms"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI column_create terms is_stop_word COLUMN_SCALAR Int8 column_create terms term COLUMN_SCALAR ShortText + table_create memos TABLE_PAT_KEY Int32 column_create memos content COLUMN_SCALAR LongText column_create memos id COLUMN_SCALAR Int32 + column_create terms content COLUMN_INDEX|WITH_POSITION memos content ALTER TABLE terms COMMENT='default_tokenizer "TokenBigram", token_filters "TokenFilterStopWord"'; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create memos TABLE_PAT_KEY Int32 column_create memos content COLUMN_SCALAR LongText column_create memos id COLUMN_SCALAR Int32 + table_create terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI --token_filters TokenFilterStopWord column_create terms is_stop_word COLUMN_SCALAR Int8 column_create terms term COLUMN_SCALAR ShortText +ALTER TABLE memos DISABLE KEYS; +ALTER TABLE memos ENABLE KEYS; +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") +table_create memos TABLE_PAT_KEY Int32 +column_create memos content COLUMN_SCALAR LongText +column_create memos id COLUMN_SCALAR Int32 + +table_create terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI --token_filters TokenFilterStopWord +column_create terms is_stop_word COLUMN_SCALAR Int8 +column_create terms term COLUMN_SCALAR ShortText + +column_create terms content COLUMN_INDEX|WITH_POSITION memos content DROP TABLE memos; DROP TABLE terms; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_disable_keys_fulltext_table.result b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_disable_keys_fulltext_table.result index c7051127946..ba6cf6c24ed 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_disable_keys_fulltext_table.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_disable_keys_fulltext_table.result @@ -8,20 +8,24 @@ id int PRIMARY KEY, content text NOT NULL, FULLTEXT INDEX content_index (content) COMMENT 'table "terms"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerAuto column_create terms term COLUMN_SCALAR ShortText + table_create memos TABLE_PAT_KEY Int32 column_create memos content COLUMN_SCALAR LongText column_create memos id COLUMN_SCALAR Int32 + column_create terms content_index COLUMN_INDEX|WITH_POSITION memos content ALTER TABLE memos DISABLE KEYS; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerAuto column_create terms term COLUMN_SCALAR ShortText + table_create memos TABLE_PAT_KEY Int32 column_create memos content COLUMN_SCALAR LongText column_create memos id COLUMN_SCALAR Int32 -DROP TABLE memos, terms; +DROP TABLE memos; +DROP TABLE terms; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_drop_primary_key.result b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_drop_primary_key.result new file mode 100644 index 00000000000..cbcf9a69b56 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_drop_primary_key.result @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS ids; +CREATE TABLE ids ( +id INT PRIMARY KEY +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES (1); +INSERT INTO ids VALUES (2); +ALTER TABLE ids DROP PRIMARY KEY; +SHOW CREATE TABLE ids; +Table Create Table +ids CREATE TABLE `ids` ( + `id` int(11) NOT NULL +) ENGINE=Mroonga DEFAULT CHARSET=utf8 +SELECT * FROM ids WHERE id = 2; +id +2 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_enable_keys_fulltext_table.result b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_enable_keys_fulltext_table.result index 97d2be3f8d5..452caa574f9 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_enable_keys_fulltext_table.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_enable_keys_fulltext_table.result @@ -9,20 +9,24 @@ content text NOT NULL, FULLTEXT INDEX content_index (content) COMMENT 'table "terms"' ) DEFAULT CHARSET=utf8; ALTER TABLE memos DISABLE KEYS; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerAuto column_create terms term COLUMN_SCALAR ShortText + table_create memos TABLE_PAT_KEY Int32 column_create memos content COLUMN_SCALAR LongText column_create memos id COLUMN_SCALAR Int32 ALTER TABLE memos ENABLE KEYS; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerAuto column_create terms term COLUMN_SCALAR ShortText + table_create memos TABLE_PAT_KEY Int32 column_create memos content COLUMN_SCALAR LongText column_create memos id COLUMN_SCALAR Int32 + column_create terms content_index COLUMN_INDEX|WITH_POSITION memos content -DROP TABLE memos, terms; +DROP TABLE memos; +DROP TABLE terms; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_spatial.result b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_spatial.result index d8ed80a157d..ed36927e7b9 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_spatial.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/alter_table_spatial.result @@ -6,115 +6,115 @@ location GEOMETRY NOT NULL ); INSERT INTO shops (name, location) VALUES ('nezu-no-taiyaki', -GeomFromText('POINT(139.762573 35.720253)')); +ST_GeomFromText('POINT(139.762573 35.720253)')); INSERT INTO shops (name, location) VALUES ('taiyaki-kataoka', -GeomFromText('POINT(139.715591 35.712521)')); +ST_GeomFromText('POINT(139.715591 35.712521)')); INSERT INTO shops (name, location) VALUES ('soba-taiyaki-ku', -GeomFromText('POINT(139.659088 35.683712)')); +ST_GeomFromText('POINT(139.659088 35.683712)')); INSERT INTO shops (name, location) VALUES ('kuruma', -GeomFromText('POINT(139.706207 35.721516)')); +ST_GeomFromText('POINT(139.706207 35.721516)')); INSERT INTO shops (name, location) VALUES ('hirose-ya', -GeomFromText('POINT(139.685608 35.714844)')); +ST_GeomFromText('POINT(139.685608 35.714844)')); INSERT INTO shops (name, location) VALUES ('sazare', -GeomFromText('POINT(139.685043 35.714653)')); +ST_GeomFromText('POINT(139.685043 35.714653)')); INSERT INTO shops (name, location) VALUES ('omede-taiyaki', -GeomFromText('POINT(139.817154 35.700516)')); +ST_GeomFromText('POINT(139.817154 35.700516)')); INSERT INTO shops (name, location) VALUES ('onaga-ya', -GeomFromText('POINT(139.81105 35.698254)')); +ST_GeomFromText('POINT(139.81105 35.698254)')); INSERT INTO shops (name, location) VALUES ('shiro-ya', -GeomFromText('POINT(139.638611 35.705517)')); +ST_GeomFromText('POINT(139.638611 35.705517)')); INSERT INTO shops (name, location) VALUES ('fuji-ya', -GeomFromText('POINT(139.637115 35.703938)')); +ST_GeomFromText('POINT(139.637115 35.703938)')); INSERT INTO shops (name, location) VALUES ('miyoshi', -GeomFromText('POINT(139.537323 35.644539)')); +ST_GeomFromText('POINT(139.537323 35.644539)')); INSERT INTO shops (name, location) VALUES ('juju-ya', -GeomFromText('POINT(139.695755 35.628922)')); +ST_GeomFromText('POINT(139.695755 35.628922)')); INSERT INTO shops (name, location) VALUES ('tatsumi-ya', -GeomFromText('POINT(139.638657 35.665501)')); +ST_GeomFromText('POINT(139.638657 35.665501)')); INSERT INTO shops (name, location) VALUES ('tetsuji', -GeomFromText('POINT(139.76857 35.680912)')); +ST_GeomFromText('POINT(139.76857 35.680912)')); INSERT INTO shops (name, location) VALUES ('gazuma-ya', -GeomFromText('POINT(139.647598 35.700817)')); +ST_GeomFromText('POINT(139.647598 35.700817)')); INSERT INTO shops (name, location) VALUES ('honma-mon', -GeomFromText('POINT(139.652573 35.722736)')); +ST_GeomFromText('POINT(139.652573 35.722736)')); INSERT INTO shops (name, location) VALUES ('naniwa-ya', -GeomFromText('POINT(139.796234 35.730061)')); +ST_GeomFromText('POINT(139.796234 35.730061)')); INSERT INTO shops (name, location) VALUES ('kuro-dai', -GeomFromText('POINT(139.704834 35.650345)')); +ST_GeomFromText('POINT(139.704834 35.650345)')); INSERT INTO shops (name, location) VALUES ('daruma', -GeomFromText('POINT(139.770599 35.681461)')); +ST_GeomFromText('POINT(139.770599 35.681461)')); INSERT INTO shops (name, location) VALUES ('yanagi-ya', -GeomFromText('POINT(139.783981 35.685341)')); +ST_GeomFromText('POINT(139.783981 35.685341)')); INSERT INTO shops (name, location) VALUES ('sharaku', -GeomFromText('POINT(139.794846 35.716969)')); +ST_GeomFromText('POINT(139.794846 35.716969)')); INSERT INTO shops (name, location) VALUES ('takane', -GeomFromText('POINT(139.560913 35.698601)')); +ST_GeomFromText('POINT(139.560913 35.698601)')); INSERT INTO shops (name, location) VALUES ('chiyoda', -GeomFromText('POINT(139.652817 35.642601)')); +ST_GeomFromText('POINT(139.652817 35.642601)')); INSERT INTO shops (name, location) VALUES ('da-ka-po', -GeomFromText('POINT(139.727356 35.627346)')); +ST_GeomFromText('POINT(139.727356 35.627346)')); INSERT INTO shops (name, location) VALUES ('matsushima-ya', -GeomFromText('POINT(139.737381 35.640556)')); +ST_GeomFromText('POINT(139.737381 35.640556)')); INSERT INTO shops (name, location) VALUES ('kazuya', -GeomFromText('POINT(139.760895 35.673508)')); +ST_GeomFromText('POINT(139.760895 35.673508)')); INSERT INTO shops (name, location) VALUES ('furuya-kogane-an', -GeomFromText('POINT(139.676071 35.680603)')); +ST_GeomFromText('POINT(139.676071 35.680603)')); INSERT INTO shops (name, location) VALUES ('hachi-no-ie', -GeomFromText('POINT(139.668106 35.608021)')); +ST_GeomFromText('POINT(139.668106 35.608021)')); INSERT INTO shops (name, location) VALUES ('azuki-chan', -GeomFromText('POINT(139.673203 35.64151)')); +ST_GeomFromText('POINT(139.673203 35.64151)')); INSERT INTO shops (name, location) VALUES ('kuriko-an', -GeomFromText('POINT(139.796829 35.712013)')); +ST_GeomFromText('POINT(139.796829 35.712013)')); INSERT INTO shops (name, location) VALUES ('yume-no-aru-machi-no-taiyaki-ya-san', -GeomFromText('POINT(139.712524 35.616199)')); +ST_GeomFromText('POINT(139.712524 35.616199)')); INSERT INTO shops (name, location) VALUES ('naze-ya', -GeomFromText('POINT(139.665833 35.609039)')); +ST_GeomFromText('POINT(139.665833 35.609039)')); INSERT INTO shops (name, location) VALUES ('sanoki-ya', -GeomFromText('POINT(139.770721 35.66592)')); +ST_GeomFromText('POINT(139.770721 35.66592)')); INSERT INTO shops (name, location) VALUES ('shigeta', -GeomFromText('POINT(139.780273 35.672626)')); +ST_GeomFromText('POINT(139.780273 35.672626)')); INSERT INTO shops (name, location) VALUES ('nishimi-ya', -GeomFromText('POINT(139.774628 35.671825)')); +ST_GeomFromText('POINT(139.774628 35.671825)')); INSERT INTO shops (name, location) VALUES ('hiiragi', -GeomFromText('POINT(139.711517 35.647701)')); +ST_GeomFromText('POINT(139.711517 35.647701)')); ALTER TABLE shops ADD SPATIAL KEY location_index (location); -SELECT id, name, AsText(location) AS location_text FROM shops -WHERE MBRContains(GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location) +SELECT id, name, ST_AsText(location) AS location_text FROM shops +WHERE MBRContains(ST_GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location) ORDER BY id; id name location_text 14 tetsuji POINT(139.76857 35.680911944444446) diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_520_ci_french.result b/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_520_ci_french.result new file mode 100644 index 00000000000..b9e22da4e2d --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_520_ci_french.result @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS diaries; +SET NAMES utf8mb4; +CREATE TABLE diaries ( +content varchar(256) COLLATE utf8mb4_unicode_520_ci, +FULLTEXT INDEX (content) +) DEFAULT CHARSET=utf8mb4; +INSERT INTO diaries VALUES ("Je suis un garçon."); +SELECT * FROM diaries WHERE MATCH (content) AGAINST ("+garcon" IN BOOLEAN MODE); +content +Je suis un garçon. +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_520_ci_japanese.result b/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_520_ci_japanese.result new file mode 100644 index 00000000000..29e55b7d9d5 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_520_ci_japanese.result @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS diaries; +SET NAMES utf8mb4; +CREATE TABLE diaries ( +content varchar(256) COLLATE utf8mb4_unicode_520_ci, +FULLTEXT INDEX (content) +) DEFAULT CHARSET=utf8mb4; +INSERT INTO diaries VALUES ("ひらがなとカタカナを覚えました。"); +SELECT * FROM diaries +WHERE MATCH (content) AGAINST ("+かたかな" IN BOOLEAN MODE); +content +ひらがなとカタカナを覚えました。 +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/column_groonga_index_fulltext_vector_other_table.result b/storage/mroonga/mysql-test/mroonga/storage/r/column_groonga_index_fulltext_vector_other_table.result index 6ac2e1937fe..724d20edd62 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/column_groonga_index_fulltext_vector_other_table.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/column_groonga_index_fulltext_vector_other_table.result @@ -12,26 +12,17 @@ tags TEXT COMMENT 'flags "COLUMN_VECTOR", type "tags"', FULLTEXT INDEX bugs_tags_index (tags) COMMENT 'table "tags"' ) DEFAULT CHARSET=utf8; INSERT INTO bugs (id, tags) VALUES (1, "Linux MySQL groonga"); -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no --dump_records no"); +mroonga_command("dump --dump_plugins no --dump_records no") table_create tags TABLE_PAT_KEY ShortText --default_tokenizer TokenDelimit column_create tags name COLUMN_SCALAR ShortText + table_create bugs TABLE_PAT_KEY UInt32 column_create bugs id COLUMN_SCALAR UInt32 + column_create bugs tags COLUMN_VECTOR tags + column_create tags bugs_tags_index COLUMN_INDEX|WITH_POSITION bugs tags -load --table tags -[ -["_key","name"], -["Linux",""], -["MySQL",""], -["groonga",""] -] -load --table bugs -[ -["_key","id","tags"], -[1,1,["Linux","MySQL","groonga"]] -] SELECT *, MATCH (tags) AGAINST ("+MySQL" IN BOOLEAN MODE) AS score FROM bugs WHERE MATCH (tags) AGAINST ("+MySQL" IN BOOLEAN MODE); diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/column_groonga_index_int_other_table.result b/storage/mroonga/mysql-test/mroonga/storage/r/column_groonga_index_int_other_table.result index e46ed71d0fe..aa7735ef780 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/column_groonga_index_int_other_table.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/column_groonga_index_int_other_table.result @@ -13,28 +13,17 @@ INDEX bugs_priority_index (priority) COMMENT 'table "priorities"' INSERT INTO bugs (id, priority) VALUES (1, 10); INSERT INTO bugs (id, priority) VALUES (2, 3); INSERT INTO bugs (id, priority) VALUES (3, -2); -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no --dump_records no"); +mroonga_command("dump --dump_plugins no --dump_records no") table_create priorities TABLE_PAT_KEY Int32 column_create priorities id COLUMN_SCALAR Int32 + table_create bugs TABLE_PAT_KEY UInt32 column_create bugs id COLUMN_SCALAR UInt32 + column_create bugs priority COLUMN_SCALAR priorities + column_create priorities bugs_priority_index COLUMN_INDEX|WITH_POSITION bugs priority -load --table priorities -[ -["_key","id"], -[-2,0], -[3,0], -[10,0] -] -load --table bugs -[ -["_key","id","priority"], -[1,1,10], -[2,2,3], -[3,3,-2] -] SELECT * FROM bugs WHERE priority = 3; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_default_tokenizer.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_default_tokenizer.result index e14c67291cd..9302037b7e1 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_default_tokenizer.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_default_tokenizer.result @@ -3,8 +3,8 @@ name VARCHAR(64) PRIMARY KEY ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='default_tokenizer "TokenDelimit"'; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create tags TABLE_PAT_KEY ShortText --default_tokenizer TokenDelimit column_create tags name COLUMN_SCALAR ShortText DROP TABLE tags; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_index_flags_none.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_index_flags_none.result new file mode 100644 index 00000000000..c9283db72bb --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_index_flags_none.result @@ -0,0 +1,9 @@ +SET NAMES utf8; +CREATE TABLE memos ( +content VARCHAR(64) NOT NULL, +FULLTEXT INDEX (content) COMMENT 'index_flags "NONE"' +) DEFAULT CHARSET=utf8; +SELECT mroonga_command("dump --dump_plugins no --dump_schema no"); +mroonga_command("dump --dump_plugins no --dump_schema no") +column_create memos-content index COLUMN_INDEX memos content +DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_index_flags_with_position_and_with_weight.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_index_flags_with_position_and_with_weight.result new file mode 100644 index 00000000000..853845d5c15 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_index_flags_with_position_and_with_weight.result @@ -0,0 +1,9 @@ +SET NAMES utf8; +CREATE TABLE memos ( +content VARCHAR(64) NOT NULL, +FULLTEXT INDEX (content) COMMENT 'index_flags "WITH_POSITION|WITH_WEIGHT"' +) DEFAULT CHARSET=utf8; +SELECT mroonga_command("dump --dump_plugins no --dump_schema no"); +mroonga_command("dump --dump_plugins no --dump_schema no") +column_create memos-content index COLUMN_INDEX|WITH_WEIGHT|WITH_POSITION memos content +DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_normalizer_primary_key_table_comment_with_using_hash.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_normalizer_primary_key_table_comment_with_using_hash.result new file mode 100644 index 00000000000..5e4f4afa377 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_normalizer_primary_key_table_comment_with_using_hash.result @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS memos; +DROP TABLE IF EXISTS terms; +SET NAMES utf8; +CREATE TABLE terms ( +term VARCHAR(64) NOT NULL, +PRIMARY KEY (term) USING HASH +) COMMENT='default_tokenizer "TokenBigram", normalizer "NormalizerAuto"' DEFAULT CHARSET=utf8; +CREATE TABLE memos ( +id INT NOT NULL PRIMARY KEY, +content TEXT NOT NULL, +FULLTEXT INDEX (content) COMMENT 'table "terms"' +) DEFAULT CHARSET=utf8; +INSERT INTO memos VALUES (1, "1日の消費㌍は約2000㌔㌍"); +SELECT * FROM memos +WHERE MATCH (content) AGAINST ("+カロリー" IN BOOLEAN MODE); +id content +1 1日の消費㌍は約2000㌔㌍ +DROP TABLE memos; +DROP TABLE terms; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_reference_type.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_reference_type.result index 68a6bc8012f..dc3f39d286e 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_reference_type.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_reference_type.result @@ -5,12 +5,14 @@ CREATE TABLE bugs ( id INT UNSIGNED PRIMARY KEY, tag VARCHAR(64) COMMENT 'type "tags"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create tags TABLE_PAT_KEY ShortText column_create tags name COLUMN_SCALAR ShortText + table_create bugs TABLE_PAT_KEY UInt32 column_create bugs id COLUMN_SCALAR UInt32 + column_create bugs tag COLUMN_SCALAR tags DROP TABLE bugs; DROP TABLE tags; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_index_comment_multiple_token_filters.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_index_comment_multiple_token_filters.result index 7d762258e6f..785c5d785b9 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_index_comment_multiple_token_filters.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_index_comment_multiple_token_filters.result @@ -6,10 +6,12 @@ CREATE TABLE memos ( content VARCHAR(64) NOT NULL, FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord,TokenFilterStopWord"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create memos TABLE_NO_KEY column_create memos content COLUMN_SCALAR ShortText + table_create memos-content TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI --token_filters TokenFilterStopWord,TokenFilterStopWord + column_create memos-content index COLUMN_INDEX|WITH_POSITION memos content DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_index_comment_one_token_filter.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_index_comment_one_token_filter.result index 6cc89a7b941..9ce60054300 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_index_comment_one_token_filter.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_index_comment_one_token_filter.result @@ -6,10 +6,12 @@ CREATE TABLE memos ( content VARCHAR(64) NOT NULL, FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create memos TABLE_NO_KEY column_create memos content COLUMN_SCALAR ShortText + table_create memos-content TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI --token_filters TokenFilterStopWord + column_create memos-content index COLUMN_INDEX|WITH_POSITION memos content DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_table_comment_multiple_token_filters.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_table_comment_multiple_token_filters.result index fe7e0059250..de1946e9214 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_table_comment_multiple_token_filters.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_table_comment_multiple_token_filters.result @@ -10,14 +10,16 @@ id INT NOT NULL PRIMARY KEY, content TEXT NOT NULL, FULLTEXT INDEX (content) COMMENT 'table "terms"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI --token_filters TokenFilterStopWord,TokenFilterStopWord column_create terms is_stop_word COLUMN_SCALAR Int8 column_create terms term COLUMN_SCALAR ShortText + table_create memos TABLE_PAT_KEY Int32 column_create memos content COLUMN_SCALAR LongText column_create memos id COLUMN_SCALAR Int32 + column_create terms content COLUMN_INDEX|WITH_POSITION memos content DROP TABLE memos; DROP TABLE terms; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_table_comment_one_token_filter.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_table_comment_one_token_filter.result index fe855b60f38..f3349980db1 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_table_comment_one_token_filter.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_token_filters_table_comment_one_token_filter.result @@ -10,14 +10,16 @@ id INT NOT NULL PRIMARY KEY, content TEXT NOT NULL, FULLTEXT INDEX (content) COMMENT 'table "terms"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI --token_filters TokenFilterStopWord column_create terms is_stop_word COLUMN_SCALAR Int8 column_create terms term COLUMN_SCALAR ShortText + table_create memos TABLE_PAT_KEY Int32 column_create memos content COLUMN_SCALAR LongText column_create memos id COLUMN_SCALAR Int32 + column_create terms content COLUMN_INDEX|WITH_POSITION memos content DROP TABLE memos; DROP TABLE terms; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_vector.result b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_vector.result index 561a266598f..af3c19e9bb0 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/create_table_vector.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/create_table_vector.result @@ -2,8 +2,8 @@ CREATE TABLE bugs ( id INT UNSIGNED PRIMARY KEY, tags TEXT COMMENT 'flags "COLUMN_VECTOR"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create bugs TABLE_PAT_KEY UInt32 column_create bugs id COLUMN_SCALAR UInt32 column_create bugs tags COLUMN_VECTOR LongText diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_boolean_mode_pragma_all.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_boolean_mode_pragma_all.result new file mode 100644 index 00000000000..c8270bc3dde --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_boolean_mode_pragma_all.result @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS memos; +SET NAMES utf8; +CREATE TABLE memos ( +id INT PRIMARY KEY, +title VARCHAR(255), +content TEXT, +FULLTEXT INDEX (title, content) +) DEFAULT CHARSET=utf8; +INSERT INTO memos VALUES(1, "Groonga", "Groonga is fast."); +INSERT INTO memos VALUES(2, "Mroonga", "Mroonga is also fast."); +INSERT INTO memos VALUES(3, "Rroonga", "Rroonga is also fast."); +SELECT *, +MATCH(title, content) +AGAINST("*W1:10,2:2DOR Groonga Mroonga" in BOOLEAN MODE) AS score +FROM memos +WHERE MATCH(title, content) +AGAINST("*W1:10,2:2DOR Groonga Mroonga" in BOOLEAN MODE); +id title content score +1 Groonga Groonga is fast. 12 +2 Mroonga Mroonga is also fast. 12 +DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_insert_select.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_insert_select.result index 6c268ff7641..c337bfcb8db 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_insert_select.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_insert_select.result @@ -43,24 +43,4 @@ select * from t2 where c2>"s" order by c1 asc; c1 c2 4 sa si su se so 5 ta ti ii ii to -select * from t1 where match(c2) against("ii") order by match(c2) against("ii") desc; -c1 c2 -3 aa ii ii ii oo -5 ta ti ii ii to -1 aa ii uu ee oo -select * from t2 where match(c2) against("ii") order by match(c2) against("ii") asc; -c1 c2 -1 aa ii uu ee oo -5 ta ti ii ii to -3 aa ii ii ii oo -select c1,c2,match(c2) against("ii") from t1 where match(c2) against("ii"); -c1 c2 match(c2) against("ii") -1 aa ii uu ee oo 174763 -3 aa ii ii ii oo 524289 -5 ta ti ii ii to 349526 -select c1,c2,match(c2) against("ii") from t2 where match(c2) against("ii"); -c1 c2 match(c2) against("ii") -1 aa ii uu ee oo 174763 -3 aa ii ii ii oo 524289 -5 ta ti ii ii to 349526 drop table t1,t2; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_different_against.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_different_against.result new file mode 100644 index 00000000000..d1f0d6bc0ab --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_different_against.result @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS diaries; +SET NAMES UTF8; +CREATE TABLE diaries( +title TEXT, +FULLTEXT KEY (title) +) DEFAULT CHARSET=utf8; +INSERT INTO diaries VALUES("Start groonga"); +INSERT INTO diaries VALUES("Start mroonga"); +INSERT INTO diaries VALUES("Start groonga and Ruby"); +SELECT * FROM diaries; +title +Start groonga +Start mroonga +Start groonga and Ruby +SELECT *, MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) AS score +FROM diaries +WHERE MATCH(title) AGAINST("groonga mroonga" IN BOOLEAN MODE) +ORDER BY MATCH(title) AGAINST("groonga" IN BOOLEAN MODE); +title score +Start mroonga 0 +Start groonga 1 +Start groonga and Ruby 1 +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_different_match.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_different_match.result new file mode 100644 index 00000000000..b4a07cd0b66 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_different_match.result @@ -0,0 +1,24 @@ +DROP TABLE IF EXISTS diaries; +SET NAMES UTF8; +CREATE TABLE diaries( +title TEXT, +body TEXT, +FULLTEXT KEY (title), +FULLTEXT KEY (body) +) DEFAULT CHARSET=utf8; +INSERT INTO diaries VALUES("Start groonga", "I read groonga's tutorial."); +INSERT INTO diaries VALUES("Start mroonga", "I read mroonga's tutorial."); +INSERT INTO diaries VALUES("Start groonga and Ruby", "I installed rroonga."); +SELECT * FROM diaries; +title body +Start groonga I read groonga's tutorial. +Start mroonga I read mroonga's tutorial. +Start groonga and Ruby I installed rroonga. +SELECT *, MATCH(body) AGAINST("groonga" IN BOOLEAN MODE) AS score +FROM diaries +WHERE MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) +ORDER BY MATCH(body) AGAINST("groonga" IN BOOLEAN MODE); +title body score +Start groonga and Ruby I installed rroonga. 0 +Start groonga I read groonga's tutorial. 1 +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_no_where.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_no_where.result new file mode 100644 index 00000000000..125b35fb96e --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_no_where.result @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS diaries; +SET NAMES UTF8; +CREATE TABLE diaries( +title TEXT, +FULLTEXT KEY (title) +) DEFAULT CHARSET=utf8; +INSERT INTO diaries VALUES("Start groonga"); +INSERT INTO diaries VALUES("Start mroonga"); +INSERT INTO diaries VALUES("Start groonga and Ruby"); +SELECT * FROM diaries; +title +Start groonga +Start mroonga +Start groonga and Ruby +SELECT *, MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) AS score +FROM diaries +ORDER BY MATCH(title) AGAINST("groonga" IN BOOLEAN MODE); +title score +Start mroonga 0 +Start groonga 1 +Start groonga and Ruby 1 +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_same_match_against.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_same_match_against.result new file mode 100644 index 00000000000..a3a668c4445 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_boolean_mode_same_match_against.result @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS diaries; +SET NAMES UTF8; +CREATE TABLE diaries( +title TEXT, +FULLTEXT KEY (title) +) DEFAULT CHARSET=utf8; +INSERT INTO diaries VALUES("Start groonga"); +INSERT INTO diaries VALUES("Start mroonga"); +INSERT INTO diaries VALUES("Start groonga and Ruby"); +SELECT * FROM diaries; +title +Start groonga +Start mroonga +Start groonga and Ruby +SELECT *, MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) AS score +FROM diaries +WHERE MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) +ORDER BY MATCH(title) AGAINST("groonga" IN BOOLEAN MODE); +title score +Start groonga 1 +Start groonga and Ruby 1 +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_asc.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_asc.result new file mode 100644 index 00000000000..7f0934d3094 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_asc.result @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS logs; +SET NAMES UTF8; +CREATE TABLE logs( +message TEXT, +FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); +SELECT * FROM logs; +message +Error Error Error +Warning Warning Warning +Error Error +Warning Warning +Error +Warning +Error Error Error Error +Warning Warning Warning Warning +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score +FROM logs +WHERE MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) +ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) ASC; +message score +Error 174763 +Error Error 349526 +Error Error Error 524289 +Error Error Error Error 699052 +DROP TABLE logs; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_desc.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_desc.result new file mode 100644 index 00000000000..fa577b17f83 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_desc.result @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS logs; +SET NAMES UTF8; +CREATE TABLE logs( +message TEXT, +FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); +SELECT * FROM logs; +message +Error Error Error +Warning Warning Warning +Error Error +Warning Warning +Error +Warning +Error Error Error Error +Warning Warning Warning Warning +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score +FROM logs +WHERE MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) +ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) DESC; +message score +Error Error Error Error 699052 +Error Error Error 524289 +Error Error 349526 +Error 174763 +DROP TABLE logs; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_different_against.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_different_against.result new file mode 100644 index 00000000000..958883e5841 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_different_against.result @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS logs; +SET NAMES UTF8; +CREATE TABLE logs( +message TEXT, +FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); +SELECT * FROM logs; +message +Error Error Error +Warning Warning Warning +Error Error +Warning Warning +Error +Warning +Error Error Error Error +Warning Warning Warning Warning +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score +FROM logs +WHERE MATCH(message) AGAINST("Error Warning" IN NATURAL LANGUAGE MODE) +ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE); +message score +Error 174763 +Error Error 349526 +Error Error Error 524289 +Error Error Error Error 699052 +DROP TABLE logs; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_different_match.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_different_match.result new file mode 100644 index 00000000000..76abb660c19 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_different_match.result @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS logs; +SET NAMES UTF8; +CREATE TABLE logs( +message TEXT, +host TEXT, +FULLTEXT KEY (message), +FULLTEXT KEY (host) +) DEFAULT CHARSET=utf8; +INSERT INTO logs VALUES("Error Error Error", "host1"); +INSERT INTO logs VALUES("Warning Warning Warning", "host1"); +INSERT INTO logs VALUES("Error Error", "host2"); +INSERT INTO logs VALUES("Warning Warning", "host2"); +INSERT INTO logs VALUES("Error", "host2"); +INSERT INTO logs VALUES("Warning", "host2"); +INSERT INTO logs VALUES("Error Error Error Error", "host2"); +INSERT INTO logs VALUES("Warning Warning Warning Warning", "host2"); +SELECT * FROM logs; +message host +Error Error Error host1 +Warning Warning Warning host1 +Error Error host2 +Warning Warning host2 +Error host2 +Warning host2 +Error Error Error Error host2 +Warning Warning Warning Warning host2 +SELECT *, MATCH(host) AGAINST("host2" IN NATURAL LANGUAGE MODE) AS score +FROM logs +WHERE MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) +ORDER BY MATCH(host) AGAINST("host2" IN NATURAL LANGUAGE MODE); +message host score +Error Error Error host1 0 +Error Error host2 116509 +Error host2 116509 +Error Error Error Error host2 116509 +DROP TABLE logs; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_no_where.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_no_where.result new file mode 100644 index 00000000000..30130e6c85b --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_no_where.result @@ -0,0 +1,37 @@ +DROP TABLE IF EXISTS logs; +SET NAMES UTF8; +CREATE TABLE logs( +message TEXT, +FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); +SELECT * FROM logs; +message +Error Error Error +Warning Warning Warning +Error Error +Warning Warning +Error +Warning +Error Error Error Error +Warning Warning Warning Warning +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score +FROM logs +ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE); +message score +Warning Warning Warning 0 +Warning Warning 0 +Warning 0 +Warning Warning Warning Warning 0 +Error 174763 +Error Error 349526 +Error Error Error 524289 +Error Error Error Error 699052 +DROP TABLE logs; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_same_match_against.result b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_same_match_against.result new file mode 100644 index 00000000000..ac2e7522db7 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/fulltext_order_natural_language_mode_same_match_against.result @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS logs; +SET NAMES UTF8; +CREATE TABLE logs( +message TEXT, +FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); +SELECT * FROM logs; +message +Error Error Error +Warning Warning Warning +Error Error +Warning Warning +Error +Warning +Error Error Error Error +Warning Warning Warning Warning +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score +FROM logs +WHERE MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) +ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE); +message score +Error 174763 +Error Error 349526 +Error Error Error 524289 +Error Error Error Error 699052 +DROP TABLE logs; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/geometry_bulk_insert_null.result b/storage/mroonga/mysql-test/mroonga/storage/r/geometry_bulk_insert_null.result index 2e9314db109..5e831860c5e 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/geometry_bulk_insert_null.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/geometry_bulk_insert_null.result @@ -6,8 +6,8 @@ INSERT INTO shops VALUES (NULL), (NULL); Warnings: Warning 1048 Column 'location' cannot be null Warning 1048 Column 'location' cannot be null -SELECT AsText(location) FROM shops; -AsText(location) +SELECT ST_AsText(location) FROM shops; +ST_AsText(location) POINT(0 0) POINT(0 0) DROP TABLE shops; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/geometry_bulk_insert_null_57.result b/storage/mroonga/mysql-test/mroonga/storage/r/geometry_bulk_insert_null_57.result new file mode 100644 index 00000000000..271cf922fd5 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/geometry_bulk_insert_null_57.result @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS shops; +CREATE TABLE shops ( +location GEOMETRY NOT NULL +); +SET SESSION sql_mode = ''; +INSERT INTO shops VALUES (NULL), (NULL); +Warnings: +Warning 1048 Column 'location' cannot be null +SET SESSION sql_mode = default; +SELECT ST_AsText(location) FROM shops; +ST_AsText(location) +POINT(0 0) +POINT(0 0) +DROP TABLE shops; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/geometry_contains.result b/storage/mroonga/mysql-test/mroonga/storage/r/geometry_contains.result index e88402a10b4..29fd989b4f4 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/geometry_contains.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/geometry_contains.result @@ -16,113 +16,113 @@ shops CREATE TABLE `shops` ( ) ENGINE=Mroonga DEFAULT CHARSET=latin1 INSERT INTO shops (name, location) VALUES ('nezu-no-taiyaki', -GeomFromText('POINT(139.762573 35.720253)')); +ST_GeomFromText('POINT(139.762573 35.720253)')); INSERT INTO shops (name, location) VALUES ('taiyaki-kataoka', -GeomFromText('POINT(139.715591 35.712521)')); +ST_GeomFromText('POINT(139.715591 35.712521)')); INSERT INTO shops (name, location) VALUES ('soba-taiyaki-ku', -GeomFromText('POINT(139.659088 35.683712)')); +ST_GeomFromText('POINT(139.659088 35.683712)')); INSERT INTO shops (name, location) VALUES ('kuruma', -GeomFromText('POINT(139.706207 35.721516)')); +ST_GeomFromText('POINT(139.706207 35.721516)')); INSERT INTO shops (name, location) VALUES ('hirose-ya', -GeomFromText('POINT(139.685608 35.714844)')); +ST_GeomFromText('POINT(139.685608 35.714844)')); INSERT INTO shops (name, location) VALUES ('sazare', -GeomFromText('POINT(139.685043 35.714653)')); +ST_GeomFromText('POINT(139.685043 35.714653)')); INSERT INTO shops (name, location) VALUES ('omede-taiyaki', -GeomFromText('POINT(139.817154 35.700516)')); +ST_GeomFromText('POINT(139.817154 35.700516)')); INSERT INTO shops (name, location) VALUES ('onaga-ya', -GeomFromText('POINT(139.81105 35.698254)')); +ST_GeomFromText('POINT(139.81105 35.698254)')); INSERT INTO shops (name, location) VALUES ('shiro-ya', -GeomFromText('POINT(139.638611 35.705517)')); +ST_GeomFromText('POINT(139.638611 35.705517)')); INSERT INTO shops (name, location) VALUES ('fuji-ya', -GeomFromText('POINT(139.637115 35.703938)')); +ST_GeomFromText('POINT(139.637115 35.703938)')); INSERT INTO shops (name, location) VALUES ('miyoshi', -GeomFromText('POINT(139.537323 35.644539)')); +ST_GeomFromText('POINT(139.537323 35.644539)')); INSERT INTO shops (name, location) VALUES ('juju-ya', -GeomFromText('POINT(139.695755 35.628922)')); +ST_GeomFromText('POINT(139.695755 35.628922)')); INSERT INTO shops (name, location) VALUES ('tatsumi-ya', -GeomFromText('POINT(139.638657 35.665501)')); +ST_GeomFromText('POINT(139.638657 35.665501)')); INSERT INTO shops (name, location) VALUES ('tetsuji', -GeomFromText('POINT(139.76857 35.680912)')); +ST_GeomFromText('POINT(139.76857 35.680912)')); INSERT INTO shops (name, location) VALUES ('gazuma-ya', -GeomFromText('POINT(139.647598 35.700817)')); +ST_GeomFromText('POINT(139.647598 35.700817)')); INSERT INTO shops (name, location) VALUES ('honma-mon', -GeomFromText('POINT(139.652573 35.722736)')); +ST_GeomFromText('POINT(139.652573 35.722736)')); INSERT INTO shops (name, location) VALUES ('naniwa-ya', -GeomFromText('POINT(139.796234 35.730061)')); +ST_GeomFromText('POINT(139.796234 35.730061)')); INSERT INTO shops (name, location) VALUES ('kuro-dai', -GeomFromText('POINT(139.704834 35.650345)')); +ST_GeomFromText('POINT(139.704834 35.650345)')); INSERT INTO shops (name, location) VALUES ('daruma', -GeomFromText('POINT(139.770599 35.681461)')); +ST_GeomFromText('POINT(139.770599 35.681461)')); INSERT INTO shops (name, location) VALUES ('yanagi-ya', -GeomFromText('POINT(139.783981 35.685341)')); +ST_GeomFromText('POINT(139.783981 35.685341)')); INSERT INTO shops (name, location) VALUES ('sharaku', -GeomFromText('POINT(139.794846 35.716969)')); +ST_GeomFromText('POINT(139.794846 35.716969)')); INSERT INTO shops (name, location) VALUES ('takane', -GeomFromText('POINT(139.560913 35.698601)')); +ST_GeomFromText('POINT(139.560913 35.698601)')); INSERT INTO shops (name, location) VALUES ('chiyoda', -GeomFromText('POINT(139.652817 35.642601)')); +ST_GeomFromText('POINT(139.652817 35.642601)')); INSERT INTO shops (name, location) VALUES ('da-ka-po', -GeomFromText('POINT(139.727356 35.627346)')); +ST_GeomFromText('POINT(139.727356 35.627346)')); INSERT INTO shops (name, location) VALUES ('matsushima-ya', -GeomFromText('POINT(139.737381 35.640556)')); +ST_GeomFromText('POINT(139.737381 35.640556)')); INSERT INTO shops (name, location) VALUES ('kazuya', -GeomFromText('POINT(139.760895 35.673508)')); +ST_GeomFromText('POINT(139.760895 35.673508)')); INSERT INTO shops (name, location) VALUES ('furuya-kogane-an', -GeomFromText('POINT(139.676071 35.680603)')); +ST_GeomFromText('POINT(139.676071 35.680603)')); INSERT INTO shops (name, location) VALUES ('hachi-no-ie', -GeomFromText('POINT(139.668106 35.608021)')); +ST_GeomFromText('POINT(139.668106 35.608021)')); INSERT INTO shops (name, location) VALUES ('azuki-chan', -GeomFromText('POINT(139.673203 35.64151)')); +ST_GeomFromText('POINT(139.673203 35.64151)')); INSERT INTO shops (name, location) VALUES ('kuriko-an', -GeomFromText('POINT(139.796829 35.712013)')); +ST_GeomFromText('POINT(139.796829 35.712013)')); INSERT INTO shops (name, location) VALUES ('yume-no-aru-machi-no-taiyaki-ya-san', -GeomFromText('POINT(139.712524 35.616199)')); +ST_GeomFromText('POINT(139.712524 35.616199)')); INSERT INTO shops (name, location) VALUES ('naze-ya', -GeomFromText('POINT(139.665833 35.609039)')); +ST_GeomFromText('POINT(139.665833 35.609039)')); INSERT INTO shops (name, location) VALUES ('sanoki-ya', -GeomFromText('POINT(139.770721 35.66592)')); +ST_GeomFromText('POINT(139.770721 35.66592)')); INSERT INTO shops (name, location) VALUES ('shigeta', -GeomFromText('POINT(139.780273 35.672626)')); +ST_GeomFromText('POINT(139.780273 35.672626)')); INSERT INTO shops (name, location) VALUES ('nishimi-ya', -GeomFromText('POINT(139.774628 35.671825)')); +ST_GeomFromText('POINT(139.774628 35.671825)')); INSERT INTO shops (name, location) VALUES ('hiiragi', -GeomFromText('POINT(139.711517 35.647701)')); -SELECT id, name, AsText(location) AS location_text FROM shops; +ST_GeomFromText('POINT(139.711517 35.647701)')); +SELECT id, name, ST_AsText(location) AS location_text FROM shops; id name location_text 1 nezu-no-taiyaki POINT(139.76257305555555 35.72025305555556) 2 taiyaki-kataoka POINT(139.7155911111111 35.712521111111116) @@ -160,8 +160,8 @@ id name location_text 34 shigeta POINT(139.78027305555557 35.67262611111111) 35 nishimi-ya POINT(139.77462805555555 35.671825) 36 hiiragi POINT(139.71151694444444 35.64770111111111) -SELECT id, name, AsText(location) AS location_text FROM shops -WHERE MBRContains(GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location) +SELECT id, name, ST_AsText(location) AS location_text FROM shops +WHERE MBRContains(ST_GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location) ORDER BY id; id name location_text 14 tetsuji POINT(139.76857 35.680911944444446) diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_asc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_asc.result new file mode 100644 index 00000000000..c2adc1cde2d --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_asc.result @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, +value INT(10), +INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); +SELECT id, value FROM ids WHERE value > 10 ORDER BY value ASC LIMIT 3; +id value +3 30 +1 50 +2 70 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_desc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_desc.result new file mode 100644 index 00000000000..715d299d608 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_desc.result @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, +value INT(10), +INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); +SELECT id, value FROM ids WHERE value > 10 ORDER BY value DESC LIMIT 3; +id value +4 90 +2 70 +1 50 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_or_equal_asc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_or_equal_asc.result new file mode 100644 index 00000000000..d562fbbb644 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_or_equal_asc.result @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, +value INT(10), +INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); +SELECT id, value FROM ids WHERE value >= 30 ORDER BY value ASC LIMIT 3; +id value +3 30 +1 50 +2 70 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_or_equal_desc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_or_equal_desc.result new file mode 100644 index 00000000000..5ecb4699fac --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_greater_than_or_equal_desc.result @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, +value INT(10), +INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); +SELECT id, value FROM ids WHERE value >= 30 ORDER BY value DESC LIMIT 3; +id value +4 90 +2 70 +1 50 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_asc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_asc.result new file mode 100644 index 00000000000..45b3fbe122a --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_asc.result @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, +value INT(10), +INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); +SELECT id, value FROM ids WHERE value < 90 ORDER BY value ASC LIMIT 3; +id value +5 10 +3 30 +1 50 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_desc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_desc.result new file mode 100644 index 00000000000..2e896e5e650 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_desc.result @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, +value INT(10), +INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); +SELECT id, value FROM ids WHERE value < 90 ORDER BY value DESC LIMIT 3; +id value +2 70 +1 50 +3 30 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_or_equal_asc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_or_equal_asc.result new file mode 100644 index 00000000000..d5313c9d0f4 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_or_equal_asc.result @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, +value INT(10), +INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); +SELECT id, value FROM ids WHERE value <= 70 ORDER BY value ASC LIMIT 3; +id value +5 10 +3 30 +1 50 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_or_equal_desc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_or_equal_desc.result new file mode 100644 index 00000000000..c56a4332f1f --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_normal_less_than_or_equal_desc.result @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, +value INT(10), +INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); +SELECT id, value FROM ids WHERE value <= 70 ORDER BY value DESC LIMIT 3; +id value +2 70 +1 50 +3 30 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_asc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_asc.result new file mode 100644 index 00000000000..1ca4b145f54 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_asc.result @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +SELECT * FROM ids WHERE ids.id > 1 ORDER BY ids.id ASC LIMIT 3; +id +2 +3 +4 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_desc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_desc.result new file mode 100644 index 00000000000..80dcb25ffe6 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_desc.result @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +SELECT * FROM ids WHERE ids.id > 3 ORDER BY ids.id DESC LIMIT 3; +id +5 +4 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_or_equal_asc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_or_equal_asc.result new file mode 100644 index 00000000000..4c1ff997d79 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_or_equal_asc.result @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +SELECT * FROM ids WHERE ids.id >= 2 ORDER BY ids.id ASC LIMIT 3; +id +2 +3 +4 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_or_equal_desc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_or_equal_desc.result new file mode 100644 index 00000000000..4998725aef5 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_greater_than_or_equal_desc.result @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +SELECT * FROM ids WHERE ids.id >= 3 ORDER BY ids.id DESC LIMIT 3; +id +5 +4 +3 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_asc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_asc.result new file mode 100644 index 00000000000..8b142f30610 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_asc.result @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +SELECT * FROM ids WHERE ids.id < 4 ORDER BY ids.id ASC LIMIT 3; +id +1 +2 +3 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_desc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_desc.result new file mode 100644 index 00000000000..eaf5b87e62c --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_desc.result @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +SELECT * FROM ids WHERE ids.id < 4 ORDER BY ids.id DESC LIMIT 3; +id +3 +2 +1 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_or_equal_asc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_or_equal_asc.result new file mode 100644 index 00000000000..e1e96c9ca8b --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_or_equal_asc.result @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +SELECT * FROM ids WHERE ids.id <= 4 ORDER BY ids.id ASC LIMIT 3; +id +1 +2 +3 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_or_equal_desc.result b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_or_equal_desc.result new file mode 100644 index 00000000000..e8124ca14c0 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/index_range_primary_less_than_or_equal_desc.result @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS ids; +SET NAMES UTF8; +CREATE TABLE ids ( +id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +SELECT * FROM ids WHERE ids.id <= 4 ORDER BY ids.id DESC LIMIT 3; +id +4 +3 +2 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_TODO_SPLIT_ME.result b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_TODO_SPLIT_ME.result new file mode 100644 index 00000000000..b66801094bd --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_TODO_SPLIT_ME.result @@ -0,0 +1,106 @@ +drop table if exists t1, t2, t3; +flush status; +create table t1 (c1 int primary key, c2 int, c3 text, key idx1(c2), fulltext index ft(c3)); +insert into t1 values(1,10,"aa ii uu ee oo"); +insert into t1 values(2,20,"ka ki ku ke ko"); +insert into t1 values(3,30,"sa si su se so"); +insert into t1 values(4,40,"ta ti tu te to"); +insert into t1 values(5,50,"aa ii uu ee oo"); +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 0 +select * from t1; +c1 c2 c3 +1 10 aa ii uu ee oo +2 20 ka ki ku ke ko +3 30 sa si su se so +4 40 ta ti tu te to +5 50 aa ii uu ee oo +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 0 +select count(*) from t1; +count(*) +5 +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 0 +select * from t1 force index(primary) where c1 between 2 and 4; +c1 c2 c3 +2 20 ka ki ku ke ko +3 30 sa si su se so +4 40 ta ti tu te to +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 0 +select count(*) from t1 force index(primary) where c1 between 2 and 4; +count(*) +3 +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 1 +select c1 from t1 force index(primary) where c1 < 3; +c1 +1 +2 +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 1 +select count(c1) from t1 force index(primary) where c1 < 3; +count(c1) +2 +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 1 +select 1 from t1 force index(primary) where c1 > 3; +1 +1 +1 +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 1 +select count(1) from t1 force index(primary) where c1 > 3; +count(1) +2 +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 2 +select * from t1 where match(c3) against("su"); +c1 c2 c3 +3 30 sa si su se so +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 2 +select count(*) from t1 where match(c3) against("su"); +count(*) +1 +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 3 +select * from t1 where match(c3) against("+su" in boolean mode); +c1 c2 c3 +3 30 sa si su se so +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 3 +select count(*) from t1 where match(c3) against("+su" in boolean mode); +count(*) +1 +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 4 +select * from t1 force index(idx1) where c2 between 20 and 40; +c1 c2 c3 +2 20 ka ki ku ke ko +3 30 sa si su se so +4 40 ta ti tu te to +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 4 +select count(*) from t1 force index(idx1) where c2 between 20 and 40; +count(*) +3 +show status like 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 5 +drop table t1; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_after_insert_multithread.result b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_after_insert_multithread.result new file mode 100644 index 00000000000..c09ec340ccb --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_after_insert_multithread.result @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS diaries; +FLUSH STATUS; +CREATE TABLE diaries ( +title TEXT, +FULLTEXT INDEX ft(title) +); +INSERT INTO diaries VALUES("Hello mroonga!"); +INSERT INTO diaries VALUES("It's funny."); +INSERT INTO diaries VALUES("Happy birthday!"); +SHOW STATUS LIKE 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 0 +SELECT COUNT(*) FROM diaries WHERE MATCH(title) AGAINST("mroonga" IN BOOLEAN MODE); +COUNT(*) +1 +SHOW STATUS LIKE 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 1 +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_after_insert_single_thread.result b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_after_insert_single_thread.result new file mode 100644 index 00000000000..c09ec340ccb --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_after_insert_single_thread.result @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS diaries; +FLUSH STATUS; +CREATE TABLE diaries ( +title TEXT, +FULLTEXT INDEX ft(title) +); +INSERT INTO diaries VALUES("Hello mroonga!"); +INSERT INTO diaries VALUES("It's funny."); +INSERT INTO diaries VALUES("Happy birthday!"); +SHOW STATUS LIKE 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 0 +SELECT COUNT(*) FROM diaries WHERE MATCH(title) AGAINST("mroonga" IN BOOLEAN MODE); +COUNT(*) +1 +SHOW STATUS LIKE 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 1 +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_disabled.result b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_disabled.result new file mode 100644 index 00000000000..9971d21e386 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_disabled.result @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS diaries; +FLUSH STATUS; +SET NAMES UTF8; +CREATE TABLE diaries ( +id INT UNSIGNED NOT NULL, +title VARCHAR(255), +content TEXT, +FULLTEXT INDEX(content) +) DEFAULT CHARSET UTF8; +SHOW CREATE TABLE diaries; +Table Create Table +diaries CREATE TABLE `diaries` ( + `id` int(10) unsigned NOT NULL, + `title` varchar(255) DEFAULT NULL, + `content` text, + FULLTEXT KEY `content` (`content`) +) ENGINE=Mroonga DEFAULT CHARSET=utf8 +INSERT INTO diaries VALUES(1, "Hello", "今日からはじめました。"); +INSERT INTO diaries VALUES(2, "天気", "明日の富士山の天気について"); +INSERT INTO diaries VALUES(3, "富士山", "今日も天気がよくてきれいに見える。"); +INSERT INTO diaries VALUES(4, "帰り道", "今日は天気がよくてよかった。"); +INSERT INTO diaries VALUES(5, "はれ", "天気がよいのは今日までみたい。"); +SET mroonga_enable_optimization=FALSE; +SELECT COUNT(*) FROM diaries +WHERE MATCH(content) AGAINST("今日" IN BOOLEAN MODE); +COUNT(*) +4 +SHOW STATUS LIKE 'mroonga_count_skip'; +Variable_name Value +Mroonga_count_skip 0 +SET mroonga_enable_optimization=TRUE; +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_index_view.result b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_index_view.result new file mode 100644 index 00000000000..407347a4fa2 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_count_skip_index_view.result @@ -0,0 +1,28 @@ +DROP TABLE IF EXISTS diaries, users; +CREATE TABLE diaries ( +id INT PRIMARY KEY AUTO_INCREMENT, +user_id INT NOT NULL, +title VARCHAR(45) NOT NULL, +KEY (user_id), +FULLTEXT INDEX title_index (title) +) DEFAULT CHARSET=UTF8; +CREATE TABLE users ( +id INT PRIMARY KEY AUTO_INCREMENT, +name VARCHAR(45) NOT NULL, +INDEX (name) +) ENGINE=InnoDB DEFAULT CHARSET=UTF8; +INSERT INTO users (id, name) VALUES (1, "Alice"), (2, "Bob"); +INSERT INTO diaries (user_id, title) VALUES (1, "survey"); +INSERT INTO diaries (user_id, title) VALUES (2, "groonga (1)"); +INSERT INTO diaries (user_id, title) VALUES (2, "groonga (2)"); +CREATE VIEW articles AS +SELECT diaries.user_id AS user_id, +diaries.title AS title, +users.name AS name +FROM diaries, users +WHERE diaries.user_id = users.id; +SELECT COUNT(*) FROM articles WHERE name = 'Bob'; +COUNT(*) +2 +DROP VIEW articles; +DROP TABLE diaries, users; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/optimization_order_limit_optimized_duplicated_order_by_columns.result b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_order_limit_optimized_duplicated_order_by_columns.result new file mode 100644 index 00000000000..9ea8ea40093 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/optimization_order_limit_optimized_duplicated_order_by_columns.result @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS ids; +FLUSH STATUS; +SET NAMES utf8mb4; +CREATE TABLE ids ( +id int PRIMARY KEY, +text varchar(32), +FULLTEXT INDEX (text) +) DEFAULT CHARSET=utf8mb4; +INSERT INTO ids VALUES (1, 'first'); +SELECT * FROM ids +WHERE MATCH(text) AGAINST('+first' IN BOOLEAN MODE) +ORDER BY id, id +LIMIT 1; +id text +1 first +SHOW STATUS LIKE 'mroonga_fast_order_limit'; +Variable_name Value +Mroonga_fast_order_limit 1 +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/partition_update.result b/storage/mroonga/mysql-test/mroonga/storage/r/partition_update.result new file mode 100644 index 00000000000..8d8208f81a9 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/partition_update.result @@ -0,0 +1,43 @@ +DROP TABLE IF EXISTS logs; +SET NAMES UTF8; +CREATE TABLE logs ( +timestamp DATETIME, +message TEXT +) DEFAULT CHARSET=UTF8 +PARTITION BY RANGE (TO_DAYS(timestamp)) ( +PARTITION p201501 VALUES LESS THAN (TO_DAYS('2015-02-01')), +PARTITION p201502 VALUES LESS THAN (TO_DAYS('2015-03-01')), +PARTITION p201503 VALUES LESS THAN (TO_DAYS('2015-04-01')), +PARTITION pfuture VALUES LESS THAN MAXVALUE +); +SHOW CREATE TABLE logs; +Table Create Table +logs CREATE TABLE `logs` ( + `timestamp` datetime DEFAULT NULL, + `message` text +) ENGINE=Mroonga DEFAULT CHARSET=utf8 +/*!50100 PARTITION BY RANGE (TO_DAYS(timestamp)) +(PARTITION p201501 VALUES LESS THAN (735995) ENGINE = Mroonga, + PARTITION p201502 VALUES LESS THAN (736023) ENGINE = Mroonga, + PARTITION p201503 VALUES LESS THAN (736054) ENGINE = Mroonga, + PARTITION pfuture VALUES LESS THAN MAXVALUE ENGINE = Mroonga) */ +INSERT INTO logs VALUES('2015-01-01 00:00:00', 'Start'); +INSERT INTO logs VALUES('2015-02-01 00:00:00', 'Start'); +INSERT INTO logs VALUES('2015-03-01 00:00:00', 'Start'); +INSERT INTO logs VALUES('2015-04-01 00:00:00', 'Start'); +SELECT * FROM logs ORDER BY timestamp; +timestamp message +2015-01-01 00:00:00 Start +2015-02-01 00:00:00 Start +2015-03-01 00:00:00 Start +2015-04-01 00:00:00 Start +UPDATE logs +SET message = 'Started' + WHERE timestamp < '2015-03-01 00:00:00'; +SELECT * FROM logs ORDER BY timestamp; +timestamp message +2015-01-01 00:00:00 Started +2015-02-01 00:00:00 Started +2015-03-01 00:00:00 Start +2015-04-01 00:00:00 Start +DROP TABLE logs; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/repair_table_no_index_file.result b/storage/mroonga/mysql-test/mroonga/storage/r/repair_table_no_index_file.result new file mode 100644 index 00000000000..0812fc1089b --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/repair_table_no_index_file.result @@ -0,0 +1,40 @@ +CREATE DATABASE repair_test; +USE repair_test; +CREATE TABLE diaries ( +id INT PRIMARY KEY AUTO_INCREMENT, +title TEXT, +body TEXT, +FULLTEXT INDEX body_index (body) +) DEFAULT CHARSET UTF8; +SHOW CREATE TABLE diaries; +Table Create Table +diaries CREATE TABLE `diaries` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `title` text, + `body` text, + PRIMARY KEY (`id`), + FULLTEXT KEY `body_index` (`body`) +) ENGINE=Mroonga DEFAULT CHARSET=utf8 +INSERT INTO diaries (title, body) VALUES ("survey", "will start Groonga!"); +INSERT INTO diaries (title, body) VALUES ("Groonga (1)", "starting Groonga..."); +INSERT INTO diaries (title, body) VALUES ("Groonga (2)", "started Groonga."); +SELECT * FROM diaries WHERE MATCH(body) AGAINST("+starting" IN BOOLEAN MODE); +id title body +2 Groonga (1) starting Groonga... +FLUSH TABLES; +SELECT * FROM diaries WHERE MATCH(body) AGAINST("+starting" IN BOOLEAN MODE); +ERROR HY000: syscall error 'repair_test.mrn.000010A.c' (No such file or directory) +REPAIR TABLE diaries; +Table Op Msg_type Msg_text +repair_test.diaries repair status OK +SELECT * FROM diaries; +id title body +1 survey will start Groonga! +2 Groonga (1) starting Groonga... +3 Groonga (2) started Groonga. +SELECT * FROM diaries WHERE MATCH(body) AGAINST("+starting" IN BOOLEAN MODE); +id title body +2 Groonga (1) starting Groonga... +DROP TABLE diaries; +DROP DATABASE repair_test; +USE test; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/replace_geometry.result b/storage/mroonga/mysql-test/mroonga/storage/r/replace_geometry.result index 94bd5167dbc..e88a799fe2f 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/replace_geometry.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/replace_geometry.result @@ -5,19 +5,19 @@ geo GEOMETRY NOT NULL, PRIMARY KEY(id) ) DEFAULT CHARSET=utf8; INSERT INTO geo_replace VALUES(1, POINT(100,100)); -SELECT id, ASTEXT(geo) FROM geo_replace; -id ASTEXT(geo) +SELECT id, ST_AsText(geo) FROM geo_replace; +id ST_AsText(geo) 1 POINT(100 100) REPLACE INTO geo_replace VALUES(1, POINT(100,200)); -SELECT id, ASTEXT(geo) FROM geo_replace; -id ASTEXT(geo) +SELECT id, ST_AsText(geo) FROM geo_replace; +id ST_AsText(geo) 1 POINT(100 200) INSERT INTO geo_replace VALUES(1, POINT(200,200)) ON DUPLICATE KEY UPDATE geo = POINT(200,200); -SELECT id, ASTEXT(geo) FROM geo_replace; -id ASTEXT(geo) +SELECT id, ST_AsText(geo) FROM geo_replace; +id ST_AsText(geo) 1 POINT(200 200) UPDATE geo_replace SET geo = POINT(200,300); -SELECT id, ASTEXT(geo) FROM geo_replace; -id ASTEXT(geo) +SELECT id, ST_AsText(geo) FROM geo_replace; +id ST_AsText(geo) 1 POINT(200 300) DROP TABLE geo_replace; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/variable_database_path_prefix.result b/storage/mroonga/mysql-test/mroonga/storage/r/variable_database_path_prefix.result index 4536dded9d7..beca93b62ad 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/variable_database_path_prefix.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/variable_database_path_prefix.result @@ -13,5 +13,5 @@ id 1 DROP TABLE counts; DROP DATABASE clean_test; -USE test; SET GLOBAL mroonga_database_path_prefix = NULL; +USE test; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/variable_max_n_records_for_estimate_global.result b/storage/mroonga/mysql-test/mroonga/storage/r/variable_max_n_records_for_estimate_global.result new file mode 100644 index 00000000000..0181fe02d84 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/variable_max_n_records_for_estimate_global.result @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS ids; +CREATE TABLE ids ( +id INT PRIMARY KEY AUTO_INCREMENT +) DEFAULT CHARSET=UTF8; +INSERT INTO ids VALUES (1); +INSERT INTO ids VALUES (2); +INSERT INTO ids VALUES (3); +INSERT INTO ids VALUES (4); +INSERT INTO ids VALUES (5); +INSERT INTO ids VALUES (6); +INSERT INTO ids VALUES (7); +INSERT INTO ids VALUES (8); +INSERT INTO ids VALUES (9); +INSERT INTO ids VALUES (10); +SET GLOBAL mroonga_max_n_records_for_estimate = 1; +EXPLAIN SELECT * FROM ids WHERE id > 5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE ids range PRIMARY PRIMARY 4 NULL 1 Using where; Using index +SET GLOBAL mroonga_max_n_records_for_estimate = DEFAULT; +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/variable_max_n_records_for_estimate_session.result b/storage/mroonga/mysql-test/mroonga/storage/r/variable_max_n_records_for_estimate_session.result new file mode 100644 index 00000000000..aca9d75c20f --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/r/variable_max_n_records_for_estimate_session.result @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS ids; +CREATE TABLE ids ( +id INT PRIMARY KEY AUTO_INCREMENT +) DEFAULT CHARSET=UTF8; +INSERT INTO ids VALUES (1); +INSERT INTO ids VALUES (2); +INSERT INTO ids VALUES (3); +INSERT INTO ids VALUES (4); +INSERT INTO ids VALUES (5); +INSERT INTO ids VALUES (6); +INSERT INTO ids VALUES (7); +INSERT INTO ids VALUES (8); +INSERT INTO ids VALUES (9); +INSERT INTO ids VALUES (10); +SET mroonga_max_n_records_for_estimate = 1; +EXPLAIN SELECT * FROM ids WHERE id > 5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE ids range PRIMARY PRIMARY 4 NULL 1 Using where; Using index +SET mroonga_max_n_records_for_estimate = DEFAULT; +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_column_with_flags.test b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_column_with_flags.test index c93ae13a19e..96c99612190 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_column_with_flags.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_column_with_flags.test @@ -30,7 +30,7 @@ CREATE TABLE tags ( ALTER TABLE tags ADD COLUMN name VARCHAR(64) COMMENT 'flags "COLUMN_VECTOR"'; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE tags; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_column_with_type.test b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_column_with_type.test index 2f39d52777a..dd05765585d 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_column_with_type.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_column_with_type.test @@ -34,7 +34,7 @@ CREATE TABLE bugs ( ALTER TABLE bugs ADD COLUMN name VARCHAR(64) COMMENT 'type "tags"'; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE bugs; DROP TABLE tags; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_index_token_filters_one_token_filter.test b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_index_token_filters_one_token_filter.test index aee355fdfc1..9c01dc27e3e 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_index_token_filters_one_token_filter.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_index_token_filters_one_token_filter.test @@ -31,11 +31,9 @@ CREATE TABLE memos ( content VARCHAR(64) NOT NULL ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); - ALTER TABLE memos ADD FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord"'; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_primary_key.test b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_primary_key.test new file mode 100644 index 00000000000..3dbfe5e24f0 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_add_primary_key.test @@ -0,0 +1,37 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +CREATE TABLE ids ( + id INT NOT NULL +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES (1); +INSERT INTO ids VALUES (2); + +ALTER TABLE ids ADD PRIMARY KEY (id); +SHOW CREATE TABLE ids; + +SELECT * FROM ids WHERE id = 2; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_change_token_filter.test b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_change_token_filter.test index fd47d2a60f0..7ec047818e3 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_change_token_filter.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_change_token_filter.test @@ -36,11 +36,16 @@ CREATE TABLE memos ( FULLTEXT INDEX (content) COMMENT 'table "terms"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); ALTER TABLE terms COMMENT='default_tokenizer "TokenBigram", token_filters "TokenFilterStopWord"'; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); + +ALTER TABLE memos DISABLE KEYS; +ALTER TABLE memos ENABLE KEYS; + +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE memos; DROP TABLE terms; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_disable_keys_fulltext_table.test b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_disable_keys_fulltext_table.test index d0c1313c784..97d2d2ed5ec 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_disable_keys_fulltext_table.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_disable_keys_fulltext_table.test @@ -35,11 +35,12 @@ CREATE TABLE memos ( FULLTEXT INDEX content_index (content) COMMENT 'table "terms"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); ALTER TABLE memos DISABLE KEYS; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); -DROP TABLE memos, terms; +DROP TABLE memos; +DROP TABLE terms; --source ../../include/mroonga/unload_mroonga_functions.inc --source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_drop_primary_key.test b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_drop_primary_key.test new file mode 100644 index 00000000000..87de166cbfd --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_drop_primary_key.test @@ -0,0 +1,37 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +CREATE TABLE ids ( + id INT PRIMARY KEY +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES (1); +INSERT INTO ids VALUES (2); + +ALTER TABLE ids DROP PRIMARY KEY; +SHOW CREATE TABLE ids; + +SELECT * FROM ids WHERE id = 2; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_enable_keys_fulltext_table.test b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_enable_keys_fulltext_table.test index b05fa6aa116..4ff4af63534 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_enable_keys_fulltext_table.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_enable_keys_fulltext_table.test @@ -36,11 +36,12 @@ CREATE TABLE memos ( ) DEFAULT CHARSET=utf8; ALTER TABLE memos DISABLE KEYS; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); ALTER TABLE memos ENABLE KEYS; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); -DROP TABLE memos, terms; +DROP TABLE memos; +DROP TABLE terms; --source ../../include/mroonga/unload_mroonga_functions.inc --source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_spatial.test b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_spatial.test index 8a41792bdae..9072d837a83 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_spatial.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/alter_table_spatial.test @@ -16,6 +16,7 @@ --source include/not_embedded.inc --source include/have_geometry.inc +--source ../../include/mroonga/have_version_56_or_later.inc --source ../../include/mroonga/have_mroonga.inc --disable_warnings @@ -30,117 +31,117 @@ CREATE TABLE shops ( INSERT INTO shops (name, location) VALUES ('nezu-no-taiyaki', - GeomFromText('POINT(139.762573 35.720253)')); + ST_GeomFromText('POINT(139.762573 35.720253)')); INSERT INTO shops (name, location) VALUES ('taiyaki-kataoka', - GeomFromText('POINT(139.715591 35.712521)')); + ST_GeomFromText('POINT(139.715591 35.712521)')); INSERT INTO shops (name, location) VALUES ('soba-taiyaki-ku', - GeomFromText('POINT(139.659088 35.683712)')); + ST_GeomFromText('POINT(139.659088 35.683712)')); INSERT INTO shops (name, location) VALUES ('kuruma', - GeomFromText('POINT(139.706207 35.721516)')); + ST_GeomFromText('POINT(139.706207 35.721516)')); INSERT INTO shops (name, location) VALUES ('hirose-ya', - GeomFromText('POINT(139.685608 35.714844)')); + ST_GeomFromText('POINT(139.685608 35.714844)')); INSERT INTO shops (name, location) VALUES ('sazare', - GeomFromText('POINT(139.685043 35.714653)')); + ST_GeomFromText('POINT(139.685043 35.714653)')); INSERT INTO shops (name, location) VALUES ('omede-taiyaki', - GeomFromText('POINT(139.817154 35.700516)')); + ST_GeomFromText('POINT(139.817154 35.700516)')); INSERT INTO shops (name, location) VALUES ('onaga-ya', - GeomFromText('POINT(139.81105 35.698254)')); + ST_GeomFromText('POINT(139.81105 35.698254)')); INSERT INTO shops (name, location) VALUES ('shiro-ya', - GeomFromText('POINT(139.638611 35.705517)')); + ST_GeomFromText('POINT(139.638611 35.705517)')); INSERT INTO shops (name, location) VALUES ('fuji-ya', - GeomFromText('POINT(139.637115 35.703938)')); + ST_GeomFromText('POINT(139.637115 35.703938)')); INSERT INTO shops (name, location) VALUES ('miyoshi', - GeomFromText('POINT(139.537323 35.644539)')); + ST_GeomFromText('POINT(139.537323 35.644539)')); INSERT INTO shops (name, location) VALUES ('juju-ya', - GeomFromText('POINT(139.695755 35.628922)')); + ST_GeomFromText('POINT(139.695755 35.628922)')); INSERT INTO shops (name, location) VALUES ('tatsumi-ya', - GeomFromText('POINT(139.638657 35.665501)')); + ST_GeomFromText('POINT(139.638657 35.665501)')); INSERT INTO shops (name, location) VALUES ('tetsuji', - GeomFromText('POINT(139.76857 35.680912)')); + ST_GeomFromText('POINT(139.76857 35.680912)')); INSERT INTO shops (name, location) VALUES ('gazuma-ya', - GeomFromText('POINT(139.647598 35.700817)')); + ST_GeomFromText('POINT(139.647598 35.700817)')); INSERT INTO shops (name, location) VALUES ('honma-mon', - GeomFromText('POINT(139.652573 35.722736)')); + ST_GeomFromText('POINT(139.652573 35.722736)')); INSERT INTO shops (name, location) VALUES ('naniwa-ya', - GeomFromText('POINT(139.796234 35.730061)')); + ST_GeomFromText('POINT(139.796234 35.730061)')); INSERT INTO shops (name, location) VALUES ('kuro-dai', - GeomFromText('POINT(139.704834 35.650345)')); + ST_GeomFromText('POINT(139.704834 35.650345)')); INSERT INTO shops (name, location) VALUES ('daruma', - GeomFromText('POINT(139.770599 35.681461)')); + ST_GeomFromText('POINT(139.770599 35.681461)')); INSERT INTO shops (name, location) VALUES ('yanagi-ya', - GeomFromText('POINT(139.783981 35.685341)')); + ST_GeomFromText('POINT(139.783981 35.685341)')); INSERT INTO shops (name, location) VALUES ('sharaku', - GeomFromText('POINT(139.794846 35.716969)')); + ST_GeomFromText('POINT(139.794846 35.716969)')); INSERT INTO shops (name, location) VALUES ('takane', - GeomFromText('POINT(139.560913 35.698601)')); + ST_GeomFromText('POINT(139.560913 35.698601)')); INSERT INTO shops (name, location) VALUES ('chiyoda', - GeomFromText('POINT(139.652817 35.642601)')); + ST_GeomFromText('POINT(139.652817 35.642601)')); INSERT INTO shops (name, location) VALUES ('da-ka-po', - GeomFromText('POINT(139.727356 35.627346)')); + ST_GeomFromText('POINT(139.727356 35.627346)')); INSERT INTO shops (name, location) VALUES ('matsushima-ya', - GeomFromText('POINT(139.737381 35.640556)')); + ST_GeomFromText('POINT(139.737381 35.640556)')); INSERT INTO shops (name, location) VALUES ('kazuya', - GeomFromText('POINT(139.760895 35.673508)')); + ST_GeomFromText('POINT(139.760895 35.673508)')); INSERT INTO shops (name, location) VALUES ('furuya-kogane-an', - GeomFromText('POINT(139.676071 35.680603)')); + ST_GeomFromText('POINT(139.676071 35.680603)')); INSERT INTO shops (name, location) VALUES ('hachi-no-ie', - GeomFromText('POINT(139.668106 35.608021)')); + ST_GeomFromText('POINT(139.668106 35.608021)')); INSERT INTO shops (name, location) VALUES ('azuki-chan', - GeomFromText('POINT(139.673203 35.64151)')); + ST_GeomFromText('POINT(139.673203 35.64151)')); INSERT INTO shops (name, location) VALUES ('kuriko-an', - GeomFromText('POINT(139.796829 35.712013)')); + ST_GeomFromText('POINT(139.796829 35.712013)')); INSERT INTO shops (name, location) VALUES ('yume-no-aru-machi-no-taiyaki-ya-san', - GeomFromText('POINT(139.712524 35.616199)')); + ST_GeomFromText('POINT(139.712524 35.616199)')); INSERT INTO shops (name, location) VALUES ('naze-ya', - GeomFromText('POINT(139.665833 35.609039)')); + ST_GeomFromText('POINT(139.665833 35.609039)')); INSERT INTO shops (name, location) VALUES ('sanoki-ya', - GeomFromText('POINT(139.770721 35.66592)')); + ST_GeomFromText('POINT(139.770721 35.66592)')); INSERT INTO shops (name, location) VALUES ('shigeta', - GeomFromText('POINT(139.780273 35.672626)')); + ST_GeomFromText('POINT(139.780273 35.672626)')); INSERT INTO shops (name, location) VALUES ('nishimi-ya', - GeomFromText('POINT(139.774628 35.671825)')); + ST_GeomFromText('POINT(139.774628 35.671825)')); INSERT INTO shops (name, location) VALUES ('hiiragi', - GeomFromText('POINT(139.711517 35.647701)')); + ST_GeomFromText('POINT(139.711517 35.647701)')); ALTER TABLE shops ADD SPATIAL KEY location_index (location); -SELECT id, name, AsText(location) AS location_text FROM shops - WHERE MBRContains(GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location) +SELECT id, name, ST_AsText(location) AS location_text FROM shops + WHERE MBRContains(ST_GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location) ORDER BY id; SHOW CREATE TABLE shops; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/collation_utf8_unicode_520_ci_french.test b/storage/mroonga/mysql-test/mroonga/storage/t/collation_utf8_unicode_520_ci_french.test new file mode 100644 index 00000000000..204b1f416ad --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/collation_utf8_unicode_520_ci_french.test @@ -0,0 +1,36 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_version_56_or_later.inc +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +SET NAMES utf8mb4; +CREATE TABLE diaries ( + content varchar(256) COLLATE utf8mb4_unicode_520_ci, + FULLTEXT INDEX (content) +) DEFAULT CHARSET=utf8mb4; + +INSERT INTO diaries VALUES ("Je suis un garçon."); + +SELECT * FROM diaries WHERE MATCH (content) AGAINST ("+garcon" IN BOOLEAN MODE); + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/collation_utf8_unicode_520_ci_japanese.test b/storage/mroonga/mysql-test/mroonga/storage/t/collation_utf8_unicode_520_ci_japanese.test new file mode 100644 index 00000000000..a6a18dc4846 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/collation_utf8_unicode_520_ci_japanese.test @@ -0,0 +1,37 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_version_56_or_later.inc +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +SET NAMES utf8mb4; +CREATE TABLE diaries ( + content varchar(256) COLLATE utf8mb4_unicode_520_ci, + FULLTEXT INDEX (content) +) DEFAULT CHARSET=utf8mb4; + +INSERT INTO diaries VALUES ("ひらがなとカタカナを覚えました。"); + +SELECT * FROM diaries + WHERE MATCH (content) AGAINST ("+かたかな" IN BOOLEAN MODE); + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/column_groonga_index_fulltext_vector_other_table.test b/storage/mroonga/mysql-test/mroonga/storage/t/column_groonga_index_fulltext_vector_other_table.test index 86ef09cc8bb..e9c9bb9b5ee 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/column_groonga_index_fulltext_vector_other_table.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/column_groonga_index_fulltext_vector_other_table.test @@ -39,7 +39,7 @@ CREATE TABLE bugs ( INSERT INTO bugs (id, tags) VALUES (1, "Linux MySQL groonga"); -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no --dump_records no"); SELECT *, MATCH (tags) AGAINST ("+MySQL" IN BOOLEAN MODE) AS score FROM bugs diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/column_groonga_index_int_other_table.test b/storage/mroonga/mysql-test/mroonga/storage/t/column_groonga_index_int_other_table.test index 91f204133dd..27cf85033cb 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/column_groonga_index_int_other_table.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/column_groonga_index_int_other_table.test @@ -40,7 +40,7 @@ INSERT INTO bugs (id, priority) VALUES (1, 10); INSERT INTO bugs (id, priority) VALUES (2, 3); INSERT INTO bugs (id, priority) VALUES (3, -2); -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no --dump_records no"); SELECT * FROM bugs diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_default_tokenizer.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_default_tokenizer.test index 7c8cd3f32a0..30b53f8eee0 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_default_tokenizer.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_default_tokenizer.test @@ -30,7 +30,7 @@ CREATE TABLE tags ( COLLATE=utf8_bin COMMENT='default_tokenizer "TokenDelimit"'; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE tags; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_index_flags_none.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_index_flags_none.test new file mode 100644 index 00000000000..039c518e673 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_index_flags_none.test @@ -0,0 +1,38 @@ +# Copyright(C) 2015 Naoya Murakami <naoya@createfield.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc +--source ../../include/mroonga/load_mroonga_functions.inc + +--disable_query_log +DROP DATABASE test; +CREATE DATABASE test; +USE test; +--enable_query_log + +SET NAMES utf8; + +CREATE TABLE memos ( + content VARCHAR(64) NOT NULL, + FULLTEXT INDEX (content) COMMENT 'index_flags "NONE"' +) DEFAULT CHARSET=utf8; + +SELECT mroonga_command("dump --dump_plugins no --dump_schema no"); + +DROP TABLE memos; + +--source ../../include/mroonga/unload_mroonga_functions.inc +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_index_flags_with_position_and_with_weight.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_index_flags_with_position_and_with_weight.test new file mode 100644 index 00000000000..9a67644d2c5 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_index_flags_with_position_and_with_weight.test @@ -0,0 +1,38 @@ +# Copyright(C) 2015 Naoya Murakami <naoya@createfield.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc +--source ../../include/mroonga/load_mroonga_functions.inc + +--disable_query_log +DROP DATABASE test; +CREATE DATABASE test; +USE test; +--enable_query_log + +SET NAMES utf8; + +CREATE TABLE memos ( + content VARCHAR(64) NOT NULL, + FULLTEXT INDEX (content) COMMENT 'index_flags "WITH_POSITION|WITH_WEIGHT"' +) DEFAULT CHARSET=utf8; + +SELECT mroonga_command("dump --dump_plugins no --dump_schema no"); + +DROP TABLE memos; + +--source ../../include/mroonga/unload_mroonga_functions.inc +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_normalizer_primary_key_table_comment_with_using_hash.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_normalizer_primary_key_table_comment_with_using_hash.test new file mode 100644 index 00000000000..14b9ea9ab87 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_normalizer_primary_key_table_comment_with_using_hash.test @@ -0,0 +1,47 @@ +# Copyright(C) 2015 Naoya Murakami <naoya@createfield.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc +--source ../../include/mroonga/load_mroonga_functions.inc + +--disable_warnings +DROP TABLE IF EXISTS memos; +DROP TABLE IF EXISTS terms; +--enable_warnings + +SET NAMES utf8; + +CREATE TABLE terms ( + term VARCHAR(64) NOT NULL, + PRIMARY KEY (term) USING HASH +) COMMENT='default_tokenizer "TokenBigram", normalizer "NormalizerAuto"' DEFAULT CHARSET=utf8; + +CREATE TABLE memos ( + id INT NOT NULL PRIMARY KEY, + content TEXT NOT NULL, + FULLTEXT INDEX (content) COMMENT 'table "terms"' +) DEFAULT CHARSET=utf8; + +INSERT INTO memos VALUES (1, "1日の消費㌍は約2000㌔㌍"); + +SELECT * FROM memos + WHERE MATCH (content) AGAINST ("+カロリー" IN BOOLEAN MODE); + +DROP TABLE memos; +DROP TABLE terms; + +--source ../../include/mroonga/unload_mroonga_functions.inc +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_reference_type.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_reference_type.test index eb1095e75a7..0a1340d24d1 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_reference_type.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_reference_type.test @@ -33,7 +33,7 @@ CREATE TABLE bugs ( tag VARCHAR(64) COMMENT 'type "tags"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE bugs; DROP TABLE tags; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_index_comment_multiple_token_filters.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_index_comment_multiple_token_filters.test index 66bf9cef699..18a2607fa0b 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_index_comment_multiple_token_filters.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_index_comment_multiple_token_filters.test @@ -32,7 +32,7 @@ CREATE TABLE memos ( FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord,TokenFilterStopWord"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_index_comment_one_token_filter.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_index_comment_one_token_filter.test index 49a23ec6b74..20df762d31b 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_index_comment_one_token_filter.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_index_comment_one_token_filter.test @@ -32,7 +32,7 @@ CREATE TABLE memos ( FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_table_comment_multiple_token_filters.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_table_comment_multiple_token_filters.test index 7d890aff9d3..e4d7d8618fa 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_table_comment_multiple_token_filters.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_table_comment_multiple_token_filters.test @@ -36,7 +36,7 @@ CREATE TABLE memos ( FULLTEXT INDEX (content) COMMENT 'table "terms"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE memos; DROP TABLE terms; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_table_comment_one_token_filter.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_table_comment_one_token_filter.test index f391647a538..905181175d0 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_table_comment_one_token_filter.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_token_filters_table_comment_one_token_filter.test @@ -36,7 +36,7 @@ CREATE TABLE memos ( FULLTEXT INDEX (content) COMMENT 'table "terms"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE memos; DROP TABLE terms; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_vector.test b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_vector.test index 8707ce0e34b..2d5498c99c8 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/create_table_vector.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/create_table_vector.test @@ -29,7 +29,7 @@ CREATE TABLE bugs ( tags TEXT COMMENT 'flags "COLUMN_VECTOR"' ) DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE bugs; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_boolean_mode_pragma_all.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_boolean_mode_pragma_all.test new file mode 100644 index 00000000000..b94172c1c72 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_boolean_mode_pragma_all.test @@ -0,0 +1,44 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS memos; +--enable_warnings + +SET NAMES utf8; +CREATE TABLE memos ( + id INT PRIMARY KEY, + title VARCHAR(255), + content TEXT, + FULLTEXT INDEX (title, content) +) DEFAULT CHARSET=utf8; + +INSERT INTO memos VALUES(1, "Groonga", "Groonga is fast."); +INSERT INTO memos VALUES(2, "Mroonga", "Mroonga is also fast."); +INSERT INTO memos VALUES(3, "Rroonga", "Rroonga is also fast."); + +SELECT *, + MATCH(title, content) + AGAINST("*W1:10,2:2DOR Groonga Mroonga" in BOOLEAN MODE) AS score + FROM memos + WHERE MATCH(title, content) + AGAINST("*W1:10,2:2DOR Groonga Mroonga" in BOOLEAN MODE); + +DROP TABLE memos; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_insert_select.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_insert_select.test index 641bbc3c4bd..3dffb2fb458 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_insert_select.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_insert_select.test @@ -36,10 +36,6 @@ select * from t1 where c1>3 order by c1 desc; select * from t2 where c1>3 order by c1 asc; select * from t1 where c2>"s" order by c2 desc; select * from t2 where c2>"s" order by c1 asc; -select * from t1 where match(c2) against("ii") order by match(c2) against("ii") desc; -select * from t2 where match(c2) against("ii") order by match(c2) against("ii") asc; -select c1,c2,match(c2) against("ii") from t1 where match(c2) against("ii"); -select c1,c2,match(c2) against("ii") from t2 where match(c2) against("ii"); drop table t1,t2; --source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_different_against.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_different_against.test new file mode 100644 index 00000000000..f2859f99dba --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_different_against.test @@ -0,0 +1,42 @@ +# Copyright(C) 2012 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE diaries( + title TEXT, + FULLTEXT KEY (title) +) DEFAULT CHARSET=utf8; + +INSERT INTO diaries VALUES("Start groonga"); +INSERT INTO diaries VALUES("Start mroonga"); +INSERT INTO diaries VALUES("Start groonga and Ruby"); + +SELECT * FROM diaries; + +SELECT *, MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) AS score + FROM diaries + WHERE MATCH(title) AGAINST("groonga mroonga" IN BOOLEAN MODE) + ORDER BY MATCH(title) AGAINST("groonga" IN BOOLEAN MODE); + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_different_match.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_different_match.test new file mode 100644 index 00000000000..dc97db87487 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_different_match.test @@ -0,0 +1,44 @@ +# Copyright(C) 2012 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE diaries( + title TEXT, + body TEXT, + FULLTEXT KEY (title), + FULLTEXT KEY (body) +) DEFAULT CHARSET=utf8; + +INSERT INTO diaries VALUES("Start groonga", "I read groonga's tutorial."); +INSERT INTO diaries VALUES("Start mroonga", "I read mroonga's tutorial."); +INSERT INTO diaries VALUES("Start groonga and Ruby", "I installed rroonga."); + +SELECT * FROM diaries; + +SELECT *, MATCH(body) AGAINST("groonga" IN BOOLEAN MODE) AS score + FROM diaries + WHERE MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) + ORDER BY MATCH(body) AGAINST("groonga" IN BOOLEAN MODE); + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_no_where.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_no_where.test new file mode 100644 index 00000000000..50dd652e2d0 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_no_where.test @@ -0,0 +1,41 @@ +# Copyright(C) 2012 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE diaries( + title TEXT, + FULLTEXT KEY (title) +) DEFAULT CHARSET=utf8; + +INSERT INTO diaries VALUES("Start groonga"); +INSERT INTO diaries VALUES("Start mroonga"); +INSERT INTO diaries VALUES("Start groonga and Ruby"); + +SELECT * FROM diaries; + +SELECT *, MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) AS score + FROM diaries + ORDER BY MATCH(title) AGAINST("groonga" IN BOOLEAN MODE); + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_same_match_against.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_same_match_against.test new file mode 100644 index 00000000000..0873849a1e7 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_boolean_mode_same_match_against.test @@ -0,0 +1,42 @@ +# Copyright(C) 2012 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE diaries( + title TEXT, + FULLTEXT KEY (title) +) DEFAULT CHARSET=utf8; + +INSERT INTO diaries VALUES("Start groonga"); +INSERT INTO diaries VALUES("Start mroonga"); +INSERT INTO diaries VALUES("Start groonga and Ruby"); + +SELECT * FROM diaries; + +SELECT *, MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) AS score + FROM diaries + WHERE MATCH(title) AGAINST("groonga" IN BOOLEAN MODE) + ORDER BY MATCH(title) AGAINST("groonga" IN BOOLEAN MODE); + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_asc.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_asc.test new file mode 100644 index 00000000000..1b29b5b9284 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_asc.test @@ -0,0 +1,47 @@ +# Copyright(C) 2015 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS logs; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE logs( + message TEXT, + FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; + +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); + +SELECT * FROM logs; + +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score + FROM logs + WHERE MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) + ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) ASC; + +DROP TABLE logs; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_desc.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_desc.test new file mode 100644 index 00000000000..761c7259057 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_desc.test @@ -0,0 +1,47 @@ +# Copyright(C) 2015 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS logs; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE logs( + message TEXT, + FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; + +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); + +SELECT * FROM logs; + +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score + FROM logs + WHERE MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) + ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) DESC; + +DROP TABLE logs; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_different_against.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_different_against.test new file mode 100644 index 00000000000..eef4e08c5d1 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_different_against.test @@ -0,0 +1,47 @@ +# Copyright(C) 2015 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS logs; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE logs( + message TEXT, + FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; + +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); + +SELECT * FROM logs; + +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score + FROM logs + WHERE MATCH(message) AGAINST("Error Warning" IN NATURAL LANGUAGE MODE) + ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE); + +DROP TABLE logs; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_different_match.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_different_match.test new file mode 100644 index 00000000000..42b621f4401 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_different_match.test @@ -0,0 +1,49 @@ +# Copyright(C) 2015 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS logs; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE logs( + message TEXT, + host TEXT, + FULLTEXT KEY (message), + FULLTEXT KEY (host) +) DEFAULT CHARSET=utf8; + +INSERT INTO logs VALUES("Error Error Error", "host1"); +INSERT INTO logs VALUES("Warning Warning Warning", "host1"); +INSERT INTO logs VALUES("Error Error", "host2"); +INSERT INTO logs VALUES("Warning Warning", "host2"); +INSERT INTO logs VALUES("Error", "host2"); +INSERT INTO logs VALUES("Warning", "host2"); +INSERT INTO logs VALUES("Error Error Error Error", "host2"); +INSERT INTO logs VALUES("Warning Warning Warning Warning", "host2"); + +SELECT * FROM logs; + +SELECT *, MATCH(host) AGAINST("host2" IN NATURAL LANGUAGE MODE) AS score + FROM logs + WHERE MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) + ORDER BY MATCH(host) AGAINST("host2" IN NATURAL LANGUAGE MODE); + +DROP TABLE logs; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_no_where.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_no_where.test new file mode 100644 index 00000000000..2078e59f042 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_no_where.test @@ -0,0 +1,46 @@ +# Copyright(C) 2015 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS logs; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE logs( + message TEXT, + FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; + +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); + +SELECT * FROM logs; + +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score + FROM logs + ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE); + +DROP TABLE logs; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_same_match_against.test b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_same_match_against.test new file mode 100644 index 00000000000..9a8c93c3470 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/fulltext_order_natural_language_mode_same_match_against.test @@ -0,0 +1,47 @@ +# Copyright(C) 2015 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS logs; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE logs( + message TEXT, + FULLTEXT KEY (message) +) DEFAULT CHARSET=utf8; + +INSERT INTO logs VALUES("Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning"); +INSERT INTO logs VALUES("Error Error"); +INSERT INTO logs VALUES("Warning Warning"); +INSERT INTO logs VALUES("Error"); +INSERT INTO logs VALUES("Warning"); +INSERT INTO logs VALUES("Error Error Error Error"); +INSERT INTO logs VALUES("Warning Warning Warning Warning"); + +SELECT * FROM logs; + +SELECT *, MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) AS score + FROM logs + WHERE MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE) + ORDER BY MATCH(message) AGAINST("Error" IN NATURAL LANGUAGE MODE); + +DROP TABLE logs; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/geometry_bulk_insert_null.test b/storage/mroonga/mysql-test/mroonga/storage/t/geometry_bulk_insert_null.test index 39462257f9d..a5f42e2c9f3 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/geometry_bulk_insert_null.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/geometry_bulk_insert_null.test @@ -15,6 +15,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA --source include/have_geometry.inc +--source ../../include/mroonga/have_version_56_or_later.inc +--source ../../include/mroonga/skip_mysql_57.inc --source ../../include/mroonga/have_mroonga.inc --disable_warnings @@ -27,7 +29,7 @@ CREATE TABLE shops ( INSERT INTO shops VALUES (NULL), (NULL); -SELECT AsText(location) FROM shops; +SELECT ST_AsText(location) FROM shops; DROP TABLE shops; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/geometry_bulk_insert_null_57.test b/storage/mroonga/mysql-test/mroonga/storage/t/geometry_bulk_insert_null_57.test new file mode 100644 index 00000000000..84ce595621e --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/geometry_bulk_insert_null_57.test @@ -0,0 +1,38 @@ +# Copyright(C) 2014 Kenji Maruyama <mmmaru777@gmail.com> +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source include/have_geometry.inc +--source ../../include/mroonga/have_version_57.inc +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS shops; +--enable_warnings + +CREATE TABLE shops ( + location GEOMETRY NOT NULL +); + +SET SESSION sql_mode = ''; +INSERT INTO shops VALUES (NULL), (NULL); +SET SESSION sql_mode = default; + +SELECT ST_AsText(location) FROM shops; + +DROP TABLE shops; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/geometry_contains.test b/storage/mroonga/mysql-test/mroonga/storage/t/geometry_contains.test index c8ec649f181..2fcf7ec5e4b 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/geometry_contains.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/geometry_contains.test @@ -15,6 +15,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA --source include/have_geometry.inc +--source ../../include/mroonga/have_version_56_or_later.inc --source ../../include/mroonga/have_mroonga.inc --disable_warnings @@ -31,116 +32,116 @@ SHOW CREATE TABLE shops; INSERT INTO shops (name, location) VALUES ('nezu-no-taiyaki', - GeomFromText('POINT(139.762573 35.720253)')); + ST_GeomFromText('POINT(139.762573 35.720253)')); INSERT INTO shops (name, location) VALUES ('taiyaki-kataoka', - GeomFromText('POINT(139.715591 35.712521)')); + ST_GeomFromText('POINT(139.715591 35.712521)')); INSERT INTO shops (name, location) VALUES ('soba-taiyaki-ku', - GeomFromText('POINT(139.659088 35.683712)')); + ST_GeomFromText('POINT(139.659088 35.683712)')); INSERT INTO shops (name, location) VALUES ('kuruma', - GeomFromText('POINT(139.706207 35.721516)')); + ST_GeomFromText('POINT(139.706207 35.721516)')); INSERT INTO shops (name, location) VALUES ('hirose-ya', - GeomFromText('POINT(139.685608 35.714844)')); + ST_GeomFromText('POINT(139.685608 35.714844)')); INSERT INTO shops (name, location) VALUES ('sazare', - GeomFromText('POINT(139.685043 35.714653)')); + ST_GeomFromText('POINT(139.685043 35.714653)')); INSERT INTO shops (name, location) VALUES ('omede-taiyaki', - GeomFromText('POINT(139.817154 35.700516)')); + ST_GeomFromText('POINT(139.817154 35.700516)')); INSERT INTO shops (name, location) VALUES ('onaga-ya', - GeomFromText('POINT(139.81105 35.698254)')); + ST_GeomFromText('POINT(139.81105 35.698254)')); INSERT INTO shops (name, location) VALUES ('shiro-ya', - GeomFromText('POINT(139.638611 35.705517)')); + ST_GeomFromText('POINT(139.638611 35.705517)')); INSERT INTO shops (name, location) VALUES ('fuji-ya', - GeomFromText('POINT(139.637115 35.703938)')); + ST_GeomFromText('POINT(139.637115 35.703938)')); INSERT INTO shops (name, location) VALUES ('miyoshi', - GeomFromText('POINT(139.537323 35.644539)')); + ST_GeomFromText('POINT(139.537323 35.644539)')); INSERT INTO shops (name, location) VALUES ('juju-ya', - GeomFromText('POINT(139.695755 35.628922)')); + ST_GeomFromText('POINT(139.695755 35.628922)')); INSERT INTO shops (name, location) VALUES ('tatsumi-ya', - GeomFromText('POINT(139.638657 35.665501)')); + ST_GeomFromText('POINT(139.638657 35.665501)')); INSERT INTO shops (name, location) VALUES ('tetsuji', - GeomFromText('POINT(139.76857 35.680912)')); + ST_GeomFromText('POINT(139.76857 35.680912)')); INSERT INTO shops (name, location) VALUES ('gazuma-ya', - GeomFromText('POINT(139.647598 35.700817)')); + ST_GeomFromText('POINT(139.647598 35.700817)')); INSERT INTO shops (name, location) VALUES ('honma-mon', - GeomFromText('POINT(139.652573 35.722736)')); + ST_GeomFromText('POINT(139.652573 35.722736)')); INSERT INTO shops (name, location) VALUES ('naniwa-ya', - GeomFromText('POINT(139.796234 35.730061)')); + ST_GeomFromText('POINT(139.796234 35.730061)')); INSERT INTO shops (name, location) VALUES ('kuro-dai', - GeomFromText('POINT(139.704834 35.650345)')); + ST_GeomFromText('POINT(139.704834 35.650345)')); INSERT INTO shops (name, location) VALUES ('daruma', - GeomFromText('POINT(139.770599 35.681461)')); + ST_GeomFromText('POINT(139.770599 35.681461)')); INSERT INTO shops (name, location) VALUES ('yanagi-ya', - GeomFromText('POINT(139.783981 35.685341)')); + ST_GeomFromText('POINT(139.783981 35.685341)')); INSERT INTO shops (name, location) VALUES ('sharaku', - GeomFromText('POINT(139.794846 35.716969)')); + ST_GeomFromText('POINT(139.794846 35.716969)')); INSERT INTO shops (name, location) VALUES ('takane', - GeomFromText('POINT(139.560913 35.698601)')); + ST_GeomFromText('POINT(139.560913 35.698601)')); INSERT INTO shops (name, location) VALUES ('chiyoda', - GeomFromText('POINT(139.652817 35.642601)')); + ST_GeomFromText('POINT(139.652817 35.642601)')); INSERT INTO shops (name, location) VALUES ('da-ka-po', - GeomFromText('POINT(139.727356 35.627346)')); + ST_GeomFromText('POINT(139.727356 35.627346)')); INSERT INTO shops (name, location) VALUES ('matsushima-ya', - GeomFromText('POINT(139.737381 35.640556)')); + ST_GeomFromText('POINT(139.737381 35.640556)')); INSERT INTO shops (name, location) VALUES ('kazuya', - GeomFromText('POINT(139.760895 35.673508)')); + ST_GeomFromText('POINT(139.760895 35.673508)')); INSERT INTO shops (name, location) VALUES ('furuya-kogane-an', - GeomFromText('POINT(139.676071 35.680603)')); + ST_GeomFromText('POINT(139.676071 35.680603)')); INSERT INTO shops (name, location) VALUES ('hachi-no-ie', - GeomFromText('POINT(139.668106 35.608021)')); + ST_GeomFromText('POINT(139.668106 35.608021)')); INSERT INTO shops (name, location) VALUES ('azuki-chan', - GeomFromText('POINT(139.673203 35.64151)')); + ST_GeomFromText('POINT(139.673203 35.64151)')); INSERT INTO shops (name, location) VALUES ('kuriko-an', - GeomFromText('POINT(139.796829 35.712013)')); + ST_GeomFromText('POINT(139.796829 35.712013)')); INSERT INTO shops (name, location) VALUES ('yume-no-aru-machi-no-taiyaki-ya-san', - GeomFromText('POINT(139.712524 35.616199)')); + ST_GeomFromText('POINT(139.712524 35.616199)')); INSERT INTO shops (name, location) VALUES ('naze-ya', - GeomFromText('POINT(139.665833 35.609039)')); + ST_GeomFromText('POINT(139.665833 35.609039)')); INSERT INTO shops (name, location) VALUES ('sanoki-ya', - GeomFromText('POINT(139.770721 35.66592)')); + ST_GeomFromText('POINT(139.770721 35.66592)')); INSERT INTO shops (name, location) VALUES ('shigeta', - GeomFromText('POINT(139.780273 35.672626)')); + ST_GeomFromText('POINT(139.780273 35.672626)')); INSERT INTO shops (name, location) VALUES ('nishimi-ya', - GeomFromText('POINT(139.774628 35.671825)')); + ST_GeomFromText('POINT(139.774628 35.671825)')); INSERT INTO shops (name, location) VALUES ('hiiragi', - GeomFromText('POINT(139.711517 35.647701)')); + ST_GeomFromText('POINT(139.711517 35.647701)')); -SELECT id, name, AsText(location) AS location_text FROM shops; -SELECT id, name, AsText(location) AS location_text FROM shops - WHERE MBRContains(GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location) +SELECT id, name, ST_AsText(location) AS location_text FROM shops; +SELECT id, name, ST_AsText(location) AS location_text FROM shops + WHERE MBRContains(ST_GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location) ORDER BY id; DROP TABLE shops; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_asc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_asc.test new file mode 100644 index 00000000000..6bab6708ba7 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_asc.test @@ -0,0 +1,41 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, + value INT(10), + INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); + +SELECT id, value FROM ids WHERE value > 10 ORDER BY value ASC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_desc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_desc.test new file mode 100644 index 00000000000..e1bbd5b338c --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_desc.test @@ -0,0 +1,41 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, + value INT(10), + INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); + +SELECT id, value FROM ids WHERE value > 10 ORDER BY value DESC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_or_equal_asc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_or_equal_asc.test new file mode 100644 index 00000000000..680a4d624a7 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_or_equal_asc.test @@ -0,0 +1,41 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, + value INT(10), + INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); + +SELECT id, value FROM ids WHERE value >= 30 ORDER BY value ASC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_or_equal_desc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_or_equal_desc.test new file mode 100644 index 00000000000..25260c48c4b --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_greater_than_or_equal_desc.test @@ -0,0 +1,41 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, + value INT(10), + INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); + +SELECT id, value FROM ids WHERE value >= 30 ORDER BY value DESC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_asc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_asc.test new file mode 100644 index 00000000000..6c183ddec21 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_asc.test @@ -0,0 +1,41 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, + value INT(10), + INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); + +SELECT id, value FROM ids WHERE value < 90 ORDER BY value ASC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_desc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_desc.test new file mode 100644 index 00000000000..aa1ba7602ee --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_desc.test @@ -0,0 +1,41 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, + value INT(10), + INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); + +SELECT id, value FROM ids WHERE value < 90 ORDER BY value DESC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_or_equal_asc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_or_equal_asc.test new file mode 100644 index 00000000000..e083afdf4f5 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_or_equal_asc.test @@ -0,0 +1,41 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, + value INT(10), + INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); + +SELECT id, value FROM ids WHERE value <= 70 ORDER BY value ASC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_or_equal_desc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_or_equal_desc.test new file mode 100644 index 00000000000..155a51506a9 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_normal_less_than_or_equal_desc.test @@ -0,0 +1,41 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT, + value INT(10), + INDEX (value) +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(1, 50); +INSERT INTO ids VALUES(2, 70); +INSERT INTO ids VALUES(3, 30); +INSERT INTO ids VALUES(4, 90); +INSERT INTO ids VALUES(5, 10); + +SELECT id, value FROM ids WHERE value <= 70 ORDER BY value DESC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_asc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_asc.test new file mode 100644 index 00000000000..a622dfbf70d --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_asc.test @@ -0,0 +1,39 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); + +SELECT * FROM ids WHERE ids.id > 1 ORDER BY ids.id ASC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_desc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_desc.test new file mode 100644 index 00000000000..63dc820ccfd --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_desc.test @@ -0,0 +1,39 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); + +SELECT * FROM ids WHERE ids.id > 3 ORDER BY ids.id DESC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_or_equal_asc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_or_equal_asc.test new file mode 100644 index 00000000000..a51e6c5da47 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_or_equal_asc.test @@ -0,0 +1,39 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); + +SELECT * FROM ids WHERE ids.id >= 2 ORDER BY ids.id ASC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_or_equal_desc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_or_equal_desc.test new file mode 100644 index 00000000000..ea5c150bf0f --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_greater_than_or_equal_desc.test @@ -0,0 +1,39 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); + +SELECT * FROM ids WHERE ids.id >= 3 ORDER BY ids.id DESC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_asc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_asc.test new file mode 100644 index 00000000000..12761c0f4d8 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_asc.test @@ -0,0 +1,39 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); + +SELECT * FROM ids WHERE ids.id < 4 ORDER BY ids.id ASC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_desc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_desc.test new file mode 100644 index 00000000000..a4f4a781ead --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_desc.test @@ -0,0 +1,39 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); + +SELECT * FROM ids WHERE ids.id < 4 ORDER BY ids.id DESC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_or_equal_asc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_or_equal_asc.test new file mode 100644 index 00000000000..2fba0d13f5f --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_or_equal_asc.test @@ -0,0 +1,39 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); + +SELECT * FROM ids WHERE ids.id <= 4 ORDER BY ids.id ASC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_or_equal_desc.test b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_or_equal_desc.test new file mode 100644 index 00000000000..e4661fc7047 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/index_range_primary_less_than_or_equal_desc.test @@ -0,0 +1,39 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +SET NAMES UTF8; + +CREATE TABLE ids ( + id INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT +) ENGINE=Mroonga DEFAULT CHARSET=utf8; + +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); +INSERT INTO ids VALUES(); + +SELECT * FROM ids WHERE ids.id <= 4 ORDER BY ids.id DESC LIMIT 3; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_TODO_SPLIT_ME.test b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_TODO_SPLIT_ME.test new file mode 100644 index 00000000000..669868c6c7f --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_TODO_SPLIT_ME.test @@ -0,0 +1,61 @@ +# Copyright(C) 2010 Kentoku SHIBA +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +drop table if exists t1, t2, t3; +--enable_warnings + +flush status; +create table t1 (c1 int primary key, c2 int, c3 text, key idx1(c2), fulltext index ft(c3)); +insert into t1 values(1,10,"aa ii uu ee oo"); +insert into t1 values(2,20,"ka ki ku ke ko"); +insert into t1 values(3,30,"sa si su se so"); +insert into t1 values(4,40,"ta ti tu te to"); +insert into t1 values(5,50,"aa ii uu ee oo"); +show status like 'mroonga_count_skip'; +select * from t1; +show status like 'mroonga_count_skip'; +select count(*) from t1; +show status like 'mroonga_count_skip'; +select * from t1 force index(primary) where c1 between 2 and 4; +show status like 'mroonga_count_skip'; +select count(*) from t1 force index(primary) where c1 between 2 and 4; +show status like 'mroonga_count_skip'; +select c1 from t1 force index(primary) where c1 < 3; +show status like 'mroonga_count_skip'; +select count(c1) from t1 force index(primary) where c1 < 3; +show status like 'mroonga_count_skip'; +select 1 from t1 force index(primary) where c1 > 3; +show status like 'mroonga_count_skip'; +select count(1) from t1 force index(primary) where c1 > 3; +show status like 'mroonga_count_skip'; +select * from t1 where match(c3) against("su"); +show status like 'mroonga_count_skip'; +select count(*) from t1 where match(c3) against("su"); +show status like 'mroonga_count_skip'; +select * from t1 where match(c3) against("+su" in boolean mode); +show status like 'mroonga_count_skip'; +select count(*) from t1 where match(c3) against("+su" in boolean mode); +show status like 'mroonga_count_skip'; +select * from t1 force index(idx1) where c2 between 20 and 40; +show status like 'mroonga_count_skip'; +select count(*) from t1 force index(idx1) where c2 between 20 and 40; +show status like 'mroonga_count_skip'; +drop table t1; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_after_insert_multithread.test b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_after_insert_multithread.test new file mode 100644 index 00000000000..bca2311ffbc --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_after_insert_multithread.test @@ -0,0 +1,45 @@ +# Copyright(C) 2012 Kentoku SHIBA +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +FLUSH STATUS; +CREATE TABLE diaries ( + title TEXT, + FULLTEXT INDEX ft(title) +); + +INSERT INTO diaries VALUES("Hello mroonga!"); +INSERT INTO diaries VALUES("It's funny."); + +CONNECT (thread2, localhost, root, ,); +CONNECTION thread2; +INSERT INTO diaries VALUES("Happy birthday!"); +DISCONNECT thread2; +CONNECTION default; + +SHOW STATUS LIKE 'mroonga_count_skip'; +SELECT COUNT(*) FROM diaries WHERE MATCH(title) AGAINST("mroonga" IN BOOLEAN MODE); +SHOW STATUS LIKE 'mroonga_count_skip'; + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_after_insert_single_thread.test b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_after_insert_single_thread.test new file mode 100644 index 00000000000..fa960be850f --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_after_insert_single_thread.test @@ -0,0 +1,40 @@ +# Copyright(C) 2012 Kentoku SHIBA +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +FLUSH STATUS; +CREATE TABLE diaries ( + title TEXT, + FULLTEXT INDEX ft(title) +); + +INSERT INTO diaries VALUES("Hello mroonga!"); +INSERT INTO diaries VALUES("It's funny."); +INSERT INTO diaries VALUES("Happy birthday!"); + +SHOW STATUS LIKE 'mroonga_count_skip'; +SELECT COUNT(*) FROM diaries WHERE MATCH(title) AGAINST("mroonga" IN BOOLEAN MODE); +SHOW STATUS LIKE 'mroonga_count_skip'; + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_disabled.test b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_disabled.test new file mode 100644 index 00000000000..e94702e8159 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_disabled.test @@ -0,0 +1,51 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +FLUSH STATUS; + +SET NAMES UTF8; +CREATE TABLE diaries ( + id INT UNSIGNED NOT NULL, + title VARCHAR(255), + content TEXT, + FULLTEXT INDEX(content) +) DEFAULT CHARSET UTF8; +SHOW CREATE TABLE diaries; + +INSERT INTO diaries VALUES(1, "Hello", "今日からはじめました。"); +INSERT INTO diaries VALUES(2, "天気", "明日の富士山の天気について"); +INSERT INTO diaries VALUES(3, "富士山", "今日も天気がよくてきれいに見える。"); +INSERT INTO diaries VALUES(4, "帰り道", "今日は天気がよくてよかった。"); +INSERT INTO diaries VALUES(5, "はれ", "天気がよいのは今日までみたい。"); + +SET mroonga_enable_optimization=FALSE; + +SELECT COUNT(*) FROM diaries + WHERE MATCH(content) AGAINST("今日" IN BOOLEAN MODE); + +SHOW STATUS LIKE 'mroonga_count_skip'; + +SET mroonga_enable_optimization=TRUE; + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_index_view.test b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_index_view.test new file mode 100644 index 00000000000..11457b1e05a --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_count_skip_index_view.test @@ -0,0 +1,56 @@ +# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source include/have_innodb.inc +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries, users; +--enable_warnings + +CREATE TABLE diaries ( + id INT PRIMARY KEY AUTO_INCREMENT, + user_id INT NOT NULL, + title VARCHAR(45) NOT NULL, + KEY (user_id), + FULLTEXT INDEX title_index (title) +) DEFAULT CHARSET=UTF8; + +CREATE TABLE users ( + id INT PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(45) NOT NULL, + INDEX (name) +) ENGINE=InnoDB DEFAULT CHARSET=UTF8; + +INSERT INTO users (id, name) VALUES (1, "Alice"), (2, "Bob"); +INSERT INTO diaries (user_id, title) VALUES (1, "survey"); +INSERT INTO diaries (user_id, title) VALUES (2, "groonga (1)"); +INSERT INTO diaries (user_id, title) VALUES (2, "groonga (2)"); + +CREATE VIEW articles AS + SELECT diaries.user_id AS user_id, + diaries.title AS title, + users.name AS name + FROM diaries, users + WHERE diaries.user_id = users.id; + + +SELECT COUNT(*) FROM articles WHERE name = 'Bob'; + +DROP VIEW articles; +DROP TABLE diaries, users; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/optimization_order_limit_optimized_duplicated_order_by_columns.test b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_order_limit_optimized_duplicated_order_by_columns.test new file mode 100644 index 00000000000..832c1c5cb27 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/optimization_order_limit_optimized_duplicated_order_by_columns.test @@ -0,0 +1,45 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# Copyright(C) 2015 GMO Media, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +FLUSH STATUS; + +SET NAMES utf8mb4; + +CREATE TABLE ids ( + id int PRIMARY KEY, + text varchar(32), + FULLTEXT INDEX (text) +) DEFAULT CHARSET=utf8mb4; + +INSERT INTO ids VALUES (1, 'first'); + +SELECT * FROM ids + WHERE MATCH(text) AGAINST('+first' IN BOOLEAN MODE) + ORDER BY id, id + LIMIT 1; + +SHOW STATUS LIKE 'mroonga_fast_order_limit'; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/partition_insert.test b/storage/mroonga/mysql-test/mroonga/storage/t/partition_insert.test index 219c3440d69..fdd45ecc3a3 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/partition_insert.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/partition_insert.test @@ -14,6 +14,8 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +--source ../../include/mroonga/skip_mariadb_100_or_later.inc +--source include/have_partition.inc --source ../../include/mroonga/have_mroonga.inc --disable_warnings diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/partition_update.test b/storage/mroonga/mysql-test/mroonga/storage/t/partition_update.test new file mode 100644 index 00000000000..8ec55268b38 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/partition_update.test @@ -0,0 +1,53 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/skip_mariadb_100_or_later.inc +--source include/have_partition.inc +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS logs; +--enable_warnings + +SET NAMES UTF8; +CREATE TABLE logs ( + timestamp DATETIME, + message TEXT +) DEFAULT CHARSET=UTF8 + PARTITION BY RANGE (TO_DAYS(timestamp)) ( + PARTITION p201501 VALUES LESS THAN (TO_DAYS('2015-02-01')), + PARTITION p201502 VALUES LESS THAN (TO_DAYS('2015-03-01')), + PARTITION p201503 VALUES LESS THAN (TO_DAYS('2015-04-01')), + PARTITION pfuture VALUES LESS THAN MAXVALUE +); +SHOW CREATE TABLE logs; + +INSERT INTO logs VALUES('2015-01-01 00:00:00', 'Start'); +INSERT INTO logs VALUES('2015-02-01 00:00:00', 'Start'); +INSERT INTO logs VALUES('2015-03-01 00:00:00', 'Start'); +INSERT INTO logs VALUES('2015-04-01 00:00:00', 'Start'); + +SELECT * FROM logs ORDER BY timestamp; + +UPDATE logs + SET message = 'Started' + WHERE timestamp < '2015-03-01 00:00:00'; + +SELECT * FROM logs ORDER BY timestamp; + +DROP TABLE logs; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/repair_table_no_index_file.test b/storage/mroonga/mysql-test/mroonga/storage/t/repair_table_no_index_file.test new file mode 100644 index 00000000000..bdbcfefcd85 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/repair_table_no_index_file.test @@ -0,0 +1,57 @@ +# Copyright(C) 2014 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source include/not_embedded.inc +--source ../../include/mroonga/have_mroonga.inc +--source ../../include/mroonga/have_mroonga_helper.inc + +CREATE DATABASE repair_test; +USE repair_test; + +CREATE TABLE diaries ( + id INT PRIMARY KEY AUTO_INCREMENT, + title TEXT, + body TEXT, + FULLTEXT INDEX body_index (body) +) DEFAULT CHARSET UTF8; +SHOW CREATE TABLE diaries; + +INSERT INTO diaries (title, body) VALUES ("survey", "will start Groonga!"); +INSERT INTO diaries (title, body) VALUES ("Groonga (1)", "starting Groonga..."); +INSERT INTO diaries (title, body) VALUES ("Groonga (2)", "started Groonga."); + +SELECT * FROM diaries WHERE MATCH(body) AGAINST("+starting" IN BOOLEAN MODE); + +--remove_file $MYSQLD_DATADIR/repair_test.mrn.000010A.c + +FLUSH TABLES; + +# Error ER_CANT_OPEN_FILE syscall error 'repair_test.mrn.000010A.c' (No such file or directory) +--error ER_CANT_OPEN_FILE +SELECT * FROM diaries WHERE MATCH(body) AGAINST("+starting" IN BOOLEAN MODE); + +REPAIR TABLE diaries; + +SELECT * FROM diaries; + +SELECT * FROM diaries WHERE MATCH(body) AGAINST("+starting" IN BOOLEAN MODE); + +DROP TABLE diaries; + +DROP DATABASE repair_test; +USE test; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/replace_geometry.test b/storage/mroonga/mysql-test/mroonga/storage/t/replace_geometry.test index 5f160af239f..65144ecc9e3 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/replace_geometry.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/replace_geometry.test @@ -14,6 +14,8 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +--source include/have_geometry.inc +--source ../../include/mroonga/have_version_56_or_later.inc --source ../../include/mroonga/have_mroonga.inc --disable_warnings @@ -26,13 +28,13 @@ CREATE TABLE geo_replace ( PRIMARY KEY(id) ) DEFAULT CHARSET=utf8; INSERT INTO geo_replace VALUES(1, POINT(100,100)); -SELECT id, ASTEXT(geo) FROM geo_replace; +SELECT id, ST_AsText(geo) FROM geo_replace; REPLACE INTO geo_replace VALUES(1, POINT(100,200)); -SELECT id, ASTEXT(geo) FROM geo_replace; +SELECT id, ST_AsText(geo) FROM geo_replace; INSERT INTO geo_replace VALUES(1, POINT(200,200)) ON DUPLICATE KEY UPDATE geo = POINT(200,200); -SELECT id, ASTEXT(geo) FROM geo_replace; +SELECT id, ST_AsText(geo) FROM geo_replace; UPDATE geo_replace SET geo = POINT(200,300); -SELECT id, ASTEXT(geo) FROM geo_replace; +SELECT id, ST_AsText(geo) FROM geo_replace; DROP TABLE geo_replace; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_column.test b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_column.test index 11ca867d82c..b2cdf053bdb 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_column.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_column.test @@ -14,6 +14,9 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# TODO: Remove the check after MariaDB 5.5.42 and MariaDB 10.0.17 are released. +--source ../../include/mroonga/have_mysql.inc + --source ../../include/mroonga/have_mroonga.inc SET @mroonga_boolean_mode_syntax_flags_backup = diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_leading_not.test b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_leading_not.test index 665682fde6e..08afc2cb39e 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_leading_not.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_leading_not.test @@ -14,6 +14,9 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# TODO: Remove the check after MariaDB 5.5.42 and MariaDB 10.0.17 are released. +--source ../../include/mroonga/have_mysql.inc + --source ../../include/mroonga/have_mroonga.inc SET @mroonga_boolean_mode_syntax_flags_backup = diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_update.test b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_update.test index 8a55c0fb0f7..4987dbe2b7a 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_update.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_allow_update.test @@ -14,6 +14,9 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# TODO: Remove the check after MariaDB 5.5.42 and MariaDB 10.0.17 are released. +--source ../../include/mroonga/have_mysql.inc + --source ../../include/mroonga/have_mroonga.inc SET @mroonga_boolean_mode_syntax_flags_backup = diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_syntax_query.test b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_syntax_query.test index 0ff2d8aed43..84cac40b16a 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_syntax_query.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_syntax_query.test @@ -14,6 +14,9 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# TODO: Remove the check after MariaDB 5.5.42 and MariaDB 10.0.17 are released. +--source ../../include/mroonga/have_mysql.inc + --source ../../include/mroonga/have_mroonga.inc SET @mroonga_boolean_mode_syntax_flags_backup = diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_syntax_script.test b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_syntax_script.test index 61d5daa63d9..dea8212bda1 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_syntax_script.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/variable_boolean_mode_syntax_flags_syntax_script.test @@ -14,6 +14,9 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# TODO: Remove the check after MariaDB 5.5.42 and MariaDB 10.0.17 are released. +--source ../../include/mroonga/have_mysql.inc + --source ../../include/mroonga/have_mroonga.inc SET @mroonga_boolean_mode_syntax_flags_backup = diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/variable_database_path_prefix.test b/storage/mroonga/mysql-test/mroonga/storage/t/variable_database_path_prefix.test index c7b2094906b..7a0c70a53b7 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/t/variable_database_path_prefix.test +++ b/storage/mroonga/mysql-test/mroonga/storage/t/variable_database_path_prefix.test @@ -1,4 +1,4 @@ -# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -35,8 +35,10 @@ SELECT * FROM counts; DROP TABLE counts; DROP DATABASE clean_test; +SET GLOBAL mroonga_database_path_prefix = NULL; + USE test; -SET GLOBAL mroonga_database_path_prefix = NULL; +--rmdir $MYSQLD_DATADIR/test/mroonga.data --source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/variable_max_n_records_for_estimate_global.result b/storage/mroonga/mysql-test/mroonga/storage/t/variable_max_n_records_for_estimate_global.result new file mode 100644 index 00000000000..0181fe02d84 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/variable_max_n_records_for_estimate_global.result @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS ids; +CREATE TABLE ids ( +id INT PRIMARY KEY AUTO_INCREMENT +) DEFAULT CHARSET=UTF8; +INSERT INTO ids VALUES (1); +INSERT INTO ids VALUES (2); +INSERT INTO ids VALUES (3); +INSERT INTO ids VALUES (4); +INSERT INTO ids VALUES (5); +INSERT INTO ids VALUES (6); +INSERT INTO ids VALUES (7); +INSERT INTO ids VALUES (8); +INSERT INTO ids VALUES (9); +INSERT INTO ids VALUES (10); +SET GLOBAL mroonga_max_n_records_for_estimate = 1; +EXPLAIN SELECT * FROM ids WHERE id > 5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE ids range PRIMARY PRIMARY 4 NULL 1 Using where; Using index +SET GLOBAL mroonga_max_n_records_for_estimate = DEFAULT; +DROP TABLE ids; diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/variable_max_n_records_for_estimate_global.test b/storage/mroonga/mysql-test/mroonga/storage/t/variable_max_n_records_for_estimate_global.test new file mode 100644 index 00000000000..264bf95fd22 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/variable_max_n_records_for_estimate_global.test @@ -0,0 +1,55 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +CREATE TABLE ids ( + id INT PRIMARY KEY AUTO_INCREMENT +) DEFAULT CHARSET=UTF8; + +INSERT INTO ids VALUES (1); +INSERT INTO ids VALUES (2); +INSERT INTO ids VALUES (3); +INSERT INTO ids VALUES (4); +INSERT INTO ids VALUES (5); +INSERT INTO ids VALUES (6); +INSERT INTO ids VALUES (7); +INSERT INTO ids VALUES (8); +INSERT INTO ids VALUES (9); +INSERT INTO ids VALUES (10); + + +SET GLOBAL mroonga_max_n_records_for_estimate = 1; + +CONNECT (new_connection, localhost, root, ,); +CONNECTION new_connection; + +EXPLAIN SELECT * FROM ids WHERE id > 5; + +CONNECTION default; +DISCONNECT new_connection; + +SET GLOBAL mroonga_max_n_records_for_estimate = DEFAULT; + + +DROP TABLE ids; + + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/storage/t/variable_max_n_records_for_estimate_session.test b/storage/mroonga/mysql-test/mroonga/storage/t/variable_max_n_records_for_estimate_session.test new file mode 100644 index 00000000000..3abc3c532d3 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/storage/t/variable_max_n_records_for_estimate_session.test @@ -0,0 +1,46 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS ids; +--enable_warnings + +CREATE TABLE ids ( + id INT PRIMARY KEY AUTO_INCREMENT +) DEFAULT CHARSET=UTF8; + +INSERT INTO ids VALUES (1); +INSERT INTO ids VALUES (2); +INSERT INTO ids VALUES (3); +INSERT INTO ids VALUES (4); +INSERT INTO ids VALUES (5); +INSERT INTO ids VALUES (6); +INSERT INTO ids VALUES (7); +INSERT INTO ids VALUES (8); +INSERT INTO ids VALUES (9); +INSERT INTO ids VALUES (10); + +SET mroonga_max_n_records_for_estimate = 1; + +EXPLAIN SELECT * FROM ids WHERE id > 5; + +SET mroonga_max_n_records_for_estimate = DEFAULT; + +DROP TABLE ids; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/alter_table_spatial.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/alter_table_spatial.result index fc38afc72d8..0e2f4c3bcca 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/alter_table_spatial.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/alter_table_spatial.result @@ -6,115 +6,115 @@ location GEOMETRY NOT NULL ) COMMENT = 'ENGINE "InnoDB"'; INSERT INTO shops (name, location) VALUES ('nezu-no-taiyaki', -GeomFromText('POINT(139.762573 35.720253)')); +ST_GeomFromText('POINT(139.762573 35.720253)')); INSERT INTO shops (name, location) VALUES ('taiyaki-kataoka', -GeomFromText('POINT(139.715591 35.712521)')); +ST_GeomFromText('POINT(139.715591 35.712521)')); INSERT INTO shops (name, location) VALUES ('soba-taiyaki-ku', -GeomFromText('POINT(139.659088 35.683712)')); +ST_GeomFromText('POINT(139.659088 35.683712)')); INSERT INTO shops (name, location) VALUES ('kuruma', -GeomFromText('POINT(139.706207 35.721516)')); +ST_GeomFromText('POINT(139.706207 35.721516)')); INSERT INTO shops (name, location) VALUES ('hirose-ya', -GeomFromText('POINT(139.685608 35.714844)')); +ST_GeomFromText('POINT(139.685608 35.714844)')); INSERT INTO shops (name, location) VALUES ('sazare', -GeomFromText('POINT(139.685043 35.714653)')); +ST_GeomFromText('POINT(139.685043 35.714653)')); INSERT INTO shops (name, location) VALUES ('omede-taiyaki', -GeomFromText('POINT(139.817154 35.700516)')); +ST_GeomFromText('POINT(139.817154 35.700516)')); INSERT INTO shops (name, location) VALUES ('onaga-ya', -GeomFromText('POINT(139.81105 35.698254)')); +ST_GeomFromText('POINT(139.81105 35.698254)')); INSERT INTO shops (name, location) VALUES ('shiro-ya', -GeomFromText('POINT(139.638611 35.705517)')); +ST_GeomFromText('POINT(139.638611 35.705517)')); INSERT INTO shops (name, location) VALUES ('fuji-ya', -GeomFromText('POINT(139.637115 35.703938)')); +ST_GeomFromText('POINT(139.637115 35.703938)')); INSERT INTO shops (name, location) VALUES ('miyoshi', -GeomFromText('POINT(139.537323 35.644539)')); +ST_GeomFromText('POINT(139.537323 35.644539)')); INSERT INTO shops (name, location) VALUES ('juju-ya', -GeomFromText('POINT(139.695755 35.628922)')); +ST_GeomFromText('POINT(139.695755 35.628922)')); INSERT INTO shops (name, location) VALUES ('tatsumi-ya', -GeomFromText('POINT(139.638657 35.665501)')); +ST_GeomFromText('POINT(139.638657 35.665501)')); INSERT INTO shops (name, location) VALUES ('tetsuji', -GeomFromText('POINT(139.76857 35.680912)')); +ST_GeomFromText('POINT(139.76857 35.680912)')); INSERT INTO shops (name, location) VALUES ('gazuma-ya', -GeomFromText('POINT(139.647598 35.700817)')); +ST_GeomFromText('POINT(139.647598 35.700817)')); INSERT INTO shops (name, location) VALUES ('honma-mon', -GeomFromText('POINT(139.652573 35.722736)')); +ST_GeomFromText('POINT(139.652573 35.722736)')); INSERT INTO shops (name, location) VALUES ('naniwa-ya', -GeomFromText('POINT(139.796234 35.730061)')); +ST_GeomFromText('POINT(139.796234 35.730061)')); INSERT INTO shops (name, location) VALUES ('kuro-dai', -GeomFromText('POINT(139.704834 35.650345)')); +ST_GeomFromText('POINT(139.704834 35.650345)')); INSERT INTO shops (name, location) VALUES ('daruma', -GeomFromText('POINT(139.770599 35.681461)')); +ST_GeomFromText('POINT(139.770599 35.681461)')); INSERT INTO shops (name, location) VALUES ('yanagi-ya', -GeomFromText('POINT(139.783981 35.685341)')); +ST_GeomFromText('POINT(139.783981 35.685341)')); INSERT INTO shops (name, location) VALUES ('sharaku', -GeomFromText('POINT(139.794846 35.716969)')); +ST_GeomFromText('POINT(139.794846 35.716969)')); INSERT INTO shops (name, location) VALUES ('takane', -GeomFromText('POINT(139.560913 35.698601)')); +ST_GeomFromText('POINT(139.560913 35.698601)')); INSERT INTO shops (name, location) VALUES ('chiyoda', -GeomFromText('POINT(139.652817 35.642601)')); +ST_GeomFromText('POINT(139.652817 35.642601)')); INSERT INTO shops (name, location) VALUES ('da-ka-po', -GeomFromText('POINT(139.727356 35.627346)')); +ST_GeomFromText('POINT(139.727356 35.627346)')); INSERT INTO shops (name, location) VALUES ('matsushima-ya', -GeomFromText('POINT(139.737381 35.640556)')); +ST_GeomFromText('POINT(139.737381 35.640556)')); INSERT INTO shops (name, location) VALUES ('kazuya', -GeomFromText('POINT(139.760895 35.673508)')); +ST_GeomFromText('POINT(139.760895 35.673508)')); INSERT INTO shops (name, location) VALUES ('furuya-kogane-an', -GeomFromText('POINT(139.676071 35.680603)')); +ST_GeomFromText('POINT(139.676071 35.680603)')); INSERT INTO shops (name, location) VALUES ('hachi-no-ie', -GeomFromText('POINT(139.668106 35.608021)')); +ST_GeomFromText('POINT(139.668106 35.608021)')); INSERT INTO shops (name, location) VALUES ('azuki-chan', -GeomFromText('POINT(139.673203 35.64151)')); +ST_GeomFromText('POINT(139.673203 35.64151)')); INSERT INTO shops (name, location) VALUES ('kuriko-an', -GeomFromText('POINT(139.796829 35.712013)')); +ST_GeomFromText('POINT(139.796829 35.712013)')); INSERT INTO shops (name, location) VALUES ('yume-no-aru-machi-no-taiyaki-ya-san', -GeomFromText('POINT(139.712524 35.616199)')); +ST_GeomFromText('POINT(139.712524 35.616199)')); INSERT INTO shops (name, location) VALUES ('naze-ya', -GeomFromText('POINT(139.665833 35.609039)')); +ST_GeomFromText('POINT(139.665833 35.609039)')); INSERT INTO shops (name, location) VALUES ('sanoki-ya', -GeomFromText('POINT(139.770721 35.66592)')); +ST_GeomFromText('POINT(139.770721 35.66592)')); INSERT INTO shops (name, location) VALUES ('shigeta', -GeomFromText('POINT(139.780273 35.672626)')); +ST_GeomFromText('POINT(139.780273 35.672626)')); INSERT INTO shops (name, location) VALUES ('nishimi-ya', -GeomFromText('POINT(139.774628 35.671825)')); +ST_GeomFromText('POINT(139.774628 35.671825)')); INSERT INTO shops (name, location) VALUES ('hiiragi', -GeomFromText('POINT(139.711517 35.647701)')); +ST_GeomFromText('POINT(139.711517 35.647701)')); ALTER TABLE shops ADD SPATIAL KEY location_index (location); -SELECT id, name, AsText(location) AS location_text FROM shops -WHERE MBRContains(GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location); +SELECT id, name, ST_AsText(location) AS location_text FROM shops +WHERE MBRContains(ST_GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location); id name location_text 14 tetsuji POINT(139.76857 35.680912) 19 daruma POINT(139.770599 35.681461) diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/count_star_with_index.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/count_star_with_index.result index 6b29be5edb7..a6d2f7862af 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/count_star_with_index.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/count_star_with_index.result @@ -1,5 +1,3 @@ -DROP TABLE IF EXISTS diaries_innodb; -DROP TABLE IF EXISTS diaries_mroonga; CREATE TABLE diaries_innodb ( id INT PRIMARY KEY AUTO_INCREMENT, body TEXT, diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_index_flags_none.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_index_flags_none.result new file mode 100644 index 00000000000..08bdd72ee03 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_index_flags_none.result @@ -0,0 +1,9 @@ +SET NAMES utf8; +CREATE TABLE memos ( +content VARCHAR(64) NOT NULL PRIMARY KEY, +FULLTEXT INDEX (content) COMMENT 'index_flags "NONE"' +) COMMENT='engine "InnoDB"' DEFAULT CHARSET=utf8; +SELECT mroonga_command("dump --dump_plugins no --dump_schema no"); +mroonga_command("dump --dump_plugins no --dump_schema no") +column_create memos-content index COLUMN_INDEX memos +DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_index_flags_with_position_and_with_weight.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_index_flags_with_position_and_with_weight.result new file mode 100644 index 00000000000..a5ac716d38e --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_index_flags_with_position_and_with_weight.result @@ -0,0 +1,9 @@ +SET NAMES utf8; +CREATE TABLE memos ( +content VARCHAR(64) NOT NULL PRIMARY KEY, +FULLTEXT INDEX (content) COMMENT 'index_flags "WITH_POSITION|WITH_WEIGHT"' +) COMMENT='engine "InnoDB"' DEFAULT CHARSET=utf8; +SELECT mroonga_command("dump --dump_plugins no --dump_schema no"); +mroonga_command("dump --dump_plugins no --dump_schema no") +column_create memos-content index COLUMN_INDEX|WITH_WEIGHT|WITH_POSITION memos +DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_token_filters_index_comment_multiple_token_filters.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_token_filters_index_comment_multiple_token_filters.result index 0c0af7a9db2..68319844df8 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_token_filters_index_comment_multiple_token_filters.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_token_filters_index_comment_multiple_token_filters.result @@ -7,9 +7,11 @@ id INT NOT NULL PRIMARY KEY, content VARCHAR(64) NOT NULL, FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord,TokenFilterStopWord"' ) COMMENT='engine "InnoDB"' DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create memos TABLE_HASH_KEY ShortText + table_create memos-content TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI --token_filters TokenFilterStopWord,TokenFilterStopWord + column_create memos-content index COLUMN_INDEX|WITH_POSITION memos DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_token_filters_index_comment_one_token_filter.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_token_filters_index_comment_one_token_filter.result index aef34c0bc26..fa3012705ad 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_token_filters_index_comment_one_token_filter.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/create_table_token_filters_index_comment_one_token_filter.result @@ -7,9 +7,11 @@ id INT NOT NULL PRIMARY KEY, content VARCHAR(64) NOT NULL, FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord"' ) COMMENT='engine "InnoDB"' DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); -mroonga_command("dump") +SELECT mroonga_command("dump --dump_plugins no"); +mroonga_command("dump --dump_plugins no") table_create memos TABLE_HASH_KEY ShortText + table_create memos-content TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerMySQLGeneralCI --token_filters TokenFilterStopWord + column_create memos-content index COLUMN_INDEX|WITH_POSITION memos DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_contains.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_contains.result index 29163384e6a..b3f5bee27d7 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_contains.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_contains.result @@ -16,113 +16,113 @@ shops CREATE TABLE `shops` ( ) ENGINE=Mroonga DEFAULT CHARSET=latin1 COMMENT='engine "innodb"' insert into shops (name, location) values ('nezu-no-taiyaki', -GeomFromText('POINT(139.762573 35.720253)')); +ST_GeomFromText('POINT(139.762573 35.720253)')); insert into shops (name, location) values ('taiyaki-kataoka', -GeomFromText('POINT(139.715591 35.712521)')); +ST_GeomFromText('POINT(139.715591 35.712521)')); insert into shops (name, location) values ('soba-taiyaki-ku', -GeomFromText('POINT(139.659088 35.683712)')); +ST_GeomFromText('POINT(139.659088 35.683712)')); insert into shops (name, location) values ('kuruma', -GeomFromText('POINT(139.706207 35.721516)')); +ST_GeomFromText('POINT(139.706207 35.721516)')); insert into shops (name, location) values ('hirose-ya', -GeomFromText('POINT(139.685608 35.714844)')); +ST_GeomFromText('POINT(139.685608 35.714844)')); insert into shops (name, location) values ('sazare', -GeomFromText('POINT(139.685043 35.714653)')); +ST_GeomFromText('POINT(139.685043 35.714653)')); insert into shops (name, location) values ('omede-taiyaki', -GeomFromText('POINT(139.817154 35.700516)')); +ST_GeomFromText('POINT(139.817154 35.700516)')); insert into shops (name, location) values ('onaga-ya', -GeomFromText('POINT(139.81105 35.698254)')); +ST_GeomFromText('POINT(139.81105 35.698254)')); insert into shops (name, location) values ('shiro-ya', -GeomFromText('POINT(139.638611 35.705517)')); +ST_GeomFromText('POINT(139.638611 35.705517)')); insert into shops (name, location) values ('fuji-ya', -GeomFromText('POINT(139.637115 35.703938)')); +ST_GeomFromText('POINT(139.637115 35.703938)')); insert into shops (name, location) values ('miyoshi', -GeomFromText('POINT(139.537323 35.644539)')); +ST_GeomFromText('POINT(139.537323 35.644539)')); insert into shops (name, location) values ('juju-ya', -GeomFromText('POINT(139.695755 35.628922)')); +ST_GeomFromText('POINT(139.695755 35.628922)')); insert into shops (name, location) values ('tatsumi-ya', -GeomFromText('POINT(139.638657 35.665501)')); +ST_GeomFromText('POINT(139.638657 35.665501)')); insert into shops (name, location) values ('tetsuji', -GeomFromText('POINT(139.76857 35.680912)')); +ST_GeomFromText('POINT(139.76857 35.680912)')); insert into shops (name, location) values ('gazuma-ya', -GeomFromText('POINT(139.647598 35.700817)')); +ST_GeomFromText('POINT(139.647598 35.700817)')); insert into shops (name, location) values ('honma-mon', -GeomFromText('POINT(139.652573 35.722736)')); +ST_GeomFromText('POINT(139.652573 35.722736)')); insert into shops (name, location) values ('naniwa-ya', -GeomFromText('POINT(139.796234 35.730061)')); +ST_GeomFromText('POINT(139.796234 35.730061)')); insert into shops (name, location) values ('kuro-dai', -GeomFromText('POINT(139.704834 35.650345)')); +ST_GeomFromText('POINT(139.704834 35.650345)')); insert into shops (name, location) values ('daruma', -GeomFromText('POINT(139.770599 35.681461)')); +ST_GeomFromText('POINT(139.770599 35.681461)')); insert into shops (name, location) values ('yanagi-ya', -GeomFromText('POINT(139.783981 35.685341)')); +ST_GeomFromText('POINT(139.783981 35.685341)')); insert into shops (name, location) values ('sharaku', -GeomFromText('POINT(139.794846 35.716969)')); +ST_GeomFromText('POINT(139.794846 35.716969)')); insert into shops (name, location) values ('takane', -GeomFromText('POINT(139.560913 35.698601)')); +ST_GeomFromText('POINT(139.560913 35.698601)')); insert into shops (name, location) values ('chiyoda', -GeomFromText('POINT(139.652817 35.642601)')); +ST_GeomFromText('POINT(139.652817 35.642601)')); insert into shops (name, location) values ('da-ka-po', -GeomFromText('POINT(139.727356 35.627346)')); +ST_GeomFromText('POINT(139.727356 35.627346)')); insert into shops (name, location) values ('matsushima-ya', -GeomFromText('POINT(139.737381 35.640556)')); +ST_GeomFromText('POINT(139.737381 35.640556)')); insert into shops (name, location) values ('kazuya', -GeomFromText('POINT(139.760895 35.673508)')); +ST_GeomFromText('POINT(139.760895 35.673508)')); insert into shops (name, location) values ('furuya-kogane-an', -GeomFromText('POINT(139.676071 35.680603)')); +ST_GeomFromText('POINT(139.676071 35.680603)')); insert into shops (name, location) values ('hachi-no-ie', -GeomFromText('POINT(139.668106 35.608021)')); +ST_GeomFromText('POINT(139.668106 35.608021)')); insert into shops (name, location) values ('azuki-chan', -GeomFromText('POINT(139.673203 35.64151)')); +ST_GeomFromText('POINT(139.673203 35.64151)')); insert into shops (name, location) values ('kuriko-an', -GeomFromText('POINT(139.796829 35.712013)')); +ST_GeomFromText('POINT(139.796829 35.712013)')); insert into shops (name, location) values ('yume-no-aru-machi-no-taiyaki-ya-san', -GeomFromText('POINT(139.712524 35.616199)')); +ST_GeomFromText('POINT(139.712524 35.616199)')); insert into shops (name, location) values ('naze-ya', -GeomFromText('POINT(139.665833 35.609039)')); +ST_GeomFromText('POINT(139.665833 35.609039)')); insert into shops (name, location) values ('sanoki-ya', -GeomFromText('POINT(139.770721 35.66592)')); +ST_GeomFromText('POINT(139.770721 35.66592)')); insert into shops (name, location) values ('shigeta', -GeomFromText('POINT(139.780273 35.672626)')); +ST_GeomFromText('POINT(139.780273 35.672626)')); insert into shops (name, location) values ('nishimi-ya', -GeomFromText('POINT(139.774628 35.671825)')); +ST_GeomFromText('POINT(139.774628 35.671825)')); insert into shops (name, location) values ('hiiragi', -GeomFromText('POINT(139.711517 35.647701)')); -select id, name, AsText(location) as location_text from shops; +ST_GeomFromText('POINT(139.711517 35.647701)')); +select id, name, ST_AsText(location) as location_text from shops; id name location_text 1 nezu-no-taiyaki POINT(139.762573 35.720253) 2 taiyaki-kataoka POINT(139.715591 35.712521) @@ -160,8 +160,8 @@ id name location_text 34 shigeta POINT(139.780273 35.672626) 35 nishimi-ya POINT(139.774628 35.671825) 36 hiiragi POINT(139.711517 35.647701) -select id, name, AsText(location) as location_text from shops -where MBRContains(GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location); +select id, name, ST_AsText(location) as location_text from shops +where MBRContains(ST_GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location); id name location_text 14 tetsuji POINT(139.76857 35.680912) 19 daruma POINT(139.770599 35.681461) diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_delete.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_delete.result index 53aa00ecbab..28b981f9c7b 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_delete.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_delete.result @@ -16,16 +16,16 @@ shops CREATE TABLE `shops` ( ) ENGINE=Mroonga DEFAULT CHARSET=latin1 COMMENT='engine "innodb"' insert into shops (name, location) values ('sazare', -GeomFromText('POINT(139.685043 35.714653)')); -select id, name, AsText(location) as location_text from shops -where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); +ST_GeomFromText('POINT(139.685043 35.714653)')); +select id, name, ST_AsText(location) as location_text from shops +where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); id name location_text 1 sazare POINT(139.685043 35.714653) delete from shops -where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); -select id, name, AsText(location) as location_text from shops -where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); +where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); +select id, name, ST_AsText(location) as location_text from shops +where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); id name location_text -select id, name, AsText(location) as location_text from shops; +select id, name, ST_AsText(location) as location_text from shops; id name location_text drop table shops; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_update.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_update.result index e982e0b95ce..20de4a32cfa 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_update.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/geometry_update.result @@ -16,21 +16,21 @@ shops CREATE TABLE `shops` ( ) ENGINE=Mroonga DEFAULT CHARSET=latin1 COMMENT='engine "innodb"' insert into shops (name, location) values ('sazare', -GeomFromText('POINT(139.685043 35.714653)')); -select id, name, AsText(location) as location_text from shops -where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); +ST_GeomFromText('POINT(139.685043 35.714653)')); +select id, name, ST_AsText(location) as location_text from shops +where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); id name location_text 1 sazare POINT(139.685043 35.714653) -select id, name, AsText(location) as location_text from shops -where MBRContains(GeomFromText('LineString(139.65659 35.57903, 139.66489 35.57262)'), location); +select id, name, ST_AsText(location) as location_text from shops +where MBRContains(ST_GeomFromText('LineString(139.65659 35.57903, 139.66489 35.57262)'), location); id name location_text -update shops set location = GeomFromText('POINT(139.66116 35.57566)') +update shops set location = ST_GeomFromText('POINT(139.66116 35.57566)') where name = 'sazare'; -select id, name, AsText(location) as location_text from shops -where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); +select id, name, ST_AsText(location) as location_text from shops +where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); id name location_text -select id, name, AsText(location) as location_text from shops -where MBRContains(GeomFromText('LineString(139.65659 35.57903, 139.66489 35.57262)'), location); +select id, name, ST_AsText(location) as location_text from shops +where MBRContains(ST_GeomFromText('LineString(139.65659 35.57903, 139.66489 35.57262)'), location); id name location_text 1 sazare POINT(139.66116 35.57566) drop table shops; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_TODO_SPLIT_ME.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_TODO_SPLIT_ME.result index 5f43aee6997..660700602b7 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_TODO_SPLIT_ME.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_TODO_SPLIT_ME.result @@ -12,20 +12,6 @@ insert into t1 values(2,20,"ka ki ku ke ko"); insert into t1 values(3,30,"ii si ii se ii"); insert into t1 values(4,40,"ta ti tu te to"); insert into t1 values(5,50,"aa ii uu ii oo"); -select *, match(c3) against("ii") from t1 -where match(c3) against("ii") order by c1 desc limit 1; -c1 c2 c3 match(c3) against("ii") -5 50 aa ii uu ii oo 349526 -show status like 'mroonga_fast_order_limit'; -Variable_name Value -Mroonga_fast_order_limit 0 -select *, match(c3) against("ii") from t1 -where match(c3) against("ii") order by c1 limit 1; -c1 c2 c3 match(c3) against("ii") -1 10 aa ii uu ee oo 174763 -show status like 'mroonga_fast_order_limit'; -Variable_name Value -Mroonga_fast_order_limit 0 select c3, match(c3) against("ii") from t1 where match(c3) against("ii") order by match(c3) against("ii") desc; c3 match(c3) against("ii") diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_no_direction.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_no_direction.result new file mode 100644 index 00000000000..9dd3e6eaa1b --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_no_direction.result @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS memos; +FLUSH STATUS; +CREATE TABLE memos ( +id int PRIMARY KEY, +content TEXT, +FULLTEXT INDEX (content) +) COMMENT = 'engine "InnoDB"'; +INSERT INTO memos VALUES(1, "Groonga is fast"); +INSERT INTO memos VALUES(2, "Mroonga is fast"); +INSERT INTO memos VALUES(3, "Mroonga is easy"); +INSERT INTO memos VALUES(4, "Mroonga is useful"); +INSERT INTO memos VALUES(5, "Mroonga is great"); +SHOW STATUS LIKE 'mroonga_fast_order_limit'; +Variable_name Value +Mroonga_fast_order_limit 0 +SELECT id, content +FROM memos +WHERE MATCH(content) AGAINST("+Mroonga" IN BOOLEAN MODE) +ORDER BY id +LIMIT 2; +id content +2 Mroonga is fast +3 Mroonga is easy +SHOW STATUS LIKE 'mroonga_fast_order_limit'; +Variable_name Value +Mroonga_fast_order_limit 1 +DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_order_by_primary_key.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_order_by_primary_key.result new file mode 100644 index 00000000000..eb5e04fcc0c --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/optimization_order_limit_order_by_primary_key.result @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS memos; +FLUSH STATUS; +CREATE TABLE memos ( +id int PRIMARY KEY, +content TEXT, +FULLTEXT INDEX (content) +) COMMENT = 'engine "InnoDB"'; +INSERT INTO memos VALUES(1, "Mroonga is fast"); +INSERT INTO memos VALUES(2, "Mroonga is easy"); +INSERT INTO memos VALUES(3, "Mroonga is useful"); +INSERT INTO memos VALUES(4, "Mroonga is great"); +INSERT INTO memos VALUES(5, "Groonga is fast"); +SHOW STATUS LIKE 'mroonga_fast_order_limit'; +Variable_name Value +Mroonga_fast_order_limit 0 +SELECT id, content +FROM memos +WHERE MATCH(content) AGAINST("+Mroonga" IN BOOLEAN MODE) +ORDER BY id DESC +LIMIT 2; +id content +4 Mroonga is great +3 Mroonga is useful +SHOW STATUS LIKE 'mroonga_fast_order_limit'; +Variable_name Value +Mroonga_fast_order_limit 1 +DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/performance_schema.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/performance_schema.result new file mode 100644 index 00000000000..c9af869f20c --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/performance_schema.result @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS diaries; +SHOW VARIABLES LIKE 'performance_schema'; +Variable_name Value +performance_schema ON +CREATE TABLE diaries ( +id INT PRIMARY KEY AUTO_INCREMENT, +content VARCHAR(255), +FULLTEXT INDEX (content) +) DEFAULT CHARSET UTF8 COMMENT = 'ENGINE "InnoDB"'; +SHOW CREATE TABLE diaries; +Table Create Table +diaries CREATE TABLE `diaries` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `content` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + FULLTEXT KEY `content` (`content`) +) ENGINE=Mroonga DEFAULT CHARSET=utf8 COMMENT='ENGINE "InnoDB"' +INSERT INTO diaries (content) VALUES ("Tommorow will be shiny day!"); +SHOW TABLES FROM performance_schema LIKE 'threads'; +Tables_in_performance_schema (threads) +threads +DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/repair_table_no_files.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/repair_table_no_files.result index 61c8e18b17d..e330fa18c89 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/repair_table_no_files.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/repair_table_no_files.result @@ -1,7 +1,5 @@ -DROP DATABASE IF EXISTS repair_test; CREATE DATABASE repair_test; USE repair_test; -DROP TABLE IF EXISTS diaries; CREATE TABLE diaries ( id INT PRIMARY KEY AUTO_INCREMENT, title TEXT, diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/repair_table_no_index_file.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/repair_table_no_index_file.result index 140b5a92b75..a5012384889 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/repair_table_no_index_file.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/repair_table_no_index_file.result @@ -1,7 +1,5 @@ -DROP DATABASE IF EXISTS repair_test; CREATE DATABASE repair_test; USE repair_test; -DROP TABLE IF EXISTS diaries; CREATE TABLE diaries ( id INT PRIMARY KEY AUTO_INCREMENT, title TEXT, @@ -25,7 +23,7 @@ id title body 2 groonga (1) starting groonga... FLUSH TABLES; SELECT * FROM diaries WHERE MATCH(body) AGAINST("starting"); -ERROR HY000: syscall error 'repair_test.mrn.0000104' (No such file or directory) +ERROR HY000: syscall error 'repair_test.mrn.0000106' (No such file or directory) REPAIR TABLE diaries; Table Op Msg_type Msg_text repair_test.diaries repair status OK diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/r/variable_match_escalation_threshold_global.result b/storage/mroonga/mysql-test/mroonga/wrapper/r/variable_match_escalation_threshold_global.result index 0cf3d49c3c3..c98847cb50a 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/r/variable_match_escalation_threshold_global.result +++ b/storage/mroonga/mysql-test/mroonga/wrapper/r/variable_match_escalation_threshold_global.result @@ -1,5 +1,4 @@ DROP TABLE IF EXISTS diaries; -SET GLOBAL mroonga_match_escalation_threshold = -1; CREATE TABLE diaries ( id INT PRIMARY KEY AUTO_INCREMENT, title TEXT, @@ -17,17 +16,12 @@ diaries CREATE TABLE `diaries` ( ) ENGINE=Mroonga DEFAULT CHARSET=utf8 COMMENT='ENGINE "InnoDB"' INSERT INTO diaries (title, tags) VALUES ("Hello groonga!", "groonga install"); INSERT INTO diaries (title, tags) VALUES ("Hello mroonga!", "mroonga install"); -SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("install" IN BOOLEAN MODE); -id title tags -1 Hello groonga! groonga install -2 Hello mroonga! mroonga install -SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("gr" IN BOOLEAN MODE); -id title tags -SET GLOBAL mroonga_match_escalation_threshold = 0; SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("gr" IN BOOLEAN MODE); id title tags -SET mroonga_match_escalation_threshold = 0; +1 Hello groonga! groonga install +SET GLOBAL mroonga_match_escalation_threshold = -1; +USE test; SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("gr" IN BOOLEAN MODE); id title tags -1 Hello groonga! groonga install +SET GLOBAL mroonga_match_escalation_threshold = DEFAULT; DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/alter_table_spatial.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/alter_table_spatial.test index 37ea8aaf149..6a62ae312b2 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/alter_table_spatial.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/alter_table_spatial.test @@ -17,6 +17,7 @@ --source include/have_innodb.inc --source include/not_embedded.inc --source include/have_geometry.inc +--source ../../include/mroonga/have_version_56_or_later.inc --source ../../include/mroonga/have_mroonga.inc --disable_warnings @@ -31,117 +32,117 @@ CREATE TABLE shops ( INSERT INTO shops (name, location) VALUES ('nezu-no-taiyaki', - GeomFromText('POINT(139.762573 35.720253)')); + ST_GeomFromText('POINT(139.762573 35.720253)')); INSERT INTO shops (name, location) VALUES ('taiyaki-kataoka', - GeomFromText('POINT(139.715591 35.712521)')); + ST_GeomFromText('POINT(139.715591 35.712521)')); INSERT INTO shops (name, location) VALUES ('soba-taiyaki-ku', - GeomFromText('POINT(139.659088 35.683712)')); + ST_GeomFromText('POINT(139.659088 35.683712)')); INSERT INTO shops (name, location) VALUES ('kuruma', - GeomFromText('POINT(139.706207 35.721516)')); + ST_GeomFromText('POINT(139.706207 35.721516)')); INSERT INTO shops (name, location) VALUES ('hirose-ya', - GeomFromText('POINT(139.685608 35.714844)')); + ST_GeomFromText('POINT(139.685608 35.714844)')); INSERT INTO shops (name, location) VALUES ('sazare', - GeomFromText('POINT(139.685043 35.714653)')); + ST_GeomFromText('POINT(139.685043 35.714653)')); INSERT INTO shops (name, location) VALUES ('omede-taiyaki', - GeomFromText('POINT(139.817154 35.700516)')); + ST_GeomFromText('POINT(139.817154 35.700516)')); INSERT INTO shops (name, location) VALUES ('onaga-ya', - GeomFromText('POINT(139.81105 35.698254)')); + ST_GeomFromText('POINT(139.81105 35.698254)')); INSERT INTO shops (name, location) VALUES ('shiro-ya', - GeomFromText('POINT(139.638611 35.705517)')); + ST_GeomFromText('POINT(139.638611 35.705517)')); INSERT INTO shops (name, location) VALUES ('fuji-ya', - GeomFromText('POINT(139.637115 35.703938)')); + ST_GeomFromText('POINT(139.637115 35.703938)')); INSERT INTO shops (name, location) VALUES ('miyoshi', - GeomFromText('POINT(139.537323 35.644539)')); + ST_GeomFromText('POINT(139.537323 35.644539)')); INSERT INTO shops (name, location) VALUES ('juju-ya', - GeomFromText('POINT(139.695755 35.628922)')); + ST_GeomFromText('POINT(139.695755 35.628922)')); INSERT INTO shops (name, location) VALUES ('tatsumi-ya', - GeomFromText('POINT(139.638657 35.665501)')); + ST_GeomFromText('POINT(139.638657 35.665501)')); INSERT INTO shops (name, location) VALUES ('tetsuji', - GeomFromText('POINT(139.76857 35.680912)')); + ST_GeomFromText('POINT(139.76857 35.680912)')); INSERT INTO shops (name, location) VALUES ('gazuma-ya', - GeomFromText('POINT(139.647598 35.700817)')); + ST_GeomFromText('POINT(139.647598 35.700817)')); INSERT INTO shops (name, location) VALUES ('honma-mon', - GeomFromText('POINT(139.652573 35.722736)')); + ST_GeomFromText('POINT(139.652573 35.722736)')); INSERT INTO shops (name, location) VALUES ('naniwa-ya', - GeomFromText('POINT(139.796234 35.730061)')); + ST_GeomFromText('POINT(139.796234 35.730061)')); INSERT INTO shops (name, location) VALUES ('kuro-dai', - GeomFromText('POINT(139.704834 35.650345)')); + ST_GeomFromText('POINT(139.704834 35.650345)')); INSERT INTO shops (name, location) VALUES ('daruma', - GeomFromText('POINT(139.770599 35.681461)')); + ST_GeomFromText('POINT(139.770599 35.681461)')); INSERT INTO shops (name, location) VALUES ('yanagi-ya', - GeomFromText('POINT(139.783981 35.685341)')); + ST_GeomFromText('POINT(139.783981 35.685341)')); INSERT INTO shops (name, location) VALUES ('sharaku', - GeomFromText('POINT(139.794846 35.716969)')); + ST_GeomFromText('POINT(139.794846 35.716969)')); INSERT INTO shops (name, location) VALUES ('takane', - GeomFromText('POINT(139.560913 35.698601)')); + ST_GeomFromText('POINT(139.560913 35.698601)')); INSERT INTO shops (name, location) VALUES ('chiyoda', - GeomFromText('POINT(139.652817 35.642601)')); + ST_GeomFromText('POINT(139.652817 35.642601)')); INSERT INTO shops (name, location) VALUES ('da-ka-po', - GeomFromText('POINT(139.727356 35.627346)')); + ST_GeomFromText('POINT(139.727356 35.627346)')); INSERT INTO shops (name, location) VALUES ('matsushima-ya', - GeomFromText('POINT(139.737381 35.640556)')); + ST_GeomFromText('POINT(139.737381 35.640556)')); INSERT INTO shops (name, location) VALUES ('kazuya', - GeomFromText('POINT(139.760895 35.673508)')); + ST_GeomFromText('POINT(139.760895 35.673508)')); INSERT INTO shops (name, location) VALUES ('furuya-kogane-an', - GeomFromText('POINT(139.676071 35.680603)')); + ST_GeomFromText('POINT(139.676071 35.680603)')); INSERT INTO shops (name, location) VALUES ('hachi-no-ie', - GeomFromText('POINT(139.668106 35.608021)')); + ST_GeomFromText('POINT(139.668106 35.608021)')); INSERT INTO shops (name, location) VALUES ('azuki-chan', - GeomFromText('POINT(139.673203 35.64151)')); + ST_GeomFromText('POINT(139.673203 35.64151)')); INSERT INTO shops (name, location) VALUES ('kuriko-an', - GeomFromText('POINT(139.796829 35.712013)')); + ST_GeomFromText('POINT(139.796829 35.712013)')); INSERT INTO shops (name, location) VALUES ('yume-no-aru-machi-no-taiyaki-ya-san', - GeomFromText('POINT(139.712524 35.616199)')); + ST_GeomFromText('POINT(139.712524 35.616199)')); INSERT INTO shops (name, location) VALUES ('naze-ya', - GeomFromText('POINT(139.665833 35.609039)')); + ST_GeomFromText('POINT(139.665833 35.609039)')); INSERT INTO shops (name, location) VALUES ('sanoki-ya', - GeomFromText('POINT(139.770721 35.66592)')); + ST_GeomFromText('POINT(139.770721 35.66592)')); INSERT INTO shops (name, location) VALUES ('shigeta', - GeomFromText('POINT(139.780273 35.672626)')); + ST_GeomFromText('POINT(139.780273 35.672626)')); INSERT INTO shops (name, location) VALUES ('nishimi-ya', - GeomFromText('POINT(139.774628 35.671825)')); + ST_GeomFromText('POINT(139.774628 35.671825)')); INSERT INTO shops (name, location) VALUES ('hiiragi', - GeomFromText('POINT(139.711517 35.647701)')); + ST_GeomFromText('POINT(139.711517 35.647701)')); ALTER TABLE shops ADD SPATIAL KEY location_index (location); -SELECT id, name, AsText(location) AS location_text FROM shops - WHERE MBRContains(GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location); +SELECT id, name, ST_AsText(location) AS location_text FROM shops + WHERE MBRContains(ST_GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location); SHOW CREATE TABLE shops; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/count_star_with_index.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/count_star_with_index.test index f8b77becdde..70c176b5116 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/count_star_with_index.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/count_star_with_index.test @@ -1,4 +1,4 @@ -# Copyright(C) 2011 Kouhei Sutou <kou@clear-code.com> +# Copyright(C) 2011-2015 Kouhei Sutou <kou@clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -18,10 +18,11 @@ --source ../../include/mroonga/have_fulltext_index_comment.inc --source ../../include/mroonga/have_mroonga.inc ---disable_warnings -DROP TABLE IF EXISTS diaries_innodb; -DROP TABLE IF EXISTS diaries_mroonga; ---enable_warnings +--disable_query_log +DROP DATABASE test; +CREATE DATABASE test; +USE test; +--enable_query_log CREATE TABLE diaries_innodb ( id INT PRIMARY KEY AUTO_INCREMENT, diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_index_flags_none.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_index_flags_none.test new file mode 100644 index 00000000000..da1e5e736ea --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_index_flags_none.test @@ -0,0 +1,39 @@ +# Copyright(C) 2015 Naoya Murakami <naoya@createfield.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source include/have_innodb.inc +--source ../../include/mroonga/have_mroonga.inc +--source ../../include/mroonga/load_mroonga_functions.inc + +--disable_query_log +DROP DATABASE test; +CREATE DATABASE test; +USE test; +--enable_query_log + +SET NAMES utf8; + +CREATE TABLE memos ( + content VARCHAR(64) NOT NULL PRIMARY KEY, + FULLTEXT INDEX (content) COMMENT 'index_flags "NONE"' +) COMMENT='engine "InnoDB"' DEFAULT CHARSET=utf8; + +SELECT mroonga_command("dump --dump_plugins no --dump_schema no"); + +DROP TABLE memos; + +--source ../../include/mroonga/unload_mroonga_functions.inc +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_index_flags_with_position_and_with_weight.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_index_flags_with_position_and_with_weight.test new file mode 100644 index 00000000000..8d0d585d56d --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_index_flags_with_position_and_with_weight.test @@ -0,0 +1,39 @@ +# Copyright(C) 2015 Naoya Murakami <naoya@createfield.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source include/have_innodb.inc +--source ../../include/mroonga/have_mroonga.inc +--source ../../include/mroonga/load_mroonga_functions.inc + +--disable_query_log +DROP DATABASE test; +CREATE DATABASE test; +USE test; +--enable_query_log + +SET NAMES utf8; + +CREATE TABLE memos ( + content VARCHAR(64) NOT NULL PRIMARY KEY, + FULLTEXT INDEX (content) COMMENT 'index_flags "WITH_POSITION|WITH_WEIGHT"' +) COMMENT='engine "InnoDB"' DEFAULT CHARSET=utf8; + +SELECT mroonga_command("dump --dump_plugins no --dump_schema no"); + +DROP TABLE memos; + +--source ../../include/mroonga/unload_mroonga_functions.inc +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_token_filters_index_comment_multiple_token_filters.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_token_filters_index_comment_multiple_token_filters.test index 9ba941f8701..9ff8e4448d9 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_token_filters_index_comment_multiple_token_filters.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_token_filters_index_comment_multiple_token_filters.test @@ -34,7 +34,7 @@ CREATE TABLE memos ( FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord,TokenFilterStopWord"' ) COMMENT='engine "InnoDB"' DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_token_filters_index_comment_one_token_filter.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_token_filters_index_comment_one_token_filter.test index 91ac84c04d5..eee08309f76 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_token_filters_index_comment_one_token_filter.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/create_table_token_filters_index_comment_one_token_filter.test @@ -34,7 +34,7 @@ CREATE TABLE memos ( FULLTEXT INDEX (content) COMMENT 'token_filters "TokenFilterStopWord"' ) COMMENT='engine "InnoDB"' DEFAULT CHARSET=utf8; -SELECT mroonga_command("dump"); +SELECT mroonga_command("dump --dump_plugins no"); DROP TABLE memos; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_contains.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_contains.test index 4676fc61331..af6242a9ad1 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_contains.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_contains.test @@ -16,6 +16,7 @@ --source include/have_innodb.inc --source include/have_geometry.inc +--source ../../include/mroonga/have_version_56_or_later.inc --source ../../include/mroonga/have_mroonga.inc --disable_warnings @@ -31,115 +32,115 @@ create table shops ( show create table shops; insert into shops (name, location) values ('nezu-no-taiyaki', - GeomFromText('POINT(139.762573 35.720253)')); + ST_GeomFromText('POINT(139.762573 35.720253)')); insert into shops (name, location) values ('taiyaki-kataoka', - GeomFromText('POINT(139.715591 35.712521)')); + ST_GeomFromText('POINT(139.715591 35.712521)')); insert into shops (name, location) values ('soba-taiyaki-ku', - GeomFromText('POINT(139.659088 35.683712)')); + ST_GeomFromText('POINT(139.659088 35.683712)')); insert into shops (name, location) values ('kuruma', - GeomFromText('POINT(139.706207 35.721516)')); + ST_GeomFromText('POINT(139.706207 35.721516)')); insert into shops (name, location) values ('hirose-ya', - GeomFromText('POINT(139.685608 35.714844)')); + ST_GeomFromText('POINT(139.685608 35.714844)')); insert into shops (name, location) values ('sazare', - GeomFromText('POINT(139.685043 35.714653)')); + ST_GeomFromText('POINT(139.685043 35.714653)')); insert into shops (name, location) values ('omede-taiyaki', - GeomFromText('POINT(139.817154 35.700516)')); + ST_GeomFromText('POINT(139.817154 35.700516)')); insert into shops (name, location) values ('onaga-ya', - GeomFromText('POINT(139.81105 35.698254)')); + ST_GeomFromText('POINT(139.81105 35.698254)')); insert into shops (name, location) values ('shiro-ya', - GeomFromText('POINT(139.638611 35.705517)')); + ST_GeomFromText('POINT(139.638611 35.705517)')); insert into shops (name, location) values ('fuji-ya', - GeomFromText('POINT(139.637115 35.703938)')); + ST_GeomFromText('POINT(139.637115 35.703938)')); insert into shops (name, location) values ('miyoshi', - GeomFromText('POINT(139.537323 35.644539)')); + ST_GeomFromText('POINT(139.537323 35.644539)')); insert into shops (name, location) values ('juju-ya', - GeomFromText('POINT(139.695755 35.628922)')); + ST_GeomFromText('POINT(139.695755 35.628922)')); insert into shops (name, location) values ('tatsumi-ya', - GeomFromText('POINT(139.638657 35.665501)')); + ST_GeomFromText('POINT(139.638657 35.665501)')); insert into shops (name, location) values ('tetsuji', - GeomFromText('POINT(139.76857 35.680912)')); + ST_GeomFromText('POINT(139.76857 35.680912)')); insert into shops (name, location) values ('gazuma-ya', - GeomFromText('POINT(139.647598 35.700817)')); + ST_GeomFromText('POINT(139.647598 35.700817)')); insert into shops (name, location) values ('honma-mon', - GeomFromText('POINT(139.652573 35.722736)')); + ST_GeomFromText('POINT(139.652573 35.722736)')); insert into shops (name, location) values ('naniwa-ya', - GeomFromText('POINT(139.796234 35.730061)')); + ST_GeomFromText('POINT(139.796234 35.730061)')); insert into shops (name, location) values ('kuro-dai', - GeomFromText('POINT(139.704834 35.650345)')); + ST_GeomFromText('POINT(139.704834 35.650345)')); insert into shops (name, location) values ('daruma', - GeomFromText('POINT(139.770599 35.681461)')); + ST_GeomFromText('POINT(139.770599 35.681461)')); insert into shops (name, location) values ('yanagi-ya', - GeomFromText('POINT(139.783981 35.685341)')); + ST_GeomFromText('POINT(139.783981 35.685341)')); insert into shops (name, location) values ('sharaku', - GeomFromText('POINT(139.794846 35.716969)')); + ST_GeomFromText('POINT(139.794846 35.716969)')); insert into shops (name, location) values ('takane', - GeomFromText('POINT(139.560913 35.698601)')); + ST_GeomFromText('POINT(139.560913 35.698601)')); insert into shops (name, location) values ('chiyoda', - GeomFromText('POINT(139.652817 35.642601)')); + ST_GeomFromText('POINT(139.652817 35.642601)')); insert into shops (name, location) values ('da-ka-po', - GeomFromText('POINT(139.727356 35.627346)')); + ST_GeomFromText('POINT(139.727356 35.627346)')); insert into shops (name, location) values ('matsushima-ya', - GeomFromText('POINT(139.737381 35.640556)')); + ST_GeomFromText('POINT(139.737381 35.640556)')); insert into shops (name, location) values ('kazuya', - GeomFromText('POINT(139.760895 35.673508)')); + ST_GeomFromText('POINT(139.760895 35.673508)')); insert into shops (name, location) values ('furuya-kogane-an', - GeomFromText('POINT(139.676071 35.680603)')); + ST_GeomFromText('POINT(139.676071 35.680603)')); insert into shops (name, location) values ('hachi-no-ie', - GeomFromText('POINT(139.668106 35.608021)')); + ST_GeomFromText('POINT(139.668106 35.608021)')); insert into shops (name, location) values ('azuki-chan', - GeomFromText('POINT(139.673203 35.64151)')); + ST_GeomFromText('POINT(139.673203 35.64151)')); insert into shops (name, location) values ('kuriko-an', - GeomFromText('POINT(139.796829 35.712013)')); + ST_GeomFromText('POINT(139.796829 35.712013)')); insert into shops (name, location) values ('yume-no-aru-machi-no-taiyaki-ya-san', - GeomFromText('POINT(139.712524 35.616199)')); + ST_GeomFromText('POINT(139.712524 35.616199)')); insert into shops (name, location) values ('naze-ya', - GeomFromText('POINT(139.665833 35.609039)')); + ST_GeomFromText('POINT(139.665833 35.609039)')); insert into shops (name, location) values ('sanoki-ya', - GeomFromText('POINT(139.770721 35.66592)')); + ST_GeomFromText('POINT(139.770721 35.66592)')); insert into shops (name, location) values ('shigeta', - GeomFromText('POINT(139.780273 35.672626)')); + ST_GeomFromText('POINT(139.780273 35.672626)')); insert into shops (name, location) values ('nishimi-ya', - GeomFromText('POINT(139.774628 35.671825)')); + ST_GeomFromText('POINT(139.774628 35.671825)')); insert into shops (name, location) values ('hiiragi', - GeomFromText('POINT(139.711517 35.647701)')); -select id, name, AsText(location) as location_text from shops; -select id, name, AsText(location) as location_text from shops - where MBRContains(GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location); + ST_GeomFromText('POINT(139.711517 35.647701)')); +select id, name, ST_AsText(location) as location_text from shops; +select id, name, ST_AsText(location) as location_text from shops + where MBRContains(ST_GeomFromText('LineString(139.7727 35.6684, 139.7038 35.7121)'), location); drop table shops; --source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_delete.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_delete.test index 25814635b22..9f3589a66d1 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_delete.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_delete.test @@ -16,6 +16,7 @@ --source include/have_innodb.inc --source include/have_geometry.inc +--source ../../include/mroonga/have_version_56_or_later.inc --source ../../include/mroonga/have_mroonga.inc --disable_warnings @@ -32,16 +33,16 @@ show create table shops; insert into shops (name, location) values ('sazare', - GeomFromText('POINT(139.685043 35.714653)')); -select id, name, AsText(location) as location_text from shops - where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); + ST_GeomFromText('POINT(139.685043 35.714653)')); +select id, name, ST_AsText(location) as location_text from shops + where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); delete from shops - where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); -select id, name, AsText(location) as location_text from shops - where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); + where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); +select id, name, ST_AsText(location) as location_text from shops + where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); -select id, name, AsText(location) as location_text from shops; +select id, name, ST_AsText(location) as location_text from shops; drop table shops; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_update.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_update.test index c6bb8a1d4e6..aff5c52c786 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_update.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/geometry_update.test @@ -16,6 +16,7 @@ --source include/have_innodb.inc --source include/have_geometry.inc +--source ../../include/mroonga/have_version_56_or_later.inc --source ../../include/mroonga/have_mroonga.inc --disable_warnings @@ -32,18 +33,18 @@ show create table shops; insert into shops (name, location) values ('sazare', - GeomFromText('POINT(139.685043 35.714653)')); -select id, name, AsText(location) as location_text from shops - where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); -select id, name, AsText(location) as location_text from shops - where MBRContains(GeomFromText('LineString(139.65659 35.57903, 139.66489 35.57262)'), location); + ST_GeomFromText('POINT(139.685043 35.714653)')); +select id, name, ST_AsText(location) as location_text from shops + where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); +select id, name, ST_AsText(location) as location_text from shops + where MBRContains(ST_GeomFromText('LineString(139.65659 35.57903, 139.66489 35.57262)'), location); -update shops set location = GeomFromText('POINT(139.66116 35.57566)') +update shops set location = ST_GeomFromText('POINT(139.66116 35.57566)') where name = 'sazare'; -select id, name, AsText(location) as location_text from shops - where MBRContains(GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); -select id, name, AsText(location) as location_text from shops - where MBRContains(GeomFromText('LineString(139.65659 35.57903, 139.66489 35.57262)'), location); +select id, name, ST_AsText(location) as location_text from shops + where MBRContains(ST_GeomFromText('LineString(139.68466 35.71592, 139.68804 35.71411)'), location); +select id, name, ST_AsText(location) as location_text from shops + where MBRContains(ST_GeomFromText('LineString(139.65659 35.57903, 139.66489 35.57262)'), location); drop table shops; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_TODO_SPLIT_ME.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_TODO_SPLIT_ME.test index 39772e5d78d..776fd27a2f2 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_TODO_SPLIT_ME.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_TODO_SPLIT_ME.test @@ -36,14 +36,6 @@ insert into t1 values(3,30,"ii si ii se ii"); insert into t1 values(4,40,"ta ti tu te to"); insert into t1 values(5,50,"aa ii uu ii oo"); -select *, match(c3) against("ii") from t1 - where match(c3) against("ii") order by c1 desc limit 1; -show status like 'mroonga_fast_order_limit'; - -select *, match(c3) against("ii") from t1 - where match(c3) against("ii") order by c1 limit 1; -show status like 'mroonga_fast_order_limit'; - select c3, match(c3) against("ii") from t1 where match(c3) against("ii") order by match(c3) against("ii") desc; show status like 'mroonga_fast_order_limit'; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_no_direction.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_no_direction.test new file mode 100644 index 00000000000..69be84e2619 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_no_direction.test @@ -0,0 +1,50 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source include/have_innodb.inc +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS memos; +--enable_warnings + +FLUSH STATUS; + +CREATE TABLE memos ( + id int PRIMARY KEY, + content TEXT, + FULLTEXT INDEX (content) +) COMMENT = 'engine "InnoDB"'; + +INSERT INTO memos VALUES(1, "Groonga is fast"); +INSERT INTO memos VALUES(2, "Mroonga is fast"); +INSERT INTO memos VALUES(3, "Mroonga is easy"); +INSERT INTO memos VALUES(4, "Mroonga is useful"); +INSERT INTO memos VALUES(5, "Mroonga is great"); + +SHOW STATUS LIKE 'mroonga_fast_order_limit'; + +SELECT id, content + FROM memos + WHERE MATCH(content) AGAINST("+Mroonga" IN BOOLEAN MODE) + ORDER BY id + LIMIT 2; + +SHOW STATUS LIKE 'mroonga_fast_order_limit'; + +DROP TABLE memos; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_order_by_primary_key.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_order_by_primary_key.test new file mode 100644 index 00000000000..f7b858e75f0 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/optimization_order_limit_order_by_primary_key.test @@ -0,0 +1,50 @@ +# Copyright(C) 2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source include/have_innodb.inc +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS memos; +--enable_warnings + +FLUSH STATUS; + +CREATE TABLE memos ( + id int PRIMARY KEY, + content TEXT, + FULLTEXT INDEX (content) +) COMMENT = 'engine "InnoDB"'; + +INSERT INTO memos VALUES(1, "Mroonga is fast"); +INSERT INTO memos VALUES(2, "Mroonga is easy"); +INSERT INTO memos VALUES(3, "Mroonga is useful"); +INSERT INTO memos VALUES(4, "Mroonga is great"); +INSERT INTO memos VALUES(5, "Groonga is fast"); + +SHOW STATUS LIKE 'mroonga_fast_order_limit'; + +SELECT id, content + FROM memos + WHERE MATCH(content) AGAINST("+Mroonga" IN BOOLEAN MODE) + ORDER BY id DESC + LIMIT 2; + +SHOW STATUS LIKE 'mroonga_fast_order_limit'; + +DROP TABLE memos; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/performance_schema-master.opt b/storage/mroonga/mysql-test/mroonga/wrapper/t/performance_schema-master.opt new file mode 100644 index 00000000000..611d08f0c78 --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/performance_schema-master.opt @@ -0,0 +1 @@ +--performance-schema diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/performance_schema.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/performance_schema.test new file mode 100644 index 00000000000..31c3e0a26ff --- /dev/null +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/performance_schema.test @@ -0,0 +1,41 @@ +# Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +--source include/have_innodb.inc +--source include/not_embedded.inc +--source include/have_perfschema.inc +--source ../../include/mroonga/have_mroonga.inc + +--disable_warnings +DROP TABLE IF EXISTS diaries; +--enable_warnings + +SHOW VARIABLES LIKE 'performance_schema'; + +CREATE TABLE diaries ( + id INT PRIMARY KEY AUTO_INCREMENT, + content VARCHAR(255), + FULLTEXT INDEX (content) +) DEFAULT CHARSET UTF8 COMMENT = 'ENGINE "InnoDB"'; +SHOW CREATE TABLE diaries; + +INSERT INTO diaries (content) VALUES ("Tommorow will be shiny day!"); + +SHOW TABLES FROM performance_schema LIKE 'threads'; + +DROP TABLE diaries; + +--source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/repair_table_no_files.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/repair_table_no_files.test index b333991d27e..8f0d01d52fb 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/repair_table_no_files.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/repair_table_no_files.test @@ -19,14 +19,9 @@ --source ../../include/mroonga/have_mroonga.inc --source ../../include/mroonga/have_mroonga_helper.inc ---disable_warnings -DROP DATABASE IF EXISTS repair_test; CREATE DATABASE repair_test; USE repair_test; -DROP TABLE IF EXISTS diaries; ---enable_warnings - CREATE TABLE diaries ( id INT PRIMARY KEY AUTO_INCREMENT, title TEXT, @@ -44,10 +39,10 @@ SELECT * FROM diaries WHERE MATCH(body) AGAINST("starting"); --remove_file $MYSQLD_DATADIR/repair_test.mrn --remove_file $MYSQLD_DATADIR/repair_test.mrn.001 --remove_file $MYSQLD_DATADIR/repair_test.mrn.0000000 ---remove_file $MYSQLD_DATADIR/repair_test.mrn.0000103 ---remove_file $MYSQLD_DATADIR/repair_test.mrn.0000104 --remove_file $MYSQLD_DATADIR/repair_test.mrn.0000105 ---remove_file $MYSQLD_DATADIR/repair_test.mrn.0000105.c +--remove_file $MYSQLD_DATADIR/repair_test.mrn.0000106 +--remove_file $MYSQLD_DATADIR/repair_test.mrn.0000107 +--remove_file $MYSQLD_DATADIR/repair_test.mrn.0000107.c FLUSH TABLES; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/repair_table_no_index_file.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/repair_table_no_index_file.test index ce6e1d23280..147d9672eae 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/repair_table_no_index_file.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/repair_table_no_index_file.test @@ -19,14 +19,9 @@ --source ../../include/mroonga/have_mroonga.inc --source ../../include/mroonga/have_mroonga_helper.inc ---disable_warnings -DROP DATABASE IF EXISTS repair_test; CREATE DATABASE repair_test; USE repair_test; -DROP TABLE IF EXISTS diaries; ---enable_warnings - CREATE TABLE diaries ( id INT PRIMARY KEY AUTO_INCREMENT, title TEXT, @@ -41,7 +36,7 @@ INSERT INTO diaries (title, body) VALUES ("groonga (2)", "started groonga."); SELECT * FROM diaries WHERE MATCH(body) AGAINST("starting"); ---remove_file $MYSQLD_DATADIR/repair_test.mrn.0000104 +--remove_file $MYSQLD_DATADIR/repair_test.mrn.0000106 FLUSH TABLES; diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/variable_match_escalation_threshold_global.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/variable_match_escalation_threshold_global.test index b272649c732..065b78d44a7 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/variable_match_escalation_threshold_global.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/variable_match_escalation_threshold_global.test @@ -1,4 +1,4 @@ -# Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> +# Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,12 +22,6 @@ DROP TABLE IF EXISTS diaries; --enable_warnings -# MySQL <= 5.5 reports wrong a warning. :< -# It has been fixed in MySQL >= 5.6 and MariaDB >= 5.3. ---disable_warnings -SET GLOBAL mroonga_match_escalation_threshold = -1; ---enable_warnings - CREATE TABLE diaries ( id INT PRIMARY KEY AUTO_INCREMENT, title TEXT, @@ -40,17 +34,23 @@ INSERT INTO diaries (title, tags) VALUES ("Hello groonga!", "groonga install"); INSERT INTO diaries (title, tags) VALUES ("Hello mroonga!", "mroonga install"); -SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("install" IN BOOLEAN MODE); - SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("gr" IN BOOLEAN MODE); -SET GLOBAL mroonga_match_escalation_threshold = 0; -SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("gr" IN BOOLEAN MODE); +# MySQL <= 5.5 reports a wrong warning. :< +# It has been fixed in MySQL >= 5.6 and MariaDB >= 5.3. +--disable_warnings +SET GLOBAL mroonga_match_escalation_threshold = -1; +--enable_warnings -SET mroonga_match_escalation_threshold = 0; +CONNECT(search_connection, localhost, root); +USE test; SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("gr" IN BOOLEAN MODE); +DISCONNECT search_connection; +CONNECTION default; +SET GLOBAL mroonga_match_escalation_threshold = DEFAULT; DROP TABLE diaries; + --source ../../include/mroonga/have_mroonga_deinit.inc diff --git a/storage/mroonga/mysql-test/mroonga/wrapper/t/variable_match_escalation_threshold_session.test b/storage/mroonga/mysql-test/mroonga/wrapper/t/variable_match_escalation_threshold_session.test index bf22001bb10..b47dcef3967 100644 --- a/storage/mroonga/mysql-test/mroonga/wrapper/t/variable_match_escalation_threshold_session.test +++ b/storage/mroonga/mysql-test/mroonga/wrapper/t/variable_match_escalation_threshold_session.test @@ -38,7 +38,7 @@ SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("install" IN BOOLEAN MODE); SELECT * FROM diaries WHERE MATCH (tags) AGAINST ("gr" IN BOOLEAN MODE); -# MySQL <= 5.5 reports wrong a warning. :< +# MySQL <= 5.5 reports a wrong warning. :< # It has been fixed in MySQL >= 5.6 and MariaDB >= 5.3. --disable_warnings SET mroonga_match_escalation_threshold = -1; diff --git a/storage/mroonga/packages/apt/Makefile.am b/storage/mroonga/packages/apt/Makefile.am index 26f5609c527..ca0e1dcb19d 100644 --- a/storage/mroonga/packages/apt/Makefile.am +++ b/storage/mroonga/packages/apt/Makefile.am @@ -1,7 +1,7 @@ REPOSITORIES_PATH = repositories DISTRIBUTIONS = debian ARCHITECTURES = i386 amd64 -CODE_NAMES = wheezy +CODE_NAMES = wheezy jessie all: diff --git a/storage/mroonga/packages/apt/Vagrantfile b/storage/mroonga/packages/apt/Vagrantfile index 4483412326a..2829da879dd 100644 --- a/storage/mroonga/packages/apt/Vagrantfile +++ b/storage/mroonga/packages/apt/Vagrantfile @@ -14,6 +14,14 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| :id => "debian-wheezy-amd64", :box_url => "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_debian-7.8_chef-provisionerless.box", }, + { + :id => "debian-jessie-i386", + :box_url => "http://packages.groonga.org/tmp/opscode_debian-8.0-i386_chef-provisionerless.box", + }, + { + :id => "debian-jessie-amd64", + :box_url => "http://packages.groonga.org/tmp/opscode_debian-8.0_chef-provisionerless.box", + }, ] vms.each do |vm| diff --git a/storage/mroonga/packages/debian/changelog b/storage/mroonga/packages/debian/changelog index 6aa6fd58d17..d3025d96100 100644 --- a/storage/mroonga/packages/debian/changelog +++ b/storage/mroonga/packages/debian/changelog @@ -1,3 +1,15 @@ +mroonga (5.02-1) unstable; urgency=low + + * New upstream release. + + -- HAYASHI Kentaro <hayashi@clear-code.com> Wed, 29 Apr 2015 00:00:00 +0900 + +mroonga (5.01-1) unstable; urgency=low + + * New upstream release. + + -- HAYASHI Kentaro <hayashi@clear-code.com> Sun, 29 Mar 2015 00:00:00 +0900 + mroonga (5.00-1) unstable; urgency=low * New upstream release. diff --git a/storage/mroonga/packages/rpm/centos/mariadb-mroonga.spec.in b/storage/mroonga/packages/rpm/centos/mariadb-mroonga.spec.in index 337eeccb1cf..ca48e59aaec 100644 --- a/storage/mroonga/packages/rpm/centos/mariadb-mroonga.spec.in +++ b/storage/mroonga/packages/rpm/centos/mariadb-mroonga.spec.in @@ -1,8 +1,8 @@ %define mariadb_epoch_default 1 %define mariadb_version_default 5.5.41 -%define mariadb_release_default 1 +%define mariadb_release_default 2 %define mariadb_dist_default .el7_0 -%define mariadb_download_base_url_default http://vault.centos.org/7.0.1406/updates/Source/SPackages +%define mariadb_download_base_url_default http://vault.centos.org/7.1.1503/os/Source/SPackages %define mariadb_spec_file_default mariadb.spec %{!?mariadb_epoch:%define mariadb_epoch %{mariadb_epoch_default}} @@ -30,7 +30,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-%(%{__id_u} -n) BuildRequires: groonga-devel >= %{groonga_required_version} BuildRequires: groonga-normalizer-mysql-devel BuildRequires: wget -BuildRequires: mariadb-devel +BuildRequires: mariadb-devel = %{mariadb_package_version} Requires: mariadb-server = %{mariadb_package_version} Requires: mariadb = %{mariadb_package_version} Requires: groonga-libs >= %{groonga_required_version} @@ -154,6 +154,12 @@ fi %doc mysql-mroonga-doc/* %changelog +* Wed Apr 29 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 5.02-1 +- new upstream release. + +* Sun Mar 29 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 5.01-1 +- new upstream release. + * Mon Feb 09 2015 <hayashi@clear-code.com> - 5.00-1 - new upstream release. diff --git a/storage/mroonga/packages/rpm/centos/mysql55-mroonga.spec.in b/storage/mroonga/packages/rpm/centos/mysql55-mroonga.spec.in index f1f5b2f4692..1fc641b9bca 100644 --- a/storage/mroonga/packages/rpm/centos/mysql55-mroonga.spec.in +++ b/storage/mroonga/packages/rpm/centos/mysql55-mroonga.spec.in @@ -39,7 +39,7 @@ BuildRequires: groonga-devel >= %{groonga_required_version} BuildRequires: groonga-normalizer-mysql-devel BuildRequires: wget BuildRequires: which -BuildRequires: mysql55-mysql-devel +BuildRequires: mysql55-mysql-devel = %{mysql_version}-%{mysql_release}.%{mysql_dist} BuildRequires: mysql55-build Requires: mysql55-mysql-server = %{mysql_version}-%{mysql_release}.%{mysql_dist} Requires: mysql55-mysql = %{mysql_version}-%{mysql_release}.%{mysql_dist} @@ -163,6 +163,12 @@ fi %doc mysql-mroonga-doc/* %changelog +* Wed Apr 29 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 5.02-1 +- new upstream release. + +* Sun Mar 29 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 5.01-1 +- new upstream release. + * Mon Feb 09 2015 <hayashi@clear-code.com> - 5.00-1 - new upstream release. diff --git a/storage/mroonga/packages/rpm/centos/mysql56-community-mroonga.spec.in b/storage/mroonga/packages/rpm/centos/mysql56-community-mroonga.spec.in index 37ae5d41b2f..53759e0c778 100644 --- a/storage/mroonga/packages/rpm/centos/mysql56-community-mroonga.spec.in +++ b/storage/mroonga/packages/rpm/centos/mysql56-community-mroonga.spec.in @@ -1,14 +1,14 @@ %{!?centos_ver:%define centos_ver 6} %if %{centos_ver} == 7 -%define mysql_version_default 5.6.23 -%define mysql_release_default 2 +%define mysql_version_default 5.6.24 +%define mysql_release_default 3 %define mysql_dist_default el7 %define mysql_download_base_url_default http://repo.mysql.com/yum/mysql-5.6-community/el/7/SRPMS %define mysql_spec_file_default mysql.spec %else -%define mysql_version_default 5.6.23 -%define mysql_release_default 2 +%define mysql_version_default 5.6.24 +%define mysql_release_default 3 %define mysql_dist_default el6 %define mysql_download_base_url_default http://repo.mysql.com/yum/mysql-5.6-community/el/6/SRPMS %define mysql_spec_file_default mysql.spec @@ -38,7 +38,7 @@ BuildRequires: groonga-normalizer-mysql-devel BuildRequires: wget BuildRequires: which BuildRequires: gcc, gcc-c++ -BuildRequires: mysql-community-devel +BuildRequires: mysql-community-devel = %{mysql_version}-%{mysql_release}.%{mysql_dist} Requires: mysql-community-server = %{mysql_version}-%{mysql_release}.%{mysql_dist} Requires: mysql-community-client = %{mysql_version}-%{mysql_release}.%{mysql_dist} Requires: groonga-libs >= %{groonga_required_version} @@ -158,7 +158,16 @@ fi %doc mysql-mroonga-doc/* %changelog -* Mon Feb 09 2015 <hayashi@clear-code.com> - 5.00-1 +* Wed Apr 29 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 5.02-1 +- new upstream release. + +* Fri Apr 10 2015 Kouhei Sutou <kou@clear-code.com> - 5.01-2 +- build against MySQL 5.6.24. + +* Sun Mar 29 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 5.01-1 +- new upstream release. + +* Mon Feb 09 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 5.00-1 - new upstream release. * Wed Feb 04 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 4.10-2 diff --git a/storage/mroonga/packages/rpm/centos/percona-server-56-mroonga.spec.in b/storage/mroonga/packages/rpm/centos/percona-server-56-mroonga.spec.in new file mode 100644 index 00000000000..f0306cace24 --- /dev/null +++ b/storage/mroonga/packages/rpm/centos/percona-server-56-mroonga.spec.in @@ -0,0 +1,156 @@ +%{!?centos_ver:%define centos_ver 6} + +%define mysql_version_default 5.6.23 +%define mysql_release_default rel72.1 +%define mysql_dist_default %{?dist} +%define mysql_download_base_url_default http://repo.percona.com/centos/%{centos_ver}/SRPMS +%define mysql_spec_file_default percona-server.spec + +%{!?mysql_version:%define mysql_version %{mysql_version_default}} +%{!?mysql_release:%define mysql_release %{mysql_release_default}} +%{!?mysql_dist:%define mysql_dist %{mysql_dist_default}} +%{!?mysql_download_base_url:%define mysql_download_base_url %{mysql_download_base_url_default}} +%{!?mysql_spec_file:%define mysql_spec_file %{mysql_spec_file_default}} + +%define groonga_required_version @REQUIRED_GROONGA_VERSION@ + +Name: percona-server-56-mroonga +Version: @VERSION@ +Release: 1%{?dist} +Summary: A fast fulltext searchable storage engine for MySQL + +Group: Applications/Databases +License: LGPLv2.1 +URL: http://mroonga.org/ +Source0: http://packages.groonga.org/source/mroonga/mroonga-%{version}.tar.gz + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-%(%{__id_u} -n) +BuildRequires: groonga-devel >= %{groonga_required_version} +BuildRequires: groonga-normalizer-mysql-devel +BuildRequires: wget +BuildRequires: which +BuildRequires: gcc +BuildRequires: gcc-c++ +BuildRequires: Percona-Server-devel-56 = %{mysql_version}-%{mysql_release}%{mysql_dist} +BuildRequires: selinux-policy-devel +Requires: Percona-Server-server-56 = %{mysql_version}-%{mysql_release}%{mysql_dist} +Requires: Percona-Server-client-56 = %{mysql_version}-%{mysql_release}%{mysql_dist} +Requires: groonga-libs >= %{groonga_required_version} +Requires: groonga-normalizer-mysql + +%description +Mroonga is a fast fulltext searchable storage plugin for MySQL. +It is based on Groonga that is a fast fulltext search engine and +column store. Groonga is good at real-time update. + +%package doc +Summary: Documentation for Mroonga +Group: Documentation +License: LGPLv2.1 + +%description doc +Documentation for Mroonga + + +%prep +%setup -q -n mroonga-%{version} + +mysql_full_version=%{mysql_version}-%{mysql_release}.generic +srpm=Percona-Server-56-${mysql_full_version}.src.rpm +if [ ! -f ../../SRPMS/$srpm ]; then + wget --continue -O ../../SRPMS/$srpm %{mysql_download_base_url}/$srpm + rpm -Uvh ../../SRPMS/$srpm +fi + +%build +mysql_source=../percona-server-%{mysql_version}-$(echo %{mysql_release} | sed -e 's/rel//') +if [ ! -d ${mysql_source} ]; then + specs_dir= + rpmbuild -bp \ + --define 'runselftest 0' \ + --define 'optflags -O0' \ + ../../SPECS/%{mysql_spec_file} +fi +%configure --disable-static --with-mysql-source=${mysql_source} \ + %{?mroonga_configure_options} +make %{?_smp_mflags} + +%install +rm -rf $RPM_BUILD_ROOT +make install DESTDIR=$RPM_BUILD_ROOT +rm $RPM_BUILD_ROOT%{_libdir}/mysql/plugin/*.la +mv $RPM_BUILD_ROOT%{_datadir}/doc/mroonga/ mysql-mroonga-doc/ + +%clean +rm -rf $RPM_BUILD_ROOT + +%post +mysql_command=`which mysql` +password_option="" +$mysql_command -u root -e "quit" +if [ $? -ne 0 ]; then + password_option="-p" +fi +current_version=0 +version=`echo %{groonga_required_version} | sed -e 's/\.//g'` +required_version=`expr $version` +version=`$mysql_command -e "SHOW VARIABLES LIKE 'mroonga_libgroonga_version'" | \ + grep mroonga | cut -f 2 | sed -e 's/\.//g'` +if [ -n "$version" ]; then + current_version=`expr $version` +fi +install_sql=%{_datadir}/mroonga/install.sql +uninstall_sql=%{_datadir}/mroonga/uninstall.sql + +if [ "$1" = 2 ] ; then + if [ $current_version -lt $required_version ]; then + command="$mysql_command -u root $password_option" + echo "run the following command after restarting MySQL server:"; + echo " $command < ${uninstall_sql}" + echo " $command < ${install_sql}" + exit 0 + else + command="$mysql_command -u root $password_option < ${uninstall_sql}" + echo $command + eval $command || \ + (echo "run the following command to unregister Mroonga:"; \ + echo " $command") + fi +fi +command="$mysql_command -u root $password_option < ${install_sql}" +echo $command +eval $command || \ + (echo "run the following command to register Mroonga:"; \ + echo " $command") + +%preun +uninstall_sql=%{_datadir}/mroonga/uninstall.sql +mysql_command=`which mysql` +if $mysql_command -u root -e "quit"; then + password_option="" +else + password_option="-p" +fi +if [ "$1" = 0 ]; then + command="$mysql_command -u root $password_option < ${uninstall_sql}" + echo $command + eval $command || \ + (echo "run the following command to unregister Mroonga:"; \ + echo " $command") +fi + +%files +%defattr(-,root,root,-) +%{_libdir}/mysql/plugin/ +%{_datadir}/mroonga/* +%{_datadir}/man/man1/* +%{_datadir}/man/*/man1/* + +%files doc +%defattr(-,root,root,-) +%doc README COPYING +%doc mysql-mroonga-doc/* + +%changelog +* Tue Mar 17 2015 Kouhei Sutou <kou@clear-code.com> - 5.00-1 +- initial release. diff --git a/storage/mroonga/packages/source/Makefile.am b/storage/mroonga/packages/source/Makefile.am index 30721406f0b..0f242c4e038 100644 --- a/storage/mroonga/packages/source/Makefile.am +++ b/storage/mroonga/packages/source/Makefile.am @@ -1,17 +1,17 @@ MROONGA_BASE = $(PACKAGE)-$(VERSION) MROONGA_TAR_GZ = $(MROONGA_BASE).tar.gz -GROONGA_VERSION = 5.0.0 +GROONGA_VERSION = 5.0.3 GROONGA_BASE = groonga-$(GROONGA_VERSION) GROONGA_TAR_GZ = $(GROONGA_BASE).tar.gz -GROONGA_NORMALIZER_MYSQL_VERSION = 1.0.8 +GROONGA_NORMALIZER_MYSQL_VERSION = 1.0.9 GROONGA_NORMALIZER_MYSQL_BASE = \ groonga-normalizer-mysql-$(GROONGA_NORMALIZER_MYSQL_VERSION) GROONGA_NORMALIZER_MYSQL_TAR_GZ = \ $(GROONGA_NORMALIZER_MYSQL_BASE).tar.gz -MARIADB_VERSION = 10.0.16 +MARIADB_VERSION = 10.0.17 MARIADB_BASE = mariadb-$(MARIADB_VERSION) MARIADB_TAR_GZ = $(MARIADB_BASE).tar.gz diff --git a/storage/mroonga/packages/ubuntu/Makefile.am b/storage/mroonga/packages/ubuntu/Makefile.am index 493419275b3..2297a50bfc9 100644 --- a/storage/mroonga/packages/ubuntu/Makefile.am +++ b/storage/mroonga/packages/ubuntu/Makefile.am @@ -1,4 +1,4 @@ -CODE_NAMES = precise,trusty,utopic +CODE_NAMES = precise,trusty,utopic,vivid SOURCE = ../$(PACKAGE)-$(VERSION).tar.gz all: diff --git a/storage/mroonga/packages/ubuntu/upload.rb b/storage/mroonga/packages/ubuntu/upload.rb index 3331de6d5eb..79331a06c8d 100755 --- a/storage/mroonga/packages/ubuntu/upload.rb +++ b/storage/mroonga/packages/ubuntu/upload.rb @@ -124,6 +124,12 @@ allow_unsigned_uploads = 0 "--distribution", code_name, "--newversion", deb_version, "Build for #{code_name}.") + case code_name + when "vivid" + run_command("sed", + "-i", "-e", "s,5\.5,5\.6,g", + "debian/rules") + end run_command("sed", "-i", "-e", "s,MYSQL_VERSION,#{@mysql_version[code_name]},", "debian/control") diff --git a/storage/mroonga/packages/windows/Makefile.am b/storage/mroonga/packages/windows/Makefile.am index a2ff8f59792..192709fac6d 100644 --- a/storage/mroonga/packages/windows/Makefile.am +++ b/storage/mroonga/packages/windows/Makefile.am @@ -1,12 +1,12 @@ EXTRA_DIST = \ README.md \ - build-vc2010.bat \ - build-vc2010-zip-32.bat \ - build-vc2010-zip-64.bat \ - build-vc2010-msi-32.bat \ - build-vc2010-msi-64.bat \ build-vc2013.bat \ build-vc2013-zip-32.bat \ build-vc2013-zip-64.bat \ build-vc2013-msi-32.bat \ - build-vc2013-msi-64.bat + build-vc2013-msi-64.bat \ + build-vc2015.bat \ + build-vc2015-zip-32.bat \ + build-vc2015-zip-64.bat \ + build-vc2015-msi-32.bat \ + build-vc2015-msi-64.bat diff --git a/storage/mroonga/packages/windows/README.md b/storage/mroonga/packages/windows/README.md index f7788ffe26b..f220634b1e7 100644 --- a/storage/mroonga/packages/windows/README.md +++ b/storage/mroonga/packages/windows/README.md @@ -6,7 +6,7 @@ TODO... ## Build with Visual C++ Express -You need to use Visual C++ 2012 or later to build Mroonga with Express +You need to use Visual C++ 2013 or later to build Mroonga with Express edition. `build-vc2013.bat` is a build batch script to build with Visual C++ Express 2013. @@ -16,5 +16,6 @@ use Professional edition or upper editions to build MSI file. ## Build with Visual C++ Professional You can build both zip file MSI file with Professional edition. -`build-vc2010.bat` is a build batch script to build with Visual C++ -Professional 2010. +But now, this feature is temporary disabled. +If you want to create MSI package, please uncomment in `build-vc2013.bat`. +And then, you can build MSI package with Visual Studio 2013 Professional. diff --git a/storage/mroonga/packages/windows/build-vc2015-msi-32.bat b/storage/mroonga/packages/windows/build-vc2015-msi-32.bat new file mode 100644 index 00000000000..4c92b5c02cc --- /dev/null +++ b/storage/mroonga/packages/windows/build-vc2015-msi-32.bat @@ -0,0 +1,8 @@ +rmdir /S /Q build-vc2015-msi-32 +mkdir build-vc2015-msi-32 +cd build-vc2015-msi-32 +cmake ..\source -G "Visual Studio 14" > config.log +cmake --build . --config RelWithDebInfo > build.log +cmake --build . --config RelWithDebInfo --target msi > msi.log +move *.msi ..\ +cd .. diff --git a/storage/mroonga/packages/windows/build-vc2015-msi-64.bat b/storage/mroonga/packages/windows/build-vc2015-msi-64.bat new file mode 100644 index 00000000000..82bc2a148ec --- /dev/null +++ b/storage/mroonga/packages/windows/build-vc2015-msi-64.bat @@ -0,0 +1,8 @@ +rmdir /S /Q build-vc2015-msi-64 +mkdir build-vc2015-msi-64 +cd build-vc2015-msi-64 +cmake ..\source -G "Visual Studio 14 Win64" > config.log +cmake --build . --config RelWithDebInfo > build.log +cmake --build . --config RelWithDebInfo --target msi > msi.log +move *.msi ..\ +cd .. diff --git a/storage/mroonga/packages/windows/build-vc2015-zip-32.bat b/storage/mroonga/packages/windows/build-vc2015-zip-32.bat new file mode 100644 index 00000000000..5cef259afe5 --- /dev/null +++ b/storage/mroonga/packages/windows/build-vc2015-zip-32.bat @@ -0,0 +1,8 @@ +rmdir /S /Q build-vc2015-zip-32 +mkdir build-vc2015-zip-32 +cd build-vc2015-zip-32 +cmake ..\source -G "Visual Studio 14" -DMRN_GROONGA_EMBED=OFF -DMRN_GROONGA_NORMALIZER_MYSQL_EMBED=OFF > config.log +cmake --build . --config RelWithDebInfo > build.log +cmake --build . --config RelWithDebInfo --target package > zip.log +move *.zip ..\ +cd .. diff --git a/storage/mroonga/packages/windows/build-vc2015-zip-64.bat b/storage/mroonga/packages/windows/build-vc2015-zip-64.bat new file mode 100644 index 00000000000..caabca179e7 --- /dev/null +++ b/storage/mroonga/packages/windows/build-vc2015-zip-64.bat @@ -0,0 +1,8 @@ +rmdir /S /Q build-vc2015-zip-64 +mkdir build-vc2015-zip-64 +cd build-vc2015-zip-64 +cmake ..\source -G "Visual Studio 14 Win64" -DMRN_GROONGA_EMBED=OFF -DMRN_GROONGA_NORMALIZER_MYSQL_EMBED=OFF > config.log +cmake --build . --config RelWithDebInfo > build.log +cmake --build . --config RelWithDebInfo --target package > zip.log +move *.zip ..\ +cd .. diff --git a/storage/mroonga/packages/windows/build-vc2015.bat b/storage/mroonga/packages/windows/build-vc2015.bat new file mode 100644 index 00000000000..f9ac1765792 --- /dev/null +++ b/storage/mroonga/packages/windows/build-vc2015.bat @@ -0,0 +1,4 @@ +build-vc2015-zip-32.bat +build-vc2015-zip-64.bat +REM build-vc2015-msi-32.bat +REM build-vc2015--msi-64.bat diff --git a/storage/mroonga/packages/yum/Makefile.am b/storage/mroonga/packages/yum/Makefile.am index b110b478a14..8321619868f 100644 --- a/storage/mroonga/packages/yum/Makefile.am +++ b/storage/mroonga/packages/yum/Makefile.am @@ -1,7 +1,7 @@ REPOSITORIES_PATH = repositories DISTRIBUTIONS = centos ARCHITECTURES = i386 x86_64 -MYSQL_VARIANTS = mysql55 mysql56-community mariadb +MYSQL_VARIANTS = mysql55 mysql56-community mariadb percona-server-56 SPEC_DIR = $(builddir)/../rpm/centos all: @@ -61,3 +61,4 @@ $(abs_top_builddir)/$(PACKAGE)-$(VERSION).tar.gz: specs: $(SPEC_DIR)/mysql55-$(PACKAGE).spec specs: $(SPEC_DIR)/mysql56-community-$(PACKAGE).spec specs: $(SPEC_DIR)/mariadb-$(PACKAGE).spec +specs: $(SPEC_DIR)/percona-server-56-$(PACKAGE).spec diff --git a/storage/mroonga/packages/yum/Vagrantfile b/storage/mroonga/packages/yum/Vagrantfile index 1a9e4584ee4..da41350eed3 100644 --- a/storage/mroonga/packages/yum/Vagrantfile +++ b/storage/mroonga/packages/yum/Vagrantfile @@ -24,7 +24,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| }, { :id => "centos-7-x86_64", - :box_url => "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_centos-7.0_chef-provisionerless.box", + :box_url => "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_centos-7.1_chef-provisionerless.box", }, ] diff --git a/storage/mroonga/packages/yum/build-in-vm.sh b/storage/mroonga/packages/yum/build-in-vm.sh index 5dc27ec38dd..cf9ef581fd0 100755 --- a/storage/mroonga/packages/yum/build-in-vm.sh +++ b/storage/mroonga/packages/yum/build-in-vm.sh @@ -38,6 +38,9 @@ for mysql_variant in ${MYSQL_VARIANTS}; do mariadb) centos_versions="7" ;; + percona-server-56) + centos_versions="6 7" + ;; esac for architecture in ${architectures}; do diff --git a/storage/mroonga/packages/yum/build-rpm.sh b/storage/mroonga/packages/yum/build-rpm.sh index 6eaa2cce02a..8661e659390 100755 --- a/storage/mroonga/packages/yum/build-rpm.sh +++ b/storage/mroonga/packages/yum/build-rpm.sh @@ -87,6 +87,12 @@ case ${distribution} in mariadb-${PACKAGE}) run yum -y install mariadb-devel ;; + percona-server-56-${PACKAGE}) + release_rpm_version=0.1-3 + release_rpm=percona-release-${release_rpm_version}.noarch.rpm + run yum -y install http://www.percona.com/downloads/percona-release/redhat/${release_rpm_version}/${release_rpm} + run yum -y install Percona-Server-devel-56 + ;; esac release_rpm=groonga-release-1.1.0-1.noarch.rpm @@ -98,6 +104,11 @@ case ${distribution} in esac run yum install -y ${DEPENDED_PACKAGES} +if [ "${package_name}" = "percona-server-56-${PACKAGE}" ]; then + if [ "${distribution_version}" = "7" ]; then + rpmbuild_options="$rpmbuild_options --define 'dist .el7'" + fi +fi if [ "${USE_MYSQLSERVICES_COMPAT}" = "yes" ]; then rpmbuild_options="$rpmbuild_options --define 'mroonga_configure_options --with-libmysqlservices-compat'" fi diff --git a/storage/mroonga/packages/yum/env.sh.in b/storage/mroonga/packages/yum/env.sh.in index 90e701ec89e..8c6d05baf5c 100644 --- a/storage/mroonga/packages/yum/env.sh.in +++ b/storage/mroonga/packages/yum/env.sh.in @@ -22,6 +22,7 @@ perl-Time-HiRes perl-Env perl-Test-Simple pam-devel +selinux-policy-devel groonga-devel groonga-normalizer-mysql-devel " diff --git a/storage/mroonga/plugin_version b/storage/mroonga/plugin_version index 6e636605163..341d0b550fd 100644 --- a/storage/mroonga/plugin_version +++ b/storage/mroonga/plugin_version @@ -1 +1 @@ -5.0
\ No newline at end of file +5.2
\ No newline at end of file diff --git a/storage/mroonga/required_groonga_normalizer_mysql_version b/storage/mroonga/required_groonga_normalizer_mysql_version index af0b7ddbffd..66c4c2263e5 100644 --- a/storage/mroonga/required_groonga_normalizer_mysql_version +++ b/storage/mroonga/required_groonga_normalizer_mysql_version @@ -1 +1 @@ -1.0.6 +1.0.9 diff --git a/storage/mroonga/required_groonga_version b/storage/mroonga/required_groonga_version index 43beb4001b8..a1ef0cae183 100644 --- a/storage/mroonga/required_groonga_version +++ b/storage/mroonga/required_groonga_version @@ -1 +1 @@ -4.0.7 +5.0.2 diff --git a/storage/mroonga/sources.am b/storage/mroonga/sources.am index 0cf4a64b504..c7ddcfa5acc 100644 --- a/storage/mroonga/sources.am +++ b/storage/mroonga/sources.am @@ -7,5 +7,6 @@ sources = \ mrn_table.hpp \ mrn_err.h \ mrn_mysql.h \ - mrn_mysql_compat.h \ + mrn_mysql_compat.h \ + mrn_variables.hpp \ ha_mroonga.def diff --git a/storage/mroonga/test/unit/test_mrn_path_mapper.cpp b/storage/mroonga/test/unit/test_mrn_path_mapper.cpp index 70009c5b32e..316c4770115 100644 --- a/storage/mroonga/test/unit/test_mrn_path_mapper.cpp +++ b/storage/mroonga/test/unit/test_mrn_path_mapper.cpp @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2012 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -112,5 +112,23 @@ namespace test_mrn_path_mapper { cppcut_assert_equal("_table", mapper.mysql_table_name()); } } + + namespace mysql_path { + void test_normal_table() { + mrn::PathMapper mapper("./db/table"); + cppcut_assert_equal("./db/table", mapper.mysql_path()); + } + + void test_temporary_table() { + mrn::PathMapper mapper("/tmp/mysqld.1/#sql27c5_1_0"); + cppcut_assert_equal("/tmp/mysqld.1/#sql27c5_1_0", + mapper.mysql_path()); + } + + void test_partition_table_path() { + mrn::PathMapper mapper("./db/table#P#p1"); + cppcut_assert_equal("./db/table", mapper.mysql_path()); + } + } } diff --git a/storage/mroonga/tools/travis/before_script.sh b/storage/mroonga/tools/travis/before_script.sh index 8fd3f78f1db..7d4d7dcec5f 100755 --- a/storage/mroonga/tools/travis/before_script.sh +++ b/storage/mroonga/tools/travis/before_script.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright(C) 2012-2014 Kouhei Sutou <kou@clear-code.com> +# Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -20,17 +20,17 @@ set -e if [ "${MROONGA_BUNDLED}" = "yes" ]; then - cmake_args=(-DCMAKE_BUILD_TYPE=Debug) - if [ "${MROONGA_TEST_EMBEDDED}" = "yes" ]; then - cmake_args=("${cmake_args[@]}" "-DWITH_EMBEDDED_SERVER=TRUE") - fi - cmake . "${cmake_args[@]}" + cmake_args=(-DCMAKE_BUILD_TYPE=Debug) + if [ "${MROONGA_TEST_EMBEDDED}" = "yes" ]; then + cmake_args=("${cmake_args[@]}" "-DWITH_EMBEDDED_SERVER=TRUE") + fi + cmake . "${cmake_args[@]}" else - ./autogen.sh + ./autogen.sh - if [ -d /opt/mysql/ ]; then - PATH=$(echo /opt/mysql/server-*/bin/):$PATH - fi - ./configure \ - --with-mysql-source=$PWD/vendor/mysql + if [ -d /opt/mysql/ ]; then + PATH=$(echo /opt/mysql/server-*/bin/):$PATH + fi + ./configure \ + --with-mysql-source=$PWD/vendor/mysql fi diff --git a/storage/mroonga/tools/travis/install.sh b/storage/mroonga/tools/travis/install.sh index 95b8b23ba19..d9b090cd154 100755 --- a/storage/mroonga/tools/travis/install.sh +++ b/storage/mroonga/tools/travis/install.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright(C) 2012-2013 Kouhei Sutou <kou@clear-code.com> +# Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -21,85 +21,94 @@ set -e mariadb_download_base=http://mirror.jmu.edu/pub/mariadb -# export GROONGA_MASTER=yes -# export GROONGA_NORMALIZER_MYSQL_MASTER=yes +export GROONGA_MASTER=yes +export GROONGA_NORMALIZER_MYSQL_MASTER=yes -curl --silent --location https://github.com/groonga/groonga/raw/master/data/travis/setup.sh | sh -curl --silent --location https://github.com/groonga/groonga-normalizer-mysql/raw/master/data/travis/setup.sh | sh -# curl --silent --location https://github.com/clear-code/cutter/raw/master/data/travis/setup.sh | sh +if [ "${MROONGA_BUNDLED}" = "yes" ]; then + mkdir -p .mroonga + mv * .mroonga/ + mv .mroonga/tools ./ + sudo apt-get -qq -y build-dep mysql-server + # Support MariaDB for now. + download_base=${mariadb_download_base}/${MYSQL_VERSION} + tar_gz=${MYSQL_VERSION}.tar.gz + curl -O ${download_base}/source/${tar_gz} + tar xzf $tar_gz + mv ${MYSQL_VERSION}/* ./ + rm -rf storage/mroonga + mv .mroonga storage/mroonga + rm -rf ${MYSQL_VERSION} + git clone --recursive --depth 1 \ + https://github.com/groonga/groonga.git \ + storage/mroonga/vendor/groonga + git clone --recursive --depth 1 \ + https://github.com/groonga/groonga-normalizer-mysql.git \ + storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql +else + curl --silent --location \ + https://github.com/groonga/groonga/raw/master/data/travis/setup.sh | sh + curl --silent --location \ + https://github.com/groonga/groonga-normalizer-mysql/raw/master/data/travis/setup.sh | sh + # curl --silent --location \ + # https://github.com/clear-code/cutter/raw/master/data/travis/setup.sh | sh -if [ ! -f /usr/lib/groonga/plugins/tokenizers/mecab.so ]; then + if [ ! -f /usr/lib/groonga/plugins/tokenizers/mecab.so ]; then sudo apt-get -qq -y install groonga-tokenizer-mecab -fi + fi -if [ "${MROONGA_BUNDLED}" = "yes" ]; then - mkdir -p .mroonga - mv * .mroonga/ - mv .mroonga/tools ./ - sudo apt-get -qq -y build-dep mysql-server - # Support MariaDB for now. - download_base=${mariadb_download_base}/${MYSQL_VERSION} - tar_gz=${MYSQL_VERSION}.tar.gz - curl -O ${download_base}/source/${tar_gz} - tar xzf $tar_gz - mv ${MYSQL_VERSION}/* ./ - rm -rf storage/mroonga - mv .mroonga storage/mroonga - rm -rf ${MYSQL_VERSION} -else - mkdir -p vendor - cd vendor + mkdir -p vendor + cd vendor - version=$(echo "$MYSQL_VERSION" | sed -e 's/^\(mysql\|mariadb\)-//') - series=$(echo "$version" | sed -e 's/\.[0-9]*\(-\?[a-z]*\)\?$//g') - case "$MYSQL_VERSION" in - mysql-*) - sudo apt-get -qq update - sudo apt-get -qq -y build-dep mysql-server - if [ "$version" = "system" ]; then - sudo apt-get -qq -y install \ - mysql-server mysql-server-5.5 mysql-server-core-5.5 \ - mysql-testsuite libmysqld-dev - apt-get -qq source mysql-server - ln -s $(find . -maxdepth 1 -type d | sort | tail -1) mysql - else - download_base="http://cdn.mysql.com/Downloads/MySQL-${series}/" - if [ "$(uname -m)" = "x86_64" ]; then - architecture=x86_64 - else - architecture=i686 - fi - deb=mysql-${version}-debian6.0-${architecture}.deb - tar_gz=mysql-${version}.tar.gz - curl -O ${download_base}${deb} & - curl -O ${download_base}${tar_gz} & - wait - sudo apt-get -qq -y install libaio1 - sudo dpkg -i $deb - tar xzf $tar_gz - ln -s mysql-${version} mysql - fi - ;; - mariadb-*) - sudo apt-get -qq -y remove --purge mysql-common + version=$(echo "$MYSQL_VERSION" | sed -e 's/^\(mysql\|mariadb\)-//') + series=$(echo "$version" | sed -e 's/\.[0-9]*\(-\?[a-z]*\)\?$//g') + case "$MYSQL_VERSION" in + mysql-*) + sudo apt-get -qq update + sudo apt-get -qq -y build-dep mysql-server + if [ "$version" = "system" ]; then + sudo apt-get -qq -y install \ + mysql-server mysql-server-5.5 mysql-server-core-5.5 \ + mysql-testsuite libmysqld-dev + apt-get -qq source mysql-server + ln -s $(find . -maxdepth 1 -type d | sort | tail -1) mysql + else + download_base="http://cdn.mysql.com/Downloads/MySQL-${series}/" + if [ "$(uname -m)" = "x86_64" ]; then + architecture=x86_64 + else + architecture=i686 + fi + deb=mysql-${version}-debian6.0-${architecture}.deb + tar_gz=mysql-${version}.tar.gz + curl -O ${download_base}${deb} & + curl -O ${download_base}${tar_gz} & + wait + sudo apt-get -qq -y install libaio1 + sudo dpkg -i $deb + tar xzf $tar_gz + ln -s mysql-${version} mysql + fi + ;; + mariadb-*) + sudo apt-get -qq -y remove --purge mysql-common - distribution=$(lsb_release --short --id | tr 'A-Z' 'a-z') - code_name=$(lsb_release --short --codename) - component=main - apt_url_base="${mariadb_download_base}/repo/${series}" - cat <<EOF | sudo tee /etc/apt/sources.list.d/mariadb.list + distribution=$(lsb_release --short --id | tr 'A-Z' 'a-z') + code_name=$(lsb_release --short --codename) + component=main + apt_url_base="${mariadb_download_base}/repo/${series}" + cat <<EOF | sudo tee /etc/apt/sources.list.d/mariadb.list deb ${apt_url_base}/${distribution}/ ${code_name} ${component} deb-src ${apt_url_base}/${distribution}/ ${code_name} ${component} EOF - sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db - sudo apt-get -qq update - sudo apt-get -qq -y build-dep mariadb-server - sudo apt-get -qq -y install \ - mariadb-server libmariadbclient-dev mariadb-test - apt-get -qq source mariadb-server - ln -s $(find . -maxdepth 1 -type d | sort | tail -1) mysql - ;; - esac + sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db + sudo apt-get -qq update + sudo apt-get -qq -y build-dep mariadb-server + sudo apt-get -qq -y install \ + mariadb-server libmariadbclient-dev mariadb-test + apt-get -qq source mariadb-server + ln -s $(find . -maxdepth 1 -type d | sort | tail -1) mysql + ;; + esac - cd .. + cd .. fi diff --git a/storage/mroonga/tools/travis/script.sh b/storage/mroonga/tools/travis/script.sh index 632a4db7a7e..6ea5c86068b 100755 --- a/storage/mroonga/tools/travis/script.sh +++ b/storage/mroonga/tools/travis/script.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright(C) 2012-2014 Kouhei Sutou <kou@clear-code.com> +# Copyright(C) 2012-2015 Kouhei Sutou <kou@clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -24,95 +24,95 @@ top_dir="${base_dir}/../.." bundled_mroonga_dir="${top_dir}/storage/mroonga" if [ -f "${bundled_mroonga_dir}/config.sh" ]; then - mroonga_dir="${bundled_mroonga_dir}" - . "${bundled_mroonga_dir}/config.sh" + mroonga_dir="${bundled_mroonga_dir}" + . "${bundled_mroonga_dir}/config.sh" else - mroonga_dir="${top_dir}" - . "${top_dir}/config.sh" + mroonga_dir="${top_dir}" + . "${top_dir}/config.sh" fi n_processors="$(grep '^processor' /proc/cpuinfo | wc -l)" max_n_processors=8 if (( $n_processors > $max_n_processors )); then - n_processors=$max_n_processors + n_processors=$max_n_processors fi build() { - if [ "${MROONGA_BUNDLED}" = "yes" ]; then - make > /dev/null - else - make -j${n_processors} > /dev/null - fi + if [ "${MROONGA_BUNDLED}" = "yes" ]; then + make -j${n_processors} > /dev/null + else + make -j${n_processors} > /dev/null + fi } run_unit_test() { - if [ "${MROONGA_BUNDLED}" != "yes" ]; then - NO_MAKE=yes ${mroonga_dir}/test/run-unit-test.sh - fi + if [ "${MROONGA_BUNDLED}" != "yes" ]; then + NO_MAKE=yes ${mroonga_dir}/test/run-unit-test.sh + fi } prepare_mysql_test_dir() { - mysql_test_dir=/usr/mysql-test - if [ -d /usr/lib/mysql-testsuite/ ]; then - sudo cp -a /usr/lib/mysql-testsuite/ ${mysql_test_dir}/ - elif [ -d /usr/share/mysql/mysql-test/ ]; then - sudo cp -a /usr/share/mysql/mysql-test/ ${mysql_test_dir}/ - elif [ -d /opt/mysql/ ]; then - mysql_test_dir=$(echo /opt/mysql/server-*/mysql-test) - else - sudo cp -a ${MYSQL_SOURCE_DIR}/mysql-test/ ${mysql_test_dir}/ - fi - sudo chown -R $(id -u):$(id -g) ${mysql_test_dir}/ + mysql_test_dir=/usr/mysql-test + if [ -d /usr/lib/mysql-testsuite/ ]; then + sudo cp -a /usr/lib/mysql-testsuite/ ${mysql_test_dir}/ + elif [ -d /usr/share/mysql/mysql-test/ ]; then + sudo cp -a /usr/share/mysql/mysql-test/ ${mysql_test_dir}/ + elif [ -d /opt/mysql/ ]; then + mysql_test_dir=$(echo /opt/mysql/server-*/mysql-test) + else + sudo cp -a ${MYSQL_SOURCE_DIR}/mysql-test/ ${mysql_test_dir}/ + fi + sudo chown -R $(id -u):$(id -g) ${mysql_test_dir}/ - cp -a ${mroonga_dir}/mysql-test/mroonga/ ${mysql_test_dir}/suite/ + cp -a ${mroonga_dir}/mysql-test/mroonga/ ${mysql_test_dir}/suite/ } collect_test_suite_names() { - cd ${mysql_test_dir}/suite/ - test_suite_names="" - for test_suite_name in $(find mroonga -type d '!' -name '[tr]'); do - if [ -n "${test_suite_names}" ]; then - test_suite_names="${test_suite_names}," - fi - test_suite_names="${test_suite_names}${test_suite_name}" - done - cd - + cd ${mysql_test_dir}/suite/ + test_suite_names="" + for test_suite_name in $(find mroonga -type d '!' -name '[tr]'); do + if [ -n "${test_suite_names}" ]; then + test_suite_names="${test_suite_names}," + fi + test_suite_names="${test_suite_names}${test_suite_name}" + done + cd - } prepare_sql_test() { - sudo make install > /dev/null - prepare_mysql_test_dir - collect_test_suite_names + sudo make install > /dev/null + prepare_mysql_test_dir + collect_test_suite_names } run_sql_test() { - test_args=() - if [ "${MROONGA_TEST_EMBEDDED}" = "yes" ]; then - test_args=("${test_args[@]}" "--embedded-server") - fi + test_args=() + if [ "${MROONGA_TEST_EMBEDDED}" = "yes" ]; then + test_args=("${test_args[@]}" "--embedded-server") + fi - if [ "${MROONGA_BUNDLED}" = "yes" ]; then - ${mroonga_dir}/test/run-sql-test.sh \ - "${test_args[@]}" \ - --parallel="${n_processors}" - else - prepare_sql_test + if [ "${MROONGA_BUNDLED}" = "yes" ]; then + ${mroonga_dir}/test/run-sql-test.sh \ + "${test_args[@]}" \ + --parallel="${n_processors}" + else + prepare_sql_test - cd ${mysql_test_dir}/ - ./mysql-test-run.pl \ - "${test_args[@]}" \ - --no-check-testcases \ - --parallel="${n_processors}" \ - --retry=1 \ - --suite="${test_suite_names}" \ - --force - fi + cd ${mysql_test_dir}/ + ./mysql-test-run.pl \ + "${test_args[@]}" \ + --no-check-testcases \ + --parallel="${n_processors}" \ + --retry=1 \ + --suite="${test_suite_names}" \ + --force + fi } build diff --git a/storage/mroonga/udf/mrn_udf_command.cpp b/storage/mroonga/udf/mrn_udf_command.cpp index 172f64896a2..d14f3ffd49d 100644 --- a/storage/mroonga/udf/mrn_udf_command.cpp +++ b/storage/mroonga/udf/mrn_udf_command.cpp @@ -25,6 +25,7 @@ #include <mrn_windows.hpp> #include <mrn_macro.hpp> #include <mrn_database_manager.hpp> +#include <mrn_variables.hpp> MRN_BEGIN_DECLS @@ -58,8 +59,8 @@ MRN_API my_bool mroonga_command_init(UDF_INIT *initid, UDF_ARGS *args, initid->maybe_null = 1; initid->const_item = 1; - info = (CommandInfo *)my_malloc(sizeof(CommandInfo), - MYF(MY_WME | MY_ZEROFILL)); + info = (CommandInfo *)mrn_my_malloc(sizeof(CommandInfo), + MYF(MY_WME | MY_ZEROFILL)); if (!info) { strcpy(message, "mroonga_command(): out of memory"); goto error; @@ -67,7 +68,7 @@ MRN_API my_bool mroonga_command_init(UDF_INIT *initid, UDF_ARGS *args, grn_ctx_init(&(info->ctx), 0); { - const char *current_db_path = current_thd->db; + const char *current_db_path = MRN_THD_DB_PATH(current_thd); const char *action; if (current_db_path) { action = "open database"; @@ -163,7 +164,7 @@ MRN_API void mroonga_command_deinit(UDF_INIT *initid) grn_obj_close(&(info->ctx), info->db); } grn_ctx_fin(&(info->ctx)); - info->result.free(); + MRN_STRING_FREE(info->result); my_free(info); } } diff --git a/storage/mroonga/udf/mrn_udf_escape.cpp b/storage/mroonga/udf/mrn_udf_escape.cpp index 36179788596..89b08a1c6d5 100644 --- a/storage/mroonga/udf/mrn_udf_escape.cpp +++ b/storage/mroonga/udf/mrn_udf_escape.cpp @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2; indent-tabs-mode: nil -*- */ /* - Copyright(C) 2013 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2013-2015 Kouhei Sutou <kou@clear-code.com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -22,6 +22,7 @@ #include <mrn_path_mapper.hpp> #include <mrn_windows.hpp> #include <mrn_macro.hpp> +#include <mrn_variables.hpp> MRN_BEGIN_DECLS @@ -62,8 +63,8 @@ MRN_API my_bool mroonga_escape_init(UDF_INIT *initid, UDF_ARGS *args, initid->maybe_null = 1; initid->const_item = 1; - info = (EscapeInfo *)my_malloc(sizeof(EscapeInfo), - MYF(MY_WME | MY_ZEROFILL)); + info = (EscapeInfo *)mrn_my_malloc(sizeof(EscapeInfo), + MYF(MY_WME | MY_ZEROFILL)); if (!info) { strcpy(message, "mroonga_escape(): out of memory"); goto error; diff --git a/storage/mroonga/udf/mrn_udf_snippet.cpp b/storage/mroonga/udf/mrn_udf_snippet.cpp index 3eaf05eed95..22ec0884014 100644 --- a/storage/mroonga/udf/mrn_udf_snippet.cpp +++ b/storage/mroonga/udf/mrn_udf_snippet.cpp @@ -26,6 +26,7 @@ #include <mrn_windows.hpp> #include <mrn_table.hpp> #include <mrn_macro.hpp> +#include <mrn_variables.hpp> MRN_BEGIN_DECLS @@ -170,8 +171,8 @@ MRN_API my_bool mroonga_snippet_init(UDF_INIT *initid, UDF_ARGS *args, char *mes initid->maybe_null = 1; initid->const_item = 1; - if (!(snip_info = (st_mrn_snip_info *) my_malloc(sizeof(st_mrn_snip_info), - MYF(MY_WME | MY_ZEROFILL)))) + if (!(snip_info = (st_mrn_snip_info *) mrn_my_malloc(sizeof(st_mrn_snip_info), + MYF(MY_WME | MY_ZEROFILL)))) { strcpy(message, "mroonga_snippet() out of memory"); goto error; @@ -292,7 +293,7 @@ MRN_API void mroonga_snippet_deinit(UDF_INIT *initid) if (snip_info->snippet) { grn_obj_close(&snip_info->ctx, snip_info->snippet); } - snip_info->result_str.free(); + MRN_STRING_FREE(snip_info->result_str); grn_obj_close(&snip_info->ctx, grn_ctx_db(&snip_info->ctx)); grn_ctx_fin(&snip_info->ctx); my_free(snip_info); diff --git a/storage/mroonga/vendor/groonga/CMakeLists.txt b/storage/mroonga/vendor/groonga/CMakeLists.txt index cddb3df28f6..85a84f91ef0 100644 --- a/storage/mroonga/vendor/groonga/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/CMakeLists.txt @@ -20,6 +20,24 @@ cmake_minimum_required(VERSION 2.6.2) set(GRN_PROJECT_NAME "groonga") project("${GRN_PROJECT_NAME}") +if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") + set(GRN_BUNDLED FALSE) +else() + set(GRN_BUNDLED TRUE) +endif() + +if(MSVC) + if(MSVC_VERSION LESS 1800) + set(GRN_OLD_MSVC_MESSAGE "Groonga supports only MSVC 2013 or later") + if(GRN_BUNDLED) + message(STATUS ${GRN_OLD_MSVC_MESSAGE}) + return() + else() + message(FATAL_ERROR ${GRN_OLD_MSVC_MESSAGE}) + endif() + endif() +endif() + if(CMAKE_C_COMPILER_ID STREQUAL "Clang") set(CMAKE_COMPILER_IS_CLANGC ON) endif() @@ -119,6 +137,12 @@ set(GRN_RELATIVE_RUBY_SCRIPTS_DIR set(GRN_RUBY_SCRIPTS_DIR "${CMAKE_INSTALL_PREFIX}/${GRN_RELATIVE_RUBY_SCRIPTS_DIR}") + +if(CMAKE_COMPILER_IS_GNUCC) + set(GRN_C_COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS} -std=gnu99") +endif() + + macro(check_cflag flag) string(REGEX REPLACE "[-=]" "_" temporary_variable_name ${flag}) string(TOUPPER "${temporary_variable_name}" temporary_variable_name) @@ -168,7 +192,8 @@ if(CMAKE_COMPILER_IS_GNUCXX) check_build_flag("-Wno-clobbered") endif() -if(GRN_EMBED) +if(NOT DEFINED CMAKE_C_COMPILE_OPTIONS_PIC) + # For old CMake if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGCXX) check_build_flag("-fPIC") endif() @@ -480,48 +505,57 @@ endif() set(GRN_WITH_MESSAGE_PACK "auto" CACHE STRING "use MessagePack for suggestion") if(NOT ${GRN_WITH_MESSAGE_PACK} STREQUAL "no") - if("${GRN_WITH_MESSAGE_PACK}" STREQUAL "yes" OR - "${GRN_WITH_MESSAGE_PACK}" STREQUAL "auto") - set(MESSAGE_PACK_INCLUDE_DIRS "") - set(MESSAGE_PACK_LIBRARY_DIRS "") - else() - set(MESSAGE_PACK_INCLUDE_DIRS "${GRN_WITH_MESSAGE_PACK}/include") - set(MESSAGE_PACK_LIBRARY_DIRS "${GRN_WITH_MESSAGE_PACK}/lib") - endif() - set(CMAKE_REQUIRED_INCLUDES_SAVE ${CMAKE_REQUIRED_INCLUDES}) - ac_check_lib(msgpack msgpack_version "${MESSAGE_PACK_LIBRARY_DIRS}") - set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_SAVE}) - if(HAVE_LIBMSGPACK) - set(MESSAGE_PACK_LIBRARIES "msgpack") + pkg_check_modules(MESSAGE_PACK msgpack) + if(MESSAGE_PACK_FOUND) set(GRN_WITH_MESSAGE_PACK TRUE) else() - if(${GRN_WITH_MESSAGE_PACK} STREQUAL "yes") - message(FATAL_ERROR "No MessagePack found") + if("${GRN_WITH_MESSAGE_PACK}" STREQUAL "yes" OR + "${GRN_WITH_MESSAGE_PACK}" STREQUAL "auto") + set(MESSAGE_PACK_INCLUDE_DIRS "") + set(MESSAGE_PACK_LIBRARY_DIRS "") + else() + set(MESSAGE_PACK_INCLUDE_DIRS "${GRN_WITH_MESSAGE_PACK}/include") + set(MESSAGE_PACK_LIBRARY_DIRS "${GRN_WITH_MESSAGE_PACK}/lib") + endif() + set(CMAKE_REQUIRED_INCLUDES_SAVE ${CMAKE_REQUIRED_INCLUDES}) + ac_check_lib(msgpack msgpack_version "${MESSAGE_PACK_LIBRARY_DIRS}") + set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_SAVE}) + if(HAVE_LIBMSGPACK) + set(MESSAGE_PACK_LIBRARIES "msgpack") + set(GRN_WITH_MESSAGE_PACK TRUE) + else() + if(${GRN_WITH_MESSAGE_PACK} STREQUAL "yes") + message(FATAL_ERROR "No MessagePack found") + endif() + set(GRN_WITH_MESSAGE_PACK FALSE) endif() - set(GRN_WITH_MESSAGE_PACK FALSE) endif() else() set(GRN_WITH_MESSAGE_PACK FALSE) endif() +find_program(RUBY NAMES "ruby2.1" "ruby21" "ruby") + option(GRN_WITH_MRUBY "use mruby" OFF) if(GRN_WITH_MRUBY) - if(CMAKE_VERSION VERSION_LESS "2.8.8") - message(FATAL_ERROR - "Your CMake (${CMAKE_VERSION}) is old. " - "CMake 2.8.8 or later is required for mruby supported build") - endif() set(MRUBY_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/vendor/mruby-source/include") - set(MRUBY_LIBS "$<TARGET_OBJECTS:mruby>") + set(MRUBY_LIBS mruby) else() set(MRUBY_INCLUDE_DIRS "") set(MRUBY_LIBS "") endif() -if(NOT GRN_EMBED) - add_subdirectory(vendor) -endif() +# TODO: Support using system Onigmo instead of bundled Onigmo. +# set(GRN_WITH_ONIGMO ON) +# set(ONIGMO_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/vendor/onigmo-source") +# set(ONIGMO_LIBS onigmo) + +# TODO: It's for Onigmo static link case. If we support system Onigmo, +# we need to remove it and add -DEXPORT to Onigmo build. +add_definitions(-DONIG_EXTERN=extern) + +add_subdirectory(vendor) add_subdirectory(lib) if(NOT GRN_EMBED) add_subdirectory(src) @@ -532,7 +566,6 @@ endif() configure_file(config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) -set(RUBY "ruby") # TODO: support customization set(GROONGA "${CMAKE_CURRENT_BINARY_DIR}/src/groonga") set(GROONGA_SUGGEST_CREATE_DATASET "${CMAKE_CURRENT_BINARY_DIR}/src/suggest/groonga-suggest-create-dataset") diff --git a/storage/mroonga/vendor/groonga/COPYING b/storage/mroonga/vendor/groonga/COPYING new file mode 100644 index 00000000000..4362b49151d --- /dev/null +++ b/storage/mroonga/vendor/groonga/COPYING @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + <one line to give the library's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + <signature of Ty Coon>, 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/storage/mroonga/vendor/groonga/Makefile.am b/storage/mroonga/vendor/groonga/Makefile.am index 65b7c928770..6f760a52b30 100644 --- a/storage/mroonga/vendor/groonga/Makefile.am +++ b/storage/mroonga/vendor/groonga/Makefile.am @@ -85,7 +85,7 @@ update-po: update-document: @if test -z "$(GROONGA_ORG_PATH)"; then \ echo "\$$(GROONGA_ORG_PATH) is missing"; \ - echo "add --with-groonga-github-com-path in configure"; \ + echo "add --with-groonga-org-path in configure"; \ exit 1; \ fi rm -rf tmp-doc diff --git a/storage/mroonga/vendor/groonga/appveyor.yml b/storage/mroonga/vendor/groonga/appveyor.yml index aa4b65fb24e..d755e6b49fd 100644 --- a/storage/mroonga/vendor/groonga/appveyor.yml +++ b/storage/mroonga/vendor/groonga/appveyor.yml @@ -1,8 +1,9 @@ version: "{build}" clone_depth: 10 build_script: - - cmake . -G "Visual Studio 12 Win64" - - cmake --build . --config RelWithDebInfo + - git submodule update --init + - cmake . -G "Visual Studio 12 2013 Win64" + - cmake --build . --config Debug notifications: - provider: Email @@ -11,7 +12,8 @@ notifications: - groonga-commit@lists.sourceforge.jp on_build_status_changed: true -before_test: - - gem install grntest -test_script: - - grntest --groonga src\groonga.exe --base-directory test\command test\command\suite +test: off +# before_test: +# - gem install grntest +# test_script: +# - grntest --groonga src\groonga.exe --base-directory test\command test\command\suite diff --git a/storage/mroonga/vendor/groonga/base_version b/storage/mroonga/vendor/groonga/base_version index 28cbf7c0aae..26611488b0a 100644 --- a/storage/mroonga/vendor/groonga/base_version +++ b/storage/mroonga/vendor/groonga/base_version @@ -1 +1 @@ -5.0.0
\ No newline at end of file +5.0.3
\ No newline at end of file diff --git a/storage/mroonga/vendor/groonga/bindings/php/groonga.dsp b/storage/mroonga/vendor/groonga/bindings/php/groonga.dsp new file mode 100644 index 00000000000..dd7c91ce736 --- /dev/null +++ b/storage/mroonga/vendor/groonga/bindings/php/groonga.dsp @@ -0,0 +1,112 @@ +# Microsoft Developer Studio Project File - Name="groonga" - Package Owner=<4> +# Microsoft Developer Studio Generated Build File, Format Version 6.00 +# ** DO NOT EDIT ** + +# TARGTYPE "Win32 (x86) Dynamic-Link Library" 0x0102 + +CFG=groonga - Win32 Debug_TS +!MESSAGE This is not a valid makefile. To build this project using NMAKE, +!MESSAGE use the Export Makefile command and run +!MESSAGE +!MESSAGE NMAKE /f "groonga.mak". +!MESSAGE +!MESSAGE You can specify a configuration when running NMAKE +!MESSAGE by defining the macro CFG on the command line. For example: +!MESSAGE +!MESSAGE NMAKE /f "groonga.mak" CFG="groonga - Win32 Debug_TS" +!MESSAGE +!MESSAGE Possible choices for configuration are: +!MESSAGE +!MESSAGE "groonga - Win32 Release_TS" (based on "Win32 (x86) Dynamic-Link Library") +!MESSAGE "groonga - Win32 Debug_TS" (based on "Win32 (x86) Dynamic-Link Library") +!MESSAGE + +# Begin Project +# PROP AllowPerConfigDependencies 0 +# PROP Scc_ProjName "" +# PROP Scc_LocalPath "" +CPP=cl.exe +MTL=midl.exe +RSC=rc.exe + +!IF "$(CFG)" == "groonga - Win32 Release_TS" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 0 +# PROP BASE Output_Dir "Release_TS" +# PROP BASE Intermediate_Dir "Release_TS" +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 0 +# PROP Output_Dir "Release_TS" +# PROP Intermediate_Dir "Release_TS" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "GROONGA_EXPORTS" /YX /FD /c +# ADD CPP /nologo /MT /W3 /GX /O2 /I "..\.." /I "..\..\Zend" /I "..\..\TSRM" /I "..\..\main" /D "WIN32" /D "PHP_EXPORTS" /D "COMPILE_DL_GROONGA" /D ZTS=1 /D HAVE_GROONGA=1 /D ZEND_DEBUG=0 /D "NDEBUG" /D "_WINDOWS" /D "ZEND_WIN32" /D "PHP_WIN32" /YX /FD /c +# ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32 +# ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32 +# ADD BASE RSC /l 0x407 /d "NDEBUG" +# ADD RSC /l 0x407 /d "NDEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 +# ADD LINK32 php4ts.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 /out:"..\..\Release_TS\php_groonga.dll" /libpath:"..\..\Release_TS" /libpath:"..\..\Release_TS_Inline" + +!ELSEIF "$(CFG)" == "groonga - Win32 Debug_TS" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 1 +# PROP BASE Output_Dir "Debug_TS" +# PROP BASE Intermediate_Dir "Debug_TS" +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 1 +# PROP Output_Dir "Debug_TS" +# PROP Intermediate_Dir "Debug_TS" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /MTd /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "GROONGA_EXPORTS" /YX /FD /GZ /c +# ADD CPP /nologo /MTd /W3 /Gm /GX /ZI /Od /I "..\.." /I "..\..\Zend" /I "..\..\TSRM" /I "..\..\main" /D ZEND_DEBUG=1 /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "PHP_EXPORTS" /D "COMPILE_DL_GROONGA" /D ZTS=1 /D "ZEND_WIN32" /D "PHP_WIN32" /D HAVE_GROONGA=1 /YX /FD /GZ /c +# ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32 +# ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32 +# ADD BASE RSC /l 0x407 /d "_DEBUG" +# ADD RSC /l 0x407 /d "_DEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /pdbtype:sept +# ADD LINK32 php4ts_debug.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /out:"..\..\Debug_TS\php_groonga.dll" /pdbtype:sept /libpath:"..\..\Debug_TS" + +!ENDIF + +# Begin Target + +# Name "groonga - Win32 Release_TS" +# Name "groonga - Win32 Debug_TS" + +# Begin Group "Source Files" + +# PROP Default_Filter "cpp;c;cxx;rc;def;r;odl;idl;hpj;bat" + +# Begin Source File + +SOURCE=./groonga.c +# End Source File + +# End Group + +# Begin Group "Header Files" + +# PROP Default_Filter "h;hpp;hxx;hm;inl" + +# Begin Source File + +SOURCE=.\php_groonga.h +# End Source File +# End Group +# End Target +# End Project diff --git a/storage/mroonga/vendor/groonga/build/ac_macros/check_functions.m4 b/storage/mroonga/vendor/groonga/build/ac_macros/check_functions.m4 index 72e3a9a545d..02eb52f30a6 100644 --- a/storage/mroonga/vendor/groonga/build/ac_macros/check_functions.m4 +++ b/storage/mroonga/vendor/groonga/build/ac_macros/check_functions.m4 @@ -5,14 +5,9 @@ AC_CHECK_FUNCS(_localtime64_s) AC_CHECK_FUNCS(_stricmp) AC_CHECK_FUNCS(_strnicmp) AC_CHECK_FUNCS(_strtoui64) -AC_CHECK_FUNCS(close) AC_CHECK_FUNCS(gmtime_r) AC_CHECK_FUNCS(localtime_r) -AC_CHECK_FUNCS(mkostemp) -AC_CHECK_FUNCS(open) -AC_CHECK_FUNCS(read) +AC_CHECK_FUNCS(mkstemp) AC_CHECK_FUNCS(strcasecmp) AC_CHECK_FUNCS(strncasecmp) AC_CHECK_FUNCS(strtoull) -AC_CHECK_FUNCS(unlink) -AC_CHECK_FUNCS(write) diff --git a/storage/mroonga/vendor/groonga/build/ac_macros/check_headers.m4 b/storage/mroonga/vendor/groonga/build/ac_macros/check_headers.m4 index 17de74c11a5..2a7e0f03acc 100644 --- a/storage/mroonga/vendor/groonga/build/ac_macros/check_headers.m4 +++ b/storage/mroonga/vendor/groonga/build/ac_macros/check_headers.m4 @@ -10,8 +10,11 @@ AC_CHECK_HEADERS(netdb.h) AC_CHECK_HEADERS(netinet/in.h) AC_CHECK_HEADERS(netinet/tcp.h) AC_CHECK_HEADERS(signal.h) +AC_CHECK_HEADERS(stdarg.h) AC_CHECK_HEADERS(stdint.h) AC_CHECK_HEADERS(stdlib.h) +AC_CHECK_HEADERS(string.h) +AC_CHECK_HEADERS(strings.h) AC_CHECK_HEADERS(sys/mman.h) AC_CHECK_HEADERS(sys/param.h) AC_CHECK_HEADERS(sys/resource.h) diff --git a/storage/mroonga/vendor/groonga/build/makefiles/gettext.am b/storage/mroonga/vendor/groonga/build/makefiles/gettext.am index 9706b485dd1..9cea8ce63c1 100644 --- a/storage/mroonga/vendor/groonga/build/makefiles/gettext.am +++ b/storage/mroonga/vendor/groonga/build/makefiles/gettext.am @@ -1,6 +1,8 @@ include $(top_srcdir)/doc/files.am include $(top_srcdir)/build/makefiles/sphinx-build.am +CLEANFILES = + EXTRA_DIST += \ $(po_files) @@ -11,9 +13,11 @@ endif if DOCUMENT_BUILDABLE BUILT_SOURCES += \ + mo-build-stamp +CLEANFILES += \ pot-build-stamp \ edit-po-build-stamp \ - $(mo_files) + mo-build-stamp endif SUFFIXES += .pot .po .mo .edit @@ -47,8 +51,8 @@ SUFFIXES += .pot .po .mo .edit msgfmt -o $@ $< if DOCUMENT_BUILDABLE -update: pot-build-stamp edit-po-build-stamp -build: update $(mo_files) +update: edit-po-build-stamp +build: mo-build-stamp else update: build: @@ -68,6 +72,15 @@ pot-build-stamp: $(absolute_source_files) $(MAKE) gettext @touch $@ -edit-po-build-stamp: $(absolute_source_files) +edit-po-build-stamp: pot-build-stamp $(MAKE) $(edit_po_files) @touch $@ + +mo_build_stamp_dependencies = edit-po-build-stamp +if DOCUMENT_BUILDABLE +mo_build_stamp_dependencies += $(edit_po_files) +endif + +mo-build-stamp: $(mo_build_stamp_dependencies) + $(MAKE) $(mo_files) + @touch $@ diff --git a/storage/mroonga/vendor/groonga/build/makefiles/sphinx.am b/storage/mroonga/vendor/groonga/build/makefiles/sphinx.am index f84fb23b739..c68f62e26ec 100644 --- a/storage/mroonga/vendor/groonga/build/makefiles/sphinx.am +++ b/storage/mroonga/vendor/groonga/build/makefiles/sphinx.am @@ -106,7 +106,7 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" man: man-build-stamp -html: html-build-stamp +html: html-recursive html-build-stamp dirhtml: dirhtml-build-stamp pickle: pickle-build-stamp json: json-build-stamp diff --git a/storage/mroonga/vendor/groonga/config.h.cmake b/storage/mroonga/vendor/groonga/config.h.cmake index e6482f070c7..cd8e4f6cc2d 100644 --- a/storage/mroonga/vendor/groonga/config.h.cmake +++ b/storage/mroonga/vendor/groonga/config.h.cmake @@ -88,6 +88,7 @@ #cmakedefine GRN_WITH_MESSAGE_PACK #cmakedefine GRN_WITH_MRUBY #cmakedefine GRN_WITH_NFKC +#cmakedefine GRN_WITH_ONIGMO #cmakedefine GRN_WITH_ZEROMQ #cmakedefine GRN_WITH_ZLIB @@ -105,6 +106,7 @@ #cmakedefine HAVE_NETINET_TCP_H #cmakedefine HAVE_PTHREAD_H #cmakedefine HAVE_SIGNAL_H +#cmakedefine HAVE_STDARG_H #cmakedefine HAVE_STDINT_H #cmakedefine HAVE_STDLIB_H #cmakedefine HAVE_STRINGS_H @@ -142,17 +144,12 @@ #cmakedefine HAVE_BACKTRACE #cmakedefine HAVE_CLOCK #cmakedefine HAVE_CLOCK_GETTIME -#cmakedefine HAVE_CLOSE #cmakedefine HAVE_FPCLASSIFY #cmakedefine HAVE_GMTIME_R #cmakedefine HAVE_LOCALTIME_R -#cmakedefine HAVE_MKOSTEMP -#cmakedefine HAVE_OPEN -#cmakedefine HAVE_READ +#cmakedefine HAVE_MKSTEMP #cmakedefine HAVE_STRCASECMP #cmakedefine HAVE_STRNCASECMP #cmakedefine HAVE_STRTOULL -#cmakedefine HAVE_UNLINK -#cmakedefine HAVE_WRITE #cmakedefine HAVE_PTHREAD_MUTEXATTR_SETPSHARED #cmakedefine HAVE_PTHREAD_CONDATTR_SETPSHARED diff --git a/storage/mroonga/vendor/groonga/configure.ac b/storage/mroonga/vendor/groonga/configure.ac index 1d74bac30f4..17f3abcc2ed 100644 --- a/storage/mroonga/vendor/groonga/configure.ac +++ b/storage/mroonga/vendor/groonga/configure.ac @@ -141,6 +141,7 @@ AC_DEFUN([REMOVE_CXXFLAG], [ TEST_CFLAGS="" TEST_CXXFLAGS="" NO_STRICT_ALIASING_CFLAGS="" +NO_FLOAT_EQUAL_CFLAGS="" if test "$GCC" = "yes"; then CHECK_BUILD_FLAG([-Wall]) CHECK_BUILD_FLAG([-Wextra]) @@ -160,6 +161,9 @@ if test "$GCC" = "yes"; then fi CHECK_BUILD_FLAG([-Wdisabled-optimization]) CHECK_BUILD_FLAG([-Wfloat-equal]) + if test "x$check_cflag" = "xyes"; then + NO_FLOAT_EQUAL_CFLAGS="-Wno-float-equal" + fi CHECK_BUILD_FLAG([-Wpointer-arith]) CHECK_CFLAG([-Wbad-function-cast]) if test "$CLANG" = "no"; then @@ -192,6 +196,7 @@ fi AC_SUBST(TEST_CFLAGS) AC_SUBST(TEST_CXXFLAGS) AC_SUBST(NO_STRICT_ALIASING_CFLAGS) +AC_SUBST(NO_FLOAT_EQUAL_CFLAGS) AC_LIBTOOL_WIN32_DLL AM_PROG_LIBTOOL @@ -210,7 +215,7 @@ GRN_DLL_FILENAME="libgroonga-\$(LT_CURRENT).dll" AC_SUBST(GRN_DLL_FILENAME) if test "$srcdir/version.sh"; then - source "$srcdir/version.sh" + . "$srcdir/version.sh" AC_SUBST(GRN_VERSION) AC_DEFINE_UNQUOTED(GRN_VERSION, ["$GRN_VERSION"], [groonga version]) fi @@ -226,6 +231,10 @@ AC_CONFIG_FILES([ lib/dat/Makefile lib/mrb/Makefile lib/mrb/scripts/Makefile + lib/mrb/scripts/command_line/Makefile + lib/mrb/scripts/context/Makefile + lib/mrb/scripts/initialize/Makefile + lib/mrb/scripts/logger/Makefile include/Makefile include/groonga/Makefile plugins/Makefile @@ -236,6 +245,7 @@ AC_CONFIG_FILES([ plugins/ruby/Makefile plugins/token_filters/Makefile plugins/sharding/Makefile + plugins/functions/Makefile examples/Makefile examples/dictionary/Makefile examples/dictionary/edict/Makefile @@ -1160,30 +1170,48 @@ AC_SUBST(LIBEVENT_CFLAGS) AC_SUBST(LIBEVENT_LIBS) # MessagePack -AC_ARG_WITH(message-pack, - [AS_HELP_STRING([--without-message-pack], - [Disable MessagePack used for suggestion. [default=/usr]])], - [with_message_pack="$withval"], - [with_message_pack="/usr"]) -if test "x$with_message_pack" != "xno"; then - _SAVE_CFLAGS="$CFLAGS" - _SAVE_LDFLAGS="$LDFLAGS" - _SAVE_LIBS="$LIBS" - CFLAGS="$CFLAGS -I$with_message_pack/include" - LDFLAGS="$LDFLAGS -L$with_message_pack/lib" - AC_SEARCH_LIBS(msgpack_version, msgpack, - [message_pack_available=yes], - [message_pack_available=no]) - CFLAGS="$_SAVE_CFLAGS" - LDFLAGS="$_SAVE_LDFLAGS" - LIBS="$_SAVE_LIBS" +AC_ARG_ENABLE(message-pack, + [AS_HELP_STRING([--disable-message-pack], + [Disable MessagePack support. [default=auto-detect]])], + [enable_message_pack="$enableval"], + [enable_message_pack="auto"]) +if test "x$enable_message_pack" != "xno"; then + m4_ifdef([PKG_CHECK_MODULES], [ + PKG_CHECK_MODULES([MESSAGE_PACK], [msgpack], + [message_pack_available=yes], + [message_pack_available=no]) + ], + [message_pack_vailable=no]) + + if test "$message_pack_available" = "no"; then + AC_ARG_WITH(message-pack, + [AS_HELP_STRING([--with-message-pack], + [Specify prefix where MessagePack is installed. [default=/usr]])], + [with_message_pack="$withval"], + [with_message_pack="/usr"]) + _SAVE_CFLAGS="$CFLAGS" + _SAVE_LDFLAGS="$LDFLAGS" + _SAVE_LIBS="$LIBS" + CFLAGS="$CFLAGS -I$with_message_pack/include" + LDFLAGS="$LDFLAGS -L$with_message_pack/lib" + AC_SEARCH_LIBS(msgpack_version, msgpack, + [message_pack_available=yes], + [message_pack_available=no]) + CFLAGS="$_SAVE_CFLAGS" + LDFLAGS="$_SAVE_LDFLAGS" + LIBS="$_SAVE_LIBS" + + if test "x$message_pack_available" = "xyes"; then + MESSAGE_PACK_CFLAGS="-I$with_message_pack/include" + MESSAGE_PACK_LIBS="-L$with_message_pack/lib -lmsgpack" + fi + fi + if test "x$message_pack_available" = "xyes"; then AC_DEFINE(GRN_WITH_MESSAGE_PACK, [1], [Define to 1 if MessagePack is available.]) - MESSAGE_PACK_CFLAGS="-I$with_message_pack/include" - MESSAGE_PACK_LIBS="-L$with_message_pack/lib -lmsgpack" else - if test "x$with_message_pack" = "xyes"; then + if test "x$enable_message_pack" = "xyes"; then AC_MSG_ERROR("No MessagePack found") fi fi @@ -1347,6 +1375,9 @@ AC_SUBST(token_filter_pluginsdir) sharding_pluginsdir="\${pluginsdir}/sharding" AC_SUBST(sharding_pluginsdir) +function_pluginsdir="\${pluginsdir}/functions" +AC_SUBST(function_pluginsdir) + AC_MSG_CHECKING(for the suffix of plugin shared libraries) shrext_cmds=$(./libtool --config | grep '^shrext_cmds=') eval $shrext_cmds @@ -1455,9 +1486,9 @@ fi AC_MSG_RESULT($enable_mruby) if test "$enable_mruby" = "yes"; then - if test ! -f "$srcdir/vendor/mruby/mruby-build.timestamp" -a \ + if test ! -f "$srcdir/vendor/mruby/mruby_build.timestamp" -a \ "$ac_cv_ruby_available" != "yes"; then - AC_MSG_ERROR(--enable-ruby requires --with-ruby) + AC_MSG_ERROR(--enable-mruby requires --with-ruby) fi AC_DEFINE(GRN_WITH_MRUBY, [1], [Define to 1 if mruby is enabled.]) MRUBY_CFLAGS="-I\$(top_srcdir)/vendor/mruby-source/include" @@ -1475,6 +1506,7 @@ AC_ARG_ENABLE(shared-onigmo, [enable_shared_onigmo="no"]) AM_CONDITIONAL(WITH_SHARED_ONIGMO, test "$enable_shared_onigmo" = "yes") +# TODO: Support using system Onigmo instead of bundled Onigmo. AC_DEFINE(GRN_WITH_ONIGMO, [1], [Use Onigmo.]) AC_CONFIG_SUBDIRS([vendor/onigmo]) diff --git a/storage/mroonga/vendor/groonga/include/groonga.h b/storage/mroonga/vendor/groonga/include/groonga.h index 96384287270..db2303a32c6 100644 --- a/storage/mroonga/vendor/groonga/include/groonga.h +++ b/storage/mroonga/vendor/groonga/include/groonga.h @@ -18,6 +18,7 @@ #ifndef GROONGA_H #define GROONGA_H +#include "groonga/portability.h" #include "groonga/groonga.h" #include "groonga/obj.h" #include "groonga/ii.h" diff --git a/storage/mroonga/vendor/groonga/include/groonga/Makefile.am b/storage/mroonga/vendor/groonga/include/groonga/Makefile.am index 63dcc1e7ba9..37a2b6f45a8 100644 --- a/storage/mroonga/vendor/groonga/include/groonga/Makefile.am +++ b/storage/mroonga/vendor/groonga/include/groonga/Makefile.am @@ -7,7 +7,9 @@ groonga_include_HEADERS = \ obj.h \ output.h \ plugin.h \ + portability.h \ request_canceler.h \ + scorer.h \ token.h \ tokenizer.h \ token_filter.h \ diff --git a/storage/mroonga/vendor/groonga/include/groonga/expr.h b/storage/mroonga/vendor/groonga/include/groonga/expr.h index cf8fb3519ca..63cbb6ebc7a 100644 --- a/storage/mroonga/vendor/groonga/include/groonga/expr.h +++ b/storage/mroonga/vendor/groonga/include/groonga/expr.h @@ -101,6 +101,8 @@ GRN_API grn_rc grn_expr_snip_add_conditions(grn_ctx *ctx, const char **closetags, unsigned int *closetag_lens); +GRN_API unsigned int grn_expr_estimate_size(grn_ctx *ctx, grn_obj *expr); + #ifdef __cplusplus } #endif diff --git a/storage/mroonga/vendor/groonga/include/groonga/groonga.h b/storage/mroonga/vendor/groonga/include/groonga/groonga.h index 16bc424b8c1..6f2aae79c01 100644 --- a/storage/mroonga/vendor/groonga/include/groonga/groonga.h +++ b/storage/mroonga/vendor/groonga/include/groonga/groonga.h @@ -18,12 +18,13 @@ #ifndef GROONGA_GROONGA_H #define GROONGA_GROONGA_H +#include <stdarg.h> +#include <sys/types.h> + #ifdef __cplusplus extern "C" { #endif -#include <stdarg.h> - #ifndef GRN_API # if defined(_WIN32) || defined(_WIN64) # define GRN_API __declspec(dllimport) @@ -182,7 +183,8 @@ typedef enum { GRN_CONTENT_TSV, GRN_CONTENT_JSON, GRN_CONTENT_XML, - GRN_CONTENT_MSGPACK + GRN_CONTENT_MSGPACK, + GRN_CONTENT_GROONGA_COMMAND_LIST } grn_content_type; typedef struct _grn_obj grn_obj; @@ -470,9 +472,12 @@ GRN_API grn_obj *grn_type_create(grn_ctx *ctx, const char *name, unsigned int na grn_obj_flags flags, unsigned int size); GRN_API grn_rc grn_plugin_register(grn_ctx *ctx, const char *name); +GRN_API grn_rc grn_plugin_unregister(grn_ctx *ctx, const char *name); GRN_API grn_rc grn_plugin_register_by_path(grn_ctx *ctx, const char *path); +GRN_API grn_rc grn_plugin_unregister_by_path(grn_ctx *ctx, const char *path); GRN_API const char *grn_plugin_get_system_plugins_dir(void); GRN_API const char *grn_plugin_get_suffix(void); +GRN_API const char *grn_plugin_get_ruby_suffix(void); typedef struct { const char *name; @@ -710,7 +715,8 @@ typedef enum { GRN_OP_TABLE_SORT, GRN_OP_TABLE_GROUP, GRN_OP_JSON_PUT, - GRN_OP_GET_MEMBER + GRN_OP_GET_MEMBER, + GRN_OP_REGEXP } grn_operator; GRN_API const char *grn_operator_to_string(grn_operator op); @@ -723,6 +729,12 @@ GRN_API grn_bool grn_operator_exec_less_equal(grn_ctx *ctx, grn_obj *x, grn_obj *y); GRN_API grn_bool grn_operator_exec_greater_equal(grn_ctx *ctx, grn_obj *x, grn_obj *y); +GRN_API grn_bool grn_operator_exec_match(grn_ctx *ctx, + grn_obj *target, grn_obj *sub_text); +GRN_API grn_bool grn_operator_exec_prefix(grn_ctx *ctx, + grn_obj *target, grn_obj *prefix); +GRN_API grn_bool grn_operator_exec_regexp(grn_ctx *ctx, + grn_obj *target, grn_obj *pattern); struct _grn_table_group_result { grn_obj *table; @@ -912,6 +924,8 @@ struct _grn_search_optarg { grn_obj *proc; int max_size; grn_obj *scorer; + grn_obj *scorer_args_expr; + unsigned int scorer_args_expr_offset; }; GRN_API grn_rc grn_obj_search(grn_ctx *ctx, grn_obj *obj, grn_obj *query, @@ -977,9 +991,22 @@ GRN_API grn_rc grn_obj_delete_hook(grn_ctx *ctx, grn_obj *obj, grn_hook_entry en GRN_API grn_obj *grn_obj_open(grn_ctx *ctx, unsigned char type, grn_obj_flags flags, grn_id domain); +/* Deprecated since 5.0.1. Use grn_column_find_index_data() instead. */ GRN_API int grn_column_index(grn_ctx *ctx, grn_obj *column, grn_operator op, grn_obj **indexbuf, int buf_size, int *section); +/* @since 5.0.1. */ +typedef struct _grn_index_datum { + grn_obj *index; + unsigned int section; +} grn_index_datum; + +/* @since 5.0.1. */ +GRN_API unsigned int grn_column_find_index_data(grn_ctx *ctx, grn_obj *column, + grn_operator op, + grn_index_datum *index_data, + unsigned int n_index_data); + GRN_API grn_rc grn_obj_delete_by_id(grn_ctx *ctx, grn_obj *db, grn_id id, grn_bool removep); GRN_API grn_rc grn_obj_path_by_id(grn_ctx *ctx, grn_obj *db, grn_id id, char *buffer); @@ -1128,10 +1155,10 @@ GRN_API void grn_logger_set_max_level(grn_ctx *ctx, grn_log_level max_level); GRN_API grn_log_level grn_logger_get_max_level(grn_ctx *ctx); #ifdef __GNUC__ -#define GRN_ATTRIBUTE_PRINTF(fmt_pos) \ +# define GRN_ATTRIBUTE_PRINTF(fmt_pos) \ __attribute__ ((format(printf, fmt_pos, fmt_pos + 1))) #else -#define GRN_ATTRIBUTE_PRINTF(fmt_pos) +# define GRN_ATTRIBUTE_PRINTF(fmt_pos) #endif /* __GNUC__ */ GRN_API void grn_logger_put(grn_ctx *ctx, grn_log_level level, @@ -1141,13 +1168,15 @@ GRN_API void grn_logger_reopen(grn_ctx *ctx); GRN_API grn_bool grn_logger_pass(grn_ctx *ctx, grn_log_level level); #ifndef GRN_LOG_DEFAULT_LEVEL -#define GRN_LOG_DEFAULT_LEVEL GRN_LOG_NOTICE +# define GRN_LOG_DEFAULT_LEVEL GRN_LOG_NOTICE #endif /* GRN_LOG_DEFAULT_LEVEL */ GRN_API void grn_default_logger_set_max_level(grn_log_level level); GRN_API grn_log_level grn_default_logger_get_max_level(void); GRN_API void grn_default_logger_set_path(const char *path); GRN_API const char *grn_default_logger_get_path(void); +GRN_API void grn_default_logger_set_rotate_threshold_size(off_t threshold); +GRN_API off_t grn_default_logger_get_rotate_threshold_size(void); #define GRN_LOG(ctx,level,...) do {\ if (grn_logger_pass(ctx, level)) {\ @@ -1180,6 +1209,8 @@ GRN_API void grn_default_query_logger_set_flags(unsigned int flags); GRN_API unsigned int grn_default_query_logger_get_flags(void); GRN_API void grn_default_query_logger_set_path(const char *path); GRN_API const char *grn_default_query_logger_get_path(void); +GRN_API void grn_default_query_logger_set_rotate_threshold_size(off_t threshold); +GRN_API off_t grn_default_query_logger_get_rotate_threshold_size(void); #define GRN_QUERY_LOG(ctx, flag, mark, format, ...) do {\ if (grn_query_logger_pass(ctx, flag)) {\ diff --git a/storage/mroonga/vendor/groonga/include/groonga/ii.h b/storage/mroonga/vendor/groonga/include/groonga/ii.h index 1bd8ce25309..176e137480c 100644 --- a/storage/mroonga/vendor/groonga/include/groonga/ii.h +++ b/storage/mroonga/vendor/groonga/include/groonga/ii.h @@ -28,6 +28,13 @@ typedef struct _grn_ii grn_ii; typedef struct _grn_ii_buffer grn_ii_buffer; GRN_API unsigned int grn_ii_estimate_size(grn_ctx *ctx, grn_ii *ii, grn_id tid); +GRN_API unsigned int grn_ii_estimate_size_for_query(grn_ctx *ctx, grn_ii *ii, + const char *query, + unsigned int query_len, + grn_search_optarg *optarg); +GRN_API unsigned int grn_ii_estimate_size_for_lexicon_cursor(grn_ctx *ctx, + grn_ii *ii, + grn_table_cursor *lexicon_cursor); GRN_API grn_ii_buffer *grn_ii_buffer_open(grn_ctx *ctx, grn_ii *ii, long long unsigned int update_buffer_size); diff --git a/storage/mroonga/vendor/groonga/include/groonga/portability.h b/storage/mroonga/vendor/groonga/include/groonga/portability.h new file mode 100644 index 00000000000..1d763765c2c --- /dev/null +++ b/storage/mroonga/vendor/groonga/include/groonga/portability.h @@ -0,0 +1,158 @@ +/* + Copyright(C) 2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ +#ifndef GROONGA_PORTABILITY_H +#define GROONGA_PORTABILITY_H + +#ifdef WIN32 +# ifdef __cplusplus +# define grn_memcpy(dest, src, n) ::memcpy_s((dest), (n), (src), (n)) +# else /* __cplusplus */ +# define grn_memcpy(dest, src, n) memcpy_s((dest), (n), (src), (n)) +# endif /* __cplusplus */ +#else /* WIN32 */ +# ifdef __cplusplus +# define grn_memcpy(dest, src, n) std::memcpy((dest), (src), (n)) +# else /* __cplusplus */ +# define grn_memcpy(dest, src, n) memcpy((dest), (src), (n)) +# endif /* __cplusplus */ +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_memmove(dest, src, n) memmove_s((dest), (n), (src), (n)) +#else /* WIN32 */ +# define grn_memmove(dest, src, n) memmove((dest), (src), (n)) +#endif /* WIN32 */ + +#define GRN_ENV_BUFFER_SIZE 1024 + +#ifdef WIN32 +# define grn_getenv(name, dest, dest_size) do { \ + char *dest_ = (dest); \ + size_t dest_size_ = (dest_size); \ + if (dest_size_ > 0) { \ + DWORD env_size; \ + env_size = GetEnvironmentVariableA((name), dest_, dest_size_); \ + if (env_size == 0 || env_size > dest_size_) { \ + dest_[0] = '\0'; \ + } \ + } \ + } while (0) +#else /* WIN32 */ +# define grn_getenv(name, dest, dest_size) do { \ + const char *env_value = getenv((name)); \ + char *dest_ = (dest); \ + size_t dest_size_ = (dest_size); \ + if (dest_size_ > 0) { \ + if (env_value) { \ + strncpy(dest_, env_value, dest_size_ - 1); \ + } else { \ + dest_[0] = '\0'; \ + } \ + } \ + } while (0) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_fopen(name, mode) _fsopen((name), (mode), _SH_DENYNO) +#else /* WIN32 */ +# define grn_fopen(name, mode) fopen((name), (mode)) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_strdup_raw(string) _strdup((string)) +#else /* WIN32 */ +# define grn_strdup_raw(string) strdup((string)) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_unlink(filename) _unlink((filename)) +#else /* WIN32 */ +# define grn_unlink(filename) unlink((filename)) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_strncat(dest, dest_size, src, n) \ + strncat_s((dest), (dest_size), (src), (n)) +#else /* WIN32 */ +# define grn_strncat(dest, dest_size, src, n) \ + strncat((dest), (src), (n)) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_strcpy(dest, dest_size, src) \ + strcpy_s((dest), (dest_size), (src)) +#else /* WIN32 */ +# define grn_strcpy(dest, dest_size, src) \ + strcpy((dest), (src)) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_strncpy(dest, dest_size, src, n) \ + strncpy_s((dest), (dest_size), (src), (n)) +#else /* WIN32 */ +# define grn_strncpy(dest, dest_size, src, n) \ + strncpy((dest), (src), (n)) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_strcat(dest, dest_size, src) \ + strcat_s((dest), (dest_size), (src)) +#else /* WIN32 */ +# define grn_strcat(dest, dest_size, src) \ + strcat((dest), (src)) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_snprintf(dest, dest_size, n, format, ...) \ + _snprintf_s((dest), (dest_size), (n) - 1, (format), __VA_ARGS__) +#else /* WIN32 */ +# define grn_snprintf(dest, dest_size, n, format, ...) \ + snprintf((dest), (n), (format), __VA_ARGS__) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_write(fd, buf, count) _write((fd), (buf), (count)) +#else /* WIN32 */ +# define grn_write(fd, buf, count) write((fd), (buf), (count)) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_read(fd, buf, count) _read((fd), (buf), (count)) +#else /* WIN32 */ +# define grn_read(fd, buf, count) read((fd), (buf), (count)) +#endif /* WIN32 */ + +#ifdef WIN32 +# define GRN_OPEN_CREATE_MODE (_S_IREAD | _S_IWRITE) +# define GRN_OPEN_FLAG_BINARY _O_BINARY +# define grn_open(fd, pathname, flags) \ + _sopen_s(&(fd), (pathname), (flags), _SH_DENYNO, GRN_OPEN_CREATE_MODE) +#else /* WIN32 */ +# define GRN_OPEN_CREATE_MODE (S_IRUSR | S_IWUSR | S_IRGRP) +# define GRN_OPEN_FLAG_BINARY 0 +# define grn_open(fd, pathname, flags) \ + (fd) = open((pathname), (flags), GRN_OPEN_CREATE_MODE) +#endif /* WIN32 */ + +#ifdef WIN32 +# define grn_close(fd) _close((fd)) +#else /* WIN32 */ +# define grn_close(fd) close((fd)) +#endif /* WIN32 */ + +#endif /* GROONGA_PORTABILITY_H */ diff --git a/storage/mroonga/vendor/groonga/include/groonga/scorer.h b/storage/mroonga/vendor/groonga/include/groonga/scorer.h index a50d0d8cd71..6f9c589affb 100644 --- a/storage/mroonga/vendor/groonga/include/groonga/scorer.h +++ b/storage/mroonga/vendor/groonga/include/groonga/scorer.h @@ -59,6 +59,13 @@ GRN_API unsigned int GRN_API int grn_scorer_matched_record_get_weight(grn_ctx *ctx, grn_scorer_matched_record *record); +GRN_API grn_obj * + grn_scorer_matched_record_get_arg(grn_ctx *ctx, + grn_scorer_matched_record *record, + unsigned int i); +GRN_API unsigned int + grn_scorer_matched_record_get_n_args(grn_ctx *ctx, + grn_scorer_matched_record *record); diff --git a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt index 2103436545f..8959f883ca3 100644 --- a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt @@ -19,6 +19,7 @@ add_definitions( include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/dat + ${ONIGMO_INCLUDE_DIRS} ${MRUBY_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS}) link_directories( @@ -42,10 +43,13 @@ set_source_files_properties(dat.cpp ${LIBGRNDAT_SOURCES} set(GRN_ALL_SOURCES ${LIBGROONGA_SOURCES} ${LIBGRNDAT_SOURCES} - ${LIBGRNMRB_SOURCES} - ${MRUBY_LIBS}) + ${LIBGRNMRB_SOURCES}) if(GRN_EMBED) add_library(libgroonga STATIC ${GRN_ALL_SOURCES}) + set_target_properties( + libgroonga + PROPERTIES + POSITION_INDEPENDENT_CODE ON) else() add_library(libgroonga SHARED ${GRN_ALL_SOURCES}) endif() @@ -59,7 +63,9 @@ set(GRN_ALL_LIBRARIES ${LIBLZ4_LIBRARIES} ${DL_LIBS} ${M_LIBS} - ${WS2_32_LIBS}) + ${WS2_32_LIBS} + ${MRUBY_LIBS} + ${ONIGMO_LIBS}) if(GRN_EMBED) target_link_libraries(libgroonga ${GRN_ALL_LIBRARIES} @@ -73,7 +79,7 @@ else() RUNTIME DESTINATION "${BIN_DIR}") endif() -if(GRN_WITH_MRUBY AND NOT GRN_EMBED) +if(GRN_WITH_MRUBY) read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/mrb/scripts/sources.am RUBY_SCRIPTS) string(REGEX REPLACE "([^;]+)" "mrb/scripts/\\1" @@ -81,4 +87,36 @@ if(GRN_WITH_MRUBY AND NOT GRN_EMBED) install( FILES ${RUBY_SCRIPTS} DESTINATION "${GRN_RELATIVE_RUBY_SCRIPTS_DIR}") + + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/mrb/scripts/command_line/sources.am + COMMANE_LINE_RUBY_SCRIPTS) + string(REGEX REPLACE "([^;]+)" "mrb/scripts/command_line/\\1" + COMMANE_LINE_RUBY_SCRIPTS "${COMMANE_LINE_RUBY_SCRIPTS}") + install( + FILES ${COMMANE_LINE_RUBY_SCRIPTS} + DESTINATION "${GRN_RELATIVE_RUBY_SCRIPTS_DIR}/command_line") + + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/mrb/scripts/context/sources.am + CONTEXT_RUBY_SCRIPTS) + string(REGEX REPLACE "([^;]+)" "mrb/scripts/context/\\1" + CONTEXT_RUBY_SCRIPTS "${CONTEXT_RUBY_SCRIPTS}") + install( + FILES ${CONTEXT_RUBY_SCRIPTS} + DESTINATION "${GRN_RELATIVE_RUBY_SCRIPTS_DIR}/context") + + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/mrb/scripts/initialize/sources.am + INITIALIZE_RUBY_SCRIPTS) + string(REGEX REPLACE "([^;]+)" "mrb/scripts/initialize/\\1" + INITIALIZE_RUBY_SCRIPTS "${INITIALIZE_RUBY_SCRIPTS}") + install( + FILES ${INITIALIZE_RUBY_SCRIPTS} + DESTINATION "${GRN_RELATIVE_RUBY_SCRIPTS_DIR}/initialize") + + read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/mrb/scripts/logger/sources.am + LOGGER_RUBY_SCRIPTS) + string(REGEX REPLACE "([^;]+)" "mrb/scripts/logger/\\1" + LOGGER_RUBY_SCRIPTS "${LOGGER_RUBY_SCRIPTS}") + install( + FILES ${LOGGER_RUBY_SCRIPTS} + DESTINATION "${GRN_RELATIVE_RUBY_SCRIPTS_DIR}/logger") endif() diff --git a/storage/mroonga/vendor/groonga/lib/com.c b/storage/mroonga/vendor/groonga/lib/com.c index a17d3a931de..eb1cacfd4c1 100644 --- a/storage/mroonga/vendor/groonga/lib/com.c +++ b/storage/mroonga/vendor/groonga/lib/com.c @@ -328,10 +328,10 @@ grn_com_event_fin(grn_ctx *ctx, grn_com_event *ev) #ifndef USE_SELECT if (ev->events) { GRN_FREE(ev->events); } # ifdef USE_EPOLL - GRN_CLOSE(ev->epfd); + grn_close(ev->epfd); # endif /* USE_EPOLL */ # ifdef USE_KQUEUE - GRN_CLOSE(ev->kqfd); + grn_close(ev->kqfd); # endif /* USE_KQUEUE*/ #endif /* USE_SELECT */ return GRN_SUCCESS; @@ -528,7 +528,7 @@ grn_com_receiver(grn_ctx *ctx, grn_com *com) grn_msg *msg = (grn_msg *)grn_msg_open(ctx, com, &ev->recv_old); grn_com_recv(ctx, msg->u.peer, &msg->header, (grn_obj *)msg); if (msg->u.peer /* is_edge_request(msg)*/) { - memcpy(&msg->edge_id, &ev->curr_edge_id, sizeof(grn_com_addr)); + grn_memcpy(&msg->edge_id, &ev->curr_edge_id, sizeof(grn_com_addr)); if (!com->has_sid) { com->has_sid = 1; com->sid = ev->curr_edge_id.sid++; @@ -560,11 +560,26 @@ grn_com_event_poll(grn_ctx *ctx, grn_com_event *ev, int timeout) FD_ZERO(&wfds); ctx->errlvl = GRN_OK; ctx->rc = GRN_SUCCESS; - GRN_HASH_EACH(ctx, ev->hash, eh, &pfd, &dummy, &com, { - if ((com->events & GRN_COM_POLLIN)) { FD_SET(*pfd, &rfds); } - if ((com->events & GRN_COM_POLLOUT)) { FD_SET(*pfd, &wfds); } - if (*pfd > nfds) { nfds = *pfd; } - }); + { + grn_hash_cursor *cursor; + cursor = grn_hash_cursor_open(ctx, ev->hash, NULL, 0, NULL, 0, 0, -1, 0); + if (cursor) { + grn_id id; + while ((id = grn_hash_cursor_next(ctx, cursor))) { + grn_hash_cursor_get_key_value(ctx, + cursor, + (void **)(&pfd), + &dummy, + (void **)(&com)); + if ((com->events & GRN_COM_POLLIN)) { FD_SET(*pfd, &rfds); } + if ((com->events & GRN_COM_POLLOUT)) { FD_SET(*pfd, &wfds); } +# ifndef WIN32 + if (*pfd > nfds) { nfds = *pfd; } +# endif /* WIN32 */ + } + grn_hash_cursor_close(ctx, cursor); + } + } nevents = select(nfds + 1, &rfds, &wfds, NULL, (timeout >= 0) ? &tv : NULL); if (nevents < 0) { SOERR("select"); @@ -926,7 +941,8 @@ grn_com_copen(grn_ctx *ctx, grn_com_event *ev, const char *dest, int port) #ifdef AI_NUMERICSERV hints.ai_flags = AI_NUMERICSERV; #endif - snprintf(port_string, sizeof(port_string), "%d", port); + grn_snprintf(port_string, sizeof(port_string), sizeof(port_string), + "%d", port); getaddrinfo_result = getaddrinfo(dest, port_string, &hints, &addrinfo_list); if (getaddrinfo_result != 0) { @@ -1034,7 +1050,8 @@ grn_com_sopen(grn_ctx *ctx, grn_com_event *ev, if (!bind_address) { bind_address = "0.0.0.0"; } - snprintf(port_string, sizeof(port_string), "%d", port); + grn_snprintf(port_string, sizeof(port_string), sizeof(port_string), + "%d", port); memset(&hints, 0, sizeof(struct addrinfo)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; @@ -1069,7 +1086,7 @@ grn_com_sopen(grn_ctx *ctx, grn_com_event *ev, SOERR("socket"); goto exit; } - memcpy(&ev->curr_edge_id.addr, he->h_addr, he->h_length); + grn_memcpy(&ev->curr_edge_id.addr, he->h_addr, he->h_length); ev->curr_edge_id.port = htons(port); ev->curr_edge_id.sid = 0; { diff --git a/storage/mroonga/vendor/groonga/lib/command.c b/storage/mroonga/vendor/groonga/lib/command.c index b0f75229744..e4be0d1ff92 100644 --- a/storage/mroonga/vendor/groonga/lib/command.c +++ b/storage/mroonga/vendor/groonga/lib/command.c @@ -57,7 +57,7 @@ grn_command_input_close(grn_ctx *ctx, grn_command_input *input) GRN_API_ENTER; /* TODO: Free input->arguments by self. */ - grn_expr_clear_vars(ctx, input->command); + /* grn_expr_clear_vars(ctx, input->command); */ GRN_FREE(input); GRN_API_RETURN(ctx->rc); diff --git a/storage/mroonga/vendor/groonga/lib/ctx.c b/storage/mroonga/vendor/groonga/lib/ctx.c index 5408832dfcc..85878036dc2 100644 --- a/storage/mroonga/vendor/groonga/lib/ctx.c +++ b/storage/mroonga/vendor/groonga/lib/ctx.c @@ -27,13 +27,26 @@ #include "grn_output.h" #include "grn_normalizer.h" #include "grn_ctx_impl_mrb.h" +#include "grn_logger.h" #include <stdio.h> #include <stdarg.h> #include <time.h> #ifdef HAVE_NETINET_IN_H -#include <netinet/in.h> +# include <netinet/in.h> #endif /* HAVE_NETINET_IN_H */ +#ifdef WIN32 +# include <share.h> +#endif /* WIN32 */ + +#if defined(HAVE__LOCALTIME64_S) && defined(__GNUC__) +# ifdef _WIN64 +# define localtime_s(tm, time) _localtime64_s(tm, time) +# else /* _WIN64 */ +# define localtime_s(tm, time) _localtime32_s(tm, time) +# endif /* _WIN64 */ +#endif /* defined(HAVE__LOCALTIME64_S) && defined(__GNUC__) */ + #define GRN_CTX_INITIALIZER(enc) \ { GRN_SUCCESS, 0, enc, 0, GRN_LOG_NOTICE,\ GRN_CTX_FIN, 0, 0, 0, 0, {0}, NULL, NULL, NULL, NULL, NULL } @@ -129,34 +142,48 @@ grn_time_now(grn_ctx *ctx, grn_obj *obj) GRN_TIME_NSEC_TO_USEC(tv.tv_nsec))); } -grn_rc -grn_timeval2str(grn_ctx *ctx, grn_timeval *tv, char *buf) +struct tm * +grn_timeval2tm(grn_ctx *ctx, grn_timeval *tv, struct tm *tm_buffer) { struct tm *ltm; const char *function_name; #ifdef HAVE__LOCALTIME64_S - struct tm tm; time_t t = tv->tv_sec; function_name = "localtime_s"; - ltm = (localtime_s(&tm, &t) == 0) ? &tm : NULL; + ltm = (localtime_s(tm_buffer, &t) == 0) ? tm_buffer : NULL; #else /* HAVE__LOCALTIME64_S */ # ifdef HAVE_LOCALTIME_R - struct tm tm; time_t t = tv->tv_sec; function_name = "localtime_r"; - ltm = localtime_r(&t, &tm); + ltm = localtime_r(&t, tm_buffer); # else /* HAVE_LOCALTIME_R */ time_t tvsec = (time_t) tv->tv_sec; function_name = "localtime"; ltm = localtime(&tvsec); # endif /* HAVE_LOCALTIME_R */ #endif /* HAVE__LOCALTIME64_S */ - if (!ltm) { SERR(function_name); } - snprintf(buf, GRN_TIMEVAL_STR_SIZE - 1, GRN_TIMEVAL_STR_FORMAT, - ltm->tm_year + 1900, ltm->tm_mon + 1, ltm->tm_mday, - ltm->tm_hour, ltm->tm_min, ltm->tm_sec, - (int)(GRN_TIME_NSEC_TO_USEC(tv->tv_nsec))); - buf[GRN_TIMEVAL_STR_SIZE - 1] = '\0'; + if (!ltm) { + SERR(function_name); + } + return ltm; +} + +grn_rc +grn_timeval2str(grn_ctx *ctx, grn_timeval *tv, char *buf, size_t buf_size) +{ + struct tm tm; + struct tm *ltm; + ltm = grn_timeval2tm(ctx, tv, &tm); + grn_snprintf(buf, buf_size, GRN_TIMEVAL_STR_SIZE, + GRN_TIMEVAL_STR_FORMAT, + ltm->tm_year + 1900, ltm->tm_mon + 1, ltm->tm_mday, + ltm->tm_hour, ltm->tm_min, ltm->tm_sec, + (int)(GRN_TIME_NSEC_TO_USEC(tv->tv_nsec))); + if (buf_size > GRN_TIMEVAL_STR_SIZE) { + buf[GRN_TIMEVAL_STR_SIZE - 1] = '\0'; + } else { + buf[buf_size - 1] = '\0'; + } return ctx->rc; } @@ -235,7 +262,7 @@ grn_alloc_info_set_backtrace(char *buffer, size_t size) if (symbol_length + 2 > rest) { break; } - memcpy(buffer, symbols[i], symbol_length); + grn_memcpy(buffer, symbols[i], symbol_length); buffer += symbol_length; rest -= symbol_length; buffer[0] = '\n'; @@ -453,7 +480,7 @@ grn_ctx_loader_clear(grn_ctx *ctx) #ifdef GRN_WITH_MESSAGE_PACK static int -grn_msgpack_buffer_write(void *data, const char *buf, unsigned int len) +grn_msgpack_buffer_write(void *data, const char *buf, msgpack_size_t len) { grn_ctx *ctx = (grn_ctx *)data; return grn_bulk_write(ctx, ctx->impl->outbuf, buf, len); @@ -550,7 +577,7 @@ grn_ctx_set_next_expr(grn_ctx *ctx, grn_obj *expr) } static void -grn_ctx_impl_clear_n_same_error_mssagges(grn_ctx *ctx) +grn_ctx_impl_clear_n_same_error_messagges(grn_ctx *ctx) { if (ctx->impl->n_same_error_messages == 0) { return; @@ -583,8 +610,8 @@ grn_ctx_impl_set_current_error_message(grn_ctx *ctx) return; } - grn_ctx_impl_clear_n_same_error_mssagges(ctx); - strcpy(ctx->impl->previous_errbuf, ctx->errbuf); + grn_ctx_impl_clear_n_same_error_messagges(ctx); + grn_strcpy(ctx->impl->previous_errbuf, GRN_CTX_MSGSIZE, ctx->errbuf); } static grn_rc @@ -594,8 +621,14 @@ grn_ctx_init_internal(grn_ctx *ctx, int flags) // if (ctx->stat != GRN_CTX_FIN) { return GRN_INVALID_ARGUMENT; } ERRCLR(ctx); ctx->flags = flags; - if (getenv("GRN_CTX_PER_DB") && strcmp(getenv("GRN_CTX_PER_DB"), "yes") == 0) { - ctx->flags |= GRN_CTX_PER_DB; + { + char grn_ctx_per_db_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_CTX_PER_DB", + grn_ctx_per_db_env, + GRN_ENV_BUFFER_SIZE); + if (grn_ctx_per_db_env[0] && strcmp(grn_ctx_per_db_env, "yes") == 0) { + ctx->flags |= GRN_CTX_PER_DB; + } } if (ERRP(ctx, GRN_ERROR)) { return ctx->rc; } ctx->stat = GRN_CTX_INITED; @@ -661,7 +694,7 @@ grn_ctx_fin(grn_ctx *ctx) CRITICAL_SECTION_LEAVE(grn_glock); } if (ctx->impl) { - grn_ctx_impl_clear_n_same_error_mssagges(ctx); + grn_ctx_impl_clear_n_same_error_messagges(ctx); if (ctx->impl->finalizer) { ctx->impl->finalizer(ctx, 0, NULL, &(ctx->user_data)); } @@ -756,474 +789,12 @@ grn_ctx_set_finalizer(grn_ctx *ctx, grn_proc_func *finalizer) grn_timeval grn_starttime; -static char *default_logger_path = NULL; -static FILE *default_logger_file = NULL; -static grn_critical_section default_logger_lock; - -static void -default_logger_log(grn_ctx *ctx, grn_log_level level, - const char *timestamp, const char *title, - const char *message, const char *location, void *user_data) -{ - const char slev[] = " EACewnid-"; - if (default_logger_path) { - CRITICAL_SECTION_ENTER(default_logger_lock); - if (!default_logger_file) { - default_logger_file = fopen(default_logger_path, "a"); - } - if (default_logger_file) { - if (location && *location) { - fprintf(default_logger_file, "%s|%c|%s %s %s\n", - timestamp, *(slev + level), title, message, location); - } else { - fprintf(default_logger_file, "%s|%c|%s %s\n", timestamp, - *(slev + level), title, message); - } - fflush(default_logger_file); - } - CRITICAL_SECTION_LEAVE(default_logger_lock); - } -} - -static void -default_logger_reopen(grn_ctx *ctx, void *user_data) -{ - GRN_LOG(ctx, GRN_LOG_NOTICE, "log will be closed."); - CRITICAL_SECTION_ENTER(default_logger_lock); - if (default_logger_file) { - fclose(default_logger_file); - default_logger_file = NULL; - } - CRITICAL_SECTION_LEAVE(default_logger_lock); - GRN_LOG(ctx, GRN_LOG_NOTICE, "log opened."); -} - -static void -default_logger_fin(grn_ctx *ctx, void *user_data) -{ - CRITICAL_SECTION_ENTER(default_logger_lock); - if (default_logger_file) { - fclose(default_logger_file); - default_logger_file = NULL; - } - CRITICAL_SECTION_LEAVE(default_logger_lock); -} - -static grn_logger default_logger = { - GRN_LOG_DEFAULT_LEVEL, - GRN_LOG_TIME|GRN_LOG_MESSAGE, - NULL, - default_logger_log, - default_logger_reopen, - default_logger_fin -}; - -static grn_logger current_logger = { - GRN_LOG_DEFAULT_LEVEL, - GRN_LOG_TIME|GRN_LOG_MESSAGE, - NULL, - NULL, - NULL, - NULL -}; - -void -grn_default_logger_set_max_level(grn_log_level max_level) -{ - default_logger.max_level = max_level; - if (current_logger.log == default_logger_log) { - current_logger.max_level = max_level; - } -} - -grn_log_level -grn_default_logger_get_max_level(void) -{ - return default_logger.max_level; -} - -void -grn_default_logger_set_path(const char *path) -{ - if (default_logger_path) { - free(default_logger_path); - } - - if (path) { - default_logger_path = strdup(path); - } else { - default_logger_path = NULL; - } -} - -const char * -grn_default_logger_get_path(void) -{ - return default_logger_path; -} - -void -grn_logger_reopen(grn_ctx *ctx) -{ - if (current_logger.reopen) { - current_logger.reopen(ctx, current_logger.user_data); - } -} - -static void -grn_logger_fin(grn_ctx *ctx) -{ - if (current_logger.fin) { - current_logger.fin(ctx, current_logger.user_data); - } -} - -static void -logger_info_func_wrapper(grn_ctx *ctx, grn_log_level level, - const char *timestamp, const char *title, - const char *message, const char *location, - void *user_data) -{ - grn_logger_info *info = user_data; - info->func(level, timestamp, title, message, location, info->func_arg); -} - -/* Deprecated since 2.1.2. */ -grn_rc -grn_logger_info_set(grn_ctx *ctx, const grn_logger_info *info) -{ - if (info) { - grn_logger logger; - - memset(&logger, 0, sizeof(grn_logger)); - logger.max_level = info->max_level; - logger.flags = info->flags; - if (info->func) { - logger.log = logger_info_func_wrapper; - logger.user_data = (grn_logger_info *)info; - } else { - logger.log = default_logger_log; - logger.reopen = default_logger_reopen; - logger.fin = default_logger_fin; - } - return grn_logger_set(ctx, &logger); - } else { - return grn_logger_set(ctx, NULL); - } -} - -grn_rc -grn_logger_set(grn_ctx *ctx, const grn_logger *logger) -{ - grn_logger_fin(ctx); - if (logger) { - current_logger = *logger; - } else { - current_logger = default_logger; - } - return GRN_SUCCESS; -} - -void -grn_logger_set_max_level(grn_ctx *ctx, grn_log_level max_level) -{ - current_logger.max_level = max_level; -} - -grn_log_level -grn_logger_get_max_level(grn_ctx *ctx) -{ - return current_logger.max_level; -} - -grn_bool -grn_logger_pass(grn_ctx *ctx, grn_log_level level) -{ - return level <= current_logger.max_level; -} - -#define TBUFSIZE GRN_TIMEVAL_STR_SIZE -#define MBUFSIZE 0x1000 -#define LBUFSIZE 0x400 - -void -grn_logger_put(grn_ctx *ctx, grn_log_level level, - const char *file, int line, const char *func, const char *fmt, ...) -{ - if (level <= current_logger.max_level && current_logger.log) { - char tbuf[TBUFSIZE]; - char mbuf[MBUFSIZE]; - char lbuf[LBUFSIZE]; - tbuf[0] = '\0'; - if (current_logger.flags & GRN_LOG_TIME) { - grn_timeval tv; - grn_timeval_now(ctx, &tv); - grn_timeval2str(ctx, &tv, tbuf); - } - if (current_logger.flags & GRN_LOG_MESSAGE) { - va_list argp; - va_start(argp, fmt); - vsnprintf(mbuf, MBUFSIZE - 1, fmt, argp); - va_end(argp); - mbuf[MBUFSIZE - 1] = '\0'; - } else { - mbuf[0] = '\0'; - } - if (current_logger.flags & GRN_LOG_LOCATION) { - snprintf(lbuf, LBUFSIZE - 1, "%d %s:%d %s()", getpid(), file, line, func); - lbuf[LBUFSIZE - 1] = '\0'; - } else { - lbuf[0] = '\0'; - } - current_logger.log(ctx, level, tbuf, "", mbuf, lbuf, - current_logger.user_data); - } -} - -static void -logger_init(void) -{ - if (!default_logger_path) { - default_logger_path = strdup(GRN_LOG_PATH); - } - memcpy(¤t_logger, &default_logger, sizeof(grn_logger)); - CRITICAL_SECTION_INIT(default_logger_lock); -} - -static void -logger_fin(grn_ctx *ctx) -{ - grn_logger_fin(ctx); - if (default_logger_path) { - free(default_logger_path); - default_logger_path = NULL; - } - CRITICAL_SECTION_FIN(default_logger_lock); -} - - -static char *default_query_logger_path = NULL; -static FILE *default_query_logger_file = NULL; -static grn_critical_section default_query_logger_lock; - -static void -default_query_logger_log(grn_ctx *ctx, unsigned int flag, - const char *timestamp, const char *info, - const char *message, void *user_data) -{ - if (default_query_logger_path) { - CRITICAL_SECTION_ENTER(default_query_logger_lock); - if (!default_query_logger_file) { - default_query_logger_file = fopen(default_query_logger_path, "a"); - } - if (default_query_logger_file) { - fprintf(default_query_logger_file, "%s|%s%s\n", timestamp, info, message); - fflush(default_query_logger_file); - } - CRITICAL_SECTION_LEAVE(default_query_logger_lock); - } -} - -static void -default_query_logger_close(grn_ctx *ctx, void *user_data) -{ - GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_DESTINATION, " ", - "query log will be closed: <%s>", default_query_logger_path); - CRITICAL_SECTION_ENTER(default_query_logger_lock); - if (default_query_logger_file) { - fclose(default_query_logger_file); - default_query_logger_file = NULL; - } - CRITICAL_SECTION_LEAVE(default_query_logger_lock); -} - -static void -default_query_logger_reopen(grn_ctx *ctx, void *user_data) -{ - default_query_logger_close(ctx, user_data); - if (default_query_logger_path) { - GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_DESTINATION, " ", - "query log is opened: <%s>", default_query_logger_path); - } -} - -static void -default_query_logger_fin(grn_ctx *ctx, void *user_data) -{ - if (default_query_logger_file) { - default_query_logger_close(ctx, user_data); - } -} - -static grn_query_logger default_query_logger = { - GRN_QUERY_LOG_DEFAULT, - NULL, - default_query_logger_log, - default_query_logger_reopen, - default_query_logger_fin -}; - -static grn_query_logger current_query_logger = { - GRN_QUERY_LOG_DEFAULT, - NULL, - NULL, - NULL, - NULL -}; - -void -grn_default_query_logger_set_flags(unsigned int flags) -{ - default_query_logger.flags = flags; - if (current_query_logger.log == default_query_logger_log) { - current_query_logger.flags = flags; - } -} - -unsigned int -grn_default_query_logger_get_flags(void) -{ - return default_query_logger.flags; -} - -void -grn_default_query_logger_set_path(const char *path) -{ - if (default_query_logger_path) { - free(default_query_logger_path); - } - - if (path) { - default_query_logger_path = strdup(path); - } else { - default_query_logger_path = NULL; - } -} - -const char * -grn_default_query_logger_get_path(void) -{ - return default_query_logger_path; -} - -void -grn_query_logger_reopen(grn_ctx *ctx) -{ - if (current_query_logger.reopen) { - current_query_logger.reopen(ctx, current_query_logger.user_data); - } -} - -static void -grn_query_logger_fin(grn_ctx *ctx) -{ - if (current_query_logger.fin) { - current_query_logger.fin(ctx, current_query_logger.user_data); - } -} - -grn_rc -grn_query_logger_set(grn_ctx *ctx, const grn_query_logger *logger) -{ - grn_query_logger_fin(ctx); - if (logger) { - current_query_logger = *logger; - } else { - current_query_logger = default_query_logger; - } - return GRN_SUCCESS; -} - -grn_bool -grn_query_logger_pass(grn_ctx *ctx, unsigned int flag) -{ - return current_query_logger.flags & flag; -} - -#define TIMESTAMP_BUFFER_SIZE TBUFSIZE -/* 8+a(%p) + 1(|) + 1(mark) + 15(elapsed time) = 25+a */ -#define INFO_BUFFER_SIZE 40 - -void -grn_query_logger_put(grn_ctx *ctx, unsigned int flag, const char *mark, - const char *format, ...) -{ - char timestamp[TIMESTAMP_BUFFER_SIZE]; - char info[INFO_BUFFER_SIZE]; - grn_obj *message = &ctx->impl->query_log_buf; - - if (!current_query_logger.log) { - return; - } - - { - grn_timeval tv; - timestamp[0] = '\0'; - grn_timeval_now(ctx, &tv); - grn_timeval2str(ctx, &tv, timestamp); - } - - if (flag & (GRN_QUERY_LOG_COMMAND | GRN_QUERY_LOG_DESTINATION)) { - snprintf(info, INFO_BUFFER_SIZE - 1, "%p|%s", ctx, mark); - info[INFO_BUFFER_SIZE - 1] = '\0'; - } else { - grn_timeval tv; - uint64_t elapsed_time; - grn_timeval_now(ctx, &tv); - elapsed_time = - (uint64_t)(tv.tv_sec - ctx->impl->tv.tv_sec) * GRN_TIME_NSEC_PER_SEC + - (tv.tv_nsec - ctx->impl->tv.tv_nsec); - - snprintf(info, INFO_BUFFER_SIZE - 1, - "%p|%s%015" GRN_FMT_INT64U " ", ctx, mark, elapsed_time); - info[INFO_BUFFER_SIZE - 1] = '\0'; - } - - { - va_list args; - - va_start(args, format); - GRN_BULK_REWIND(message); - grn_text_vprintf(ctx, message, format, args); - va_end(args); - GRN_TEXT_PUTC(ctx, message, '\0'); - } - - current_query_logger.log(ctx, flag, timestamp, info, GRN_TEXT_VALUE(message), - current_query_logger.user_data); -} - -static void -query_logger_init(void) -{ - memcpy(¤t_query_logger, &default_query_logger, sizeof(grn_query_logger)); - CRITICAL_SECTION_INIT(default_query_logger_lock); -} - -static void -query_logger_fin(grn_ctx *ctx) -{ - grn_query_logger_fin(ctx); - if (default_query_logger_path) { - free(default_query_logger_path); - } - CRITICAL_SECTION_FIN(default_query_logger_lock); -} - -void -grn_log_reopen(grn_ctx *ctx) -{ - grn_logger_reopen(ctx); - grn_query_logger_reopen(ctx); -} - - static void check_overcommit_memory(grn_ctx *ctx) { FILE *file; int value; - file = fopen("/proc/sys/vm/overcommit_memory", "r"); + file = grn_fopen("/proc/sys/vm/overcommit_memory", "r"); if (!file) { return; } value = fgetc(file); if (value != '1') { @@ -1245,25 +816,13 @@ check_overcommit_memory(grn_ctx *ctx) fclose(file); } -static void -check_grn_ja_skip_same_value_put(grn_ctx *ctx) -{ - const char *grn_ja_skip_same_value_put_env; - - grn_ja_skip_same_value_put_env = getenv("GRN_JA_SKIP_SAME_VALUE_PUT"); - if (grn_ja_skip_same_value_put_env && - strcmp(grn_ja_skip_same_value_put_env, "no") == 0) { - grn_ja_skip_same_value_put = GRN_FALSE; - } -} - grn_rc grn_init(void) { grn_rc rc; grn_ctx *ctx = &grn_gctx; - logger_init(); - query_logger_init(); + grn_logger_init(); + grn_query_logger_init(); CRITICAL_SECTION_INIT(grn_glock); grn_gtick = 0; ctx->next = ctx; @@ -1288,22 +847,50 @@ grn_init(void) } // expand_stack(); #ifdef USE_FAIL_MALLOC - if (getenv("GRN_FMALLOC_PROB")) { - grn_fmalloc_prob = strtod(getenv("GRN_FMALLOC_PROB"), 0) * RAND_MAX; - if (getenv("GRN_FMALLOC_SEED")) { - srand((unsigned int)atoi(getenv("GRN_FMALLOC_SEED"))); - } else { - srand((unsigned int)time(NULL)); + { + char grn_fmalloc_prob_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_FMALLOC_PROB", + grn_fmalloc_prob_env, + GRN_ENV_BUFFER_SIZE); + if (grn_fmalloc_prob_env[0]) { + char grn_fmalloc_seed_env[GRN_ENV_BUFFER_SIZE]; + grn_fmalloc_prob = strtod(grn_fmalloc_prob_env, 0) * RAND_MAX; + grn_getenv("GRN_FMALLOC_SEED", + grn_fmalloc_seed_env, + GRN_ENV_BUFFER_SIZE); + if (grn_fmalloc_seed_env[0]) { + srand((unsigned int)atoi(grn_fmalloc_seed_env)); + } else { + srand((unsigned int)time(NULL)); + } } } - if (getenv("GRN_FMALLOC_FUNC")) { - grn_fmalloc_func = getenv("GRN_FMALLOC_FUNC"); + { + static char grn_fmalloc_func_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_FMALLOC_FUNC", + grn_fmalloc_func_env, + GRN_ENV_BUFFER_SIZE); + if (grn_fmalloc_func_env[0]) { + grn_fmalloc_func = grn_fmalloc_func_env; + } } - if (getenv("GRN_FMALLOC_FILE")) { - grn_fmalloc_file = getenv("GRN_FMALLOC_FILE"); + { + static char grn_fmalloc_file_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_FMALLOC_FILE", + grn_fmalloc_file_env, + GRN_ENV_BUFFER_SIZE); + if (grn_fmalloc_file_env[0]) { + grn_fmalloc_file = grn_fmalloc_file_env; + } } - if (getenv("GRN_FMALLOC_LINE")) { - grn_fmalloc_line = atoi(getenv("GRN_FMALLOC_LINE")); + { + char grn_fmalloc_line_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_FMALLOC_LINE", + grn_fmalloc_line_env, + GRN_ENV_BUFFER_SIZE); + if (grn_fmalloc_line_env[0]) { + grn_fmalloc_line = atoi(grn_fmalloc_line_env); + } } #endif /* USE_FAIL_MALLOC */ if ((rc = grn_com_init())) { @@ -1343,7 +930,6 @@ grn_init(void) } GRN_LOG(ctx, GRN_LOG_NOTICE, "grn_init"); check_overcommit_memory(ctx); - check_grn_ja_skip_same_value_put(ctx); return rc; } @@ -1426,7 +1012,7 @@ grn_fin(void) GRN_GFREE(ctx); } } - query_logger_fin(ctx); + grn_query_logger_fin(ctx); grn_request_canceler_fin(); grn_cache_fin(); grn_tokenizers_fin(); @@ -1436,7 +1022,7 @@ grn_fin(void) grn_ctx_fin(ctx); grn_com_fin(); GRN_LOG(ctx, GRN_LOG_NOTICE, "grn_fin (%d)", alloc_count); - logger_fin(ctx); + grn_logger_fin(ctx); CRITICAL_SECTION_FIN(grn_glock); return GRN_SUCCESS; } @@ -1511,6 +1097,26 @@ grn_ctx_set_output_type(grn_ctx *ctx, grn_content_type type) if (ctx->impl) { ctx->impl->output_type = type; + switch (ctx->impl->output_type) { + case GRN_CONTENT_NONE : + ctx->impl->mime_type = "application/octet-stream"; + break; + case GRN_CONTENT_TSV : + ctx->impl->mime_type = "text/tab-separated-values"; + break; + case GRN_CONTENT_JSON : + ctx->impl->mime_type = "application/json"; + break; + case GRN_CONTENT_XML : + ctx->impl->mime_type = "text/xml"; + break; + case GRN_CONTENT_MSGPACK : + ctx->impl->mime_type = "application/x-msgpack"; + break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + ctx->impl->mime_type = "text/x-groonga-command-list"; + break; + } } else { rc = GRN_INVALID_ARGUMENT; } @@ -1630,7 +1236,7 @@ get_content_mime_type(grn_ctx *ctx, const char *p, const char *pe) ctx->impl->mime_type = "text/plain"; } else if (p + 3 == pe && !memcmp(p, "tsv", 3)) { ctx->impl->output_type = GRN_CONTENT_TSV; - ctx->impl->mime_type = "text/plain"; + ctx->impl->mime_type = "text/tab-separated-values"; } break; case 'x': @@ -1870,6 +1476,7 @@ grn_rc grn_ctx_sendv(grn_ctx *ctx, int argc, char **argv, int flags) { grn_obj buf; + GRN_API_ENTER; GRN_TEXT_INIT(&buf, 0); while (argc--) { // todo : encode into json like syntax @@ -1879,7 +1486,7 @@ grn_ctx_sendv(grn_ctx *ctx, int argc, char **argv, int flags) } grn_ctx_send(ctx, GRN_TEXT_VALUE(&buf), GRN_TEXT_LEN(&buf), flags); GRN_OBJ_FIN(ctx, &buf); - return ctx->rc; + GRN_API_RETURN(ctx->rc); } static int @@ -2103,7 +1710,7 @@ grn_cache_open(grn_ctx *ctx) cache->next = (grn_cache_entry *)cache; cache->prev = (grn_cache_entry *)cache; - cache->hash = grn_hash_create(&grn_gctx, NULL, GRN_TABLE_MAX_KEY_SIZE, + cache->hash = grn_hash_create(&grn_gctx, NULL, GRN_CACHE_MAX_KEY_SIZE, sizeof(grn_cache_entry), GRN_OBJ_KEY_VAR_SIZE); MUTEX_INIT(cache->mutex); cache->max_nentries = GRN_CACHE_DEFAULT_MAX_N_ENTRIES; @@ -2157,10 +1764,18 @@ grn_cache_init(void) grn_rc grn_cache_set_max_n_entries(grn_ctx *ctx, grn_cache *cache, unsigned int n) { + uint32_t current_max_n_entries; + if (!cache) { return GRN_INVALID_ARGUMENT; } + + current_max_n_entries = cache->max_nentries; cache->max_nentries = n; + if (n < current_max_n_entries) { + grn_cache_expire(cache, current_max_n_entries - n); + } + return GRN_SUCCESS; } @@ -2412,7 +2027,7 @@ grn_ctx_realloc(grn_ctx *ctx, void *ptr, size_t size, if (res && ptr) { int32_t *header = &((int32_t *)ptr)[-2]; size_t size_ = header[1]; - memcpy(res, ptr, size_ > size ? size : size_); + grn_memcpy(res, ptr, size_ > size ? size : size_); grn_ctx_free(ctx, ptr, file, line, func); } } else { @@ -2428,7 +2043,7 @@ grn_ctx_strdup(grn_ctx *ctx, const char *s, const char* file, int line, const ch if (s) { size_t size = strlen(s) + 1; if ((res = grn_ctx_alloc(ctx, size, 0, file, line, func))) { - memcpy(res, s, size); + grn_memcpy(res, s, size); } } return res; @@ -2669,6 +2284,20 @@ grn_ctx_set_strdup(grn_ctx *ctx, grn_strdup_func strdup_func) ctx->impl->strdup_func = strdup_func; } +grn_free_func +grn_ctx_get_free(grn_ctx *ctx) +{ + if (!ctx || !ctx->impl) { return NULL; } + return ctx->impl->free_func; +} + +void +grn_ctx_set_free(grn_ctx *ctx, grn_free_func free_func) +{ + if (!ctx || !ctx->impl) { return; } + ctx->impl->free_func = free_func; +} + void * grn_malloc(grn_ctx *ctx, size_t size, const char* file, int line, const char *func) { @@ -2708,6 +2337,16 @@ grn_strdup(grn_ctx *ctx, const char *string, const char* file, int line, const c return grn_strdup_default(ctx, string, file, line, func); } } + +void +grn_free(grn_ctx *ctx, void *ptr, const char* file, int line, const char *func) +{ + if (ctx && ctx->impl && ctx->impl->free_func) { + return ctx->impl->free_func(ctx, ptr, file, line, func); + } else { + return grn_free_default(ctx, ptr, file, line, func); + } +} #endif void * @@ -2810,12 +2449,12 @@ grn_strdup_default(grn_ctx *ctx, const char *s, const char* file, int line, cons { if (!ctx) { return NULL; } { - char *res = strdup(s); + char *res = grn_strdup_raw(s); if (res) { GRN_ADD_ALLOC_COUNT(1); grn_alloc_info_add(res, file, line, func); } else { - if (!(res = strdup(s))) { + if (!(res = grn_strdup_raw(s))) { MERR("strdup(%p)=%p (%s:%d) <%d>", s, res, file, line, alloc_count); } else { GRN_ADD_ALLOC_COUNT(1); @@ -2847,7 +2486,8 @@ grn_malloc_fail(grn_ctx *ctx, size_t size, const char* file, int line, const cha if (grn_fail_malloc_check(size, file, line, func)) { return grn_malloc_default(ctx, size, file, line, func); } else { - MERR("fail_malloc (%d) (%s:%d@%s) <%d>", size, file, line, func, alloc_count); + MERR("fail_malloc (%" GRN_FMT_SIZE ") (%s:%d@%s) <%d>", + size, file, line, func, alloc_count); return NULL; } } @@ -2858,7 +2498,8 @@ grn_calloc_fail(grn_ctx *ctx, size_t size, const char* file, int line, const cha if (grn_fail_malloc_check(size, file, line, func)) { return grn_calloc_default(ctx, size, file, line, func); } else { - MERR("fail_calloc (%d) (%s:%d@%s) <%d>", size, file, line, func, alloc_count); + MERR("fail_calloc (%" GRN_FMT_SIZE ") (%s:%d@%s) <%d>", + size, file, line, func, alloc_count); return NULL; } } diff --git a/storage/mroonga/vendor/groonga/lib/ctx_impl_mrb.c b/storage/mroonga/vendor/groonga/lib/ctx_impl_mrb.c index 05692eee13e..4c1a2a3b4f9 100644 --- a/storage/mroonga/vendor/groonga/lib/ctx_impl_mrb.c +++ b/storage/mroonga/vendor/groonga/lib/ctx_impl_mrb.c @@ -33,6 +33,7 @@ # include "mrb/mrb_void.h" # include "mrb/mrb_bulk.h" # include "mrb/mrb_object.h" +# include "mrb/mrb_object_flags.h" # include "mrb/mrb_database.h" # include "mrb/mrb_table.h" # include "mrb/mrb_array.h" @@ -52,6 +53,7 @@ # include "mrb/mrb_command_input.h" # include "mrb/mrb_table_cursor.h" # include "mrb/mrb_table_cursor_flags.h" +# include "mrb/mrb_content_type.h" # include "mrb/mrb_writer.h" # include <mruby/array.h> @@ -116,6 +118,7 @@ grn_ctx_impl_mrb_init_bindings(grn_ctx *ctx) grn_mrb_void_init(ctx); grn_mrb_bulk_init(ctx); grn_mrb_object_init(ctx); + grn_mrb_object_flags_init(ctx); grn_mrb_database_init(ctx); grn_mrb_table_init(ctx); grn_mrb_array_init(ctx); @@ -135,6 +138,7 @@ grn_ctx_impl_mrb_init_bindings(grn_ctx *ctx) grn_mrb_command_input_init(ctx); grn_mrb_table_cursor_init(ctx); grn_mrb_table_cursor_flags_init(ctx); + grn_mrb_content_type_init(ctx); grn_mrb_writer_init(ctx); grn_mrb_load(ctx, "initialize/post.rb"); @@ -143,9 +147,11 @@ grn_ctx_impl_mrb_init_bindings(grn_ctx *ctx) void grn_ctx_impl_mrb_init(grn_ctx *ctx) { - const char *grn_mruby_enabled; - grn_mruby_enabled = getenv("GRN_MRUBY_ENABLED"); - if (grn_mruby_enabled && strcmp(grn_mruby_enabled, "no") == 0) { + char grn_mruby_enabled[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_MRUBY_ENABLED", + grn_mruby_enabled, + GRN_ENV_BUFFER_SIZE); + if (grn_mruby_enabled[0] && strcmp(grn_mruby_enabled, "no") == 0) { ctx->impl->mrb.state = NULL; ctx->impl->mrb.base_directory[0] = '\0'; ctx->impl->mrb.module = NULL; @@ -153,6 +159,7 @@ grn_ctx_impl_mrb_init(grn_ctx *ctx) ctx->impl->mrb.checked_procs = NULL; ctx->impl->mrb.registered_plugins = NULL; ctx->impl->mrb.builtin.time_class = NULL; + ctx->impl->mrb.groonga.operator_class = NULL; } else { mrb_state *mrb; diff --git a/storage/mroonga/vendor/groonga/lib/dat.cpp b/storage/mroonga/vendor/groonga/lib/dat.cpp index 4a3db12d67d..60588d55710 100644 --- a/storage/mroonga/vendor/groonga/lib/dat.cpp +++ b/storage/mroonga/vendor/groonga/lib/dat.cpp @@ -70,7 +70,7 @@ bool grn_dat_remove_file(grn_ctx *ctx, const char *path) { struct stat stat; - return !::stat(path, &stat) && !unlink(path); + return !::stat(path, &stat) && !grn_unlink(path); } grn_rc @@ -144,7 +144,7 @@ grn_dat_generate_trie_path(const char *base_path, char *trie_path, uint32_t file return; } const size_t len = std::strlen(base_path); - std::memcpy(trie_path, base_path, len); + grn_memcpy(trie_path, base_path, len); trie_path[len] = '.'; grn_itoh(file_id % (1U << (4 * FILE_ID_LENGTH)), trie_path + len + 1, FILE_ID_LENGTH); @@ -502,7 +502,7 @@ grn_dat_get_key(grn_ctx *ctx, grn_dat *dat, grn_id id, void *keybuf, int bufsize return 0; } if (keybuf && (bufsize >= (int)key.length())) { - std::memcpy(keybuf, key.ptr(), key.length()); + grn_memcpy(keybuf, key.ptr(), key.length()); } return (int)key.length(); } diff --git a/storage/mroonga/vendor/groonga/lib/dat/dat.hpp b/storage/mroonga/vendor/groonga/lib/dat/dat.hpp index d2b2264752b..c941bf25523 100644 --- a/storage/mroonga/vendor/groonga/lib/dat/dat.hpp +++ b/storage/mroonga/vendor/groonga/lib/dat/dat.hpp @@ -42,6 +42,12 @@ # endif // WIN32 #endif // GRN_DAT_API +#ifdef WIN32 +# define grn_memcpy(dest, src, n) ::memcpy_s((dest), (n), (src), (n)) +#else // WIN32 +# define grn_memcpy(dest, src, n) std::memcpy((dest), (src), (n)) +#endif // WIN32 + namespace grn { namespace dat { diff --git a/storage/mroonga/vendor/groonga/lib/dat/key-cursor.cpp b/storage/mroonga/vendor/groonga/lib/dat/key-cursor.cpp index 90ba25eca53..6bf864dba55 100644 --- a/storage/mroonga/vendor/groonga/lib/dat/key-cursor.cpp +++ b/storage/mroonga/vendor/groonga/lib/dat/key-cursor.cpp @@ -133,7 +133,7 @@ void KeyCursor::ascending_init(const String &min_str, const String &max_str) { if (max_str.ptr() != NULL) { if (max_str.length() != 0) { end_buf_ = new UInt8[max_str.length()]; - std::memcpy(end_buf_, max_str.ptr(), max_str.length()); + grn_memcpy(end_buf_, max_str.ptr(), max_str.length()); end_str_.assign(end_buf_, max_str.length()); } } @@ -206,7 +206,7 @@ void KeyCursor::descending_init(const String &min_str, const String &max_str) { if (min_str.ptr() != NULL) { if (min_str.length() != 0) { end_buf_ = new UInt8[min_str.length()]; - std::memcpy(end_buf_, min_str.ptr(), min_str.length()); + grn_memcpy(end_buf_, min_str.ptr(), min_str.length()); end_str_.assign(end_buf_, min_str.length()); } } diff --git a/storage/mroonga/vendor/groonga/lib/db.c b/storage/mroonga/vendor/groonga/lib/db.c index b9cea065a6a..357df82e314 100644 --- a/storage/mroonga/vendor/groonga/lib/db.c +++ b/storage/mroonga/vendor/groonga/lib/db.c @@ -32,7 +32,6 @@ #include "grn_normalizer.h" #include "grn_util.h" #include <string.h> -#include <float.h> typedef struct { grn_id id; @@ -90,7 +89,7 @@ inline static void gen_pathname(const char *path, char *buffer, int fno) { size_t len = strlen(path); - memcpy(buffer, path, len); + grn_memcpy(buffer, path, len); if (fno >= 0) { buffer[len] = '.'; grn_itoh(fno, buffer + len + 1, 7); @@ -156,11 +155,15 @@ grn_db_create(grn_ctx *ctx, const char *path, grn_db_create_optarg *optarg) if ((s = GRN_MALLOC(sizeof(grn_db)))) { grn_bool use_default_db_key = GRN_TRUE; grn_bool use_pat_as_db_keys = GRN_FALSE; - if (getenv("GRN_DB_KEY")) { - if (!strcmp(getenv("GRN_DB_KEY"), "pat")) { + char grn_db_key_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_DB_KEY", + grn_db_key_env, + GRN_ENV_BUFFER_SIZE); + if (grn_db_key_env[0]) { + if (!strcmp(grn_db_key_env, "pat")) { use_default_db_key = GRN_FALSE; use_pat_as_db_keys = GRN_TRUE; - } else if (!strcmp(getenv("GRN_DB_KEY"), "dat")) { + } else if (!strcmp(grn_db_key_env, "dat")) { use_default_db_key = GRN_FALSE; } } @@ -1919,9 +1922,11 @@ grn_table_truncate(grn_ctx *ctx, grn_obj *table) } grn_hash_close(ctx, cols); } - grn_table_get_info(ctx, table, NULL, NULL, &tokenizer, &normalizer, NULL); - GRN_PTR_INIT(&token_filters, GRN_OBJ_VECTOR, GRN_ID_NIL); - grn_obj_get_info(ctx, table, GRN_INFO_TOKEN_FILTERS, &token_filters); + if (table->header.type != GRN_TABLE_NO_KEY) { + grn_table_get_info(ctx, table, NULL, NULL, &tokenizer, &normalizer, NULL); + GRN_PTR_INIT(&token_filters, GRN_OBJ_VECTOR, GRN_ID_NIL); + grn_obj_get_info(ctx, table, GRN_INFO_TOKEN_FILTERS, &token_filters); + } switch (table->header.type) { case GRN_TABLE_PAT_KEY : for (hooks = DB_OBJ(table)->hooks[GRN_HOOK_INSERT]; hooks; hooks = hooks->next) { @@ -1954,10 +1959,12 @@ grn_table_truncate(grn_ctx *ctx, grn_obj *table) rc = grn_array_truncate(ctx, (grn_array *)table); break; } - grn_obj_set_info(ctx, table, GRN_INFO_DEFAULT_TOKENIZER, tokenizer); - grn_obj_set_info(ctx, table, GRN_INFO_NORMALIZER, normalizer); - grn_obj_set_info(ctx, table, GRN_INFO_TOKEN_FILTERS, &token_filters); - GRN_OBJ_FIN(ctx, &token_filters); + if (table->header.type != GRN_TABLE_NO_KEY) { + grn_obj_set_info(ctx, table, GRN_INFO_DEFAULT_TOKENIZER, tokenizer); + grn_obj_set_info(ctx, table, GRN_INFO_NORMALIZER, normalizer); + grn_obj_set_info(ctx, table, GRN_INFO_TOKEN_FILTERS, &token_filters); + GRN_OBJ_FIN(ctx, &token_filters); + } if (rc == GRN_SUCCESS) { grn_obj_touch(ctx, table, NULL); } @@ -2060,7 +2067,7 @@ subrecs_push(byte *subrecs, int size, int n_subrecs, double score, void *body, i } v = subrecs + n * (GRN_RSET_SCORE_SIZE + size); *((double *)v) = score; - memcpy(v + GRN_RSET_SCORE_SIZE, body, size); + grn_memcpy(v + GRN_RSET_SCORE_SIZE, body, size); } inline static void @@ -2094,8 +2101,8 @@ subrecs_replace_min(byte *subrecs, int size, int n_subrecs, double score, void * } } v = subrecs + n * (GRN_RSET_SCORE_SIZE + size); - memcpy(v, &score, GRN_RSET_SCORE_SIZE); - memcpy(v + GRN_RSET_SCORE_SIZE, body, size); + grn_memcpy(v, &score, GRN_RSET_SCORE_SIZE); + grn_memcpy(v + GRN_RSET_SCORE_SIZE, body, size); } inline static void @@ -2784,16 +2791,8 @@ grn_accessor_resolve(grn_ctx *ctx, grn_obj *accessor, int deep, { grn_id *tid; - grn_obj *domain; grn_obj *next_res; - grn_search_optarg next_optarg; grn_rset_recinfo *recinfo; - if (optarg) { - next_optarg = *optarg; - next_optarg.mode = GRN_OP_EXACT; - } else { - memset(&next_optarg, 0, sizeof(grn_search_optarg)); - } { grn_obj *range = grn_ctx_at(ctx, DB_OBJ(index)->range); next_res = grn_table_create(ctx, NULL, 0, NULL, @@ -2808,29 +2807,33 @@ grn_accessor_resolve(grn_ctx *ctx, grn_obj *accessor, int deep, break; } } - domain = grn_ctx_at(ctx, index->header.domain); GRN_HASH_EACH(ctx, (grn_hash *)current_res, id, &tid, NULL, &recinfo, { - next_optarg.weight_vector = NULL; - next_optarg.vector_size = recinfo->score; - if (domain->header.type == GRN_TABLE_NO_KEY) { - rc = grn_ii_sel(ctx, (grn_ii *)index, - (const char *)tid, sizeof(grn_id), - (grn_hash *)next_res, GRN_OP_OR, - &next_optarg); - } else { - char key[GRN_TABLE_MAX_KEY_SIZE]; - int key_len; - key_len = grn_table_get_key(ctx, domain, *tid, - key, GRN_TABLE_MAX_KEY_SIZE); - rc = grn_ii_sel(ctx, (grn_ii *)index, key, key_len, - (grn_hash *)next_res, GRN_OP_OR, - &next_optarg); + grn_ii *ii = (grn_ii *)index; + grn_ii_cursor *ii_cursor; + grn_ii_posting *posting; + + ii_cursor = grn_ii_cursor_open(ctx, ii, *tid, + GRN_ID_NIL, GRN_ID_MAX, + ii->n_elements, + 0); + if (!ii_cursor) { + continue; } + + while ((posting = grn_ii_cursor_next(ctx, ii_cursor))) { + grn_ii_posting add_posting = *posting; + add_posting.weight += recinfo->score - 1; + grn_ii_posting_add(ctx, + &add_posting, + (grn_hash *)next_res, + GRN_OP_OR); + } + grn_ii_cursor_close(ctx, ii_cursor); + if (rc != GRN_SUCCESS) { break; } }); - grn_obj_unlink(ctx, domain); if (current_res != base_res) { grn_obj_unlink(ctx, current_res); } @@ -3789,7 +3792,7 @@ grn_table_setoperation(grn_ctx *ctx, grn_obj *table1, grn_obj *table2, grn_obj * GRN_TABLE_EACH(ctx, table2, 0, 0, id, &key, &key_size, &value2, { if (grn_table_add_v_inline(ctx, table1, key, key_size, &value1, &added)) { if (added) { - memcpy(value1, value2, value_size); + grn_memcpy(value1, value2, value_size); } else { grn_rset_recinfo *ri1 = value1; grn_rset_recinfo *ri2 = value2; @@ -3800,7 +3803,7 @@ grn_table_setoperation(grn_ctx *ctx, grn_obj *table1, grn_obj *table2, grn_obj * } else { GRN_TABLE_EACH(ctx, table2, 0, 0, id, &key, &key_size, &value2, { if (grn_table_add_v_inline(ctx, table1, key, key_size, &value1, NULL)) { - memcpy(value1, value2, value_size); + grn_memcpy(value1, value2, value_size); } }); } @@ -3832,7 +3835,7 @@ grn_table_setoperation(grn_ctx *ctx, grn_obj *table1, grn_obj *table2, grn_obj * case GRN_OP_ADJUST : GRN_TABLE_EACH(ctx, table2, 0, 0, id, &key, &key_size, &value2, { if (grn_table_get_v(ctx, table1, key, key_size, &value1)) { - memcpy(value1, value2, value_size); + grn_memcpy(value1, value2, value_size); } }); break; @@ -3881,7 +3884,7 @@ grn_obj_column_(grn_ctx *ctx, grn_obj *table, const char *name, unsigned int nam if (len) { buf[len++] = GRN_DB_DELIMITER; if (len + name_size <= GRN_TABLE_MAX_KEY_SIZE) { - memcpy(buf + len, name, name_size); + grn_memcpy(buf + len, name, name_size); column = grn_ctx_get(ctx, buf, len + name_size); } else { ERR(GRN_INVALID_ARGUMENT, "name is too long"); @@ -4027,7 +4030,7 @@ grn_column_create(grn_ctx *ctx, grn_obj *table, goto exit; } fullname[len] = GRN_DB_DELIMITER; - memcpy(fullname + len + 1, name, name_size); + grn_memcpy(fullname + len + 1, name, name_size); name_size += len + 1; } else { ERR(GRN_FUNCTION_NOT_IMPLEMENTED, @@ -4155,7 +4158,7 @@ grn_column_open(grn_ctx *ctx, grn_obj *table, goto exit; } fullname[len] = GRN_DB_DELIMITER; - memcpy(fullname + len + 1, name, name_size); + grn_memcpy(fullname + len + 1, name, name_size); name_size += len + 1; } else { ERR(GRN_INVALID_ARGUMENT, "todo : not supported yet"); @@ -4219,7 +4222,7 @@ default_column_set_value(grn_ctx *ctx, grn_proc_ctx *pctx, grn_obj *in, grn_obj ERR(GRN_NO_MEMORY_AVAILABLE, "ra get failed"); return GRN_NO_MEMORY_AVAILABLE; } - memcpy(v, in->u.p.ptr, value_size); + grn_memcpy(v, in->u.p.ptr, value_size); grn_ra_unref(ctx, (grn_ra *)pctx->obj, arg->id); } break; @@ -5240,7 +5243,11 @@ grn_obj_is_persistent(grn_ctx *ctx, grn_obj *obj) if (GRN_BULK_VSIZE(p_key)) {\ id = addp ? grn_table_add_by_key(ctx, table, p_key, NULL)\ : grn_table_get_by_key(ctx, table, p_key);\ - if (id) { GRN_RECORD_SET(ctx, dest, id); }\ + if (id) {\ + GRN_RECORD_SET(ctx, dest, id);\ + } else {\ + rc = GRN_INVALID_ARGUMENT;\ + }\ } else {\ GRN_RECORD_SET(ctx, dest, GRN_ID_NIL);\ }\ @@ -5250,7 +5257,11 @@ grn_obj_is_persistent(grn_ctx *ctx, grn_obj *obj) GRN_UINT32_INIT(&record_id, 0);\ grn_obj_cast(ctx, src, &record_id, GRN_TRUE);\ id = GRN_UINT32_VALUE(&record_id);\ - if (id) { GRN_RECORD_SET(ctx, dest, id); }\ + if (id) {\ + GRN_RECORD_SET(ctx, dest, id);\ + } else {\ + rc = GRN_INVALID_ARGUMENT;\ + }\ }\ } else {\ rc = GRN_FUNCTION_NOT_IMPLEMENTED;\ @@ -6605,13 +6616,13 @@ grn_obj_set_value_column_fix_size(grn_ctx *ctx, grn_obj *obj, grn_id id, } else { void *b; if ((b = GRN_CALLOC(element_size))) { - memcpy(b, v, s); - memcpy(p, b, element_size); + grn_memcpy(b, v, s); + grn_memcpy(p, b, element_size); GRN_FREE(b); } } } else { - memcpy(p, v, s); + grn_memcpy(p, v, s); } rc = GRN_SUCCESS; break; @@ -7197,6 +7208,7 @@ build_index(grn_ctx *ctx, grn_obj *obj) break; default : use_grn_ii_build = GRN_FALSE; + break; } if ((ii->header->flags & GRN_OBJ_WITH_WEIGHT)) { use_grn_ii_build = GRN_FALSE; @@ -7214,10 +7226,14 @@ build_index(grn_ctx *ctx, grn_obj *obj) } if (use_grn_ii_build) { uint64_t sparsity = 10; - if (getenv("GRN_INDEX_SPARSITY")) { + char grn_index_sparsity_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_INDEX_SPARSITY", + grn_index_sparsity_env, + GRN_ENV_BUFFER_SIZE); + if (grn_index_sparsity_env[0]) { uint64_t v; errno = 0; - v = strtoull(getenv("GRN_INDEX_SPARSITY"), NULL, 0); + v = strtoull(grn_index_sparsity_env, NULL, 0); if (!errno) { sparsity = v; } } grn_ii_build(ctx, ii, sparsity); @@ -7385,7 +7401,7 @@ grn_hook_unpack(grn_ctx *ctx, grn_db_obj *obj, const char *buf, uint32_t buf_siz new->proc = NULL; } if ((new->hld_size = hld_size)) { - memcpy(NEXT_ADDR(new), p, hld_size); + grn_memcpy(NEXT_ADDR(new), p, hld_size); p += hld_size; } *last = new; @@ -7482,8 +7498,10 @@ grn_obj_set_info_source_validate_report_error(grn_ctx *ctx, source_name, GRN_TABLE_MAX_KEY_SIZE); if (GRN_OBJ_TABLEP(source)) { source_name[source_name_size] = '\0'; - strncat(source_name, "._key", - GRN_TABLE_MAX_KEY_SIZE - source_name_size - 1); + grn_strncat(source_name, + GRN_TABLE_MAX_KEY_SIZE, + "._key", + GRN_TABLE_MAX_KEY_SIZE - source_name_size - 1); source_name_size = strlen(source_name); } table_domain_name_size = grn_obj_name(ctx, table_domain, @@ -7496,7 +7514,10 @@ grn_obj_set_info_source_validate_report_error(grn_ctx *ctx, GRN_TABLE_MAX_KEY_SIZE); grn_obj_unlink(ctx, source_type); } else { - strncpy(source_type_name, "(nil)", GRN_TABLE_MAX_KEY_SIZE); + grn_strncpy(source_type_name, + GRN_TABLE_MAX_KEY_SIZE, + "(nil)", + GRN_TABLE_MAX_KEY_SIZE); source_type_name_size = strlen(source_type_name); } ERR(GRN_INVALID_ARGUMENT, @@ -7619,7 +7640,7 @@ grn_obj_set_info_source_update(grn_ctx *ctx, grn_obj *obj, grn_obj *value) if (!v2) { return ctx->rc; } - memcpy(v2, v, s); + grn_memcpy(v2, v, s); if (DB_OBJ(obj)->source) { GRN_FREE(DB_OBJ(obj)->source); } DB_OBJ(obj)->source = v2; DB_OBJ(obj)->source_size = s; @@ -7741,7 +7762,7 @@ grn_obj_set_info(grn_ctx *ctx, grn_obj *obj, grn_info_type type, grn_obj *value) switch (DB_OBJ(obj)->header.type) { case GRN_TABLE_HASH_KEY : ((grn_hash *)obj)->tokenizer = value; - ((grn_hash *)obj)->header->tokenizer = grn_obj_id(ctx, value); + ((grn_hash *)obj)->header.common->tokenizer = grn_obj_id(ctx, value); rc = GRN_SUCCESS; break; case GRN_TABLE_PAT_KEY : @@ -7762,7 +7783,7 @@ grn_obj_set_info(grn_ctx *ctx, grn_obj *obj, grn_info_type type, grn_obj *value) switch (DB_OBJ(obj)->header.type) { case GRN_TABLE_HASH_KEY : ((grn_hash *)obj)->normalizer = value; - ((grn_hash *)obj)->header->normalizer = grn_obj_id(ctx, value); + ((grn_hash *)obj)->header.common->normalizer = grn_obj_id(ctx, value); rc = GRN_SUCCESS; break; case GRN_TABLE_PAT_KEY : @@ -7839,7 +7860,7 @@ grn_obj_add_hook(grn_ctx *ctx, grn_obj *obj, grn_hook_entry entry, new->proc = (grn_proc *)proc; new->hld_size = hld_size; if (hld_size) { - memcpy(NEXT_ADDR(new), hld_value, hld_size); + grn_memcpy(NEXT_ADDR(new), hld_value, hld_size); } for (i = 0; i != offset && *last; i++) { last = &(*last)->next; } new->next = *last; @@ -8347,7 +8368,8 @@ _grn_obj_remove(grn_ctx *ctx, grn_obj *obj) const char *n = _grn_table_key(ctx, ctx->impl->db, DB_OBJ(obj)->id, &s); GRN_LOG(ctx, GRN_LOG_NOTICE, "DDL:obj_remove %.*s", s, n); } - if ((io_path = grn_obj_path(ctx, obj)) && *io_path != '\0') { + if (obj->header.type != GRN_PROC && + (io_path = grn_obj_path(ctx, obj)) && *io_path != '\0') { if (!(path = GRN_STRDUP(io_path))) { ERR(GRN_NO_MEMORY_AVAILABLE, "cannot duplicate path: <%s>", io_path); return; @@ -8495,7 +8517,7 @@ grn_table_rename(grn_ctx *ctx, grn_obj *table, const char *name, unsigned int na if (!(rc = grn_obj_rename(ctx, table, name, name_size))) { grn_id *key; char fullname[GRN_TABLE_MAX_KEY_SIZE]; - memcpy(fullname, name, name_size); + grn_memcpy(fullname, name, name_size); fullname[name_size] = GRN_DB_DELIMITER; GRN_HASH_EACH(ctx, cols, id, &key, NULL, NULL, { grn_obj *col = grn_ctx_at(ctx, *key); @@ -8536,7 +8558,7 @@ grn_column_rename(grn_ctx *ctx, grn_obj *column, const char *name, unsigned int goto exit; } fullname[len] = GRN_DB_DELIMITER; - memcpy(fullname + len + 1, name, name_size); + grn_memcpy(fullname + len + 1, name, name_size); name_size += len + 1; rc = grn_obj_rename(ctx, column, fullname, name_size); } @@ -8678,7 +8700,7 @@ grn_db_obj_init(grn_ctx *ctx, grn_obj *db, grn_id id, grn_db_obj *obj) NULL,\ NULL);\ if (size > PATH_MAX) { ERR(GRN_FILENAME_TOO_LONG, "too long path"); }\ - memcpy(buffer, path, size);\ + grn_memcpy(buffer, path, size);\ buffer[size] = '\0';\ } else {\ gen_pathname(grn_obj_io(s->keys)->path, buffer, id); \ @@ -8700,7 +8722,7 @@ grn_db_obj_init(grn_ctx *ctx, grn_obj *db, grn_id id, grn_db_obj *obj) NULL);\ if (size) {\ if ((r->source = GRN_MALLOC(size))) {\ - memcpy(r->source, p, size);\ + grn_memcpy(r->source, p, size);\ r->source_size = size;\ }\ }\ @@ -9308,7 +9330,7 @@ grn_column_name(grn_ctx *ctx, grn_obj *obj, char *namebuf, int buf_size) } len = pe - p0; if (len && len <= buf_size) { - memcpy(namebuf, p0, len); + grn_memcpy(namebuf, p0, len); } } } @@ -9354,7 +9376,7 @@ grn_column_name(grn_ctx *ctx, grn_obj *obj, char *namebuf, int buf_size) if (name) { len = strlen(name); if (len <= buf_size) { - memcpy(namebuf, name, len); + grn_memcpy(namebuf, name, len); } } } @@ -10450,18 +10472,24 @@ grn_db_init_builtin_types(grn_ctx *ctx) #define MULTI_COLUMN_INDEXP(i) (DB_OBJ(i)->source_size > sizeof(grn_id)) static inline int -grn_column_index_column_equal(grn_ctx *ctx, grn_obj *obj, grn_operator op, - grn_obj **indexbuf, int buf_size, int *section) +grn_column_find_index_data_column_equal(grn_ctx *ctx, grn_obj *obj, + grn_operator op, + grn_index_datum *index_data, + unsigned int n_index_data, + grn_obj **index_buf, int buf_size, + int *section_buf) { int n = 0; - grn_obj **ip = indexbuf; + grn_obj **ip = index_buf; grn_hook *hooks; for (hooks = DB_OBJ(obj)->hooks[GRN_HOOK_SET]; hooks; hooks = hooks->next) { default_set_value_hook_data *data = (void *)NEXT_ADDR(hooks); grn_obj *target = grn_ctx_at(ctx, data->target); + int section; if (target->header.type != GRN_COLUMN_INDEX) { continue; } - if (section) { *section = (MULTI_COLUMN_INDEXP(target)) ? data->section : 0; } + section = (MULTI_COLUMN_INDEXP(target)) ? data->section : 0; + if (section_buf) { *section_buf = section; } if (obj->header.type != GRN_COLUMN_FIX_SIZE) { grn_obj *tokenizer, *lexicon = grn_ctx_at(ctx, target->header.domain); if (!lexicon) { continue; } @@ -10471,18 +10499,47 @@ grn_column_index_column_equal(grn_ctx *ctx, grn_obj *obj, grn_operator op, if (n < buf_size) { *ip++ = target; } + if (n < n_index_data) { + index_data[n].index = target; + index_data[n].section = section; + } n++; } return n; } +static inline grn_bool +is_valid_regexp_index(grn_ctx *ctx, grn_obj *index_column) +{ + grn_obj *tokenizer; + grn_obj *lexicon; + + lexicon = grn_ctx_at(ctx, index_column->header.domain); + if (!lexicon) { + return GRN_FALSE; + } + + grn_table_get_info(ctx, lexicon, NULL, NULL, &tokenizer, NULL, NULL); + grn_obj_unlink(ctx, lexicon); + if (!tokenizer) { + return GRN_FALSE; + } + + /* TODO: Restrict to TokenRegexp? */ + return GRN_TRUE; +} + static inline int -grn_column_index_column_match(grn_ctx *ctx, grn_obj *obj, grn_operator op, - grn_obj **indexbuf, int buf_size, int *section) +grn_column_find_index_data_column_match(grn_ctx *ctx, grn_obj *obj, + grn_operator op, + grn_index_datum *index_data, + unsigned int n_index_data, + grn_obj **index_buf, int buf_size, + int *section_buf) { int n = 0; - grn_obj **ip = indexbuf; + grn_obj **ip = index_buf; grn_hook_entry hook_entry; grn_hook *hooks; @@ -10501,11 +10558,20 @@ grn_column_index_column_match(grn_ctx *ctx, grn_obj *obj, grn_operator op, for (hooks = DB_OBJ(obj)->hooks[hook_entry]; hooks; hooks = hooks->next) { default_set_value_hook_data *data = (void *)NEXT_ADDR(hooks); grn_obj *target = grn_ctx_at(ctx, data->target); + int section; if (target->header.type != GRN_COLUMN_INDEX) { continue; } - if (section) { *section = (MULTI_COLUMN_INDEXP(target)) ? data->section : 0; } + if (op == GRN_OP_REGEXP && !is_valid_regexp_index(ctx, target)) { + continue; + } + section = (MULTI_COLUMN_INDEXP(target)) ? data->section : 0; + if (section_buf) { *section_buf = section; } if (n < buf_size) { *ip++ = target; } + if (n < n_index_data) { + index_data[n].index = target; + index_data[n].section = section; + } n++; } @@ -10513,11 +10579,15 @@ grn_column_index_column_match(grn_ctx *ctx, grn_obj *obj, grn_operator op, } static inline int -grn_column_index_column_range(grn_ctx *ctx, grn_obj *obj, grn_operator op, - grn_obj **indexbuf, int buf_size, int *section) +grn_column_find_index_data_column_range(grn_ctx *ctx, grn_obj *obj, + grn_operator op, + grn_index_datum *index_data, + unsigned int n_index_data, + grn_obj **index_buf, int buf_size, + int *section_buf) { int n = 0; - grn_obj **ip = indexbuf; + grn_obj **ip = index_buf; grn_hook_entry hook_entry; grn_hook *hooks; @@ -10536,8 +10606,10 @@ grn_column_index_column_range(grn_ctx *ctx, grn_obj *obj, grn_operator op, for (hooks = DB_OBJ(obj)->hooks[hook_entry]; hooks; hooks = hooks->next) { default_set_value_hook_data *data = (void *)NEXT_ADDR(hooks); grn_obj *target = grn_ctx_at(ctx, data->target); + int section; if (target->header.type != GRN_COLUMN_INDEX) { continue; } - if (section) { *section = (MULTI_COLUMN_INDEXP(target)) ? data->section : 0; } + section = (MULTI_COLUMN_INDEXP(target)) ? data->section : 0; + if (section_buf) { *section_buf = section; } { grn_obj *tokenizer, *lexicon = grn_ctx_at(ctx, target->header.domain); if (!lexicon) { continue; } @@ -10549,6 +10621,10 @@ grn_column_index_column_range(grn_ctx *ctx, grn_obj *obj, grn_operator op, if (n < buf_size) { *ip++ = target; } + if (n < n_index_data) { + index_data[n].index = target; + index_data[n].section = section; + } n++; } @@ -10599,6 +10675,9 @@ is_valid_index(grn_ctx *ctx, grn_obj *index_column, grn_operator op) case GRN_OP_CALL : return is_valid_range_index(ctx, index_column); break; + case GRN_OP_REGEXP : + return is_valid_regexp_index(ctx, index_column); + break; default : return GRN_FALSE; break; @@ -10629,12 +10708,16 @@ find_section(grn_ctx *ctx, grn_obj *index_column, grn_obj *indexed_column) } static int -grn_column_index_accessor_index_column(grn_ctx *ctx, grn_accessor *a, - grn_operator op, - grn_obj **indexbuf, int buf_size, - int *section) +grn_column_find_index_data_accessor_index_column(grn_ctx *ctx, grn_accessor *a, + grn_operator op, + grn_index_datum *index_data, + unsigned int n_index_data, + grn_obj **index_buf, + int buf_size, + int *section_buf) { grn_obj *index_column = a->obj; + int section = 0; if (!is_valid_index(ctx, index_column, op)) { return 0; @@ -10651,23 +10734,32 @@ grn_column_index_accessor_index_column(grn_ctx *ctx, grn_accessor *a, if (is_invalid_section) { return 0; } - if (section) { - *section = specified_section; + section = specified_section; + if (section_buf) { + *section_buf = section; } } if (buf_size > 0) { - *indexbuf = index_column; + *index_buf = index_column; + } + if (n_index_data > 0) { + index_data[0].index = index_column; + index_data[0].section = section; } return 1; } static inline int -grn_column_index_accessor(grn_ctx *ctx, grn_obj *obj, grn_operator op, - grn_obj **indexbuf, int buf_size, int *section) +grn_column_find_index_data_accessor_match(grn_ctx *ctx, grn_obj *obj, + grn_operator op, + grn_index_datum *index_data, + unsigned n_index_data, + grn_obj **index_buf, int buf_size, + int *section_buf) { int n = 0; - grn_obj **ip = indexbuf; + grn_obj **ip = index_buf; grn_accessor *a = (grn_accessor *)obj; while (a) { @@ -10677,8 +10769,12 @@ grn_column_index_accessor(grn_ctx *ctx, grn_obj *obj, grn_operator op, if (a->action == GRN_ACCESSOR_GET_COLUMN_VALUE && GRN_OBJ_INDEX_COLUMNP(a->obj)) { - return grn_column_index_accessor_index_column(ctx, a, op, indexbuf, - buf_size, section); + return grn_column_find_index_data_accessor_index_column(ctx, a, op, + index_data, + n_index_data, + index_buf, + buf_size, + section_buf); } switch (a->action) { @@ -10704,16 +10800,23 @@ grn_column_index_accessor(grn_ctx *ctx, grn_obj *obj, grn_operator op, found = GRN_TRUE; if (!a->next) { + int section; + if (!is_valid_index(ctx, target, op)) { continue; } - if (section) { - *section = (MULTI_COLUMN_INDEXP(target)) ? data->section : 0; + section = (MULTI_COLUMN_INDEXP(target)) ? data->section : 0; + if (section_buf) { + *section_buf = section; } if (n < buf_size) { *ip++ = target; } + if (n < n_index_data) { + index_data[n].index = target; + index_data[n].section = section; + } n++; } } @@ -10727,17 +10830,102 @@ grn_column_index_accessor(grn_ctx *ctx, grn_obj *obj, grn_operator op, return n; } +static inline int +grn_column_find_index_data_accessor(grn_ctx *ctx, grn_obj *obj, + grn_operator op, + grn_index_datum *index_data, + unsigned n_index_data, + grn_obj **index_buf, int buf_size, + int *section_buf) +{ + int n = 0; + + if (section_buf) { + *section_buf = 0; + } + switch (op) { + case GRN_OP_EQUAL : + case GRN_OP_TERM_EXTRACT : + if (buf_size > 0) { + index_buf[n] = obj; + } + if (n_index_data > 0) { + index_data[n].index = obj; + index_data[n].section = 0; + } + n++; + break; + case GRN_OP_PREFIX : + { + grn_accessor *a = (grn_accessor *)obj; + if (a->action == GRN_ACCESSOR_GET_KEY) { + if (a->obj->header.type == GRN_TABLE_PAT_KEY) { + if (buf_size > 0) { + index_buf[n] = obj; + } + if (n_index_data > 0) { + index_data[n].index = obj; + index_data[n].section = 0; + } + n++; + } + /* FIXME: GRN_TABLE_DAT_KEY should be supported */ + } + } + break; + case GRN_OP_SUFFIX : + { + grn_accessor *a = (grn_accessor *)obj; + if (a->action == GRN_ACCESSOR_GET_KEY) { + if (a->obj->header.type == GRN_TABLE_PAT_KEY && + a->obj->header.flags & GRN_OBJ_KEY_WITH_SIS) { + if (buf_size > 0) { + index_buf[n] = obj; + } + if (n_index_data > 0) { + index_data[n].index = obj; + index_data[n].section = 0; + } + n++; + } + } + } + break; + case GRN_OP_MATCH : + case GRN_OP_NEAR : + case GRN_OP_NEAR2 : + case GRN_OP_SIMILAR : + case GRN_OP_LESS : + case GRN_OP_GREATER : + case GRN_OP_LESS_EQUAL : + case GRN_OP_GREATER_EQUAL : + case GRN_OP_CALL : + case GRN_OP_REGEXP : + n = grn_column_find_index_data_accessor_match(ctx, obj, op, + index_data, n_index_data, + index_buf, buf_size, + section_buf); + break; + default : + break; + } + + return n; +} + int grn_column_index(grn_ctx *ctx, grn_obj *obj, grn_operator op, - grn_obj **indexbuf, int buf_size, int *section) + grn_obj **index_buf, int buf_size, int *section_buf) { int n = 0; GRN_API_ENTER; if (GRN_DB_OBJP(obj)) { switch (op) { case GRN_OP_EQUAL : - n = grn_column_index_column_equal(ctx, obj, op, - indexbuf, buf_size, section); + n = grn_column_find_index_data_column_equal(ctx, obj, op, + NULL, 0, + index_buf, buf_size, + section_buf); break; case GRN_OP_PREFIX : case GRN_OP_SUFFIX : @@ -10745,68 +10933,75 @@ grn_column_index(grn_ctx *ctx, grn_obj *obj, grn_operator op, case GRN_OP_NEAR : case GRN_OP_NEAR2 : case GRN_OP_SIMILAR : - n = grn_column_index_column_match(ctx, obj, op, - indexbuf, buf_size, section); + case GRN_OP_REGEXP : + n = grn_column_find_index_data_column_match(ctx, obj, op, + NULL, 0, + index_buf, buf_size, + section_buf); break; case GRN_OP_LESS : case GRN_OP_GREATER : case GRN_OP_LESS_EQUAL : case GRN_OP_GREATER_EQUAL : case GRN_OP_CALL : - n = grn_column_index_column_range(ctx, obj, op, - indexbuf, buf_size, section); + n = grn_column_find_index_data_column_range(ctx, obj, op, + NULL, 0, + index_buf, buf_size, + section_buf); break; default : break; } } else if (GRN_ACCESSORP(obj)) { - if (section) { - *section = 0; - } + n = grn_column_find_index_data_accessor(ctx, obj, op, + NULL, 0, + index_buf, buf_size, + section_buf); + } + GRN_API_RETURN(n); +} + +unsigned int +grn_column_find_index_data(grn_ctx *ctx, grn_obj *obj, grn_operator op, + grn_index_datum *index_data, + unsigned int n_index_data) +{ + unsigned int n = 0; + GRN_API_ENTER; + if (GRN_DB_OBJP(obj)) { switch (op) { case GRN_OP_EQUAL : - case GRN_OP_TERM_EXTRACT : - if (buf_size) { indexbuf[n] = obj; } - n++; + n = grn_column_find_index_data_column_equal(ctx, obj, op, + index_data, n_index_data, + NULL, 0, NULL); break; case GRN_OP_PREFIX : - { - grn_accessor *a = (grn_accessor *)obj; - if (a->action == GRN_ACCESSOR_GET_KEY) { - if (a->obj->header.type == GRN_TABLE_PAT_KEY) { - if (buf_size) { indexbuf[n] = obj; } - n++; - } - /* FIXME: GRN_TABLE_DAT_KEY should be supported */ - } - } - break; case GRN_OP_SUFFIX : - { - grn_accessor *a = (grn_accessor *)obj; - if (a->action == GRN_ACCESSOR_GET_KEY) { - if (a->obj->header.type == GRN_TABLE_PAT_KEY && - a->obj->header.flags & GRN_OBJ_KEY_WITH_SIS) { - if (buf_size) { indexbuf[n] = obj; } - n++; - } - } - } - break; case GRN_OP_MATCH : case GRN_OP_NEAR : case GRN_OP_NEAR2 : case GRN_OP_SIMILAR : + case GRN_OP_REGEXP : + n = grn_column_find_index_data_column_match(ctx, obj, op, + index_data, n_index_data, + NULL, 0, NULL); + break; case GRN_OP_LESS : case GRN_OP_GREATER : case GRN_OP_LESS_EQUAL : case GRN_OP_GREATER_EQUAL : case GRN_OP_CALL : - n = grn_column_index_accessor(ctx, obj, op, indexbuf, buf_size, section); + n = grn_column_find_index_data_column_range(ctx, obj, op, + index_data, n_index_data, + NULL, 0, NULL); break; default : break; } + } else if (GRN_ACCESSORP(obj)) { + n = grn_column_find_index_data_accessor(ctx, obj, op, + index_data, n_index_data, + NULL, 0, NULL); } GRN_API_RETURN(n); } @@ -11659,7 +11854,6 @@ json_read(grn_ctx *ctx, grn_loader *loader, const char *str, unsigned int str_le case GRN_LOADER_BEGIN : if ((len = grn_isspace(str, ctx->encoding))) { str += len; - c = *str; continue; } switch (c) { @@ -11679,7 +11873,6 @@ json_read(grn_ctx *ctx, grn_loader *loader, const char *str, unsigned int str_le case GRN_LOADER_TOKEN : if ((len = grn_isspace(str, ctx->encoding))) { str += len; - c = *str; continue; } switch (c) { @@ -12073,6 +12266,7 @@ grn_load_(grn_ctx *ctx, grn_content_type input_type, case GRN_CONTENT_TSV : case GRN_CONTENT_XML : case GRN_CONTENT_MSGPACK : + case GRN_CONTENT_GROONGA_COMMAND_LIST : ERR(GRN_FUNCTION_NOT_IMPLEMENTED, "unsupported input_type"); // todo break; diff --git a/storage/mroonga/vendor/groonga/lib/expr.c b/storage/mroonga/vendor/groonga/lib/expr.c index ef477eb77ee..f13502a0f14 100644 --- a/storage/mroonga/vendor/groonga/lib/expr.c +++ b/storage/mroonga/vendor/groonga/lib/expr.c @@ -19,12 +19,10 @@ #include "grn_db.h" #include "grn_ctx_impl.h" #include <string.h> -#include <float.h> #include "grn_ii.h" #include "grn_geo.h" #include "grn_expr.h" #include "grn_util.h" -#include "grn_normalizer.h" #include "grn_mrb.h" #include "mrb/mrb_expr.h" @@ -357,7 +355,7 @@ grn_expr_open(grn_ctx *ctx, grn_obj_spec *spec, const uint8_t *p, const uint8_t { grn_expr *expr = NULL; if ((expr = GRN_MALLOCN(grn_expr, 1))) { - int size = 256; + int size = GRN_STACK_SIZE; expr->consts = NULL; expr->nconsts = 0; GRN_TEXT_INIT(&expr->name_buf, 0); @@ -722,6 +720,21 @@ grn_expr_get_var_by_offset(grn_ctx *ctx, grn_obj *expr, unsigned int offset) DFI_PUT(e, type, domain, code); \ } while (0) +static void +grn_expr_append_obj_resolve_const(grn_ctx *ctx, + grn_obj *obj, + grn_id to_domain) +{ + grn_obj dest; + + GRN_OBJ_INIT(&dest, GRN_BULK, 0, to_domain); + if (!grn_obj_cast(ctx, obj, &dest, GRN_FALSE)) { + grn_obj_reinit(ctx, obj, to_domain, 0); + grn_bulk_write(ctx, obj, GRN_BULK_HEAD(&dest), GRN_BULK_VSIZE(&dest)); + } + GRN_OBJ_FIN(ctx, &dest); +} + grn_obj * grn_expr_append_obj(grn_ctx *ctx, grn_obj *expr, grn_obj *obj, grn_operator op, int nargs) { @@ -854,26 +867,14 @@ grn_expr_append_obj(grn_ctx *ctx, grn_obj *expr, grn_obj *obj, grn_operator op, if (CONSTP(y)) { /* todo */ } else { - grn_obj dest; if (xd != yd) { - GRN_OBJ_INIT(&dest, GRN_BULK, 0, yd); - if (!grn_obj_cast(ctx, x, &dest, GRN_FALSE)) { - grn_obj_reinit(ctx, x, yd, 0); - grn_bulk_write(ctx, x, GRN_BULK_HEAD(&dest), GRN_BULK_VSIZE(&dest)); - } - GRN_OBJ_FIN(ctx, &dest); + grn_expr_append_obj_resolve_const(ctx, x, yd); } } } else { if (CONSTP(y)) { - grn_obj dest; if (xd != yd) { - GRN_OBJ_INIT(&dest, GRN_BULK, 0, xd); - if (!grn_obj_cast(ctx, y, &dest, GRN_FALSE)) { - grn_obj_reinit(ctx, y, xd, 0); - grn_bulk_write(ctx, y, GRN_BULK_HEAD(&dest), GRN_BULK_VSIZE(&dest)); - } - GRN_OBJ_FIN(ctx, &dest); + grn_expr_append_obj_resolve_const(ctx, y, xd); } } } @@ -908,6 +909,7 @@ grn_expr_append_obj(grn_ctx *ctx, grn_obj *expr, grn_obj *obj, grn_operator op, case GRN_OP_GET_REF : case GRN_OP_ADJUST : case GRN_OP_TERM_EXTRACT : + case GRN_OP_REGEXP : PUSH_CODE(e, op, obj, nargs, code); if (nargs) { int i = nargs - 1; @@ -2223,274 +2225,6 @@ grn_proc_call(grn_ctx *ctx, grn_obj *proc, int nargs, grn_obj *caller) } \ } while (0) -static grn_bool -string_is_contained(grn_ctx *ctx, - const char *text, unsigned int text_len, - const char *sub_text, unsigned int sub_text_len) -{ - /* TODO: Use more fast algorithm such as Boyer-Moore algorithm that - * is used in snip.c. */ - const char *text_end = text + text_len; - unsigned int sub_text_current = 0; - - for (; text < text_end; text++) { - if (text[0] == sub_text[sub_text_current]) { - sub_text_current++; - if (sub_text_current == sub_text_len) { - return GRN_TRUE; - } - } else { - sub_text_current = 0; - } - } - - return GRN_FALSE; -} - -static grn_bool -pseudo_query_scan_raw_text_raw_text(grn_ctx *ctx, - const char *x, unsigned int x_len, - const char *y, unsigned int y_len) -{ - grn_obj *normalizer; - grn_obj *norm_x; - grn_obj *norm_y; - const char *norm_x_raw; - const char *norm_y_raw; - unsigned int norm_x_raw_length_in_bytes; - unsigned int norm_y_raw_length_in_bytes; - grn_bool matched = GRN_FALSE; - - if (x_len == 0 || y_len == 0) { - return GRN_FALSE; - } - - normalizer = grn_ctx_get(ctx, GRN_NORMALIZER_AUTO_NAME, -1); - norm_x = grn_string_open(ctx, x, x_len, normalizer, 0); - norm_y = grn_string_open(ctx, y, y_len, normalizer, 0); - grn_string_get_normalized(ctx, norm_x, - &norm_x_raw, &norm_x_raw_length_in_bytes, - NULL); - grn_string_get_normalized(ctx, norm_y, - &norm_y_raw, &norm_y_raw_length_in_bytes, - NULL); - matched = string_is_contained(ctx, - norm_x_raw, norm_x_raw_length_in_bytes, - norm_y_raw, norm_y_raw_length_in_bytes); - - grn_obj_close(ctx, norm_x); - grn_obj_close(ctx, norm_y); - grn_obj_unlink(ctx, normalizer); - - return matched; -} - -static grn_bool -pseudo_query_scan_record_text(grn_ctx *ctx, grn_obj *record, grn_obj *table, - grn_obj *y) -{ - grn_obj *normalizer; - char x_key[GRN_TABLE_MAX_KEY_SIZE]; - int x_key_len; - grn_bool matched = GRN_FALSE; - - if (table->header.domain != GRN_DB_SHORT_TEXT) { - return GRN_FALSE; - } - - x_key_len = grn_table_get_key(ctx, table, GRN_RECORD_VALUE(record), - x_key, GRN_TABLE_MAX_KEY_SIZE); - grn_table_get_info(ctx, table, NULL, NULL, NULL, &normalizer, NULL); - if (normalizer) { - grn_obj *norm_y; - const char *norm_y_raw; - unsigned int norm_y_raw_length_in_bytes; - norm_y = grn_string_open(ctx, GRN_TEXT_VALUE(y), GRN_TEXT_LEN(y), - normalizer, 0); - grn_string_get_normalized(ctx, norm_y, - &norm_y_raw, &norm_y_raw_length_in_bytes, - NULL); - matched = string_is_contained(ctx, - x_key, x_key_len, - norm_y_raw, norm_y_raw_length_in_bytes); - grn_obj_close(ctx, norm_y); - } else { - matched = pseudo_query_scan_raw_text_raw_text(ctx, - x_key, - x_key_len, - GRN_TEXT_VALUE(y), - GRN_TEXT_LEN(y)); - } - - return matched; -} - -static grn_bool -pseudo_query_scan_text_text(grn_ctx *ctx, grn_obj *x, grn_obj *y) -{ - return pseudo_query_scan_raw_text_raw_text(ctx, - GRN_TEXT_VALUE(x), - GRN_TEXT_LEN(x), - GRN_TEXT_VALUE(y), - GRN_TEXT_LEN(y)); -} - -static grn_bool -pseudo_query_scan(grn_ctx *ctx, grn_obj *x, grn_obj *y) -{ - switch (x->header.domain) { - case GRN_DB_SHORT_TEXT : - case GRN_DB_TEXT : - case GRN_DB_LONG_TEXT : - switch (y->header.domain) { - case GRN_DB_SHORT_TEXT : - case GRN_DB_TEXT : - case GRN_DB_LONG_TEXT : - return pseudo_query_scan_text_text(ctx, x, y); - default : - break; - } - return GRN_FALSE; - default: - { - grn_obj *domain; - domain = grn_ctx_at(ctx, x->header.domain); - if (GRN_OBJ_TABLEP(domain)) { - switch (y->header.domain) { - case GRN_DB_SHORT_TEXT : - case GRN_DB_TEXT : - case GRN_DB_LONG_TEXT : - return pseudo_query_scan_record_text(ctx, x, domain, y); - default : - break; - } - } - } - return GRN_FALSE; - } -} - -static grn_bool -pseudo_prefix_search_match(grn_ctx *ctx, - const char *x, unsigned int x_len, - const char *y, unsigned int y_len) -{ - return (x_len >= y_len && strncmp(x, y, y_len) == 0); -} - -static grn_bool -pseudo_prefix_search_raw_text_raw_text(grn_ctx *ctx, - const char *x, unsigned int x_len, - const char *y, unsigned int y_len) -{ - grn_obj *normalizer; - grn_obj *norm_x; - grn_obj *norm_y; - const char *norm_x_raw; - const char *norm_y_raw; - unsigned int norm_x_raw_len; - unsigned int norm_y_raw_len; - grn_bool matched = GRN_FALSE; - - normalizer = grn_ctx_get(ctx, GRN_NORMALIZER_AUTO_NAME, -1); - norm_x = grn_string_open(ctx, x, x_len, normalizer, 0); - norm_y = grn_string_open(ctx, y, y_len, normalizer, 0); - grn_string_get_normalized(ctx, norm_x, &norm_x_raw, &norm_x_raw_len, NULL); - grn_string_get_normalized(ctx, norm_y, &norm_y_raw, &norm_y_raw_len, NULL); - matched = pseudo_prefix_search_match(ctx, - norm_x_raw, norm_x_raw_len, - norm_y_raw, norm_y_raw_len); - - grn_obj_close(ctx, norm_x); - grn_obj_close(ctx, norm_y); - grn_obj_unlink(ctx, normalizer); - - return matched; -} - -static grn_bool -pseudo_prefix_search_record_text(grn_ctx *ctx, grn_obj *record, grn_obj *table, - grn_obj *y) -{ - grn_obj *normalizer; - char x_key[GRN_TABLE_MAX_KEY_SIZE]; - int x_key_len; - grn_bool matched = GRN_FALSE; - - if (table->header.domain != GRN_DB_SHORT_TEXT) { - return GRN_FALSE; - } - - x_key_len = grn_table_get_key(ctx, table, GRN_RECORD_VALUE(record), - x_key, GRN_TABLE_MAX_KEY_SIZE); - grn_table_get_info(ctx, table, NULL, NULL, NULL, &normalizer, NULL); - if (normalizer) { - grn_obj *norm_y; - const char *norm_y_raw; - unsigned int norm_y_raw_len; - norm_y = grn_string_open(ctx, GRN_TEXT_VALUE(y), GRN_TEXT_LEN(y), - normalizer, 0); - grn_string_get_normalized(ctx, norm_y, &norm_y_raw, &norm_y_raw_len, NULL); - matched = pseudo_prefix_search_match(ctx, - x_key, x_key_len, - norm_y_raw, norm_y_raw_len); - grn_obj_close(ctx, norm_y); - } else { - matched = pseudo_prefix_search_raw_text_raw_text(ctx, - x_key, - x_key_len, - GRN_TEXT_VALUE(y), - GRN_TEXT_LEN(y)); - } - - return matched; -} - -static grn_bool -pseudo_prefix_search_text_text(grn_ctx *ctx, grn_obj *x, grn_obj *y) -{ - return pseudo_prefix_search_raw_text_raw_text(ctx, - GRN_TEXT_VALUE(x), - GRN_TEXT_LEN(x), - GRN_TEXT_VALUE(y), - GRN_TEXT_LEN(y)); -} - -static grn_bool -pseudo_prefix_search(grn_ctx *ctx, grn_obj *x, grn_obj *y) -{ - switch (x->header.domain) { - case GRN_DB_SHORT_TEXT : - case GRN_DB_TEXT : - case GRN_DB_LONG_TEXT : - switch (y->header.domain) { - case GRN_DB_SHORT_TEXT : - case GRN_DB_TEXT : - case GRN_DB_LONG_TEXT : - return pseudo_prefix_search_text_text(ctx, x, y); - default : - break; - } - return GRN_FALSE; - default: - { - grn_obj *domain; - domain = grn_ctx_at(ctx, x->header.domain); - if (GRN_OBJ_TABLEP(domain)) { - switch (y->header.domain) { - case GRN_DB_SHORT_TEXT : - case GRN_DB_TEXT : - case GRN_DB_LONG_TEXT : - return pseudo_prefix_search_record_text(ctx, x, domain, y); - default : - break; - } - } - } - return GRN_FALSE; - } -} - inline static void grn_expr_exec_get_member(grn_ctx *ctx, grn_obj *expr, @@ -2534,12 +2268,45 @@ grn_expr_exec_get_member(grn_ctx *ctx, GRN_OBJ_FIN(ctx, &values); } +static inline grn_bool +grn_expr_exec_is_simple_expr(grn_ctx *ctx, grn_obj *expr) +{ + grn_expr *e = (grn_expr *)expr; + + if (expr->header.type != GRN_EXPR) { + return GRN_FALSE; + } + + if (e->codes_curr != 1) { + return GRN_FALSE; + } + + switch (e->codes[0].op) { + case GRN_OP_PUSH : + return GRN_TRUE; + default : + return GRN_FALSE; + } +} + +static inline grn_obj * +grn_expr_exec_simple(grn_ctx *ctx, grn_obj *expr) +{ + grn_expr *e = (grn_expr *)expr; + + return e->codes[0].value; +} + grn_obj * grn_expr_exec(grn_ctx *ctx, grn_obj *expr, int nargs) { grn_obj *val = NULL; uint32_t stack_curr = ctx->impl->stack_curr; GRN_API_ENTER; + if (grn_expr_exec_is_simple_expr(ctx, expr)) { + val = grn_expr_exec_simple(ctx, expr); + GRN_API_RETURN(val); + } if (expr->header.type == GRN_PROC) { grn_proc *proc = (grn_proc *)expr; if (proc->type == GRN_PROC_COMMAND) { @@ -3170,7 +2937,7 @@ grn_expr_exec(grn_ctx *ctx, grn_obj *expr, int nargs) POP1(y); POP1(x); WITH_SPSAVE({ - matched = pseudo_query_scan(ctx, x, y); + matched = grn_operator_exec_match(ctx, x, y); }); ALLOC1(res); grn_obj_reinit(ctx, res, GRN_DB_INT32, 0); @@ -3207,7 +2974,7 @@ grn_expr_exec(grn_ctx *ctx, grn_obj *expr, int nargs) POP1(y); POP1(x); WITH_SPSAVE({ - matched = pseudo_prefix_search(ctx, x, y); + matched = grn_operator_exec_prefix(ctx, x, y); }); ALLOC1(res); grn_obj_reinit(ctx, res, GRN_DB_INT32, 0); @@ -3697,6 +3464,21 @@ grn_expr_exec(grn_ctx *ctx, grn_obj *expr, int nargs) code++; } break; + case GRN_OP_REGEXP : + { + grn_obj *target, *pattern; + grn_bool matched; + POP1(pattern); + POP1(target); + WITH_SPSAVE({ + matched = grn_operator_exec_regexp(ctx, target, pattern); + }); + ALLOC1(res); + grn_obj_reinit(ctx, res, GRN_DB_BOOL, 0); + GRN_BOOL_SET(ctx, res, matched); + } + code++; + break; default : ERR(GRN_FUNCTION_NOT_IMPLEMENTED, "not implemented operator assigned"); goto exit; @@ -3753,12 +3535,17 @@ struct _grn_scan_info { grn_obj *args[GRN_SCAN_INFO_MAX_N_ARGS]; int max_interval; int similarity_threshold; - grn_obj *scorer; + grn_obj scorers; + grn_obj scorer_args_exprs; + grn_obj scorer_args_expr_offsets; }; #define SI_FREE(si) do {\ GRN_OBJ_FIN(ctx, &(si)->wv);\ GRN_OBJ_FIN(ctx, &(si)->index);\ + GRN_OBJ_FIN(ctx, &(si)->scorers);\ + GRN_OBJ_FIN(ctx, &(si)->scorer_args_exprs);\ + GRN_OBJ_FIN(ctx, &(si)->scorer_args_expr_offsets);\ GRN_FREE(si);\ } while (0) @@ -3777,7 +3564,9 @@ struct _grn_scan_info { (si)->max_interval = DEFAULT_MAX_INTERVAL;\ (si)->similarity_threshold = DEFAULT_SIMILARITY_THRESHOLD;\ (si)->start = (st);\ - (si)->scorer = NULL;\ + GRN_PTR_INIT(&(si)->scorers, GRN_OBJ_VECTOR, GRN_ID_NIL);\ + GRN_PTR_INIT(&(si)->scorer_args_exprs, GRN_OBJ_VECTOR, GRN_ID_NIL);\ + GRN_UINT32_INIT(&(si)->scorer_args_expr_offsets, GRN_OBJ_VECTOR);\ } while (0) static scan_info ** @@ -3821,9 +3610,9 @@ put_logical_op(grn_ctx *ctx, scan_info **sis, int *ip, grn_operator op, int star } else { s_->flags &= ~SCAN_PUSH; s_->logical_op = op; - memcpy(&sis[i], &sis[j], sizeof(scan_info *) * (r - j)); - memmove(&sis[j], &sis[r], sizeof(scan_info *) * (i - r)); - memcpy(&sis[i + j - r], &sis[i], sizeof(scan_info *) * (r - j)); + grn_memcpy(&sis[i], &sis[j], sizeof(scan_info *) * (r - j)); + grn_memmove(&sis[j], &sis[r], sizeof(scan_info *) * (i - r)); + grn_memcpy(&sis[i + j - r], &sis[i], sizeof(scan_info *) * (r - j)); } break; } @@ -4003,11 +3792,18 @@ grn_expr_inspect_internal(grn_ctx *ctx, grn_obj *buf, grn_obj *expr) static void -scan_info_put_index(grn_ctx *ctx, scan_info *si, grn_obj *index, uint32_t sid, int32_t weight) +scan_info_put_index(grn_ctx *ctx, scan_info *si, + grn_obj *index, uint32_t sid, int32_t weight, + grn_obj *scorer, + grn_obj *scorer_args_expr, + uint32_t scorer_args_expr_offset) { GRN_PTR_PUT(ctx, &si->index, index); GRN_UINT32_PUT(ctx, &si->wv, sid); GRN_INT32_PUT(ctx, &si->wv, weight); + GRN_PTR_PUT(ctx, &si->scorers, scorer); + GRN_PTR_PUT(ctx, &si->scorer_args_exprs, scorer_args_expr); + GRN_UINT32_PUT(ctx, &si->scorer_args_expr_offsets, scorer_args_expr_offset); { int i, ni = (GRN_BULK_VSIZE(&si->index) / sizeof(grn_obj *)) - 1; grn_obj **pi = &GRN_PTR_VALUE_AT(&si->index, ni); @@ -4015,10 +3811,10 @@ scan_info_put_index(grn_ctx *ctx, scan_info *si, grn_obj *index, uint32_t sid, i if (index == pi[-1]) { if (i) { int32_t *pw = &GRN_INT32_VALUE_AT(&si->wv, (ni - i) * 2); - memmove(pw + 2, pw, sizeof(int32_t) * 2 * i); + grn_memmove(pw + 2, pw, sizeof(int32_t) * 2 * i); pw[0] = (int32_t) sid; pw[1] = weight; - memmove(pi + 1, pi, sizeof(grn_obj *) * i); + grn_memmove(pi + 1, pi, sizeof(grn_obj *) * i); pi[0] = index; } return; @@ -4028,10 +3824,13 @@ scan_info_put_index(grn_ctx *ctx, scan_info *si, grn_obj *index, uint32_t sid, i } static int32_t -get_weight(grn_ctx *ctx, grn_expr_code *ec) +get_weight(grn_ctx *ctx, grn_expr_code *ec, uint32_t *offset) { if (ec->modify == 2 && ec[2].op == GRN_OP_STAR && ec[1].value && ec[1].value->header.type == GRN_BULK) { + if (offset) { + *offset = 2; + } if (ec[1].value->header.domain == GRN_DB_INT32 || ec[1].value->header.domain == GRN_DB_UINT32) { return GRN_INT32_VALUE(ec[1].value); @@ -4046,6 +3845,9 @@ get_weight(grn_ctx *ctx, grn_expr_code *ec) return weight; } } else { + if (offset) { + *offset = 0; + } return 1; } } @@ -4067,7 +3869,9 @@ grn_scan_info_open(grn_ctx *ctx, int start) si->max_interval = DEFAULT_MAX_INTERVAL; si->similarity_threshold = DEFAULT_SIMILARITY_THRESHOLD; si->start = start; - si->scorer = NULL; + GRN_PTR_INIT(&si->scorers, GRN_OBJ_VECTOR, GRN_ID_NIL); + GRN_PTR_INIT(&si->scorer_args_exprs, GRN_OBJ_VECTOR, GRN_ID_NIL); + GRN_UINT32_INIT(&si->scorer_args_expr_offsets, GRN_OBJ_VECTOR); return si; } @@ -4079,9 +3883,16 @@ grn_scan_info_close(grn_ctx *ctx, scan_info *si) } void -grn_scan_info_put_index(grn_ctx *ctx, scan_info *si, grn_obj *index, uint32_t sid, int32_t weight) +grn_scan_info_put_index(grn_ctx *ctx, scan_info *si, + grn_obj *index, uint32_t sid, int32_t weight, + grn_obj *scorer, + grn_obj *scorer_args_expr, + uint32_t scorer_args_expr_offset) { - scan_info_put_index(ctx, si, index, sid, weight); + scan_info_put_index(ctx, si, index, sid, weight, + scorer, + scorer_args_expr, + scorer_args_expr_offset); } scan_info ** @@ -4092,9 +3903,9 @@ grn_scan_info_put_logical_op(grn_ctx *ctx, scan_info **sis, int *ip, } int32_t -grn_expr_code_get_weight(grn_ctx *ctx, grn_expr_code *ec) +grn_expr_code_get_weight(grn_ctx *ctx, grn_expr_code *ec, uint32_t *offset) { - return get_weight(ctx, ec); + return get_weight(ctx, ec, offset); } int @@ -4169,18 +3980,6 @@ grn_scan_info_set_similarity_threshold(scan_info *si, int similarity_threshold) si->similarity_threshold = similarity_threshold; } -grn_obj * -grn_scan_info_get_scorer(scan_info *si) -{ - return si->scorer; -} - -void -grn_scan_info_set_scorer(scan_info *si, grn_obj *scorer) -{ - si->scorer = scorer; -} - grn_bool grn_scan_info_push_arg(scan_info *si, grn_obj *arg) { @@ -4202,32 +4001,64 @@ grn_scan_info_get_arg(grn_ctx *ctx, scan_info *si, int i) } static uint32_t -scan_info_build_find_index_column_index(grn_ctx *ctx, - scan_info *si, - grn_expr_code *ec, - uint32_t n_rest_codes, - grn_operator op) +scan_info_build_match_expr_codes_find_index(grn_ctx *ctx, scan_info *si, + grn_expr *expr, uint32_t i, + grn_obj **index, + int *sid) { - uint32_t offset = 0; - grn_obj *index; - int sid = 0; - int32_t weight; - - index = ec->value; - if (n_rest_codes >= 2 && - ec[1].value && - (ec[1].value->header.domain == GRN_DB_INT32 || - ec[1].value->header.domain == GRN_DB_UINT32) && - ec[2].op == GRN_OP_GET_MEMBER) { - if (ec[1].value->header.domain == GRN_DB_INT32) { - sid = GRN_INT32_VALUE(ec[1].value) + 1; - } else { - sid = GRN_UINT32_VALUE(ec[1].value) + 1; + grn_expr_code *ec; + uint32_t offset = 1; + grn_index_datum index_datum; + unsigned int n_index_data = 0; + + ec = &(expr->codes[i]); + switch (ec->value->header.type) { + case GRN_ACCESSOR : + n_index_data = grn_column_find_index_data(ctx, ec->value, si->op, + &index_datum, 1); + if (n_index_data > 0) { + grn_accessor *a = (grn_accessor *)(ec->value); + *sid = index_datum.section; + if (a->next && a->obj != index_datum.index) { + *index = ec->value; + } else { + *index = index_datum.index; + } + } + break; + case GRN_COLUMN_FIX_SIZE : + case GRN_COLUMN_VAR_SIZE : + n_index_data = grn_column_find_index_data(ctx, ec->value, si->op, + &index_datum, 1); + if (n_index_data > 0) { + *index = index_datum.index; + *sid = index_datum.section; + } + break; + case GRN_COLUMN_INDEX : + { + uint32_t n_rest_codes; + + *index = ec->value; + + n_rest_codes = expr->codes_curr - i; + if (n_rest_codes >= 2 && + ec[1].value && + (ec[1].value->header.domain == GRN_DB_INT32 || + ec[1].value->header.domain == GRN_DB_UINT32) && + ec[2].op == GRN_OP_GET_MEMBER) { + if (ec[1].value->header.domain == GRN_DB_INT32) { + *sid = GRN_INT32_VALUE(ec[1].value) + 1; + } else { + *sid = GRN_UINT32_VALUE(ec[1].value) + 1; + } + offset += 2; + } } - offset = 2; + break; + default : + break; } - weight = get_weight(ctx, ec + offset); - scan_info_put_index(ctx, si, index, sid, weight); return offset; } @@ -4237,8 +4068,9 @@ scan_info_build_match_expr_codes(grn_ctx *ctx, scan_info *si, grn_expr *expr, uint32_t i) { grn_expr_code *ec; - grn_obj *index; - int sid; + grn_obj *index = NULL; + int sid = 0; + uint32_t offset = 0; ec = &(expr->codes[i]); if (!ec->value) { @@ -4247,29 +4079,19 @@ scan_info_build_match_expr_codes(grn_ctx *ctx, scan_info *si, switch (ec->value->header.type) { case GRN_ACCESSOR : - if (grn_column_index(ctx, ec->value, si->op, &index, 1, &sid)) { - int32_t weight = get_weight(ctx, ec); - si->flags |= SCAN_ACCESSOR; - if (((grn_accessor *)ec->value)->next) { - scan_info_put_index(ctx, si, ec->value, sid, weight); - } else { - scan_info_put_index(ctx, si, index, sid, weight); - } - } - break; case GRN_COLUMN_FIX_SIZE : case GRN_COLUMN_VAR_SIZE : - if (grn_column_index(ctx, ec->value, si->op, &index, 1, &sid)) { - scan_info_put_index(ctx, si, index, sid, get_weight(ctx, ec)); - } - break; case GRN_COLUMN_INDEX : - { - uint32_t n_rest_codes; - uint32_t offset; - n_rest_codes = expr->codes_curr - i; - offset = scan_info_build_find_index_column_index(ctx, si, ec, - n_rest_codes, si->op); + offset = scan_info_build_match_expr_codes_find_index(ctx, si, expr, i, + &index, &sid); + i += offset - 1; + if (index) { + if (ec->value->header.type == GRN_ACCESSOR) { + si->flags |= SCAN_ACCESSOR; + } + scan_info_put_index(ctx, si, index, sid, + get_weight(ctx, &(expr->codes[i]), &offset), + NULL, NULL, 0); i += offset; } break; @@ -4285,24 +4107,27 @@ scan_info_build_match_expr_codes(grn_ctx *ctx, scan_info *si, GRN_OBJ_FIN(ctx, &inspected); return expr->codes_curr; } - si->scorer = ec->value; - i = scan_info_build_match_expr_codes(ctx, si, expr, i + 1); - if (expr->codes[i].op != GRN_OP_CALL) { - grn_obj inspected; - GRN_TEXT_INIT(&inspected, 0); - grn_inspect(ctx, &inspected, si->scorer); - ERR(GRN_INVALID_ARGUMENT, - "scorer must have only one argument: <%.*s>", - (int)GRN_TEXT_LEN(&inspected), - GRN_TEXT_VALUE(&inspected)); - GRN_OBJ_FIN(ctx, &inspected); - return expr->codes_curr; + i++; + offset = scan_info_build_match_expr_codes_find_index(ctx, si, expr, i, + &index, &sid); + i += offset; + if (index) { + uint32_t scorer_args_expr_offset = 0; + if (expr->codes[i].op != GRN_OP_CALL) { + scorer_args_expr_offset = i; + } + while (i < expr->codes_curr && expr->codes[i].op != GRN_OP_CALL) { + i++; + } + scan_info_put_index(ctx, si, index, sid, + get_weight(ctx, &(expr->codes[i]), &offset), + ec->value, + (grn_obj *)expr, + scorer_args_expr_offset); + i += offset; } break; - case GRN_TABLE_NO_KEY : - case GRN_TABLE_HASH_KEY : - case GRN_TABLE_PAT_KEY : - case GRN_TABLE_DAT_KEY : + default : { char name[GRN_TABLE_MAX_KEY_SIZE]; int name_size; @@ -4328,26 +4153,144 @@ scan_info_build_match_expr(grn_ctx *ctx, scan_info *si, grn_expr *expr) } } +static grn_bool +is_index_searchable_regexp(grn_ctx *ctx, grn_obj *regexp) +{ + const char *regexp_raw; + const char *regexp_raw_end; + grn_bool escaping = GRN_FALSE; + + if (!(regexp->header.domain == GRN_DB_SHORT_TEXT || + regexp->header.domain == GRN_DB_TEXT || + regexp->header.domain == GRN_DB_LONG_TEXT)) { + return GRN_FALSE; + } + + regexp_raw = GRN_TEXT_VALUE(regexp); + regexp_raw_end = regexp_raw + GRN_TEXT_LEN(regexp); + + while (regexp_raw < regexp_raw_end) { + unsigned int char_len; + + char_len = grn_charlen(ctx, regexp_raw, regexp_raw_end); + if (char_len == 0) { + return GRN_FALSE; + } + + if (char_len == 1) { + if (escaping) { + escaping = GRN_FALSE; + switch (regexp_raw[0]) { + case 'Z' : + case 'b' : + case 'B' : + case 'd' : + case 'D' : + case 'h' : + case 'H' : + case 'p' : + case 's' : + case 'S' : + case 'w' : + case 'W' : + case 'X' : + case 'k' : + case 'g' : + case '1' : + case '2' : + case '3' : + case '4' : + case '5' : + case '6' : + case '7' : + case '8' : + case '9' : + return GRN_FALSE; + default : + break; + } + } else { + switch (regexp_raw[0]) { + case '.' : + case '[' : + case ']' : + case '|' : + case '?' : + case '+' : + case '*' : + case '{' : + case '}' : + case '^' : + case '$' : + case '(' : + case ')' : + escaping = GRN_FALSE; + return GRN_FALSE; + case '\\' : + escaping = GRN_TRUE; + break; + default : + escaping = GRN_FALSE; + break; + } + } + } else { + escaping = GRN_FALSE; + } + + regexp_raw += char_len; + } + + return GRN_TRUE; +} + static void scan_info_build_match(grn_ctx *ctx, scan_info *si) { - int sid; - grn_obj *index, **p = si->args, **pe = si->args + si->nargs; + grn_obj **p, **pe; + + if (si->op == GRN_OP_REGEXP) { + p = si->args; + pe = si->args + si->nargs; + for (; p < pe; p++) { + if ((*p)->header.type == GRN_BULK && + !is_index_searchable_regexp(ctx, *p)) { + return; + } + } + } + + p = si->args; + pe = si->args + si->nargs; for (; p < pe; p++) { if ((*p)->header.type == GRN_EXPR) { scan_info_build_match_expr(ctx, si, (grn_expr *)(*p)); } else if (GRN_DB_OBJP(*p)) { - if (grn_column_index(ctx, *p, si->op, &index, 1, &sid)) { - scan_info_put_index(ctx, si, index, sid, 1); + grn_index_datum index_datum; + unsigned int n_index_data; + n_index_data = grn_column_find_index_data(ctx, *p, si->op, + &index_datum, 1); + if (n_index_data > 0) { + scan_info_put_index(ctx, si, + index_datum.index, index_datum.section, 1, + NULL, NULL, 0); } } else if (GRN_ACCESSORP(*p)) { + grn_index_datum index_datum; + unsigned int n_index_data; si->flags |= SCAN_ACCESSOR; - if (grn_column_index(ctx, *p, si->op, &index, 1, &sid)) { + n_index_data = grn_column_find_index_data(ctx, *p, si->op, + &index_datum, 1); + if (n_index_data > 0) { + grn_obj *index; if (((grn_accessor *)(*p))->next) { - scan_info_put_index(ctx, si, *p, sid, 1); + index = *p; } else { - scan_info_put_index(ctx, si, index, sid, 1); + index = index_datum.index; } + scan_info_put_index(ctx, si, + index, index_datum.section, 1, + NULL, NULL, 0); } } else { switch (si->op) { @@ -4412,6 +4355,7 @@ scan_info_build(grn_ctx *ctx, grn_obj *expr, int *n, case GRN_OP_GEO_WITHINP6 : case GRN_OP_GEO_WITHINP8 : case GRN_OP_TERM_EXTRACT : + case GRN_OP_REGEXP : if (stat < SCAN_COL1 || SCAN_CONST < stat) { return NULL; } stat = SCAN_START; m++; @@ -4492,6 +4436,7 @@ scan_info_build(grn_ctx *ctx, grn_obj *expr, int *n, case GRN_OP_GEO_WITHINP6 : case GRN_OP_GEO_WITHINP8 : case GRN_OP_TERM_EXTRACT : + case GRN_OP_REGEXP : stat = SCAN_START; si->op = c->op; si->end = c - e->codes; @@ -4572,17 +4517,28 @@ scan_info_build(grn_ctx *ctx, grn_obj *expr, int *n, sis[i++] = si; /* better index resolving framework for functions should be implemented */ { - int sid; - grn_obj *index, **p = si->args, **pe = si->args + si->nargs; + grn_obj **p = si->args, **pe = si->args + si->nargs; for (; p < pe; p++) { if (GRN_DB_OBJP(*p)) { - if (grn_column_index(ctx, *p, c->op, &index, 1, &sid)) { - scan_info_put_index(ctx, si, index, sid, 1); + grn_index_datum index_datum; + unsigned int n_index_data; + n_index_data = grn_column_find_index_data(ctx, *p, c->op, + &index_datum, 1); + if (n_index_data > 0) { + scan_info_put_index(ctx, si, + index_datum.index, index_datum.section, 1, + NULL, NULL, 0); } } else if (GRN_ACCESSORP(*p)) { + grn_index_datum index_datum; + unsigned int n_index_data; si->flags |= SCAN_ACCESSOR; - if (grn_column_index(ctx, *p, c->op, &index, 1, &sid)) { - scan_info_put_index(ctx, si, index, sid, 1); + n_index_data = grn_column_find_index_data(ctx, *p, c->op, + &index_datum, 1); + if (n_index_data > 0) { + scan_info_put_index(ctx, si, + index_datum.index, index_datum.section, 1, + NULL, NULL, 0); } } else { si->query = *p; @@ -4924,10 +4880,18 @@ grn_table_select_index_range_accessor(grn_ctx *ctx, grn_obj *table, accessor = (grn_accessor *)GRN_PTR_VALUE_AT(accessor_stack, i - 1); target = accessor->obj; - if (grn_column_index(ctx, target, GRN_OP_EQUAL, &index, 1, §ion) == 0) { - grn_obj_unlink(ctx, current_res); - current_res = NULL; - break; + { + grn_index_datum index_datum; + unsigned int n_index_data; + n_index_data = grn_column_find_index_data(ctx, target, GRN_OP_EQUAL, + &index_datum, 1); + if (n_index_data == 0) { + grn_obj_unlink(ctx, current_res); + current_res = NULL; + break; + } + index = index_datum.index; + section = index_datum.section; } if (section > 0) { @@ -5100,7 +5064,10 @@ grn_table_select_index(grn_ctx *ctx, grn_obj *table, scan_info *si, GRN_BULK_HEAD(si->query), GRN_BULK_VSIZE(si->query)); } - grn_ii_at(ctx, (grn_ii *)index, tid, (grn_hash *)res, si->logical_op); + if (tid != GRN_ID_NIL) { + grn_ii_at(ctx, (grn_ii *)index, tid, (grn_hash *)res, + si->logical_op); + } } grn_ii_resolve_sel_and(ctx, (grn_hash *)res, si->logical_op); processed = GRN_TRUE; @@ -5188,9 +5155,11 @@ grn_table_select_index(grn_ctx *ctx, grn_obj *table, scan_info *si, case GRN_OP_NEAR : case GRN_OP_NEAR2 : case GRN_OP_SIMILAR : + case GRN_OP_REGEXP : { grn_obj wv, **ip = &GRN_PTR_VALUE(&si->index); - int j = GRN_BULK_VSIZE(&si->index)/sizeof(grn_obj *); + int j; + int n_indexes = GRN_BULK_VSIZE(&si->index)/sizeof(grn_obj *); int32_t *wp = &GRN_INT32_VALUE(&si->wv); grn_search_optarg optarg; GRN_INT32_INIT(&wv, GRN_OBJ_VECTOR); @@ -5217,21 +5186,31 @@ grn_table_select_index(grn_ctx *ctx, grn_obj *table, scan_info *si, optarg.vector_size = 1; optarg.proc = NULL; optarg.max_size = 0; - optarg.scorer = si->scorer; ctx->flags |= GRN_CTX_TEMPORARY_DISABLE_II_RESOLVE_SEL_AND; - for (; j--; ip++, wp += 2) { + for (j = 0; j < n_indexes; j++, ip++, wp += 2) { uint32_t sid = (uint32_t) wp[0]; int32_t weight = wp[1]; if (sid) { int weight_index = sid - 1; - GRN_INT32_SET_AT(ctx, &wv, weight_index, weight); + int current_vector_size; + current_vector_size = GRN_BULK_VSIZE(&wv)/sizeof(int32_t); + if (weight_index < current_vector_size) { + ((int *)GRN_BULK_HEAD(&wv))[weight_index] = weight; + } else { + GRN_INT32_SET_AT(ctx, &wv, weight_index, weight); + } optarg.weight_vector = &GRN_INT32_VALUE(&wv); optarg.vector_size = GRN_BULK_VSIZE(&wv)/sizeof(int32_t); } else { optarg.weight_vector = NULL; optarg.vector_size = weight; } - if (j) { + optarg.scorer = GRN_PTR_VALUE_AT(&(si->scorers), j); + optarg.scorer_args_expr = + GRN_PTR_VALUE_AT(&(si->scorer_args_exprs), j); + optarg.scorer_args_expr_offset = + GRN_UINT32_VALUE_AT(&(si->scorer_args_expr_offsets), j); + if (j < n_indexes - 1) { if (sid && ip[0] == ip[1]) { continue; } } else { ctx->flags &= ~GRN_CTX_TEMPORARY_DISABLE_II_RESOLVE_SEL_AND; @@ -5475,239 +5454,6 @@ skip_space(grn_ctx *ctx, efs_info *q) } } -static grn_rc get_expr(grn_ctx *ctx, efs_info *q, grn_obj *column, grn_operator mode); -static grn_rc get_token(grn_ctx *ctx, efs_info *q, efs_op *op, grn_obj *column, grn_operator mode); - -static grn_rc -get_phrase(grn_ctx *ctx, efs_info *q, grn_obj *column, int mode, int option) -{ - const char *start, *s; - start = s = q->cur; - GRN_BULK_REWIND(&q->buf); - while (1) { - unsigned int len; - if (s >= q->str_end) { - q->cur = s; - break; - } - len = grn_charlen(ctx, s, q->str_end); - if (len == 0) { - /* invalid string containing malformed multibyte char */ - return GRN_END_OF_DATA; - } else if (len == 1) { - if (*s == GRN_QUERY_QUOTER) { - q->cur = s + 1; - break; - } else if (*s == GRN_QUERY_ESCAPE && s + 1 < q->str_end) { - s++; - len = grn_charlen(ctx, s, q->str_end); - } - } - GRN_TEXT_PUT(ctx, &q->buf, s, len); - s += len; - } - grn_expr_append_obj(ctx, q->e, q->v, GRN_OP_PUSH, 1); - grn_expr_append_const(ctx, q->e, column, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GET_VALUE, 2); - grn_expr_append_const(ctx, q->e, &q->buf, GRN_OP_PUSH, 1); - if (mode == GRN_OP_MATCH || mode == GRN_OP_EXACT) { - grn_expr_append_op(ctx, q->e, mode, 2); - } else { - grn_expr_append_const_int(ctx, q->e, option, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, mode, 3); - } - return GRN_SUCCESS; -} - -static grn_rc -get_geocond(grn_ctx *ctx, efs_info *q, grn_obj *longitude, grn_obj *latitude) -{ - unsigned int len; - const char *start = q->cur, *end; - for (end = q->cur;; ) { - /* null check and length check */ - if (!(len = grn_charlen(ctx, end, q->str_end))) { - q->cur = q->str_end; - break; - } - if (grn_isspace(end, ctx->encoding) || - *end == GRN_QUERY_PARENR) { - q->cur = end; - break; - } - } - { - const char *tokbuf[8]; - int32_t lng0, lat0, lng1, lat1, lng2, lat2, r; - int32_t n = grn_str_tok((char *)start, end - start, ',', tokbuf, 8, NULL); - switch (n) { - case 3 : - lng0 = grn_atoi(start, tokbuf[0], NULL); - lat0 = grn_atoi(tokbuf[0] + 1, tokbuf[1], NULL); - r = grn_atoi(tokbuf[1] + 1, tokbuf[2], NULL); - grn_expr_append_obj(ctx, q->e, q->v, GRN_OP_PUSH, 1); - grn_expr_append_const(ctx, q->e, longitude, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GET_VALUE, 2); - grn_expr_append_obj(ctx, q->e, q->v, GRN_OP_PUSH, 1); - grn_expr_append_const(ctx, q->e, latitude, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GET_VALUE, 2); - grn_expr_append_const_int(ctx, q->e, lng0, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, lat0, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, r, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GEO_WITHINP5, 5); - break; - case 4 : - lng0 = grn_atoi(start, tokbuf[0], NULL); - lat0 = grn_atoi(tokbuf[0] + 1, tokbuf[1], NULL); - lng1 = grn_atoi(tokbuf[1] + 1, tokbuf[2], NULL); - lat1 = grn_atoi(tokbuf[2] + 1, tokbuf[3], NULL); - grn_expr_append_obj(ctx, q->e, q->v, GRN_OP_PUSH, 1); - grn_expr_append_const(ctx, q->e, longitude, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GET_VALUE, 2); - grn_expr_append_obj(ctx, q->e, q->v, GRN_OP_PUSH, 1); - grn_expr_append_const(ctx, q->e, latitude, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GET_VALUE, 2); - grn_expr_append_const_int(ctx, q->e, lng0, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, lat0, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, lng1, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, lat1, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GEO_WITHINP6, 6); - break; - case 6 : - lng0 = grn_atoi(start, tokbuf[0], NULL); - lat0 = grn_atoi(tokbuf[0] + 1, tokbuf[1], NULL); - lng1 = grn_atoi(tokbuf[1] + 1, tokbuf[2], NULL); - lat1 = grn_atoi(tokbuf[2] + 1, tokbuf[3], NULL); - lng2 = grn_atoi(tokbuf[3] + 1, tokbuf[4], NULL); - lat2 = grn_atoi(tokbuf[4] + 1, tokbuf[5], NULL); - grn_expr_append_obj(ctx, q->e, q->v, GRN_OP_PUSH, 1); - grn_expr_append_const(ctx, q->e, longitude, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GET_VALUE, 2); - grn_expr_append_obj(ctx, q->e, q->v, GRN_OP_PUSH, 1); - grn_expr_append_const(ctx, q->e, latitude, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GET_VALUE, 2); - grn_expr_append_const_int(ctx, q->e, lng0, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, lat0, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, lng1, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, lat1, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, lng2, GRN_OP_PUSH, 1); - grn_expr_append_const_int(ctx, q->e, lat2, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GEO_WITHINP8, 8); - break; - default : - ERR(GRN_INVALID_ARGUMENT, "invalid geocond"); - break; - } - } - return ctx->rc; -} - -static grn_rc -get_word(grn_ctx *ctx, efs_info *q, grn_obj *column, int mode, int option) -{ - const char *start = q->cur, *end; - unsigned int len; - for (end = q->cur;; ) { - /* null check and length check */ - if (!(len = grn_charlen(ctx, end, q->str_end))) { - q->cur = q->str_end; - break; - } - if (grn_isspace(end, ctx->encoding) || - *end == GRN_QUERY_PARENR) { - q->cur = end; - break; - } - if (*end == GRN_QUERY_COLUMN) { - grn_obj *c = grn_obj_column(ctx, q->table, start, end - start); - if (c && end + 1 < q->str_end) { - efs_op op; - switch (end[1]) { - case '!' : - mode = GRN_OP_NOT_EQUAL; - q->cur = end + 2; - break; - case '=' : - if (q->flags & GRN_EXPR_ALLOW_UPDATE) { - mode = GRN_OP_ASSIGN; - q->cur = end + 2; - } else { - get_token(ctx, q, &op, c, mode); - } - break; - case '<' : - if (end + 2 < q->str_end && end[2] == '=') { - mode = GRN_OP_LESS_EQUAL; - q->cur = end + 3; - } else { - mode = GRN_OP_LESS; - q->cur = end + 2; - } - break; - case '>' : - if (end + 2 < q->str_end && end[2] == '=') { - mode = GRN_OP_GREATER_EQUAL; - q->cur = end + 3; - } else { - mode = GRN_OP_GREATER; - q->cur = end + 2; - } - break; - case '%' : - mode = GRN_OP_MATCH; - q->cur = end + 2; - break; - case '@' : - q->cur = end + 2; - return get_geocond(ctx, q, column, c); - break; - default : - mode = GRN_OP_EQUAL; - q->cur = end + 1; - break; - } - return get_token(ctx, q, &op, c, mode); - } else { - ERR(GRN_INVALID_ARGUMENT, "column lookup failed"); - return ctx->rc; - } - } else if (*end == GRN_QUERY_PREFIX) { - mode = GRN_OP_PREFIX; - q->cur = end + 1; - break; - } - end += len; - } - if (!column) { - ERR(GRN_INVALID_ARGUMENT, "column missing"); - return ctx->rc; - } - if (mode == GRN_OP_ASSIGN) { - grn_expr_append_obj(ctx, q->e, q->v, GRN_OP_PUSH, 1); - grn_expr_append_const(ctx, q->e, column, GRN_OP_PUSH, 1); - grn_expr_append_const_str(ctx, q->e, start, end - start, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_ASSIGN, 2); - } else { - grn_expr_append_obj(ctx, q->e, q->v, GRN_OP_PUSH, 1); - grn_expr_append_const(ctx, q->e, column, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, GRN_OP_GET_VALUE, 2); - grn_expr_append_const_str(ctx, q->e, start, end - start, GRN_OP_PUSH, 1); - switch (mode) { - case GRN_OP_NEAR : - case GRN_OP_NEAR2 : - case GRN_OP_SIMILAR : - case GRN_OP_TERM_EXTRACT : - grn_expr_append_const_int(ctx, q->e, option, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, mode, 3); - break; - default : - grn_expr_append_op(ctx, q->e, mode, 2); - break; - } - } - return GRN_SUCCESS; -} - static grn_bool get_op(efs_info *q, efs_op *op, grn_operator *mode, int *option) { @@ -5756,90 +5502,6 @@ get_op(efs_info *q, efs_op *op, grn_operator *mode, int *option) return found; } -static grn_rc -get_token(grn_ctx *ctx, efs_info *q, efs_op *op, grn_obj *column, grn_operator mode) -{ - int option = 0; - op->op = q->default_op; - op->weight = DEFAULT_WEIGHT; - for (;;) { - skip_space(ctx, q); - if (q->cur >= q->str_end) { return GRN_END_OF_DATA; } - switch (*q->cur) { - case '\0' : - return GRN_END_OF_DATA; - break; - case GRN_QUERY_PARENR : - q->cur++; - return GRN_END_OF_DATA; - break; - case GRN_QUERY_QUOTEL : - q->cur++; - return get_phrase(ctx, q, column, mode, option); - break; - case GRN_QUERY_PREFIX : - q->cur++; - get_op(q, op, &mode, &option); - break; - case GRN_QUERY_AND : - q->cur++; - op->op = GRN_OP_AND; - break; - case GRN_QUERY_AND_NOT : - q->cur++; - op->op = GRN_OP_AND_NOT; - break; - case GRN_QUERY_ADJ_INC : - q->cur++; - if (op->weight < 127) { op->weight++; } - op->op = GRN_OP_ADJUST; - break; - case GRN_QUERY_ADJ_DEC : - q->cur++; - if (op->weight > -128) { op->weight--; } - op->op = GRN_OP_ADJUST; - break; - case GRN_QUERY_ADJ_NEG : - q->cur++; - op->op = GRN_OP_ADJUST; - op->weight = -1; - break; - case GRN_QUERY_PARENL : - q->cur++; - return get_expr(ctx, q, column, mode); - break; - case 'O' : - if (q->cur[1] == 'R' && q->cur[2] == ' ') { - q->cur += 2; - op->op = GRN_OP_OR; - break; - } - /* fallthru */ - default : - return get_word(ctx, q, column, mode, option); - break; - } - } - return GRN_SUCCESS; -} - -static grn_rc -get_expr(grn_ctx *ctx, efs_info *q, grn_obj *column, grn_operator mode) -{ - efs_op op; - grn_rc rc = get_token(ctx, q, &op, column, mode); - if (rc) { return rc; } - while (!(rc = get_token(ctx, q, &op, column, mode))) { - if (op.op == GRN_OP_ADJUST) { - grn_expr_append_const_int(ctx, q->e, op.weight, GRN_OP_PUSH, 1); - grn_expr_append_op(ctx, q->e, op.op, 3); - } else { - grn_expr_append_op(ctx, q->e, op.op, 2); - } - } - return rc; -} - #define DISABLE_UNUSED_CODE 1 #ifndef DISABLE_UNUSED_CODE static const char * @@ -6090,6 +5752,10 @@ get_word_(grn_ctx *ctx, efs_info *q) mode = GRN_OP_SUFFIX; q->cur = end + 2; break; + case '~' : + mode = GRN_OP_REGEXP; + q->cur = end + 2; + break; default : mode = GRN_OP_EQUAL; q->cur = end + 1; @@ -6502,6 +6168,10 @@ parse_script(grn_ctx *ctx, efs_info *q) PARSE(GRN_EXPR_TOKEN_SUFFIX); q->cur += 2; break; + case '~' : + PARSE(GRN_EXPR_TOKEN_REGEXP); + q->cur += 2; + break; default : PARSE(GRN_EXPR_TOKEN_MATCH); q->cur++; @@ -7195,3 +6865,42 @@ grn_expr_dump_plan(grn_ctx *ctx, grn_obj *expr, grn_obj *buffer) } GRN_API_RETURN(GRN_SUCCESS); } + +static unsigned int +grn_expr_estimate_size_raw(grn_ctx *ctx, grn_obj *expr, grn_obj *table) +{ + return grn_table_size(ctx, table); +} + +unsigned int +grn_expr_estimate_size(grn_ctx *ctx, grn_obj *expr) +{ + grn_obj *table; + grn_obj *variable; + unsigned int size; + + variable = grn_expr_get_var_by_offset(ctx, expr, 0); + if (!variable) { + ERR(GRN_INVALID_ARGUMENT, "at least one variable must be defined"); + return 0; + } + + table = grn_ctx_at(ctx, variable->header.domain); + if (!table) { + ERR(GRN_INVALID_ARGUMENT, + "variable refers unknown domain: <%u>", variable->header.domain); + return 0; + } + + GRN_API_ENTER; +#ifdef GRN_WITH_MRUBY + if (ctx->impl->mrb.state) { + size = grn_mrb_expr_estimate_size(ctx, expr, table); + } else { + size = grn_expr_estimate_size_raw(ctx, expr, table); + } +#else + size = grn_expr_estimate_size_raw(ctx, expr, table); +#endif + GRN_API_RETURN(size); +} diff --git a/storage/mroonga/vendor/groonga/lib/geo.c b/storage/mroonga/vendor/groonga/lib/geo.c index 21b50c4a1a4..d2eb049ce21 100644 --- a/storage/mroonga/vendor/groonga/lib/geo.c +++ b/storage/mroonga/vendor/groonga/lib/geo.c @@ -91,18 +91,18 @@ compute_min_and_max_key(uint8_t *key_base, int diff_bit, diff_bit_mask = 0xff >> (diff_bit % 8); if (diff_byte == sizeof(grn_geo_point)) { - if (key_min) { memcpy(key_min, key_base, diff_byte); } - if (key_max) { memcpy(key_max, key_base, diff_byte); } + if (key_min) { grn_memcpy(key_min, key_base, diff_byte); } + if (key_max) { grn_memcpy(key_max, key_base, diff_byte); } } else { if (key_min) { - memcpy(key_min, key_base, diff_byte + 1); + grn_memcpy(key_min, key_base, diff_byte + 1); key_min[diff_byte] &= ~diff_bit_mask; memset(key_min + diff_byte + 1, 0, sizeof(grn_geo_point) - diff_byte - 1); } if (key_max) { - memcpy(key_max, key_base, diff_byte + 1); + grn_memcpy(key_max, key_base, diff_byte + 1); key_max[diff_byte] |= diff_bit_mask; memset(key_max + diff_byte + 1, 0xff, sizeof(grn_geo_point) - diff_byte - 1); @@ -274,7 +274,7 @@ grn_geo_table_sort_detect_far_point(grn_ctx *ctx, grn_obj *table, grn_obj *index grn_gton(geo_key_curr, base_point, sizeof(grn_geo_point)); *diff_bit = sizeof(grn_geo_point) * 8; diff_bit_current = sizeof(grn_geo_point) * 8; - memcpy(&point, base_point, sizeof(grn_geo_point)); + grn_memcpy(&point, base_point, sizeof(grn_geo_point)); ep = entries; inspect_mesh(ctx, &point, *diff_bit, -1); while ((tid = grn_pat_cursor_next(ctx, pc))) { @@ -862,7 +862,7 @@ grn_geo_select_in_circle(grn_ctx *ctx, grn_obj *index, name_size = grn_obj_name(ctx, domain_object, name, GRN_TABLE_MAX_KEY_SIZE); grn_obj_unlink(ctx, domain_object); } else { - strcpy(name, "(null)"); + grn_strcpy(name, GRN_TABLE_MAX_KEY_SIZE, "(null)"); name_size = strlen(name); } ERR(GRN_INVALID_ARGUMENT, @@ -1034,7 +1034,7 @@ in_rectangle_data_fill(grn_ctx *ctx, grn_obj *index, name_size = grn_obj_name(ctx, domain_object, name, GRN_TABLE_MAX_KEY_SIZE); grn_obj_unlink(ctx, domain_object); } else { - strcpy(name, "(null)"); + grn_strcpy(name, GRN_TABLE_MAX_KEY_SIZE, "(null)"); name_size = strlen(name); } ERR(GRN_INVALID_ARGUMENT, @@ -1419,8 +1419,8 @@ grn_geo_cursor_area_init(grn_ctx *ctx, } area->current_entry = 0; - memcpy(&(area->top_left), &area_top_left, sizeof(grn_geo_point)); - memcpy(&(area->bottom_right), &area_bottom_right, sizeof(grn_geo_point)); + grn_memcpy(&(area->top_left), &area_top_left, sizeof(grn_geo_point)); + grn_memcpy(&(area->bottom_right), &area_bottom_right, sizeof(grn_geo_point)); grn_gton(area->top_left_key, &area_top_left, sizeof(grn_geo_point)); grn_gton(area->bottom_right_key, &area_bottom_right, sizeof(grn_geo_point)); @@ -1430,7 +1430,7 @@ grn_geo_cursor_area_init(grn_ctx *ctx, &area_bottom_right, &data); entry->target_bit = data.rectangle_common_bit; - memcpy(entry->key, data.rectangle_common_key, sizeof(grn_geo_point)); + grn_memcpy(entry->key, data.rectangle_common_key, sizeof(grn_geo_point)); entry->status_flags = GRN_GEO_CURSOR_ENTRY_STATUS_TOP_INCLUDED | GRN_GEO_CURSOR_ENTRY_STATUS_BOTTOM_INCLUDED | @@ -1474,8 +1474,8 @@ grn_geo_cursor_open_in_rectangle(grn_ctx *ctx, cursor->pat = data.pat; cursor->index = index; - memcpy(&(cursor->top_left), data.top_left, sizeof(grn_geo_point)); - memcpy(&(cursor->bottom_right), data.bottom_right, sizeof(grn_geo_point)); + grn_memcpy(&(cursor->top_left), data.top_left, sizeof(grn_geo_point)); + grn_memcpy(&(cursor->bottom_right), data.bottom_right, sizeof(grn_geo_point)); cursor->pat_cursor = NULL; cursor->ii_cursor = NULL; cursor->offset = offset; @@ -1494,10 +1494,12 @@ grn_geo_cursor_open_in_rectangle(grn_ctx *ctx, } } { - const char *minimum_reduce_bit_env; + char minimum_reduce_bit_env[GRN_ENV_BUFFER_SIZE]; cursor->minimum_reduce_bit = 0; - minimum_reduce_bit_env = getenv("GRN_GEO_IN_RECTANGLE_MINIMUM_REDUCE_BIT"); - if (minimum_reduce_bit_env) { + grn_getenv("GRN_GEO_IN_RECTANGLE_MINIMUM_REDUCE_BIT", + minimum_reduce_bit_env, + GRN_ENV_BUFFER_SIZE); + if (minimum_reduce_bit_env[0]) { cursor->minimum_reduce_bit = atoi(minimum_reduce_bit_env); } if (cursor->minimum_reduce_bit < 1) { @@ -1544,7 +1546,7 @@ grn_geo_cursor_entry_next_push(grn_ctx *ctx, grn_geo_cursor_area *area; area = &(cursor->areas[cursor->current_area]); next_entry = &(area->entries[++area->current_entry]); - memcpy(next_entry, entry, sizeof(grn_geo_cursor_entry)); + grn_memcpy(next_entry, entry, sizeof(grn_geo_cursor_entry)); pushed = GRN_TRUE; } grn_table_cursor_close(ctx, pat_cursor); @@ -1578,9 +1580,9 @@ grn_geo_cursor_entry_next(grn_ctx *ctx, top_left_key = area->top_left_key; bottom_right_key = area->bottom_right_key; - memcpy(entry, - &(area->entries[area->current_entry--]), - sizeof(grn_geo_cursor_entry)); + grn_memcpy(entry, + &(area->entries[area->current_entry--]), + sizeof(grn_geo_cursor_entry)); while (GRN_TRUE) { grn_geo_cursor_entry next_entry0, next_entry1; grn_bool pushed = GRN_FALSE; @@ -1663,9 +1665,9 @@ grn_geo_cursor_entry_next(grn_ctx *ctx, break; } - memcpy(&next_entry0, entry, sizeof(grn_geo_cursor_entry)); + grn_memcpy(&next_entry0, entry, sizeof(grn_geo_cursor_entry)); next_entry0.target_bit++; - memcpy(&next_entry1, entry, sizeof(grn_geo_cursor_entry)); + grn_memcpy(&next_entry1, entry, sizeof(grn_geo_cursor_entry)); next_entry1.target_bit++; SET_N_BIT(next_entry1.key, next_entry1.target_bit); @@ -1767,9 +1769,9 @@ grn_geo_cursor_entry_next(grn_ctx *ctx, print_key_mark(ctx, stack_entry->target_bit); } #endif - memcpy(entry, - &(area->entries[area->current_entry--]), - sizeof(grn_geo_cursor_entry)); + grn_memcpy(entry, + &(area->entries[area->current_entry--]), + sizeof(grn_geo_cursor_entry)); #ifdef GEO_DEBUG printf("%d: pop entry\n", entry->target_bit); #endif @@ -1979,7 +1981,7 @@ geo_point_get(grn_ctx *ctx, grn_obj *pat, int flags, grn_geo_point *geo_point) void *key; int key_size; key_size = grn_table_cursor_get_key(ctx, cursor, &key); - memcpy(geo_point, key, key_size); + grn_memcpy(geo_point, key, key_size); } exit: diff --git a/storage/mroonga/vendor/groonga/lib/grn.h b/storage/mroonga/vendor/groonga/lib/grn.h index c33ad7f7ec4..f4ee6e962a9 100644 --- a/storage/mroonga/vendor/groonga/lib/grn.h +++ b/storage/mroonga/vendor/groonga/lib/grn.h @@ -1,5 +1,5 @@ /* -*- c-basic-offset: 2 -*- */ -/* Copyright(C) 2009-2014 Brazil +/* Copyright(C) 2009-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -23,6 +23,10 @@ # include <config.h> #endif /* HAVE_CONFIG_H */ +#if defined(WIN32) && defined(__GNUC__) +# define __MINGW_MSVC_COMPAT_WARNINGS +#endif /* defined(WIN32) && defined(__GNUC__) */ + #ifdef __cplusplus # define __STDC_LIMIT_MACROS #endif @@ -67,30 +71,6 @@ # define GRN_VAR extern #endif -#ifdef HAVE_OPEN -# define GRN_OPEN(pathname, ...) open(pathname, __VA_ARGS__) -#else -# define GRN_OPEN(pathname, ...) _open(pathname, __VA_ARGS__) -#endif /* HAVE_OPEN */ - -#ifdef HAVE_CLOSE -# define GRN_CLOSE(fd) close(fd) -#else -# define GRN_CLOSE(fd) _close(fd) -#endif /* HAVE_CLOSE */ - -#ifdef HAVE_READ -# define GRN_READ(fd, buf, count) read(fd, buf, count) -#else -# define GRN_READ(fd, buf, count) _read(fd, buf, count) -#endif /* HAVE_READ */ - -#ifdef HAVE_WRITE -# define GRN_WRITE(fd, buf, count) write(fd, buf, count) -#else -# define GRN_WRITE(fd, buf, count) _write(fd, buf, count) -#endif /* HAVE_WRITE */ - #ifdef WIN32 # if defined(__GNUC__) && !defined(WINVER) @@ -116,15 +96,9 @@ # endif # endif -# ifndef __GNUC__ -# define snprintf(str, size, ...) _snprintf(str, size, __VA_ARGS__) -# endif /* __GNUC__ */ # if !defined(__GNUC__) && _MSC_VER < 1500 # define vsnprintf(str, size, format, ap) _vsnprintf(str, size, format, ap) # endif /* !defined(__GNUC__) && _MSC_VER < 1500 */ -# ifndef HAVE_UNLINK -# define unlink(pathname) _unlink(pathname) -# endif # define getpid() _getpid() # if !defined(__GNUC__) && _MSC_VER < 1400 # define fstat(fd, buf) _fstat(fd, buf) @@ -479,12 +453,14 @@ typedef int grn_cond; # else /* WIN64 */ # define GRN_FMT_SOCKET "u" # endif /* WIN64 */ +# define GRN_FMT_OFF64_T GRN_FMT_LLD #else /* WIN32 */ # define GRN_FMT_LLD "lld" # define GRN_FMT_LLU "llu" # define GRN_FMT_SIZE "zu" # define GRN_FMT_SSIZE "zd" # define GRN_FMT_SOCKET "d" +# define GRN_FMT_OFF64_T "jd" #endif /* WIN32 */ #ifdef __GNUC__ @@ -549,13 +525,6 @@ typedef int grn_cond; (*(p) = (v)) # endif /* ATOMIC 64BIT SET */ -# ifdef HAVE_MKOSTEMP -# define GRN_MKOSTEMP(template,flags,mode) mkostemp(template,flags) -# else /* HAVE_MKOSTEMP */ -# define GRN_MKOSTEMP(template,flags,mode) \ - (mktemp(template), GRN_OPEN((template),((flags)|O_RDWR|O_CREAT|O_EXCL),mode)) -# endif /* HAVE_MKOSTEMP */ - #elif (defined(WIN32) || defined (_WIN64)) /* __GNUC__ */ # define GRN_ATOMIC_ADD_EX(p,i,r) \ @@ -585,9 +554,6 @@ typedef int grn_cond; # define GRN_BIT_SCAN_REV(v,r) for (r = 31; r && !((1 << r) & v); r--) # define GRN_BIT_SCAN_REV0(v,r) GRN_BIT_SCAN_REV(v,r) -# define GRN_MKOSTEMP(template,flags,mode) \ - (mktemp(template), GRN_OPEN((template),((flags)|O_RDWR|O_CREAT),mode)) - #else /* __GNUC__ */ # if (defined(__sun) && defined(__SVR4)) /* ATOMIC ADD */ @@ -603,9 +569,6 @@ typedef int grn_cond; # define GRN_BIT_SCAN_REV(v,r) for (r = 31; r && !((1 << r) & v); r--) # define GRN_BIT_SCAN_REV0(v,r) GRN_BIT_SCAN_REV(v,r) -# define GRN_MKOSTEMP(template,flags,mode) \ - (mktemp(template), GRN_OPEN((template),flags,mode)) - #endif /* __GNUC__ */ typedef uint8_t byte; @@ -712,7 +675,7 @@ grn_str_greater(const uint8_t *ap, uint32_t as, const uint8_t *bp, uint32_t bs) lo_ = (lo_ | (lo_ << 1)) & 0x5555555555555555ULL;\ result_ = (la_ << 1) | lo_;\ grn_hton_uint64(result_, result_);\ - memcpy(keybuf, &result_, sizeof(result_));\ + grn_memcpy(keybuf, &result_, sizeof(result_));\ } while (0) #define grn_ntog(keybuf,key,size) do {\ diff --git a/storage/mroonga/vendor/groonga/lib/grn_ctx.h b/storage/mroonga/vendor/groonga/lib/grn_ctx.h index dbf461e668e..2f052f5f1c5 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_ctx.h +++ b/storage/mroonga/vendor/groonga/lib/grn_ctx.h @@ -329,6 +329,58 @@ GRN_API void grn_ctx_impl_set_current_error_message(grn_ctx *ctx); ERR(rc, "socket error '%s' (%s)[%d]", str, m, e);\ } while (0) +#define ERRNO_ERR(str) do {\ + grn_rc rc;\ + int errno_keep = errno;\ + grn_bool show_errno = GRN_FALSE;\ + char system_message[SYSTEM_ERROR_MESSAGE_BUFFER_SIZE];\ + strerror_s(system_message, SYSTEM_ERROR_MESSAGE_BUFFER_SIZE, errno);\ + switch (errno_keep) {\ + case EPERM : rc = GRN_OPERATION_NOT_PERMITTED; break;\ + case ENOENT : rc = GRN_NO_SUCH_FILE_OR_DIRECTORY; break;\ + case ESRCH : rc = GRN_NO_SUCH_PROCESS; break;\ + case EINTR : rc = GRN_INTERRUPTED_FUNCTION_CALL; break;\ + case EIO : rc = GRN_INPUT_OUTPUT_ERROR; break;\ + case E2BIG : rc = GRN_ARG_LIST_TOO_LONG; break;\ + case ENOEXEC : rc = GRN_EXEC_FORMAT_ERROR; break;\ + case EBADF : rc = GRN_BAD_FILE_DESCRIPTOR; break;\ + case ECHILD : rc = GRN_NO_CHILD_PROCESSES; break;\ + case EAGAIN: rc = GRN_OPERATION_WOULD_BLOCK; break;\ + case ENOMEM : rc = GRN_NO_MEMORY_AVAILABLE; break;\ + case EACCES : rc = GRN_PERMISSION_DENIED; break;\ + case EFAULT : rc = GRN_BAD_ADDRESS; break;\ + case EEXIST : rc = GRN_FILE_EXISTS; break;\ + /* case EXDEV : */\ + case ENODEV : rc = GRN_NO_SUCH_DEVICE; break;\ + case ENOTDIR : rc = GRN_NOT_A_DIRECTORY; break;\ + case EISDIR : rc = GRN_IS_A_DIRECTORY; break;\ + case EINVAL : rc = GRN_INVALID_ARGUMENT; break;\ + case EMFILE : rc = GRN_TOO_MANY_OPEN_FILES; break;\ + case ENOTTY : rc = GRN_INAPPROPRIATE_I_O_CONTROL_OPERATION; break;\ + case EFBIG : rc = GRN_FILE_TOO_LARGE; break;\ + case ENOSPC : rc = GRN_NO_SPACE_LEFT_ON_DEVICE; break;\ + case ESPIPE : rc = GRN_INVALID_SEEK; break;\ + case EROFS : rc = GRN_READ_ONLY_FILE_SYSTEM; break;\ + case EMLINK : rc = GRN_TOO_MANY_LINKS; break;\ + case EPIPE : rc = GRN_BROKEN_PIPE; break;\ + case EDOM : rc = GRN_DOMAIN_ERROR; break;\ + case ERANGE : rc = GRN_RANGE_ERROR; break;\ + case EDEADLOCK : rc = GRN_RESOURCE_DEADLOCK_AVOIDED; break;\ + case ENAMETOOLONG : rc = GRN_FILENAME_TOO_LONG; break;\ + case EILSEQ : rc = GRN_ILLEGAL_BYTE_SEQUENCE; break;\ + /* case STRUNCATE : */\ + default :\ + rc = GRN_UNKNOWN_ERROR;\ + show_errno = GRN_TRUE;\ + break;\ + }\ + if (show_errno) {\ + ERR(rc, "syscall error '%s' (%s)[%d]", str, system_message, errno_keep);\ + } else {\ + ERR(rc, "syscall error '%s' (%s)", str, system_message);\ + }\ +} while (0) + #else /* WIN32 */ #define SERR(str) do {\ grn_rc rc;\ @@ -387,7 +439,7 @@ GRN_API void grn_ctx_impl_set_current_error_message(grn_ctx *ctx); break;\ }\ if (show_errno) {\ - ERR(rc, "syscall error '%s' (%s)[%d]", str, system_message, errno_keep); \ + ERR(rc, "syscall error '%s' (%s)[%d]", str, system_message, errno_keep);\ } else {\ ERR(rc, "syscall error '%s' (%s)", str, system_message);\ }\ @@ -395,6 +447,8 @@ GRN_API void grn_ctx_impl_set_current_error_message(grn_ctx *ctx); #define SOERR(str) SERR(str) +#define ERRNO_ERR(str) SERR(str) + #endif /* WIN32 */ #define GERR(rc,...) ERRSET(&grn_gctx, GRN_ERROR, (rc), __VA_ARGS__) @@ -450,6 +504,8 @@ typedef void *(*grn_realloc_func) (grn_ctx *ctx, void *ptr, size_t size, const char *file, int line, const char *func); typedef char *(*grn_strdup_func) (grn_ctx *ctx, const char *string, const char *file, int line, const char *func); +typedef void (*grn_free_func) (grn_ctx *ctx, void *ptr, + const char *file, int line, const char *func); grn_malloc_func grn_ctx_get_malloc(grn_ctx *ctx); void grn_ctx_set_malloc(grn_ctx *ctx, grn_malloc_func malloc_func); grn_calloc_func grn_ctx_get_calloc(grn_ctx *ctx); @@ -458,11 +514,14 @@ grn_realloc_func grn_ctx_get_realloc(grn_ctx *ctx); void grn_ctx_set_realloc(grn_ctx *ctx, grn_realloc_func realloc_func); grn_strdup_func grn_ctx_get_strdup(grn_ctx *ctx); void grn_ctx_set_strdup(grn_ctx *ctx, grn_strdup_func strdup_func); +grn_free_func grn_ctx_get_free(grn_ctx *ctx); +void grn_ctx_set_free(grn_ctx *ctx, grn_free_func free_func); void *grn_malloc(grn_ctx *ctx, size_t size, const char* file, int line, const char *func); void *grn_calloc(grn_ctx *ctx, size_t size, const char* file, int line, const char *func); void *grn_realloc(grn_ctx *ctx, void *ptr, size_t size, const char* file, int line, const char *func); char *grn_strdup(grn_ctx *ctx, const char *s, const char* file, int line, const char *func); +void grn_free(grn_ctx *ctx, void *ptr, const char *file, int line, const char *func); #else # define grn_malloc grn_malloc_default # define grn_calloc grn_calloc_default @@ -518,7 +577,8 @@ extern grn_timeval grn_starttime; #define GRN_TIME_USEC_TO_NSEC(usec) ((usec) * GRN_TIME_NSEC_PER_USEC) GRN_API grn_rc grn_timeval_now(grn_ctx *ctx, grn_timeval *tv); -GRN_API grn_rc grn_timeval2str(grn_ctx *ctx, grn_timeval *tv, char *buf); +GRN_API grn_rc grn_timeval2str(grn_ctx *ctx, grn_timeval *tv, char *buf, size_t buf_size); +struct tm *grn_timeval2tm(grn_ctx *ctx, grn_timeval *tv, struct tm *tm_buffer); grn_rc grn_str2timeval(const char *str, uint32_t str_len, grn_timeval *tv); GRN_API void grn_ctx_log(grn_ctx *ctx, const char *fmt, ...) GRN_ATTRIBUTE_PRINTF(2); @@ -582,6 +642,8 @@ typedef struct { /**** cache ****/ +#define GRN_CACHE_MAX_KEY_SIZE GRN_HASH_MAX_KEY_SIZE_LARGE + typedef struct { uint32_t nentries; uint32_t max_nentries; diff --git a/storage/mroonga/vendor/groonga/lib/grn_ctx_impl.h b/storage/mroonga/vendor/groonga/lib/grn_ctx_impl.h index 9e0b5677497..b653f35015b 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_ctx_impl.h +++ b/storage/mroonga/vendor/groonga/lib/grn_ctx_impl.h @@ -19,16 +19,14 @@ #define GRN_CTX_IMPL_H #ifndef GRN_CTX_H -#include "grn_ctx.h" +# include "grn_ctx.h" #endif /* GRN_CTX_H */ #ifndef GRN_COM_H -#include "grn_com.h" +# include "grn_com.h" #endif /* GRN_COM_H */ -#ifdef GRN_WITH_MESSAGE_PACK -#include <msgpack.h> -#endif +#include "grn_msgpack.h" #ifdef GRN_WITH_MRUBY # include <mruby.h> @@ -111,6 +109,9 @@ struct _grn_mrb_data { struct { struct RClass *time_class; } builtin; + struct { + struct RClass *operator_class; + } groonga; }; #endif @@ -129,6 +130,7 @@ struct _grn_ctx_impl { grn_calloc_func calloc_func; grn_realloc_func realloc_func; grn_strdup_func strdup_func; + grn_free_func free_func; #endif #ifdef USE_MEMORY_DEBUG diff --git a/storage/mroonga/vendor/groonga/lib/grn_db.h b/storage/mroonga/vendor/groonga/lib/grn_db.h index 0eeab145e0e..cbc2cccaf4e 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_db.h +++ b/storage/mroonga/vendor/groonga/lib/grn_db.h @@ -26,6 +26,8 @@ #include <groonga/token_filter.h> #include <groonga/scorer.h> +#include <float.h> + #ifdef __cplusplus extern "C" { #endif diff --git a/storage/mroonga/vendor/groonga/lib/grn_ecmascript.c b/storage/mroonga/vendor/groonga/lib/grn_ecmascript.c index 80c596bc6b3..b3e4da09095 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_ecmascript.c +++ b/storage/mroonga/vendor/groonga/lib/grn_ecmascript.c @@ -58,13 +58,13 @@ ** defined, then do no error processing. */ #define YYCODETYPE unsigned char -#define YYNOCODE 113 +#define YYNOCODE 114 #define YYACTIONTYPE unsigned short int #define grn_expr_parserTOKENTYPE int typedef union { int yyinit; grn_expr_parserTOKENTYPE yy0; - void * yy81; + void * yy165; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -73,8 +73,8 @@ typedef union { #define grn_expr_parserARG_PDECL , efs_info *efsi #define grn_expr_parserARG_FETCH efs_info *efsi = yypParser->efsi #define grn_expr_parserARG_STORE yypParser->efsi = efsi -#define YYNSTATE 223 -#define YYNRULE 131 +#define YYNSTATE 225 +#define YYNRULE 132 #define YY_NO_ACTION (YYNSTATE+YYNRULE+2) #define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) #define YY_ERROR_ACTION (YYNSTATE+YYNRULE) @@ -143,392 +143,399 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. */ -#define YY_ACTTAB_COUNT (1610) +#define YY_ACTTAB_COUNT (1639) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2, 70, 52, 51, 50, 220, 1, 75, 79, 123, - /* 10 */ 4, 219, 69, 355, 76, 107, 78, 150, 219, 189, - /* 20 */ 192, 213, 84, 121, 120, 133, 132, 131, 115, 85, - /* 30 */ 98, 111, 99, 178, 209, 195, 73, 188, 184, 188, - /* 40 */ 184, 220, 71, 25, 79, 138, 9, 32, 69, 64, - /* 50 */ 63, 215, 33, 28, 67, 66, 65, 62, 61, 60, - /* 60 */ 59, 58, 57, 183, 182, 181, 180, 179, 3, 75, - /* 70 */ 113, 34, 6, 219, 189, 192, 213, 84, 121, 120, - /* 80 */ 133, 132, 131, 115, 85, 98, 111, 99, 178, 209, - /* 90 */ 195, 73, 187, 105, 188, 184, 220, 1, 28, 79, - /* 100 */ 123, 4, 122, 69, 31, 30, 189, 192, 213, 84, - /* 110 */ 121, 120, 133, 132, 131, 115, 85, 98, 111, 99, - /* 120 */ 178, 209, 195, 73, 139, 127, 188, 184, 36, 35, - /* 130 */ 110, 68, 56, 55, 8, 24, 129, 197, 196, 29, - /* 140 */ 64, 63, 54, 53, 124, 67, 66, 65, 62, 61, - /* 150 */ 60, 59, 58, 57, 183, 182, 181, 180, 179, 3, - /* 160 */ 7, 26, 165, 185, 83, 142, 127, 176, 189, 166, - /* 170 */ 213, 84, 121, 120, 133, 132, 131, 115, 85, 98, - /* 180 */ 111, 99, 178, 209, 195, 73, 173, 130, 188, 184, - /* 190 */ 11, 82, 81, 80, 77, 220, 71, 148, 79, 138, - /* 200 */ 9, 171, 69, 64, 63, 174, 28, 72, 67, 66, - /* 210 */ 65, 62, 61, 60, 59, 58, 57, 183, 182, 181, - /* 220 */ 180, 179, 3, 177, 7, 194, 193, 185, 83, 106, - /* 230 */ 126, 176, 189, 144, 213, 84, 121, 120, 133, 132, - /* 240 */ 131, 115, 85, 98, 111, 99, 178, 209, 195, 73, - /* 250 */ 164, 224, 188, 184, 141, 171, 23, 171, 10, 110, - /* 260 */ 143, 226, 191, 140, 221, 28, 218, 64, 63, 125, - /* 270 */ 356, 356, 67, 66, 65, 62, 61, 60, 59, 58, - /* 280 */ 57, 183, 182, 181, 180, 179, 3, 170, 7, 122, - /* 290 */ 217, 185, 83, 189, 192, 213, 84, 121, 120, 133, - /* 300 */ 132, 131, 115, 85, 98, 111, 99, 178, 209, 195, - /* 310 */ 73, 74, 216, 188, 184, 225, 49, 48, 47, 46, - /* 320 */ 45, 44, 43, 42, 41, 40, 39, 38, 37, 5, - /* 330 */ 149, 64, 63, 146, 222, 356, 67, 66, 65, 62, - /* 340 */ 61, 60, 59, 58, 57, 183, 182, 181, 180, 179, - /* 350 */ 3, 116, 356, 145, 356, 189, 192, 213, 84, 121, - /* 360 */ 120, 133, 132, 131, 115, 85, 98, 111, 99, 178, - /* 370 */ 209, 195, 73, 113, 356, 188, 184, 189, 192, 213, - /* 380 */ 84, 121, 120, 133, 132, 131, 115, 85, 98, 111, - /* 390 */ 99, 178, 209, 195, 73, 356, 356, 188, 184, 223, - /* 400 */ 356, 356, 81, 80, 77, 220, 71, 356, 79, 138, - /* 410 */ 9, 356, 69, 189, 162, 213, 84, 121, 120, 133, - /* 420 */ 132, 131, 115, 85, 98, 111, 99, 178, 209, 195, - /* 430 */ 73, 356, 7, 188, 184, 185, 83, 356, 356, 167, - /* 440 */ 109, 189, 144, 213, 84, 121, 120, 133, 132, 131, - /* 450 */ 115, 85, 98, 111, 99, 178, 209, 195, 73, 356, - /* 460 */ 7, 188, 184, 185, 83, 356, 356, 356, 356, 147, - /* 470 */ 356, 356, 356, 356, 356, 64, 63, 356, 356, 356, - /* 480 */ 67, 66, 65, 62, 61, 60, 59, 58, 57, 183, - /* 490 */ 182, 181, 180, 179, 3, 356, 356, 356, 356, 356, - /* 500 */ 356, 356, 356, 64, 63, 356, 356, 167, 67, 66, - /* 510 */ 65, 62, 61, 60, 59, 58, 57, 183, 182, 181, - /* 520 */ 180, 179, 3, 189, 214, 213, 84, 121, 120, 133, - /* 530 */ 132, 131, 115, 85, 98, 111, 99, 178, 209, 195, - /* 540 */ 73, 356, 356, 188, 184, 189, 212, 213, 84, 121, - /* 550 */ 120, 133, 132, 131, 115, 85, 98, 111, 99, 178, - /* 560 */ 209, 195, 73, 168, 356, 188, 184, 189, 137, 213, - /* 570 */ 84, 121, 120, 133, 132, 131, 115, 85, 98, 111, - /* 580 */ 99, 178, 209, 195, 73, 356, 356, 188, 184, 356, - /* 590 */ 189, 211, 213, 84, 121, 120, 133, 132, 131, 115, - /* 600 */ 85, 98, 111, 99, 178, 209, 195, 73, 356, 356, - /* 610 */ 188, 184, 189, 172, 213, 84, 121, 120, 133, 132, - /* 620 */ 131, 115, 85, 98, 111, 99, 178, 209, 195, 73, - /* 630 */ 356, 356, 188, 184, 189, 163, 213, 84, 121, 120, - /* 640 */ 133, 132, 131, 115, 85, 98, 111, 99, 178, 209, - /* 650 */ 195, 73, 356, 356, 188, 184, 189, 161, 213, 84, - /* 660 */ 121, 120, 133, 132, 131, 115, 85, 98, 111, 99, - /* 670 */ 178, 209, 195, 73, 356, 356, 188, 184, 189, 160, - /* 680 */ 213, 84, 121, 120, 133, 132, 131, 115, 85, 98, - /* 690 */ 111, 99, 178, 209, 195, 73, 356, 356, 188, 184, - /* 700 */ 189, 159, 213, 84, 121, 120, 133, 132, 131, 115, - /* 710 */ 85, 98, 111, 99, 178, 209, 195, 73, 356, 356, - /* 720 */ 188, 184, 189, 158, 213, 84, 121, 120, 133, 132, - /* 730 */ 131, 115, 85, 98, 111, 99, 178, 209, 195, 73, - /* 740 */ 356, 356, 188, 184, 189, 157, 213, 84, 121, 120, - /* 750 */ 133, 132, 131, 115, 85, 98, 111, 99, 178, 209, - /* 760 */ 195, 73, 356, 356, 188, 184, 189, 156, 213, 84, - /* 770 */ 121, 120, 133, 132, 131, 115, 85, 98, 111, 99, - /* 780 */ 178, 209, 195, 73, 356, 356, 188, 184, 189, 155, - /* 790 */ 213, 84, 121, 120, 133, 132, 131, 115, 85, 98, - /* 800 */ 111, 99, 178, 209, 195, 73, 356, 356, 188, 184, - /* 810 */ 189, 154, 213, 84, 121, 120, 133, 132, 131, 115, - /* 820 */ 85, 98, 111, 99, 178, 209, 195, 73, 356, 356, - /* 830 */ 188, 184, 189, 153, 213, 84, 121, 120, 133, 132, - /* 840 */ 131, 115, 85, 98, 111, 99, 178, 209, 195, 73, - /* 850 */ 356, 356, 188, 184, 189, 152, 213, 84, 121, 120, - /* 860 */ 133, 132, 131, 115, 85, 98, 111, 99, 178, 209, - /* 870 */ 195, 73, 356, 356, 188, 184, 189, 151, 213, 84, - /* 880 */ 121, 120, 133, 132, 131, 115, 85, 98, 111, 99, - /* 890 */ 178, 209, 195, 73, 356, 356, 188, 184, 189, 175, - /* 900 */ 213, 84, 121, 120, 133, 132, 131, 115, 85, 98, - /* 910 */ 111, 99, 178, 209, 195, 73, 356, 356, 188, 184, - /* 920 */ 189, 169, 213, 84, 121, 120, 133, 132, 131, 115, - /* 930 */ 85, 98, 111, 99, 178, 209, 195, 73, 356, 189, - /* 940 */ 188, 184, 117, 356, 108, 133, 132, 131, 115, 85, - /* 950 */ 98, 111, 99, 178, 209, 195, 73, 356, 189, 188, - /* 960 */ 184, 117, 356, 356, 136, 132, 131, 115, 85, 98, - /* 970 */ 111, 99, 178, 209, 195, 73, 356, 356, 188, 184, - /* 980 */ 189, 356, 356, 117, 356, 356, 128, 132, 131, 115, - /* 990 */ 85, 98, 111, 99, 178, 209, 195, 73, 356, 356, - /* 1000 */ 188, 184, 189, 356, 356, 117, 356, 356, 356, 135, - /* 1010 */ 131, 115, 85, 98, 111, 99, 178, 209, 195, 73, - /* 1020 */ 356, 356, 188, 184, 356, 27, 22, 21, 20, 19, - /* 1030 */ 18, 17, 16, 15, 14, 13, 12, 189, 356, 356, - /* 1040 */ 117, 356, 356, 356, 356, 134, 115, 85, 98, 111, - /* 1050 */ 99, 178, 209, 195, 73, 356, 356, 188, 184, 356, - /* 1060 */ 189, 356, 356, 117, 356, 356, 197, 196, 356, 119, - /* 1070 */ 85, 98, 111, 99, 178, 209, 195, 73, 356, 189, - /* 1080 */ 188, 184, 117, 7, 356, 356, 185, 83, 356, 87, - /* 1090 */ 98, 111, 99, 178, 209, 195, 73, 356, 189, 188, - /* 1100 */ 184, 117, 356, 356, 356, 356, 356, 356, 86, 98, - /* 1110 */ 111, 99, 178, 209, 195, 73, 356, 189, 188, 184, - /* 1120 */ 117, 356, 356, 356, 356, 356, 356, 356, 104, 111, - /* 1130 */ 99, 178, 209, 195, 73, 356, 189, 188, 184, 117, - /* 1140 */ 183, 182, 181, 180, 179, 3, 356, 102, 111, 99, - /* 1150 */ 178, 209, 195, 73, 356, 189, 188, 184, 117, 356, - /* 1160 */ 356, 356, 356, 356, 356, 356, 100, 111, 99, 178, - /* 1170 */ 209, 195, 73, 356, 189, 188, 184, 117, 356, 356, - /* 1180 */ 356, 356, 356, 356, 356, 97, 111, 99, 178, 209, - /* 1190 */ 195, 73, 356, 189, 188, 184, 117, 356, 356, 356, - /* 1200 */ 356, 356, 356, 356, 96, 111, 99, 178, 209, 195, - /* 1210 */ 73, 356, 189, 188, 184, 117, 356, 356, 356, 356, - /* 1220 */ 356, 356, 356, 95, 111, 99, 178, 209, 195, 73, - /* 1230 */ 356, 189, 188, 184, 117, 356, 356, 356, 356, 356, - /* 1240 */ 356, 356, 94, 111, 99, 178, 209, 195, 73, 356, - /* 1250 */ 189, 188, 184, 117, 356, 356, 356, 356, 356, 356, - /* 1260 */ 356, 93, 111, 99, 178, 209, 195, 73, 356, 189, - /* 1270 */ 188, 184, 117, 356, 356, 356, 356, 356, 356, 356, - /* 1280 */ 92, 111, 99, 178, 209, 195, 73, 356, 189, 188, - /* 1290 */ 184, 117, 356, 356, 356, 356, 356, 356, 356, 91, - /* 1300 */ 111, 99, 178, 209, 195, 73, 356, 189, 188, 184, - /* 1310 */ 117, 356, 356, 356, 356, 356, 356, 356, 90, 111, - /* 1320 */ 99, 178, 209, 195, 73, 356, 189, 188, 184, 117, - /* 1330 */ 356, 356, 356, 356, 356, 356, 356, 89, 111, 99, - /* 1340 */ 178, 209, 195, 73, 356, 189, 188, 184, 117, 356, - /* 1350 */ 356, 356, 356, 356, 356, 356, 88, 111, 99, 178, - /* 1360 */ 209, 195, 73, 356, 189, 188, 184, 117, 356, 356, - /* 1370 */ 356, 356, 356, 356, 356, 356, 118, 99, 178, 209, - /* 1380 */ 195, 73, 356, 189, 188, 184, 117, 356, 356, 356, - /* 1390 */ 356, 356, 356, 356, 356, 114, 99, 178, 209, 195, - /* 1400 */ 73, 356, 189, 188, 184, 117, 356, 356, 356, 356, - /* 1410 */ 356, 356, 356, 356, 112, 99, 178, 209, 195, 73, - /* 1420 */ 356, 189, 188, 184, 117, 356, 356, 356, 356, 356, - /* 1430 */ 189, 356, 356, 117, 103, 178, 209, 195, 73, 356, - /* 1440 */ 356, 188, 184, 101, 178, 209, 195, 73, 356, 189, - /* 1450 */ 188, 184, 117, 356, 356, 356, 356, 356, 189, 356, - /* 1460 */ 356, 117, 356, 210, 209, 195, 73, 356, 189, 188, - /* 1470 */ 184, 117, 208, 209, 195, 73, 356, 189, 188, 184, - /* 1480 */ 117, 356, 207, 209, 195, 73, 356, 189, 188, 184, - /* 1490 */ 117, 206, 209, 195, 73, 356, 189, 188, 184, 117, - /* 1500 */ 356, 205, 209, 195, 73, 356, 189, 188, 184, 117, - /* 1510 */ 204, 209, 195, 73, 356, 189, 188, 184, 117, 356, - /* 1520 */ 203, 209, 195, 73, 356, 189, 188, 184, 117, 202, - /* 1530 */ 209, 195, 73, 356, 189, 188, 184, 117, 356, 201, - /* 1540 */ 209, 195, 73, 356, 356, 188, 184, 356, 200, 209, - /* 1550 */ 195, 73, 356, 189, 188, 184, 117, 356, 356, 356, - /* 1560 */ 356, 189, 356, 356, 117, 356, 356, 199, 209, 195, - /* 1570 */ 73, 356, 356, 188, 184, 198, 209, 195, 73, 356, - /* 1580 */ 189, 188, 184, 117, 356, 356, 356, 356, 189, 356, - /* 1590 */ 356, 117, 356, 356, 190, 209, 195, 73, 356, 356, - /* 1600 */ 188, 184, 186, 209, 195, 73, 356, 356, 188, 184, + /* 0 */ 2, 71, 53, 52, 51, 222, 1, 76, 80, 125, + /* 10 */ 4, 221, 70, 358, 77, 109, 28, 152, 221, 191, + /* 20 */ 194, 215, 88, 123, 122, 135, 134, 133, 117, 85, + /* 30 */ 100, 113, 101, 180, 211, 197, 74, 190, 186, 190, + /* 40 */ 186, 222, 72, 79, 80, 140, 9, 189, 70, 25, + /* 50 */ 65, 64, 217, 28, 28, 68, 67, 66, 63, 62, + /* 60 */ 61, 60, 59, 58, 185, 184, 183, 182, 181, 3, + /* 70 */ 76, 115, 6, 193, 221, 191, 194, 215, 88, 123, + /* 80 */ 122, 135, 134, 133, 117, 85, 100, 113, 101, 180, + /* 90 */ 211, 197, 74, 166, 107, 190, 186, 222, 1, 23, + /* 100 */ 80, 125, 4, 124, 70, 31, 30, 191, 194, 215, + /* 110 */ 88, 123, 122, 135, 134, 133, 117, 85, 100, 113, + /* 120 */ 101, 180, 211, 197, 74, 141, 129, 190, 186, 36, + /* 130 */ 35, 112, 69, 57, 56, 8, 32, 131, 55, 54, + /* 140 */ 34, 29, 65, 64, 176, 33, 73, 68, 67, 66, + /* 150 */ 63, 62, 61, 60, 59, 58, 185, 184, 183, 182, + /* 160 */ 181, 3, 7, 26, 128, 187, 84, 199, 198, 178, + /* 170 */ 191, 168, 215, 88, 123, 122, 135, 134, 133, 117, + /* 180 */ 85, 100, 113, 101, 180, 211, 197, 74, 144, 129, + /* 190 */ 190, 186, 11, 83, 82, 81, 78, 222, 72, 150, + /* 200 */ 80, 140, 9, 173, 70, 24, 65, 64, 228, 169, + /* 210 */ 167, 68, 67, 66, 63, 62, 61, 60, 59, 58, + /* 220 */ 185, 184, 183, 182, 181, 3, 179, 7, 196, 195, + /* 230 */ 187, 84, 108, 143, 178, 191, 146, 215, 88, 123, + /* 240 */ 122, 135, 134, 133, 117, 85, 100, 113, 101, 180, + /* 250 */ 211, 197, 74, 226, 227, 190, 186, 126, 173, 75, + /* 260 */ 173, 175, 132, 145, 142, 112, 170, 28, 5, 10, + /* 270 */ 223, 65, 64, 220, 127, 219, 68, 67, 66, 63, + /* 280 */ 62, 61, 60, 59, 58, 185, 184, 183, 182, 181, + /* 290 */ 3, 172, 7, 124, 218, 187, 84, 191, 194, 215, + /* 300 */ 88, 123, 122, 135, 134, 133, 117, 85, 100, 113, + /* 310 */ 101, 180, 211, 197, 74, 151, 224, 190, 186, 359, + /* 320 */ 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, + /* 330 */ 40, 39, 38, 37, 359, 359, 65, 64, 148, 359, + /* 340 */ 359, 68, 67, 66, 63, 62, 61, 60, 59, 58, + /* 350 */ 185, 184, 183, 182, 181, 3, 118, 359, 147, 359, + /* 360 */ 191, 194, 215, 88, 123, 122, 135, 134, 133, 117, + /* 370 */ 85, 100, 113, 101, 180, 211, 197, 74, 115, 359, + /* 380 */ 190, 186, 191, 194, 215, 88, 123, 122, 135, 134, + /* 390 */ 133, 117, 85, 100, 113, 101, 180, 211, 197, 74, + /* 400 */ 359, 359, 190, 186, 225, 359, 359, 82, 81, 78, + /* 410 */ 222, 72, 359, 80, 140, 9, 359, 70, 359, 191, + /* 420 */ 164, 215, 88, 123, 122, 135, 134, 133, 117, 85, + /* 430 */ 100, 113, 101, 180, 211, 197, 74, 359, 7, 190, + /* 440 */ 186, 187, 84, 359, 359, 169, 111, 191, 146, 215, + /* 450 */ 88, 123, 122, 135, 134, 133, 117, 85, 100, 113, + /* 460 */ 101, 180, 211, 197, 74, 359, 7, 190, 186, 187, + /* 470 */ 84, 359, 359, 359, 359, 149, 359, 359, 359, 359, + /* 480 */ 359, 359, 65, 64, 359, 359, 359, 68, 67, 66, + /* 490 */ 63, 62, 61, 60, 59, 58, 185, 184, 183, 182, + /* 500 */ 181, 3, 359, 359, 359, 359, 359, 359, 359, 359, + /* 510 */ 65, 64, 359, 359, 359, 68, 67, 66, 63, 62, + /* 520 */ 61, 60, 59, 58, 185, 184, 183, 182, 181, 3, + /* 530 */ 191, 216, 215, 88, 123, 122, 135, 134, 133, 117, + /* 540 */ 85, 100, 113, 101, 180, 211, 197, 74, 359, 359, + /* 550 */ 190, 186, 191, 214, 215, 88, 123, 122, 135, 134, + /* 560 */ 133, 117, 85, 100, 113, 101, 180, 211, 197, 74, + /* 570 */ 359, 359, 190, 186, 191, 139, 215, 88, 123, 122, + /* 580 */ 135, 134, 133, 117, 85, 100, 113, 101, 180, 211, + /* 590 */ 197, 74, 359, 359, 190, 186, 359, 359, 191, 213, + /* 600 */ 215, 88, 123, 122, 135, 134, 133, 117, 85, 100, + /* 610 */ 113, 101, 180, 211, 197, 74, 359, 359, 190, 186, + /* 620 */ 191, 174, 215, 88, 123, 122, 135, 134, 133, 117, + /* 630 */ 85, 100, 113, 101, 180, 211, 197, 74, 359, 359, + /* 640 */ 190, 186, 191, 165, 215, 88, 123, 122, 135, 134, + /* 650 */ 133, 117, 85, 100, 113, 101, 180, 211, 197, 74, + /* 660 */ 359, 359, 190, 186, 191, 163, 215, 88, 123, 122, + /* 670 */ 135, 134, 133, 117, 85, 100, 113, 101, 180, 211, + /* 680 */ 197, 74, 359, 359, 190, 186, 191, 162, 215, 88, + /* 690 */ 123, 122, 135, 134, 133, 117, 85, 100, 113, 101, + /* 700 */ 180, 211, 197, 74, 359, 359, 190, 186, 191, 161, + /* 710 */ 215, 88, 123, 122, 135, 134, 133, 117, 85, 100, + /* 720 */ 113, 101, 180, 211, 197, 74, 359, 359, 190, 186, + /* 730 */ 191, 160, 215, 88, 123, 122, 135, 134, 133, 117, + /* 740 */ 85, 100, 113, 101, 180, 211, 197, 74, 359, 359, + /* 750 */ 190, 186, 191, 159, 215, 88, 123, 122, 135, 134, + /* 760 */ 133, 117, 85, 100, 113, 101, 180, 211, 197, 74, + /* 770 */ 359, 359, 190, 186, 191, 158, 215, 88, 123, 122, + /* 780 */ 135, 134, 133, 117, 85, 100, 113, 101, 180, 211, + /* 790 */ 197, 74, 359, 359, 190, 186, 191, 157, 215, 88, + /* 800 */ 123, 122, 135, 134, 133, 117, 85, 100, 113, 101, + /* 810 */ 180, 211, 197, 74, 359, 359, 190, 186, 191, 156, + /* 820 */ 215, 88, 123, 122, 135, 134, 133, 117, 85, 100, + /* 830 */ 113, 101, 180, 211, 197, 74, 359, 359, 190, 186, + /* 840 */ 191, 155, 215, 88, 123, 122, 135, 134, 133, 117, + /* 850 */ 85, 100, 113, 101, 180, 211, 197, 74, 359, 359, + /* 860 */ 190, 186, 191, 154, 215, 88, 123, 122, 135, 134, + /* 870 */ 133, 117, 85, 100, 113, 101, 180, 211, 197, 74, + /* 880 */ 359, 359, 190, 186, 191, 153, 215, 88, 123, 122, + /* 890 */ 135, 134, 133, 117, 85, 100, 113, 101, 180, 211, + /* 900 */ 197, 74, 359, 359, 190, 186, 191, 177, 215, 88, + /* 910 */ 123, 122, 135, 134, 133, 117, 85, 100, 113, 101, + /* 920 */ 180, 211, 197, 74, 359, 359, 190, 186, 191, 171, + /* 930 */ 215, 88, 123, 122, 135, 134, 133, 117, 85, 100, + /* 940 */ 113, 101, 180, 211, 197, 74, 359, 191, 190, 186, + /* 950 */ 119, 359, 110, 135, 134, 133, 117, 85, 100, 113, + /* 960 */ 101, 180, 211, 197, 74, 359, 191, 190, 186, 119, + /* 970 */ 359, 359, 138, 134, 133, 117, 85, 100, 113, 101, + /* 980 */ 180, 211, 197, 74, 359, 359, 190, 186, 191, 359, + /* 990 */ 359, 119, 359, 359, 130, 134, 133, 117, 85, 100, + /* 1000 */ 113, 101, 180, 211, 197, 74, 359, 359, 190, 186, + /* 1010 */ 191, 359, 359, 119, 359, 359, 359, 137, 133, 117, + /* 1020 */ 85, 100, 113, 101, 180, 211, 197, 74, 359, 359, + /* 1030 */ 190, 186, 359, 27, 22, 21, 20, 19, 18, 17, + /* 1040 */ 16, 15, 14, 13, 12, 191, 359, 359, 119, 359, + /* 1050 */ 359, 359, 359, 136, 117, 85, 100, 113, 101, 180, + /* 1060 */ 211, 197, 74, 359, 359, 190, 186, 359, 359, 191, + /* 1070 */ 359, 359, 119, 359, 359, 199, 198, 359, 121, 85, + /* 1080 */ 100, 113, 101, 180, 211, 197, 74, 359, 191, 190, + /* 1090 */ 186, 119, 7, 359, 359, 187, 84, 359, 87, 100, + /* 1100 */ 113, 101, 180, 211, 197, 74, 359, 191, 190, 186, + /* 1110 */ 119, 359, 359, 359, 359, 359, 359, 86, 100, 113, + /* 1120 */ 101, 180, 211, 197, 74, 359, 191, 190, 186, 119, + /* 1130 */ 359, 359, 359, 359, 359, 359, 359, 106, 113, 101, + /* 1140 */ 180, 211, 197, 74, 359, 191, 190, 186, 119, 359, + /* 1150 */ 185, 184, 183, 182, 181, 3, 104, 113, 101, 180, + /* 1160 */ 211, 197, 74, 359, 191, 190, 186, 119, 359, 359, + /* 1170 */ 359, 359, 359, 359, 359, 102, 113, 101, 180, 211, + /* 1180 */ 197, 74, 359, 191, 190, 186, 119, 359, 359, 359, + /* 1190 */ 359, 359, 359, 359, 99, 113, 101, 180, 211, 197, + /* 1200 */ 74, 359, 191, 190, 186, 119, 359, 359, 359, 359, + /* 1210 */ 359, 359, 359, 98, 113, 101, 180, 211, 197, 74, + /* 1220 */ 359, 191, 190, 186, 119, 359, 359, 359, 359, 359, + /* 1230 */ 359, 359, 97, 113, 101, 180, 211, 197, 74, 359, + /* 1240 */ 191, 190, 186, 119, 359, 359, 359, 359, 359, 359, + /* 1250 */ 359, 96, 113, 101, 180, 211, 197, 74, 359, 191, + /* 1260 */ 190, 186, 119, 359, 359, 359, 359, 359, 359, 359, + /* 1270 */ 95, 113, 101, 180, 211, 197, 74, 359, 191, 190, + /* 1280 */ 186, 119, 359, 359, 359, 359, 359, 359, 359, 94, + /* 1290 */ 113, 101, 180, 211, 197, 74, 359, 191, 190, 186, + /* 1300 */ 119, 359, 359, 359, 359, 359, 359, 359, 93, 113, + /* 1310 */ 101, 180, 211, 197, 74, 359, 191, 190, 186, 119, + /* 1320 */ 359, 359, 359, 359, 359, 359, 359, 92, 113, 101, + /* 1330 */ 180, 211, 197, 74, 359, 191, 190, 186, 119, 359, + /* 1340 */ 359, 359, 359, 359, 359, 359, 91, 113, 101, 180, + /* 1350 */ 211, 197, 74, 359, 191, 190, 186, 119, 359, 359, + /* 1360 */ 359, 359, 359, 359, 359, 90, 113, 101, 180, 211, + /* 1370 */ 197, 74, 359, 191, 190, 186, 119, 359, 359, 359, + /* 1380 */ 359, 359, 359, 359, 89, 113, 101, 180, 211, 197, + /* 1390 */ 74, 359, 191, 190, 186, 119, 359, 359, 359, 359, + /* 1400 */ 359, 359, 359, 359, 120, 101, 180, 211, 197, 74, + /* 1410 */ 359, 191, 190, 186, 119, 359, 359, 359, 359, 359, + /* 1420 */ 359, 359, 359, 116, 101, 180, 211, 197, 74, 359, + /* 1430 */ 191, 190, 186, 119, 359, 359, 359, 359, 359, 359, + /* 1440 */ 359, 359, 114, 101, 180, 211, 197, 74, 359, 191, + /* 1450 */ 190, 186, 119, 359, 359, 359, 359, 359, 191, 359, + /* 1460 */ 359, 119, 105, 180, 211, 197, 74, 359, 359, 190, + /* 1470 */ 186, 103, 180, 211, 197, 74, 359, 191, 190, 186, + /* 1480 */ 119, 359, 359, 359, 359, 359, 359, 191, 359, 359, + /* 1490 */ 119, 212, 211, 197, 74, 359, 191, 190, 186, 119, + /* 1500 */ 359, 210, 211, 197, 74, 359, 191, 190, 186, 119, + /* 1510 */ 209, 211, 197, 74, 359, 191, 190, 186, 119, 359, + /* 1520 */ 208, 211, 197, 74, 359, 191, 190, 186, 119, 207, + /* 1530 */ 211, 197, 74, 359, 191, 190, 186, 119, 359, 206, + /* 1540 */ 211, 197, 74, 359, 191, 190, 186, 119, 205, 211, + /* 1550 */ 197, 74, 359, 191, 190, 186, 119, 359, 204, 211, + /* 1560 */ 197, 74, 359, 191, 190, 186, 119, 203, 211, 197, + /* 1570 */ 74, 359, 359, 190, 186, 359, 359, 202, 211, 197, + /* 1580 */ 74, 359, 191, 190, 186, 119, 359, 359, 359, 359, + /* 1590 */ 191, 359, 359, 119, 359, 359, 201, 211, 197, 74, + /* 1600 */ 359, 359, 190, 186, 200, 211, 197, 74, 359, 191, + /* 1610 */ 190, 186, 119, 359, 359, 359, 359, 191, 359, 359, + /* 1620 */ 119, 359, 359, 192, 211, 197, 74, 359, 359, 190, + /* 1630 */ 186, 188, 211, 197, 74, 359, 359, 190, 186, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 1, 2, 47, 48, 49, 6, 7, 76, 9, 10, - /* 10 */ 11, 80, 13, 75, 76, 77, 9, 81, 80, 81, - /* 20 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, - /* 30 */ 92, 93, 94, 95, 96, 97, 98, 101, 102, 101, - /* 40 */ 102, 6, 7, 28, 9, 10, 11, 29, 13, 50, - /* 50 */ 51, 12, 30, 14, 55, 56, 57, 58, 59, 60, - /* 60 */ 61, 62, 63, 64, 65, 66, 67, 68, 69, 76, - /* 70 */ 77, 31, 7, 80, 81, 82, 83, 84, 85, 86, + /* 0 */ 1, 2, 48, 49, 50, 6, 7, 77, 9, 10, + /* 10 */ 11, 81, 13, 76, 77, 78, 14, 82, 81, 82, + /* 20 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, + /* 30 */ 93, 94, 95, 96, 97, 98, 99, 102, 103, 102, + /* 40 */ 103, 6, 7, 9, 9, 10, 11, 8, 13, 28, + /* 50 */ 51, 52, 12, 14, 14, 56, 57, 58, 59, 60, + /* 60 */ 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + /* 70 */ 77, 78, 7, 71, 81, 82, 83, 84, 85, 86, /* 80 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, - /* 90 */ 97, 98, 8, 79, 101, 102, 6, 7, 14, 9, - /* 100 */ 10, 11, 77, 13, 3, 4, 81, 82, 83, 84, + /* 90 */ 97, 98, 99, 8, 80, 102, 103, 6, 7, 14, + /* 100 */ 9, 10, 11, 78, 13, 3, 4, 82, 83, 84, /* 110 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 120 */ 95, 96, 97, 98, 110, 111, 101, 102, 32, 33, - /* 130 */ 105, 52, 53, 54, 69, 28, 71, 56, 57, 5, - /* 140 */ 50, 51, 50, 51, 39, 55, 56, 57, 58, 59, - /* 150 */ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - /* 160 */ 7, 27, 10, 10, 11, 110, 111, 14, 81, 82, - /* 170 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - /* 180 */ 93, 94, 95, 96, 97, 98, 106, 107, 101, 102, - /* 190 */ 103, 104, 3, 4, 5, 6, 7, 8, 9, 10, - /* 200 */ 11, 10, 13, 50, 51, 12, 14, 14, 55, 56, - /* 210 */ 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - /* 220 */ 67, 68, 69, 70, 7, 99, 100, 10, 11, 78, - /* 230 */ 52, 14, 81, 82, 83, 84, 85, 86, 87, 88, - /* 240 */ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, - /* 250 */ 8, 0, 101, 102, 64, 64, 14, 66, 103, 105, - /* 260 */ 109, 0, 70, 66, 80, 14, 80, 50, 51, 10, - /* 270 */ 112, 112, 55, 56, 57, 58, 59, 60, 61, 62, - /* 280 */ 63, 64, 65, 66, 67, 68, 69, 70, 7, 77, - /* 290 */ 80, 10, 11, 81, 82, 83, 84, 85, 86, 87, - /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - /* 310 */ 98, 50, 80, 101, 102, 0, 34, 35, 36, 37, - /* 320 */ 38, 39, 40, 41, 42, 43, 44, 45, 46, 14, - /* 330 */ 80, 50, 51, 52, 80, 112, 55, 56, 57, 58, - /* 340 */ 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - /* 350 */ 69, 77, 112, 72, 112, 81, 82, 83, 84, 85, - /* 360 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 370 */ 96, 97, 98, 77, 112, 101, 102, 81, 82, 83, - /* 380 */ 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 390 */ 94, 95, 96, 97, 98, 112, 112, 101, 102, 0, - /* 400 */ 112, 112, 3, 4, 5, 6, 7, 112, 9, 10, - /* 410 */ 11, 112, 13, 81, 82, 83, 84, 85, 86, 87, - /* 420 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - /* 430 */ 98, 112, 7, 101, 102, 10, 11, 112, 112, 14, - /* 440 */ 108, 81, 82, 83, 84, 85, 86, 87, 88, 89, - /* 450 */ 90, 91, 92, 93, 94, 95, 96, 97, 98, 112, - /* 460 */ 7, 101, 102, 10, 11, 112, 112, 112, 112, 109, - /* 470 */ 112, 112, 112, 112, 112, 50, 51, 112, 112, 112, - /* 480 */ 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, - /* 490 */ 65, 66, 67, 68, 69, 112, 112, 112, 112, 112, - /* 500 */ 112, 112, 112, 50, 51, 112, 112, 14, 55, 56, - /* 510 */ 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - /* 520 */ 67, 68, 69, 81, 82, 83, 84, 85, 86, 87, - /* 530 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - /* 540 */ 98, 112, 112, 101, 102, 81, 82, 83, 84, 85, - /* 550 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 560 */ 96, 97, 98, 70, 112, 101, 102, 81, 82, 83, - /* 570 */ 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 580 */ 94, 95, 96, 97, 98, 112, 112, 101, 102, 112, - /* 590 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 600 */ 91, 92, 93, 94, 95, 96, 97, 98, 112, 112, - /* 610 */ 101, 102, 81, 82, 83, 84, 85, 86, 87, 88, - /* 620 */ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, - /* 630 */ 112, 112, 101, 102, 81, 82, 83, 84, 85, 86, - /* 640 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, - /* 650 */ 97, 98, 112, 112, 101, 102, 81, 82, 83, 84, - /* 660 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 670 */ 95, 96, 97, 98, 112, 112, 101, 102, 81, 82, - /* 680 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - /* 690 */ 93, 94, 95, 96, 97, 98, 112, 112, 101, 102, - /* 700 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 710 */ 91, 92, 93, 94, 95, 96, 97, 98, 112, 112, - /* 720 */ 101, 102, 81, 82, 83, 84, 85, 86, 87, 88, - /* 730 */ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, - /* 740 */ 112, 112, 101, 102, 81, 82, 83, 84, 85, 86, - /* 750 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, - /* 760 */ 97, 98, 112, 112, 101, 102, 81, 82, 83, 84, - /* 770 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 780 */ 95, 96, 97, 98, 112, 112, 101, 102, 81, 82, - /* 790 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - /* 800 */ 93, 94, 95, 96, 97, 98, 112, 112, 101, 102, - /* 810 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 820 */ 91, 92, 93, 94, 95, 96, 97, 98, 112, 112, - /* 830 */ 101, 102, 81, 82, 83, 84, 85, 86, 87, 88, - /* 840 */ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, - /* 850 */ 112, 112, 101, 102, 81, 82, 83, 84, 85, 86, - /* 860 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, - /* 870 */ 97, 98, 112, 112, 101, 102, 81, 82, 83, 84, - /* 880 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 890 */ 95, 96, 97, 98, 112, 112, 101, 102, 81, 82, - /* 900 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - /* 910 */ 93, 94, 95, 96, 97, 98, 112, 112, 101, 102, - /* 920 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 930 */ 91, 92, 93, 94, 95, 96, 97, 98, 112, 81, - /* 940 */ 101, 102, 84, 112, 86, 87, 88, 89, 90, 91, - /* 950 */ 92, 93, 94, 95, 96, 97, 98, 112, 81, 101, - /* 960 */ 102, 84, 112, 112, 87, 88, 89, 90, 91, 92, - /* 970 */ 93, 94, 95, 96, 97, 98, 112, 112, 101, 102, - /* 980 */ 81, 112, 112, 84, 112, 112, 87, 88, 89, 90, - /* 990 */ 91, 92, 93, 94, 95, 96, 97, 98, 112, 112, - /* 1000 */ 101, 102, 81, 112, 112, 84, 112, 112, 112, 88, - /* 1010 */ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, - /* 1020 */ 112, 112, 101, 102, 112, 15, 16, 17, 18, 19, - /* 1030 */ 20, 21, 22, 23, 24, 25, 26, 81, 112, 112, - /* 1040 */ 84, 112, 112, 112, 112, 89, 90, 91, 92, 93, - /* 1050 */ 94, 95, 96, 97, 98, 112, 112, 101, 102, 112, - /* 1060 */ 81, 112, 112, 84, 112, 112, 56, 57, 112, 90, - /* 1070 */ 91, 92, 93, 94, 95, 96, 97, 98, 112, 81, - /* 1080 */ 101, 102, 84, 7, 112, 112, 10, 11, 112, 91, - /* 1090 */ 92, 93, 94, 95, 96, 97, 98, 112, 81, 101, - /* 1100 */ 102, 84, 112, 112, 112, 112, 112, 112, 91, 92, - /* 1110 */ 93, 94, 95, 96, 97, 98, 112, 81, 101, 102, - /* 1120 */ 84, 112, 112, 112, 112, 112, 112, 112, 92, 93, - /* 1130 */ 94, 95, 96, 97, 98, 112, 81, 101, 102, 84, - /* 1140 */ 64, 65, 66, 67, 68, 69, 112, 92, 93, 94, - /* 1150 */ 95, 96, 97, 98, 112, 81, 101, 102, 84, 112, - /* 1160 */ 112, 112, 112, 112, 112, 112, 92, 93, 94, 95, - /* 1170 */ 96, 97, 98, 112, 81, 101, 102, 84, 112, 112, - /* 1180 */ 112, 112, 112, 112, 112, 92, 93, 94, 95, 96, - /* 1190 */ 97, 98, 112, 81, 101, 102, 84, 112, 112, 112, - /* 1200 */ 112, 112, 112, 112, 92, 93, 94, 95, 96, 97, - /* 1210 */ 98, 112, 81, 101, 102, 84, 112, 112, 112, 112, - /* 1220 */ 112, 112, 112, 92, 93, 94, 95, 96, 97, 98, - /* 1230 */ 112, 81, 101, 102, 84, 112, 112, 112, 112, 112, - /* 1240 */ 112, 112, 92, 93, 94, 95, 96, 97, 98, 112, - /* 1250 */ 81, 101, 102, 84, 112, 112, 112, 112, 112, 112, - /* 1260 */ 112, 92, 93, 94, 95, 96, 97, 98, 112, 81, - /* 1270 */ 101, 102, 84, 112, 112, 112, 112, 112, 112, 112, - /* 1280 */ 92, 93, 94, 95, 96, 97, 98, 112, 81, 101, - /* 1290 */ 102, 84, 112, 112, 112, 112, 112, 112, 112, 92, - /* 1300 */ 93, 94, 95, 96, 97, 98, 112, 81, 101, 102, - /* 1310 */ 84, 112, 112, 112, 112, 112, 112, 112, 92, 93, - /* 1320 */ 94, 95, 96, 97, 98, 112, 81, 101, 102, 84, - /* 1330 */ 112, 112, 112, 112, 112, 112, 112, 92, 93, 94, - /* 1340 */ 95, 96, 97, 98, 112, 81, 101, 102, 84, 112, - /* 1350 */ 112, 112, 112, 112, 112, 112, 92, 93, 94, 95, - /* 1360 */ 96, 97, 98, 112, 81, 101, 102, 84, 112, 112, - /* 1370 */ 112, 112, 112, 112, 112, 112, 93, 94, 95, 96, - /* 1380 */ 97, 98, 112, 81, 101, 102, 84, 112, 112, 112, - /* 1390 */ 112, 112, 112, 112, 112, 93, 94, 95, 96, 97, - /* 1400 */ 98, 112, 81, 101, 102, 84, 112, 112, 112, 112, - /* 1410 */ 112, 112, 112, 112, 93, 94, 95, 96, 97, 98, - /* 1420 */ 112, 81, 101, 102, 84, 112, 112, 112, 112, 112, - /* 1430 */ 81, 112, 112, 84, 94, 95, 96, 97, 98, 112, - /* 1440 */ 112, 101, 102, 94, 95, 96, 97, 98, 112, 81, - /* 1450 */ 101, 102, 84, 112, 112, 112, 112, 112, 81, 112, - /* 1460 */ 112, 84, 112, 95, 96, 97, 98, 112, 81, 101, - /* 1470 */ 102, 84, 95, 96, 97, 98, 112, 81, 101, 102, - /* 1480 */ 84, 112, 95, 96, 97, 98, 112, 81, 101, 102, - /* 1490 */ 84, 95, 96, 97, 98, 112, 81, 101, 102, 84, - /* 1500 */ 112, 95, 96, 97, 98, 112, 81, 101, 102, 84, - /* 1510 */ 95, 96, 97, 98, 112, 81, 101, 102, 84, 112, - /* 1520 */ 95, 96, 97, 98, 112, 81, 101, 102, 84, 95, - /* 1530 */ 96, 97, 98, 112, 81, 101, 102, 84, 112, 95, - /* 1540 */ 96, 97, 98, 112, 112, 101, 102, 112, 95, 96, - /* 1550 */ 97, 98, 112, 81, 101, 102, 84, 112, 112, 112, - /* 1560 */ 112, 81, 112, 112, 84, 112, 112, 95, 96, 97, - /* 1570 */ 98, 112, 112, 101, 102, 95, 96, 97, 98, 112, - /* 1580 */ 81, 101, 102, 84, 112, 112, 112, 112, 81, 112, - /* 1590 */ 112, 84, 112, 112, 95, 96, 97, 98, 112, 112, - /* 1600 */ 101, 102, 95, 96, 97, 98, 112, 112, 101, 102, + /* 120 */ 95, 96, 97, 98, 99, 111, 112, 102, 103, 32, + /* 130 */ 33, 106, 53, 54, 55, 70, 29, 72, 51, 52, + /* 140 */ 31, 5, 51, 52, 12, 30, 14, 56, 57, 58, + /* 150 */ 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + /* 160 */ 69, 70, 7, 27, 53, 10, 11, 57, 58, 14, + /* 170 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + /* 180 */ 92, 93, 94, 95, 96, 97, 98, 99, 111, 112, + /* 190 */ 102, 103, 104, 105, 3, 4, 5, 6, 7, 8, + /* 200 */ 9, 10, 11, 10, 13, 28, 51, 52, 0, 14, + /* 210 */ 10, 56, 57, 58, 59, 60, 61, 62, 63, 64, + /* 220 */ 65, 66, 67, 68, 69, 70, 71, 7, 100, 101, + /* 230 */ 10, 11, 79, 65, 14, 82, 83, 84, 85, 86, + /* 240 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + /* 250 */ 97, 98, 99, 0, 0, 102, 103, 39, 65, 51, + /* 260 */ 67, 107, 108, 110, 67, 106, 71, 14, 14, 104, + /* 270 */ 81, 51, 52, 81, 10, 81, 56, 57, 58, 59, + /* 280 */ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + /* 290 */ 70, 71, 7, 78, 81, 10, 11, 82, 83, 84, + /* 300 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + /* 310 */ 95, 96, 97, 98, 99, 81, 81, 102, 103, 113, + /* 320 */ 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + /* 330 */ 44, 45, 46, 47, 113, 113, 51, 52, 53, 113, + /* 340 */ 113, 56, 57, 58, 59, 60, 61, 62, 63, 64, + /* 350 */ 65, 66, 67, 68, 69, 70, 78, 113, 73, 113, + /* 360 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + /* 370 */ 92, 93, 94, 95, 96, 97, 98, 99, 78, 113, + /* 380 */ 102, 103, 82, 83, 84, 85, 86, 87, 88, 89, + /* 390 */ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + /* 400 */ 113, 113, 102, 103, 0, 113, 113, 3, 4, 5, + /* 410 */ 6, 7, 113, 9, 10, 11, 113, 13, 113, 82, + /* 420 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, + /* 430 */ 93, 94, 95, 96, 97, 98, 99, 113, 7, 102, + /* 440 */ 103, 10, 11, 113, 113, 14, 109, 82, 83, 84, + /* 450 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + /* 460 */ 95, 96, 97, 98, 99, 113, 7, 102, 103, 10, + /* 470 */ 11, 113, 113, 113, 113, 110, 113, 113, 113, 113, + /* 480 */ 113, 113, 51, 52, 113, 113, 113, 56, 57, 58, + /* 490 */ 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + /* 500 */ 69, 70, 113, 113, 113, 113, 113, 113, 113, 113, + /* 510 */ 51, 52, 113, 113, 113, 56, 57, 58, 59, 60, + /* 520 */ 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + /* 530 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + /* 540 */ 92, 93, 94, 95, 96, 97, 98, 99, 113, 113, + /* 550 */ 102, 103, 82, 83, 84, 85, 86, 87, 88, 89, + /* 560 */ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + /* 570 */ 113, 113, 102, 103, 82, 83, 84, 85, 86, 87, + /* 580 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + /* 590 */ 98, 99, 113, 113, 102, 103, 113, 113, 82, 83, + /* 600 */ 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 610 */ 94, 95, 96, 97, 98, 99, 113, 113, 102, 103, + /* 620 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + /* 630 */ 92, 93, 94, 95, 96, 97, 98, 99, 113, 113, + /* 640 */ 102, 103, 82, 83, 84, 85, 86, 87, 88, 89, + /* 650 */ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + /* 660 */ 113, 113, 102, 103, 82, 83, 84, 85, 86, 87, + /* 670 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + /* 680 */ 98, 99, 113, 113, 102, 103, 82, 83, 84, 85, + /* 690 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 700 */ 96, 97, 98, 99, 113, 113, 102, 103, 82, 83, + /* 710 */ 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 720 */ 94, 95, 96, 97, 98, 99, 113, 113, 102, 103, + /* 730 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + /* 740 */ 92, 93, 94, 95, 96, 97, 98, 99, 113, 113, + /* 750 */ 102, 103, 82, 83, 84, 85, 86, 87, 88, 89, + /* 760 */ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + /* 770 */ 113, 113, 102, 103, 82, 83, 84, 85, 86, 87, + /* 780 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + /* 790 */ 98, 99, 113, 113, 102, 103, 82, 83, 84, 85, + /* 800 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 810 */ 96, 97, 98, 99, 113, 113, 102, 103, 82, 83, + /* 820 */ 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 830 */ 94, 95, 96, 97, 98, 99, 113, 113, 102, 103, + /* 840 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + /* 850 */ 92, 93, 94, 95, 96, 97, 98, 99, 113, 113, + /* 860 */ 102, 103, 82, 83, 84, 85, 86, 87, 88, 89, + /* 870 */ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + /* 880 */ 113, 113, 102, 103, 82, 83, 84, 85, 86, 87, + /* 890 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + /* 900 */ 98, 99, 113, 113, 102, 103, 82, 83, 84, 85, + /* 910 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 920 */ 96, 97, 98, 99, 113, 113, 102, 103, 82, 83, + /* 930 */ 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 940 */ 94, 95, 96, 97, 98, 99, 113, 82, 102, 103, + /* 950 */ 85, 113, 87, 88, 89, 90, 91, 92, 93, 94, + /* 960 */ 95, 96, 97, 98, 99, 113, 82, 102, 103, 85, + /* 970 */ 113, 113, 88, 89, 90, 91, 92, 93, 94, 95, + /* 980 */ 96, 97, 98, 99, 113, 113, 102, 103, 82, 113, + /* 990 */ 113, 85, 113, 113, 88, 89, 90, 91, 92, 93, + /* 1000 */ 94, 95, 96, 97, 98, 99, 113, 113, 102, 103, + /* 1010 */ 82, 113, 113, 85, 113, 113, 113, 89, 90, 91, + /* 1020 */ 92, 93, 94, 95, 96, 97, 98, 99, 113, 113, + /* 1030 */ 102, 103, 113, 15, 16, 17, 18, 19, 20, 21, + /* 1040 */ 22, 23, 24, 25, 26, 82, 113, 113, 85, 113, + /* 1050 */ 113, 113, 113, 90, 91, 92, 93, 94, 95, 96, + /* 1060 */ 97, 98, 99, 113, 113, 102, 103, 113, 113, 82, + /* 1070 */ 113, 113, 85, 113, 113, 57, 58, 113, 91, 92, + /* 1080 */ 93, 94, 95, 96, 97, 98, 99, 113, 82, 102, + /* 1090 */ 103, 85, 7, 113, 113, 10, 11, 113, 92, 93, + /* 1100 */ 94, 95, 96, 97, 98, 99, 113, 82, 102, 103, + /* 1110 */ 85, 113, 113, 113, 113, 113, 113, 92, 93, 94, + /* 1120 */ 95, 96, 97, 98, 99, 113, 82, 102, 103, 85, + /* 1130 */ 113, 113, 113, 113, 113, 113, 113, 93, 94, 95, + /* 1140 */ 96, 97, 98, 99, 113, 82, 102, 103, 85, 113, + /* 1150 */ 65, 66, 67, 68, 69, 70, 93, 94, 95, 96, + /* 1160 */ 97, 98, 99, 113, 82, 102, 103, 85, 113, 113, + /* 1170 */ 113, 113, 113, 113, 113, 93, 94, 95, 96, 97, + /* 1180 */ 98, 99, 113, 82, 102, 103, 85, 113, 113, 113, + /* 1190 */ 113, 113, 113, 113, 93, 94, 95, 96, 97, 98, + /* 1200 */ 99, 113, 82, 102, 103, 85, 113, 113, 113, 113, + /* 1210 */ 113, 113, 113, 93, 94, 95, 96, 97, 98, 99, + /* 1220 */ 113, 82, 102, 103, 85, 113, 113, 113, 113, 113, + /* 1230 */ 113, 113, 93, 94, 95, 96, 97, 98, 99, 113, + /* 1240 */ 82, 102, 103, 85, 113, 113, 113, 113, 113, 113, + /* 1250 */ 113, 93, 94, 95, 96, 97, 98, 99, 113, 82, + /* 1260 */ 102, 103, 85, 113, 113, 113, 113, 113, 113, 113, + /* 1270 */ 93, 94, 95, 96, 97, 98, 99, 113, 82, 102, + /* 1280 */ 103, 85, 113, 113, 113, 113, 113, 113, 113, 93, + /* 1290 */ 94, 95, 96, 97, 98, 99, 113, 82, 102, 103, + /* 1300 */ 85, 113, 113, 113, 113, 113, 113, 113, 93, 94, + /* 1310 */ 95, 96, 97, 98, 99, 113, 82, 102, 103, 85, + /* 1320 */ 113, 113, 113, 113, 113, 113, 113, 93, 94, 95, + /* 1330 */ 96, 97, 98, 99, 113, 82, 102, 103, 85, 113, + /* 1340 */ 113, 113, 113, 113, 113, 113, 93, 94, 95, 96, + /* 1350 */ 97, 98, 99, 113, 82, 102, 103, 85, 113, 113, + /* 1360 */ 113, 113, 113, 113, 113, 93, 94, 95, 96, 97, + /* 1370 */ 98, 99, 113, 82, 102, 103, 85, 113, 113, 113, + /* 1380 */ 113, 113, 113, 113, 93, 94, 95, 96, 97, 98, + /* 1390 */ 99, 113, 82, 102, 103, 85, 113, 113, 113, 113, + /* 1400 */ 113, 113, 113, 113, 94, 95, 96, 97, 98, 99, + /* 1410 */ 113, 82, 102, 103, 85, 113, 113, 113, 113, 113, + /* 1420 */ 113, 113, 113, 94, 95, 96, 97, 98, 99, 113, + /* 1430 */ 82, 102, 103, 85, 113, 113, 113, 113, 113, 113, + /* 1440 */ 113, 113, 94, 95, 96, 97, 98, 99, 113, 82, + /* 1450 */ 102, 103, 85, 113, 113, 113, 113, 113, 82, 113, + /* 1460 */ 113, 85, 95, 96, 97, 98, 99, 113, 113, 102, + /* 1470 */ 103, 95, 96, 97, 98, 99, 113, 82, 102, 103, + /* 1480 */ 85, 113, 113, 113, 113, 113, 113, 82, 113, 113, + /* 1490 */ 85, 96, 97, 98, 99, 113, 82, 102, 103, 85, + /* 1500 */ 113, 96, 97, 98, 99, 113, 82, 102, 103, 85, + /* 1510 */ 96, 97, 98, 99, 113, 82, 102, 103, 85, 113, + /* 1520 */ 96, 97, 98, 99, 113, 82, 102, 103, 85, 96, + /* 1530 */ 97, 98, 99, 113, 82, 102, 103, 85, 113, 96, + /* 1540 */ 97, 98, 99, 113, 82, 102, 103, 85, 96, 97, + /* 1550 */ 98, 99, 113, 82, 102, 103, 85, 113, 96, 97, + /* 1560 */ 98, 99, 113, 82, 102, 103, 85, 96, 97, 98, + /* 1570 */ 99, 113, 113, 102, 103, 113, 113, 96, 97, 98, + /* 1580 */ 99, 113, 82, 102, 103, 85, 113, 113, 113, 113, + /* 1590 */ 82, 113, 113, 85, 113, 113, 96, 97, 98, 99, + /* 1600 */ 113, 113, 102, 103, 96, 97, 98, 99, 113, 82, + /* 1610 */ 102, 103, 85, 113, 113, 113, 113, 82, 113, 113, + /* 1620 */ 85, 113, 113, 96, 97, 98, 99, 113, 113, 102, + /* 1630 */ 103, 96, 97, 98, 99, 113, 113, 102, 103, }; -#define YY_SHIFT_USE_DFLT (-46) -#define YY_SHIFT_COUNT (138) -#define YY_SHIFT_MIN (-45) -#define YY_SHIFT_MAX (1076) +#define YY_SHIFT_USE_DFLT (-47) +#define YY_SHIFT_COUNT (140) +#define YY_SHIFT_MIN (-46) +#define YY_SHIFT_MAX (1085) static const short yy_shift_ofst[] = { - /* 0 */ -1, 90, 281, 425, 453, 281, 453, 453, 453, 453, - /* 10 */ 217, 153, 453, 453, 453, 453, 453, 453, 453, 453, - /* 20 */ 453, 453, 453, 453, 453, 453, 453, 453, 453, 453, - /* 30 */ 453, 453, 453, 453, 453, 453, 453, 453, 453, 453, - /* 40 */ 453, 453, 453, 453, 453, 453, 453, 453, 453, 453, - /* 50 */ 453, 453, 453, 453, 453, 453, 453, 453, 453, 453, - /* 60 */ 453, 453, 453, 453, 453, 453, 453, 453, 453, 1076, - /* 70 */ 259, 35, 191, 65, 259, 189, 399, 35, 35, 35, - /* 80 */ 35, 35, 493, -46, 1010, 282, 282, 282, -45, -45, - /* 90 */ -45, -45, -45, -45, -45, -45, -45, -45, -45, 79, - /* 100 */ -45, 79, -45, 79, -45, 261, 315, 251, 101, 242, - /* 110 */ 193, 92, 92, 84, 92, 96, 192, 81, 92, 96, - /* 120 */ 101, 134, 39, 7, 197, 105, 190, 178, 18, 152, - /* 130 */ 107, 40, 22, 18, 40, 22, 18, 15, 7, + /* 0 */ -1, 91, 285, 431, 459, 285, 459, 459, 459, 459, + /* 10 */ 220, 155, 459, 459, 459, 459, 459, 459, 459, 459, + /* 20 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 30 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 40 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 50 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 60 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 70 */ 1085, 264, 35, 193, 65, 264, 191, 404, 35, 35, + /* 80 */ 35, 35, 35, 195, -47, 286, 286, 286, 1018, -46, + /* 90 */ -46, -46, -46, -46, -46, -46, -46, -46, -46, -46, + /* 100 */ -46, 79, -46, 79, -46, 79, -46, 208, 254, 253, + /* 110 */ 102, 85, 132, 87, 87, 39, 87, 97, 2, 110, + /* 120 */ 87, 97, 102, 136, 40, 34, 197, 218, 168, 111, + /* 130 */ 107, 200, 177, 109, 115, 107, 109, 115, 107, 21, + /* 140 */ 34, }; -#define YY_REDUCE_USE_DFLT (-70) -#define YY_REDUCE_COUNT (83) -#define YY_REDUCE_MIN (-69) -#define YY_REDUCE_MAX (1507) +#define YY_REDUCE_USE_DFLT (-71) +#define YY_REDUCE_COUNT (84) +#define YY_REDUCE_MIN (-70) +#define YY_REDUCE_MAX (1535) static const short yy_reduce_ofst[] = { - /* 0 */ -62, -7, 151, 87, 25, 360, 332, 296, 274, 212, - /* 10 */ 839, 817, 795, 773, 751, 729, 707, 685, 663, 641, - /* 20 */ 619, 597, 575, 553, 531, 509, 486, 464, 442, 858, - /* 30 */ 899, 877, 921, 956, 979, 1017, 998, 1264, 1245, 1226, - /* 40 */ 1207, 1188, 1169, 1150, 1131, 1112, 1093, 1074, 1055, 1036, - /* 50 */ 1321, 1302, 1283, 1349, 1340, 1507, 1499, 1480, 1472, 1453, - /* 60 */ 1444, 1434, 1425, 1415, 1406, 1396, 1387, 1377, 1368, -64, - /* 70 */ 14, -69, 80, 126, 55, 254, 254, 250, 232, 210, - /* 80 */ 186, 184, 155, 154, + /* 0 */ -63, -7, 153, 88, 25, 365, 337, 300, 278, 215, + /* 10 */ 846, 824, 802, 780, 758, 736, 714, 692, 670, 648, + /* 20 */ 626, 604, 582, 560, 538, 516, 492, 470, 448, 865, + /* 30 */ 906, 884, 928, 963, 987, 1025, 1006, 1291, 1272, 1253, + /* 40 */ 1234, 1215, 1196, 1177, 1158, 1139, 1120, 1101, 1082, 1063, + /* 50 */ 1044, 1348, 1329, 1310, 1376, 1367, 1535, 1527, 1508, 1500, + /* 60 */ 1481, 1471, 1462, 1452, 1443, 1433, 1424, 1414, 1405, 1395, + /* 70 */ -65, 14, -70, 154, 128, 77, 235, 235, 234, 213, + /* 80 */ 194, 192, 189, 165, 159, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 354, 354, 342, 354, 332, 354, 339, 354, 354, 354, - /* 10 */ 354, 354, 354, 354, 354, 354, 354, 354, 354, 354, - /* 20 */ 354, 354, 354, 354, 354, 354, 354, 354, 354, 354, - /* 30 */ 354, 354, 354, 354, 354, 354, 354, 354, 354, 354, - /* 40 */ 354, 354, 354, 354, 354, 354, 354, 354, 354, 354, - /* 50 */ 354, 354, 354, 354, 354, 354, 354, 354, 354, 354, - /* 60 */ 354, 354, 354, 354, 354, 354, 354, 354, 354, 354, - /* 70 */ 348, 354, 354, 310, 354, 354, 354, 354, 354, 354, - /* 80 */ 354, 354, 354, 332, 306, 266, 268, 267, 282, 281, - /* 90 */ 280, 279, 278, 277, 276, 275, 274, 273, 269, 287, - /* 100 */ 272, 289, 271, 288, 270, 354, 354, 354, 256, 354, - /* 110 */ 354, 283, 286, 354, 285, 264, 354, 306, 284, 265, - /* 120 */ 255, 253, 354, 316, 354, 354, 354, 351, 259, 354, - /* 130 */ 354, 262, 260, 257, 263, 261, 258, 354, 354, 349, - /* 140 */ 353, 352, 350, 343, 347, 346, 345, 344, 233, 231, - /* 150 */ 237, 252, 251, 250, 249, 248, 247, 246, 245, 244, - /* 160 */ 243, 242, 340, 341, 338, 337, 328, 326, 325, 330, - /* 170 */ 324, 335, 334, 333, 331, 329, 327, 323, 290, 322, - /* 180 */ 321, 320, 319, 318, 317, 316, 293, 315, 314, 312, - /* 190 */ 292, 336, 238, 313, 311, 309, 308, 307, 305, 304, - /* 200 */ 303, 302, 301, 300, 299, 298, 297, 296, 295, 294, - /* 210 */ 291, 254, 241, 240, 239, 236, 235, 234, 230, 227, - /* 220 */ 232, 229, 228, + /* 0 */ 357, 357, 345, 357, 335, 357, 342, 357, 357, 357, + /* 10 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, + /* 20 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, + /* 30 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, + /* 40 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, + /* 50 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, + /* 60 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, + /* 70 */ 357, 351, 357, 357, 313, 357, 357, 357, 357, 357, + /* 80 */ 357, 357, 357, 357, 335, 268, 270, 269, 309, 285, + /* 90 */ 284, 283, 282, 281, 280, 279, 278, 277, 276, 275, + /* 100 */ 271, 290, 274, 292, 273, 291, 272, 357, 357, 357, + /* 110 */ 258, 357, 357, 286, 289, 357, 288, 266, 357, 309, + /* 120 */ 287, 267, 257, 255, 357, 319, 357, 357, 357, 354, + /* 130 */ 261, 357, 357, 264, 262, 259, 265, 263, 260, 357, + /* 140 */ 357, 352, 356, 355, 353, 346, 350, 349, 348, 347, + /* 150 */ 235, 233, 239, 254, 253, 252, 251, 250, 249, 248, + /* 160 */ 247, 246, 245, 244, 343, 344, 341, 340, 331, 329, + /* 170 */ 328, 333, 327, 338, 337, 336, 334, 332, 330, 326, + /* 180 */ 293, 325, 324, 323, 322, 321, 320, 319, 296, 318, + /* 190 */ 317, 315, 295, 339, 240, 316, 314, 312, 311, 310, + /* 200 */ 308, 307, 306, 305, 304, 303, 302, 301, 300, 299, + /* 210 */ 298, 297, 294, 256, 243, 242, 241, 238, 237, 236, + /* 220 */ 232, 229, 234, 231, 230, }; /* The next table maps tokens into fallback tokens. If a construct @@ -632,23 +639,24 @@ static const char *const yyTokenName[] = { "EQUAL", "NOT_EQUAL", "LESS", "GREATER", "LESS_EQUAL", "GREATER_EQUAL", "IN", "MATCH", "NEAR", "NEAR2", "SIMILAR", "TERM_EXTRACT", - "LCP", "PREFIX", "SUFFIX", "SHIFTL", - "SHIFTR", "SHIFTRR", "PLUS", "MINUS", - "STAR", "SLASH", "MOD", "DELETE", - "INCR", "DECR", "NOT", "BITWISE_NOT", - "ADJUST", "EXACT", "PARTIAL", "UNSPLIT", - "DECIMAL", "HEX_INTEGER", "STRING", "BOOLEAN", - "NULL", "BRACKETL", "BRACKETR", "DOT", - "NONEXISTENT_COLUMN", "error", "suppress_unused_variable_warning", "input", - "query", "expression", "output_columns", "adjuster", - "query_element", "primary_expression", "assignment_expression", "conditional_expression", - "lefthand_side_expression", "logical_or_expression", "logical_and_expression", "bitwise_or_expression", - "bitwise_xor_expression", "bitwise_and_expression", "equality_expression", "relational_expression", - "shift_expression", "additive_expression", "multiplicative_expression", "unary_expression", - "postfix_expression", "call_expression", "member_expression", "arguments", - "member_expression_part", "object_literal", "array_literal", "elision", - "element_list", "property_name_and_value_list", "property_name_and_value", "property_name", - "argument_list", "output_column", "adjust_expression", "adjust_match_expression", + "LCP", "PREFIX", "SUFFIX", "REGEXP", + "SHIFTL", "SHIFTR", "SHIFTRR", "PLUS", + "MINUS", "STAR", "SLASH", "MOD", + "DELETE", "INCR", "DECR", "NOT", + "BITWISE_NOT", "ADJUST", "EXACT", "PARTIAL", + "UNSPLIT", "DECIMAL", "HEX_INTEGER", "STRING", + "BOOLEAN", "NULL", "BRACKETL", "BRACKETR", + "DOT", "NONEXISTENT_COLUMN", "error", "suppress_unused_variable_warning", + "input", "query", "expression", "output_columns", + "adjuster", "query_element", "primary_expression", "assignment_expression", + "conditional_expression", "lefthand_side_expression", "logical_or_expression", "logical_and_expression", + "bitwise_or_expression", "bitwise_xor_expression", "bitwise_and_expression", "equality_expression", + "relational_expression", "shift_expression", "additive_expression", "multiplicative_expression", + "unary_expression", "postfix_expression", "call_expression", "member_expression", + "arguments", "member_expression_part", "object_literal", "array_literal", + "elision", "element_list", "property_name_and_value_list", "property_name_and_value", + "property_name", "argument_list", "output_column", "adjust_expression", + "adjust_match_expression", }; #endif /* NDEBUG */ @@ -716,77 +724,78 @@ static const char *const yyRuleName[] = { /* 57 */ "relational_expression ::= relational_expression LCP shift_expression", /* 58 */ "relational_expression ::= relational_expression PREFIX shift_expression", /* 59 */ "relational_expression ::= relational_expression SUFFIX shift_expression", - /* 60 */ "shift_expression ::= additive_expression", - /* 61 */ "shift_expression ::= shift_expression SHIFTL additive_expression", - /* 62 */ "shift_expression ::= shift_expression SHIFTR additive_expression", - /* 63 */ "shift_expression ::= shift_expression SHIFTRR additive_expression", - /* 64 */ "additive_expression ::= multiplicative_expression", - /* 65 */ "additive_expression ::= additive_expression PLUS multiplicative_expression", - /* 66 */ "additive_expression ::= additive_expression MINUS multiplicative_expression", - /* 67 */ "multiplicative_expression ::= unary_expression", - /* 68 */ "multiplicative_expression ::= multiplicative_expression STAR unary_expression", - /* 69 */ "multiplicative_expression ::= multiplicative_expression SLASH unary_expression", - /* 70 */ "multiplicative_expression ::= multiplicative_expression MOD unary_expression", - /* 71 */ "unary_expression ::= postfix_expression", - /* 72 */ "unary_expression ::= DELETE unary_expression", - /* 73 */ "unary_expression ::= INCR unary_expression", - /* 74 */ "unary_expression ::= DECR unary_expression", - /* 75 */ "unary_expression ::= PLUS unary_expression", - /* 76 */ "unary_expression ::= MINUS unary_expression", - /* 77 */ "unary_expression ::= NOT unary_expression", - /* 78 */ "unary_expression ::= BITWISE_NOT unary_expression", - /* 79 */ "unary_expression ::= ADJUST unary_expression", - /* 80 */ "unary_expression ::= EXACT unary_expression", - /* 81 */ "unary_expression ::= PARTIAL unary_expression", - /* 82 */ "unary_expression ::= UNSPLIT unary_expression", - /* 83 */ "postfix_expression ::= lefthand_side_expression", - /* 84 */ "postfix_expression ::= lefthand_side_expression INCR", - /* 85 */ "postfix_expression ::= lefthand_side_expression DECR", - /* 86 */ "lefthand_side_expression ::= call_expression", - /* 87 */ "lefthand_side_expression ::= member_expression", - /* 88 */ "call_expression ::= member_expression arguments", - /* 89 */ "member_expression ::= primary_expression", - /* 90 */ "member_expression ::= member_expression member_expression_part", - /* 91 */ "primary_expression ::= object_literal", - /* 92 */ "primary_expression ::= PARENL expression PARENR", - /* 93 */ "primary_expression ::= IDENTIFIER", - /* 94 */ "primary_expression ::= array_literal", - /* 95 */ "primary_expression ::= DECIMAL", - /* 96 */ "primary_expression ::= HEX_INTEGER", - /* 97 */ "primary_expression ::= STRING", - /* 98 */ "primary_expression ::= BOOLEAN", - /* 99 */ "primary_expression ::= NULL", - /* 100 */ "array_literal ::= BRACKETL elision BRACKETR", - /* 101 */ "array_literal ::= BRACKETL element_list elision BRACKETR", - /* 102 */ "array_literal ::= BRACKETL element_list BRACKETR", - /* 103 */ "elision ::= COMMA", - /* 104 */ "elision ::= elision COMMA", - /* 105 */ "element_list ::= assignment_expression", - /* 106 */ "element_list ::= elision assignment_expression", - /* 107 */ "element_list ::= element_list elision assignment_expression", - /* 108 */ "object_literal ::= BRACEL property_name_and_value_list BRACER", - /* 109 */ "property_name_and_value_list ::=", - /* 110 */ "property_name_and_value_list ::= property_name_and_value_list COMMA property_name_and_value", - /* 111 */ "property_name_and_value ::= property_name COLON assignment_expression", - /* 112 */ "property_name ::= IDENTIFIER|STRING|DECIMAL", - /* 113 */ "member_expression_part ::= BRACKETL expression BRACKETR", - /* 114 */ "member_expression_part ::= DOT IDENTIFIER", - /* 115 */ "arguments ::= PARENL argument_list PARENR", - /* 116 */ "argument_list ::=", - /* 117 */ "argument_list ::= assignment_expression", - /* 118 */ "argument_list ::= argument_list COMMA assignment_expression", - /* 119 */ "output_columns ::=", - /* 120 */ "output_columns ::= output_column", - /* 121 */ "output_columns ::= output_columns COMMA output_column", - /* 122 */ "output_column ::= STAR", - /* 123 */ "output_column ::= NONEXISTENT_COLUMN", - /* 124 */ "output_column ::= assignment_expression", - /* 125 */ "adjuster ::=", - /* 126 */ "adjuster ::= adjust_expression", - /* 127 */ "adjuster ::= adjuster PLUS adjust_expression", - /* 128 */ "adjust_expression ::= adjust_match_expression", - /* 129 */ "adjust_expression ::= adjust_match_expression STAR DECIMAL", - /* 130 */ "adjust_match_expression ::= IDENTIFIER MATCH STRING", + /* 60 */ "relational_expression ::= relational_expression REGEXP shift_expression", + /* 61 */ "shift_expression ::= additive_expression", + /* 62 */ "shift_expression ::= shift_expression SHIFTL additive_expression", + /* 63 */ "shift_expression ::= shift_expression SHIFTR additive_expression", + /* 64 */ "shift_expression ::= shift_expression SHIFTRR additive_expression", + /* 65 */ "additive_expression ::= multiplicative_expression", + /* 66 */ "additive_expression ::= additive_expression PLUS multiplicative_expression", + /* 67 */ "additive_expression ::= additive_expression MINUS multiplicative_expression", + /* 68 */ "multiplicative_expression ::= unary_expression", + /* 69 */ "multiplicative_expression ::= multiplicative_expression STAR unary_expression", + /* 70 */ "multiplicative_expression ::= multiplicative_expression SLASH unary_expression", + /* 71 */ "multiplicative_expression ::= multiplicative_expression MOD unary_expression", + /* 72 */ "unary_expression ::= postfix_expression", + /* 73 */ "unary_expression ::= DELETE unary_expression", + /* 74 */ "unary_expression ::= INCR unary_expression", + /* 75 */ "unary_expression ::= DECR unary_expression", + /* 76 */ "unary_expression ::= PLUS unary_expression", + /* 77 */ "unary_expression ::= MINUS unary_expression", + /* 78 */ "unary_expression ::= NOT unary_expression", + /* 79 */ "unary_expression ::= BITWISE_NOT unary_expression", + /* 80 */ "unary_expression ::= ADJUST unary_expression", + /* 81 */ "unary_expression ::= EXACT unary_expression", + /* 82 */ "unary_expression ::= PARTIAL unary_expression", + /* 83 */ "unary_expression ::= UNSPLIT unary_expression", + /* 84 */ "postfix_expression ::= lefthand_side_expression", + /* 85 */ "postfix_expression ::= lefthand_side_expression INCR", + /* 86 */ "postfix_expression ::= lefthand_side_expression DECR", + /* 87 */ "lefthand_side_expression ::= call_expression", + /* 88 */ "lefthand_side_expression ::= member_expression", + /* 89 */ "call_expression ::= member_expression arguments", + /* 90 */ "member_expression ::= primary_expression", + /* 91 */ "member_expression ::= member_expression member_expression_part", + /* 92 */ "primary_expression ::= object_literal", + /* 93 */ "primary_expression ::= PARENL expression PARENR", + /* 94 */ "primary_expression ::= IDENTIFIER", + /* 95 */ "primary_expression ::= array_literal", + /* 96 */ "primary_expression ::= DECIMAL", + /* 97 */ "primary_expression ::= HEX_INTEGER", + /* 98 */ "primary_expression ::= STRING", + /* 99 */ "primary_expression ::= BOOLEAN", + /* 100 */ "primary_expression ::= NULL", + /* 101 */ "array_literal ::= BRACKETL elision BRACKETR", + /* 102 */ "array_literal ::= BRACKETL element_list elision BRACKETR", + /* 103 */ "array_literal ::= BRACKETL element_list BRACKETR", + /* 104 */ "elision ::= COMMA", + /* 105 */ "elision ::= elision COMMA", + /* 106 */ "element_list ::= assignment_expression", + /* 107 */ "element_list ::= elision assignment_expression", + /* 108 */ "element_list ::= element_list elision assignment_expression", + /* 109 */ "object_literal ::= BRACEL property_name_and_value_list BRACER", + /* 110 */ "property_name_and_value_list ::=", + /* 111 */ "property_name_and_value_list ::= property_name_and_value_list COMMA property_name_and_value", + /* 112 */ "property_name_and_value ::= property_name COLON assignment_expression", + /* 113 */ "property_name ::= IDENTIFIER|STRING|DECIMAL", + /* 114 */ "member_expression_part ::= BRACKETL expression BRACKETR", + /* 115 */ "member_expression_part ::= DOT IDENTIFIER", + /* 116 */ "arguments ::= PARENL argument_list PARENR", + /* 117 */ "argument_list ::=", + /* 118 */ "argument_list ::= assignment_expression", + /* 119 */ "argument_list ::= argument_list COMMA assignment_expression", + /* 120 */ "output_columns ::=", + /* 121 */ "output_columns ::= output_column", + /* 122 */ "output_columns ::= output_columns COMMA output_column", + /* 123 */ "output_column ::= STAR", + /* 124 */ "output_column ::= NONEXISTENT_COLUMN", + /* 125 */ "output_column ::= assignment_expression", + /* 126 */ "adjuster ::=", + /* 127 */ "adjuster ::= adjust_expression", + /* 128 */ "adjuster ::= adjuster PLUS adjust_expression", + /* 129 */ "adjust_expression ::= adjust_match_expression", + /* 130 */ "adjust_expression ::= adjust_match_expression STAR DECIMAL", + /* 131 */ "adjust_match_expression ::= IDENTIFIER MATCH STRING", }; #endif /* NDEBUG */ @@ -865,13 +874,13 @@ static void yy_destructor( ** which appear on the RHS of the rule, but which are not used ** inside the C code. */ - case 74: /* suppress_unused_variable_warning */ + case 75: /* suppress_unused_variable_warning */ { #line 11 "grn_ecmascript.lemon" (void)efsi; -#line 875 "grn_ecmascript.c" +#line 884 "grn_ecmascript.c" } break; default: break; /* If no destructor action specified: do nothing */ @@ -1109,137 +1118,138 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ unsigned char nrhs; /* Number of right-hand side symbols in the rule */ } yyRuleInfo[] = { - { 75, 1 }, - { 75, 1 }, - { 75, 2 }, - { 75, 2 }, + { 76, 1 }, { 76, 1 }, { 76, 2 }, - { 76, 3 }, - { 76, 3 }, - { 76, 3 }, - { 80, 1 }, - { 80, 3 }, - { 80, 2 }, - { 80, 3 }, - { 80, 3 }, - { 80, 2 }, + { 76, 2 }, { 77, 1 }, + { 77, 2 }, { 77, 3 }, - { 82, 1 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, - { 82, 3 }, + { 77, 3 }, + { 77, 3 }, + { 81, 1 }, + { 81, 3 }, + { 81, 2 }, + { 81, 3 }, + { 81, 3 }, + { 81, 2 }, + { 78, 1 }, + { 78, 3 }, { 83, 1 }, - { 83, 5 }, - { 85, 1 }, - { 85, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 83, 3 }, + { 84, 1 }, + { 84, 5 }, { 86, 1 }, { 86, 3 }, - { 86, 3 }, { 87, 1 }, { 87, 3 }, + { 87, 3 }, { 88, 1 }, { 88, 3 }, { 89, 1 }, { 89, 3 }, { 90, 1 }, { 90, 3 }, - { 90, 3 }, { 91, 1 }, { 91, 3 }, { 91, 3 }, - { 91, 3 }, - { 91, 3 }, - { 91, 3 }, - { 91, 3 }, - { 91, 3 }, - { 91, 3 }, - { 91, 3 }, - { 91, 3 }, - { 91, 3 }, - { 91, 3 }, - { 91, 3 }, { 92, 1 }, { 92, 3 }, { 92, 3 }, { 92, 3 }, + { 92, 3 }, + { 92, 3 }, + { 92, 3 }, + { 92, 3 }, + { 92, 3 }, + { 92, 3 }, + { 92, 3 }, + { 92, 3 }, + { 92, 3 }, + { 92, 3 }, + { 92, 3 }, { 93, 1 }, { 93, 3 }, { 93, 3 }, + { 93, 3 }, { 94, 1 }, { 94, 3 }, { 94, 3 }, - { 94, 3 }, { 95, 1 }, - { 95, 2 }, - { 95, 2 }, - { 95, 2 }, - { 95, 2 }, - { 95, 2 }, - { 95, 2 }, - { 95, 2 }, - { 95, 2 }, - { 95, 2 }, - { 95, 2 }, - { 95, 2 }, + { 95, 3 }, + { 95, 3 }, + { 95, 3 }, { 96, 1 }, { 96, 2 }, { 96, 2 }, - { 84, 1 }, - { 84, 1 }, + { 96, 2 }, + { 96, 2 }, + { 96, 2 }, + { 96, 2 }, + { 96, 2 }, + { 96, 2 }, + { 96, 2 }, + { 96, 2 }, + { 96, 2 }, + { 97, 1 }, + { 97, 2 }, { 97, 2 }, - { 98, 1 }, + { 85, 1 }, + { 85, 1 }, { 98, 2 }, - { 81, 1 }, - { 81, 3 }, - { 81, 1 }, - { 81, 1 }, - { 81, 1 }, - { 81, 1 }, - { 81, 1 }, - { 81, 1 }, - { 81, 1 }, - { 102, 3 }, - { 102, 4 }, - { 102, 3 }, - { 103, 1 }, - { 103, 2 }, + { 99, 1 }, + { 99, 2 }, + { 82, 1 }, + { 82, 3 }, + { 82, 1 }, + { 82, 1 }, + { 82, 1 }, + { 82, 1 }, + { 82, 1 }, + { 82, 1 }, + { 82, 1 }, + { 103, 3 }, + { 103, 4 }, + { 103, 3 }, { 104, 1 }, { 104, 2 }, - { 104, 3 }, - { 101, 3 }, - { 105, 0 }, + { 105, 1 }, + { 105, 2 }, { 105, 3 }, + { 102, 3 }, + { 106, 0 }, { 106, 3 }, - { 107, 1 }, - { 100, 3 }, - { 100, 2 }, - { 99, 3 }, - { 108, 0 }, + { 107, 3 }, { 108, 1 }, - { 108, 3 }, - { 78, 0 }, - { 78, 1 }, - { 78, 3 }, - { 109, 1 }, - { 109, 1 }, + { 101, 3 }, + { 101, 2 }, + { 100, 3 }, + { 109, 0 }, { 109, 1 }, + { 109, 3 }, { 79, 0 }, { 79, 1 }, { 79, 3 }, { 110, 1 }, - { 110, 3 }, + { 110, 1 }, + { 110, 1 }, + { 80, 0 }, + { 80, 1 }, + { 80, 3 }, + { 111, 1 }, { 111, 3 }, + { 112, 3 }, }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1299,7 +1309,7 @@ static void yy_reduce( { grn_expr_append_op(efsi->ctx, efsi->e, grn_int32_value_at(&efsi->op_stack, -1), 2); } -#line 1303 "grn_ecmascript.c" +#line 1313 "grn_ecmascript.c" break; case 6: /* query ::= query LOGICAL_AND query_element */ case 35: /* logical_and_expression ::= logical_and_expression LOGICAL_AND bitwise_or_expression */ yytestcase(yyruleno==35); @@ -1307,7 +1317,7 @@ static void yy_reduce( { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_AND, 2); } -#line 1311 "grn_ecmascript.c" +#line 1321 "grn_ecmascript.c" break; case 7: /* query ::= query LOGICAL_AND_NOT query_element */ case 36: /* logical_and_expression ::= logical_and_expression LOGICAL_AND_NOT bitwise_or_expression */ yytestcase(yyruleno==36); @@ -1315,7 +1325,7 @@ static void yy_reduce( { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_AND_NOT, 2); } -#line 1319 "grn_ecmascript.c" +#line 1329 "grn_ecmascript.c" break; case 8: /* query ::= query LOGICAL_OR query_element */ case 33: /* logical_or_expression ::= logical_or_expression LOGICAL_OR logical_and_expression */ yytestcase(yyruleno==33); @@ -1323,7 +1333,7 @@ static void yy_reduce( { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_OR, 2); } -#line 1327 "grn_ecmascript.c" +#line 1337 "grn_ecmascript.c" break; case 11: /* query_element ::= RELATIVE_OP query_element */ #line 62 "grn_ecmascript.lemon" @@ -1331,7 +1341,7 @@ static void yy_reduce( int mode; GRN_INT32_POP(&efsi->mode_stack, mode); } -#line 1335 "grn_ecmascript.c" +#line 1345 "grn_ecmascript.c" break; case 12: /* query_element ::= IDENTIFIER RELATIVE_OP query_element */ #line 66 "grn_ecmascript.lemon" @@ -1358,7 +1368,7 @@ static void yy_reduce( break; } } -#line 1362 "grn_ecmascript.c" +#line 1372 "grn_ecmascript.c" break; case 13: /* query_element ::= BRACEL expression BRACER */ case 14: /* query_element ::= EVAL primary_expression */ yytestcase(yyruleno==14); @@ -1366,98 +1376,98 @@ static void yy_reduce( { efsi->flags = efsi->default_flags; } -#line 1370 "grn_ecmascript.c" +#line 1380 "grn_ecmascript.c" break; case 16: /* expression ::= expression COMMA assignment_expression */ #line 97 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_COMMA, 2); } -#line 1377 "grn_ecmascript.c" +#line 1387 "grn_ecmascript.c" break; case 18: /* assignment_expression ::= lefthand_side_expression ASSIGN assignment_expression */ #line 102 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_ASSIGN, 2); } -#line 1384 "grn_ecmascript.c" +#line 1394 "grn_ecmascript.c" break; case 19: /* assignment_expression ::= lefthand_side_expression STAR_ASSIGN assignment_expression */ #line 105 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_STAR_ASSIGN, 2); } -#line 1391 "grn_ecmascript.c" +#line 1401 "grn_ecmascript.c" break; case 20: /* assignment_expression ::= lefthand_side_expression SLASH_ASSIGN assignment_expression */ #line 108 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SLASH_ASSIGN, 2); } -#line 1398 "grn_ecmascript.c" +#line 1408 "grn_ecmascript.c" break; case 21: /* assignment_expression ::= lefthand_side_expression MOD_ASSIGN assignment_expression */ #line 111 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_MOD_ASSIGN, 2); } -#line 1405 "grn_ecmascript.c" +#line 1415 "grn_ecmascript.c" break; case 22: /* assignment_expression ::= lefthand_side_expression PLUS_ASSIGN assignment_expression */ #line 114 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_PLUS_ASSIGN, 2); } -#line 1412 "grn_ecmascript.c" +#line 1422 "grn_ecmascript.c" break; case 23: /* assignment_expression ::= lefthand_side_expression MINUS_ASSIGN assignment_expression */ #line 117 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_MINUS_ASSIGN, 2); } -#line 1419 "grn_ecmascript.c" +#line 1429 "grn_ecmascript.c" break; case 24: /* assignment_expression ::= lefthand_side_expression SHIFTL_ASSIGN assignment_expression */ #line 120 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SHIFTL_ASSIGN, 2); } -#line 1426 "grn_ecmascript.c" +#line 1436 "grn_ecmascript.c" break; case 25: /* assignment_expression ::= lefthand_side_expression SHIFTR_ASSIGN assignment_expression */ #line 123 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SHIFTR_ASSIGN, 2); } -#line 1433 "grn_ecmascript.c" +#line 1443 "grn_ecmascript.c" break; case 26: /* assignment_expression ::= lefthand_side_expression SHIFTRR_ASSIGN assignment_expression */ #line 126 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SHIFTRR_ASSIGN, 2); } -#line 1440 "grn_ecmascript.c" +#line 1450 "grn_ecmascript.c" break; case 27: /* assignment_expression ::= lefthand_side_expression AND_ASSIGN assignment_expression */ #line 129 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_AND_ASSIGN, 2); } -#line 1447 "grn_ecmascript.c" +#line 1457 "grn_ecmascript.c" break; case 28: /* assignment_expression ::= lefthand_side_expression XOR_ASSIGN assignment_expression */ #line 132 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_XOR_ASSIGN, 2); } -#line 1454 "grn_ecmascript.c" +#line 1464 "grn_ecmascript.c" break; case 29: /* assignment_expression ::= lefthand_side_expression OR_ASSIGN assignment_expression */ #line 135 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_OR_ASSIGN, 2); } -#line 1461 "grn_ecmascript.c" +#line 1471 "grn_ecmascript.c" break; case 31: /* conditional_expression ::= logical_or_expression QUESTION assignment_expression COLON assignment_expression */ #line 140 "grn_ecmascript.lemon" @@ -1466,202 +1476,209 @@ static void yy_reduce( e->codes[yymsp[-3].minor.yy0].nargs = yymsp[-1].minor.yy0 - yymsp[-3].minor.yy0; e->codes[yymsp[-1].minor.yy0].nargs = e->codes_curr - yymsp[-1].minor.yy0 - 1; } -#line 1470 "grn_ecmascript.c" +#line 1480 "grn_ecmascript.c" break; case 38: /* bitwise_or_expression ::= bitwise_or_expression BITWISE_OR bitwise_xor_expression */ #line 160 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_BITWISE_OR, 2); } -#line 1477 "grn_ecmascript.c" +#line 1487 "grn_ecmascript.c" break; case 40: /* bitwise_xor_expression ::= bitwise_xor_expression BITWISE_XOR bitwise_and_expression */ #line 165 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_BITWISE_XOR, 2); } -#line 1484 "grn_ecmascript.c" +#line 1494 "grn_ecmascript.c" break; case 42: /* bitwise_and_expression ::= bitwise_and_expression BITWISE_AND equality_expression */ #line 170 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_BITWISE_AND, 2); } -#line 1491 "grn_ecmascript.c" +#line 1501 "grn_ecmascript.c" break; case 44: /* equality_expression ::= equality_expression EQUAL relational_expression */ #line 175 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_EQUAL, 2); } -#line 1498 "grn_ecmascript.c" +#line 1508 "grn_ecmascript.c" break; case 45: /* equality_expression ::= equality_expression NOT_EQUAL relational_expression */ #line 178 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_NOT_EQUAL, 2); } -#line 1505 "grn_ecmascript.c" +#line 1515 "grn_ecmascript.c" break; case 47: /* relational_expression ::= relational_expression LESS shift_expression */ #line 183 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_LESS, 2); } -#line 1512 "grn_ecmascript.c" +#line 1522 "grn_ecmascript.c" break; case 48: /* relational_expression ::= relational_expression GREATER shift_expression */ #line 186 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_GREATER, 2); } -#line 1519 "grn_ecmascript.c" +#line 1529 "grn_ecmascript.c" break; case 49: /* relational_expression ::= relational_expression LESS_EQUAL shift_expression */ #line 189 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_LESS_EQUAL, 2); } -#line 1526 "grn_ecmascript.c" +#line 1536 "grn_ecmascript.c" break; case 50: /* relational_expression ::= relational_expression GREATER_EQUAL shift_expression */ #line 192 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_GREATER_EQUAL, 2); } -#line 1533 "grn_ecmascript.c" +#line 1543 "grn_ecmascript.c" break; case 51: /* relational_expression ::= relational_expression IN shift_expression */ #line 195 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_IN, 2); } -#line 1540 "grn_ecmascript.c" +#line 1550 "grn_ecmascript.c" break; case 52: /* relational_expression ::= relational_expression MATCH shift_expression */ - case 130: /* adjust_match_expression ::= IDENTIFIER MATCH STRING */ yytestcase(yyruleno==130); + case 131: /* adjust_match_expression ::= IDENTIFIER MATCH STRING */ yytestcase(yyruleno==131); #line 198 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_MATCH, 2); } -#line 1548 "grn_ecmascript.c" +#line 1558 "grn_ecmascript.c" break; case 53: /* relational_expression ::= relational_expression NEAR shift_expression */ #line 201 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_NEAR, 2); } -#line 1555 "grn_ecmascript.c" +#line 1565 "grn_ecmascript.c" break; case 54: /* relational_expression ::= relational_expression NEAR2 shift_expression */ #line 204 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_NEAR2, 2); } -#line 1562 "grn_ecmascript.c" +#line 1572 "grn_ecmascript.c" break; case 55: /* relational_expression ::= relational_expression SIMILAR shift_expression */ #line 207 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SIMILAR, 2); } -#line 1569 "grn_ecmascript.c" +#line 1579 "grn_ecmascript.c" break; case 56: /* relational_expression ::= relational_expression TERM_EXTRACT shift_expression */ #line 210 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_TERM_EXTRACT, 2); } -#line 1576 "grn_ecmascript.c" +#line 1586 "grn_ecmascript.c" break; case 57: /* relational_expression ::= relational_expression LCP shift_expression */ #line 213 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_LCP, 2); } -#line 1583 "grn_ecmascript.c" +#line 1593 "grn_ecmascript.c" break; case 58: /* relational_expression ::= relational_expression PREFIX shift_expression */ #line 216 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_PREFIX, 2); } -#line 1590 "grn_ecmascript.c" +#line 1600 "grn_ecmascript.c" break; case 59: /* relational_expression ::= relational_expression SUFFIX shift_expression */ #line 219 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SUFFIX, 2); } -#line 1597 "grn_ecmascript.c" +#line 1607 "grn_ecmascript.c" break; - case 61: /* shift_expression ::= shift_expression SHIFTL additive_expression */ -#line 224 "grn_ecmascript.lemon" + case 60: /* relational_expression ::= relational_expression REGEXP shift_expression */ +#line 222 "grn_ecmascript.lemon" { - grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SHIFTL, 2); + grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_REGEXP, 2); } -#line 1604 "grn_ecmascript.c" +#line 1614 "grn_ecmascript.c" break; - case 62: /* shift_expression ::= shift_expression SHIFTR additive_expression */ + case 62: /* shift_expression ::= shift_expression SHIFTL additive_expression */ #line 227 "grn_ecmascript.lemon" { - grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SHIFTR, 2); + grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SHIFTL, 2); } -#line 1611 "grn_ecmascript.c" +#line 1621 "grn_ecmascript.c" break; - case 63: /* shift_expression ::= shift_expression SHIFTRR additive_expression */ + case 63: /* shift_expression ::= shift_expression SHIFTR additive_expression */ #line 230 "grn_ecmascript.lemon" { + grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SHIFTR, 2); +} +#line 1628 "grn_ecmascript.c" + break; + case 64: /* shift_expression ::= shift_expression SHIFTRR additive_expression */ +#line 233 "grn_ecmascript.lemon" +{ grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SHIFTRR, 2); } -#line 1618 "grn_ecmascript.c" +#line 1635 "grn_ecmascript.c" break; - case 65: /* additive_expression ::= additive_expression PLUS multiplicative_expression */ - case 127: /* adjuster ::= adjuster PLUS adjust_expression */ yytestcase(yyruleno==127); -#line 235 "grn_ecmascript.lemon" + case 66: /* additive_expression ::= additive_expression PLUS multiplicative_expression */ + case 128: /* adjuster ::= adjuster PLUS adjust_expression */ yytestcase(yyruleno==128); +#line 238 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_PLUS, 2); } -#line 1626 "grn_ecmascript.c" +#line 1643 "grn_ecmascript.c" break; - case 66: /* additive_expression ::= additive_expression MINUS multiplicative_expression */ -#line 238 "grn_ecmascript.lemon" + case 67: /* additive_expression ::= additive_expression MINUS multiplicative_expression */ +#line 241 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_MINUS, 2); } -#line 1633 "grn_ecmascript.c" +#line 1650 "grn_ecmascript.c" break; - case 68: /* multiplicative_expression ::= multiplicative_expression STAR unary_expression */ - case 129: /* adjust_expression ::= adjust_match_expression STAR DECIMAL */ yytestcase(yyruleno==129); -#line 243 "grn_ecmascript.lemon" + case 69: /* multiplicative_expression ::= multiplicative_expression STAR unary_expression */ + case 130: /* adjust_expression ::= adjust_match_expression STAR DECIMAL */ yytestcase(yyruleno==130); +#line 246 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_STAR, 2); } -#line 1641 "grn_ecmascript.c" +#line 1658 "grn_ecmascript.c" break; - case 69: /* multiplicative_expression ::= multiplicative_expression SLASH unary_expression */ -#line 246 "grn_ecmascript.lemon" + case 70: /* multiplicative_expression ::= multiplicative_expression SLASH unary_expression */ +#line 249 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SLASH, 2); } -#line 1648 "grn_ecmascript.c" +#line 1665 "grn_ecmascript.c" break; - case 70: /* multiplicative_expression ::= multiplicative_expression MOD unary_expression */ -#line 249 "grn_ecmascript.lemon" + case 71: /* multiplicative_expression ::= multiplicative_expression MOD unary_expression */ +#line 252 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_MOD, 2); } -#line 1655 "grn_ecmascript.c" +#line 1672 "grn_ecmascript.c" break; - case 72: /* unary_expression ::= DELETE unary_expression */ -#line 254 "grn_ecmascript.lemon" + case 73: /* unary_expression ::= DELETE unary_expression */ +#line 257 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_DELETE, 1); } -#line 1662 "grn_ecmascript.c" +#line 1679 "grn_ecmascript.c" break; - case 73: /* unary_expression ::= INCR unary_expression */ -#line 257 "grn_ecmascript.lemon" + case 74: /* unary_expression ::= INCR unary_expression */ +#line 260 "grn_ecmascript.lemon" { grn_ctx *ctx = efsi->ctx; grn_expr *e = (grn_expr *)(efsi->e); @@ -1679,10 +1696,10 @@ static void yy_reduce( grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_INCR, 1); } } -#line 1683 "grn_ecmascript.c" +#line 1700 "grn_ecmascript.c" break; - case 74: /* unary_expression ::= DECR unary_expression */ -#line 274 "grn_ecmascript.lemon" + case 75: /* unary_expression ::= DECR unary_expression */ +#line 277 "grn_ecmascript.lemon" { grn_ctx *ctx = efsi->ctx; grn_expr *e = (grn_expr *)(efsi->e); @@ -1700,66 +1717,66 @@ static void yy_reduce( grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_DECR, 1); } } -#line 1704 "grn_ecmascript.c" +#line 1721 "grn_ecmascript.c" break; - case 75: /* unary_expression ::= PLUS unary_expression */ -#line 291 "grn_ecmascript.lemon" + case 76: /* unary_expression ::= PLUS unary_expression */ +#line 294 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_PLUS, 1); } -#line 1711 "grn_ecmascript.c" +#line 1728 "grn_ecmascript.c" break; - case 76: /* unary_expression ::= MINUS unary_expression */ -#line 294 "grn_ecmascript.lemon" + case 77: /* unary_expression ::= MINUS unary_expression */ +#line 297 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_MINUS, 1); } -#line 1718 "grn_ecmascript.c" +#line 1735 "grn_ecmascript.c" break; - case 77: /* unary_expression ::= NOT unary_expression */ -#line 297 "grn_ecmascript.lemon" + case 78: /* unary_expression ::= NOT unary_expression */ +#line 300 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_NOT, 1); } -#line 1725 "grn_ecmascript.c" +#line 1742 "grn_ecmascript.c" break; - case 78: /* unary_expression ::= BITWISE_NOT unary_expression */ -#line 300 "grn_ecmascript.lemon" + case 79: /* unary_expression ::= BITWISE_NOT unary_expression */ +#line 303 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_BITWISE_NOT, 1); } -#line 1732 "grn_ecmascript.c" +#line 1749 "grn_ecmascript.c" break; - case 79: /* unary_expression ::= ADJUST unary_expression */ -#line 303 "grn_ecmascript.lemon" + case 80: /* unary_expression ::= ADJUST unary_expression */ +#line 306 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_ADJUST, 1); } -#line 1739 "grn_ecmascript.c" +#line 1756 "grn_ecmascript.c" break; - case 80: /* unary_expression ::= EXACT unary_expression */ -#line 306 "grn_ecmascript.lemon" + case 81: /* unary_expression ::= EXACT unary_expression */ +#line 309 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_EXACT, 1); } -#line 1746 "grn_ecmascript.c" +#line 1763 "grn_ecmascript.c" break; - case 81: /* unary_expression ::= PARTIAL unary_expression */ -#line 309 "grn_ecmascript.lemon" + case 82: /* unary_expression ::= PARTIAL unary_expression */ +#line 312 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_PARTIAL, 1); } -#line 1753 "grn_ecmascript.c" +#line 1770 "grn_ecmascript.c" break; - case 82: /* unary_expression ::= UNSPLIT unary_expression */ -#line 312 "grn_ecmascript.lemon" + case 83: /* unary_expression ::= UNSPLIT unary_expression */ +#line 315 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_UNSPLIT, 1); } -#line 1760 "grn_ecmascript.c" +#line 1777 "grn_ecmascript.c" break; - case 84: /* postfix_expression ::= lefthand_side_expression INCR */ -#line 317 "grn_ecmascript.lemon" + case 85: /* postfix_expression ::= lefthand_side_expression INCR */ +#line 320 "grn_ecmascript.lemon" { grn_ctx *ctx = efsi->ctx; grn_expr *e = (grn_expr *)(efsi->e); @@ -1777,10 +1794,10 @@ static void yy_reduce( grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_INCR_POST, 1); } } -#line 1781 "grn_ecmascript.c" +#line 1798 "grn_ecmascript.c" break; - case 85: /* postfix_expression ::= lefthand_side_expression DECR */ -#line 334 "grn_ecmascript.lemon" + case 86: /* postfix_expression ::= lefthand_side_expression DECR */ +#line 337 "grn_ecmascript.lemon" { grn_ctx *ctx = efsi->ctx; grn_expr *e = (grn_expr *)(efsi->e); @@ -1798,51 +1815,51 @@ static void yy_reduce( grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_DECR_POST, 1); } } -#line 1802 "grn_ecmascript.c" +#line 1819 "grn_ecmascript.c" break; - case 88: /* call_expression ::= member_expression arguments */ -#line 355 "grn_ecmascript.lemon" + case 89: /* call_expression ::= member_expression arguments */ +#line 358 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_CALL, yymsp[0].minor.yy0); } -#line 1809 "grn_ecmascript.c" +#line 1826 "grn_ecmascript.c" break; - case 113: /* member_expression_part ::= BRACKETL expression BRACKETR */ -#line 391 "grn_ecmascript.lemon" + case 114: /* member_expression_part ::= BRACKETL expression BRACKETR */ +#line 394 "grn_ecmascript.lemon" { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_GET_MEMBER, 2); } -#line 1816 "grn_ecmascript.c" +#line 1833 "grn_ecmascript.c" break; - case 115: /* arguments ::= PARENL argument_list PARENR */ -#line 396 "grn_ecmascript.lemon" + case 116: /* arguments ::= PARENL argument_list PARENR */ +#line 399 "grn_ecmascript.lemon" { yygotominor.yy0 = yymsp[-1].minor.yy0; } -#line 1821 "grn_ecmascript.c" +#line 1838 "grn_ecmascript.c" break; - case 116: /* argument_list ::= */ -#line 397 "grn_ecmascript.lemon" + case 117: /* argument_list ::= */ +#line 400 "grn_ecmascript.lemon" { yygotominor.yy0 = 0; } -#line 1826 "grn_ecmascript.c" +#line 1843 "grn_ecmascript.c" break; - case 117: /* argument_list ::= assignment_expression */ -#line 398 "grn_ecmascript.lemon" + case 118: /* argument_list ::= assignment_expression */ +#line 401 "grn_ecmascript.lemon" { yygotominor.yy0 = 1; } -#line 1831 "grn_ecmascript.c" +#line 1848 "grn_ecmascript.c" break; - case 118: /* argument_list ::= argument_list COMMA assignment_expression */ -#line 399 "grn_ecmascript.lemon" + case 119: /* argument_list ::= argument_list COMMA assignment_expression */ +#line 402 "grn_ecmascript.lemon" { yygotominor.yy0 = yymsp[-2].minor.yy0 + 1; } -#line 1836 "grn_ecmascript.c" +#line 1853 "grn_ecmascript.c" break; - case 119: /* output_columns ::= */ -#line 401 "grn_ecmascript.lemon" + case 120: /* output_columns ::= */ +#line 404 "grn_ecmascript.lemon" { yygotominor.yy0 = 0; } -#line 1843 "grn_ecmascript.c" +#line 1860 "grn_ecmascript.c" break; - case 120: /* output_columns ::= output_column */ -#line 404 "grn_ecmascript.lemon" + case 121: /* output_columns ::= output_column */ +#line 407 "grn_ecmascript.lemon" { if (yymsp[0].minor.yy0) { yygotominor.yy0 = 0; @@ -1850,10 +1867,10 @@ static void yy_reduce( yygotominor.yy0 = 1; } } -#line 1854 "grn_ecmascript.c" +#line 1871 "grn_ecmascript.c" break; - case 121: /* output_columns ::= output_columns COMMA output_column */ -#line 412 "grn_ecmascript.lemon" + case 122: /* output_columns ::= output_columns COMMA output_column */ +#line 415 "grn_ecmascript.lemon" { if (yymsp[0].minor.yy0) { yygotominor.yy0 = yymsp[-2].minor.yy0; @@ -1864,10 +1881,10 @@ static void yy_reduce( yygotominor.yy0 = 1; } } -#line 1868 "grn_ecmascript.c" +#line 1885 "grn_ecmascript.c" break; - case 122: /* output_column ::= STAR */ -#line 423 "grn_ecmascript.lemon" + case 123: /* output_column ::= STAR */ +#line 426 "grn_ecmascript.lemon" { grn_ctx *ctx = efsi->ctx; grn_obj *expr = efsi->e; @@ -1905,21 +1922,21 @@ static void yy_reduce( yygotominor.yy0 = GRN_TRUE; } } -#line 1909 "grn_ecmascript.c" +#line 1926 "grn_ecmascript.c" break; - case 123: /* output_column ::= NONEXISTENT_COLUMN */ -#line 460 "grn_ecmascript.lemon" + case 124: /* output_column ::= NONEXISTENT_COLUMN */ +#line 463 "grn_ecmascript.lemon" { yygotominor.yy0 = GRN_TRUE; } -#line 1916 "grn_ecmascript.c" +#line 1933 "grn_ecmascript.c" break; - case 124: /* output_column ::= assignment_expression */ -#line 463 "grn_ecmascript.lemon" + case 125: /* output_column ::= assignment_expression */ +#line 466 "grn_ecmascript.lemon" { yygotominor.yy0 = GRN_FALSE; } -#line 1923 "grn_ecmascript.c" +#line 1940 "grn_ecmascript.c" break; default: /* (0) input ::= query */ yytestcase(yyruleno==0); @@ -1939,41 +1956,41 @@ static void yy_reduce( /* (41) bitwise_and_expression ::= equality_expression */ yytestcase(yyruleno==41); /* (43) equality_expression ::= relational_expression */ yytestcase(yyruleno==43); /* (46) relational_expression ::= shift_expression */ yytestcase(yyruleno==46); - /* (60) shift_expression ::= additive_expression */ yytestcase(yyruleno==60); - /* (64) additive_expression ::= multiplicative_expression */ yytestcase(yyruleno==64); - /* (67) multiplicative_expression ::= unary_expression */ yytestcase(yyruleno==67); - /* (71) unary_expression ::= postfix_expression */ yytestcase(yyruleno==71); - /* (83) postfix_expression ::= lefthand_side_expression */ yytestcase(yyruleno==83); - /* (86) lefthand_side_expression ::= call_expression */ yytestcase(yyruleno==86); - /* (87) lefthand_side_expression ::= member_expression */ yytestcase(yyruleno==87); - /* (89) member_expression ::= primary_expression */ yytestcase(yyruleno==89); - /* (90) member_expression ::= member_expression member_expression_part */ yytestcase(yyruleno==90); - /* (91) primary_expression ::= object_literal */ yytestcase(yyruleno==91); - /* (92) primary_expression ::= PARENL expression PARENR */ yytestcase(yyruleno==92); - /* (93) primary_expression ::= IDENTIFIER */ yytestcase(yyruleno==93); - /* (94) primary_expression ::= array_literal */ yytestcase(yyruleno==94); - /* (95) primary_expression ::= DECIMAL */ yytestcase(yyruleno==95); - /* (96) primary_expression ::= HEX_INTEGER */ yytestcase(yyruleno==96); - /* (97) primary_expression ::= STRING */ yytestcase(yyruleno==97); - /* (98) primary_expression ::= BOOLEAN */ yytestcase(yyruleno==98); - /* (99) primary_expression ::= NULL */ yytestcase(yyruleno==99); - /* (100) array_literal ::= BRACKETL elision BRACKETR */ yytestcase(yyruleno==100); - /* (101) array_literal ::= BRACKETL element_list elision BRACKETR */ yytestcase(yyruleno==101); - /* (102) array_literal ::= BRACKETL element_list BRACKETR */ yytestcase(yyruleno==102); - /* (103) elision ::= COMMA */ yytestcase(yyruleno==103); - /* (104) elision ::= elision COMMA */ yytestcase(yyruleno==104); - /* (105) element_list ::= assignment_expression */ yytestcase(yyruleno==105); - /* (106) element_list ::= elision assignment_expression */ yytestcase(yyruleno==106); - /* (107) element_list ::= element_list elision assignment_expression */ yytestcase(yyruleno==107); - /* (108) object_literal ::= BRACEL property_name_and_value_list BRACER */ yytestcase(yyruleno==108); - /* (109) property_name_and_value_list ::= */ yytestcase(yyruleno==109); - /* (110) property_name_and_value_list ::= property_name_and_value_list COMMA property_name_and_value */ yytestcase(yyruleno==110); - /* (111) property_name_and_value ::= property_name COLON assignment_expression */ yytestcase(yyruleno==111); - /* (112) property_name ::= IDENTIFIER|STRING|DECIMAL */ yytestcase(yyruleno==112); - /* (114) member_expression_part ::= DOT IDENTIFIER */ yytestcase(yyruleno==114); - /* (125) adjuster ::= */ yytestcase(yyruleno==125); - /* (126) adjuster ::= adjust_expression */ yytestcase(yyruleno==126); - /* (128) adjust_expression ::= adjust_match_expression */ yytestcase(yyruleno==128); + /* (61) shift_expression ::= additive_expression */ yytestcase(yyruleno==61); + /* (65) additive_expression ::= multiplicative_expression */ yytestcase(yyruleno==65); + /* (68) multiplicative_expression ::= unary_expression */ yytestcase(yyruleno==68); + /* (72) unary_expression ::= postfix_expression */ yytestcase(yyruleno==72); + /* (84) postfix_expression ::= lefthand_side_expression */ yytestcase(yyruleno==84); + /* (87) lefthand_side_expression ::= call_expression */ yytestcase(yyruleno==87); + /* (88) lefthand_side_expression ::= member_expression */ yytestcase(yyruleno==88); + /* (90) member_expression ::= primary_expression */ yytestcase(yyruleno==90); + /* (91) member_expression ::= member_expression member_expression_part */ yytestcase(yyruleno==91); + /* (92) primary_expression ::= object_literal */ yytestcase(yyruleno==92); + /* (93) primary_expression ::= PARENL expression PARENR */ yytestcase(yyruleno==93); + /* (94) primary_expression ::= IDENTIFIER */ yytestcase(yyruleno==94); + /* (95) primary_expression ::= array_literal */ yytestcase(yyruleno==95); + /* (96) primary_expression ::= DECIMAL */ yytestcase(yyruleno==96); + /* (97) primary_expression ::= HEX_INTEGER */ yytestcase(yyruleno==97); + /* (98) primary_expression ::= STRING */ yytestcase(yyruleno==98); + /* (99) primary_expression ::= BOOLEAN */ yytestcase(yyruleno==99); + /* (100) primary_expression ::= NULL */ yytestcase(yyruleno==100); + /* (101) array_literal ::= BRACKETL elision BRACKETR */ yytestcase(yyruleno==101); + /* (102) array_literal ::= BRACKETL element_list elision BRACKETR */ yytestcase(yyruleno==102); + /* (103) array_literal ::= BRACKETL element_list BRACKETR */ yytestcase(yyruleno==103); + /* (104) elision ::= COMMA */ yytestcase(yyruleno==104); + /* (105) elision ::= elision COMMA */ yytestcase(yyruleno==105); + /* (106) element_list ::= assignment_expression */ yytestcase(yyruleno==106); + /* (107) element_list ::= elision assignment_expression */ yytestcase(yyruleno==107); + /* (108) element_list ::= element_list elision assignment_expression */ yytestcase(yyruleno==108); + /* (109) object_literal ::= BRACEL property_name_and_value_list BRACER */ yytestcase(yyruleno==109); + /* (110) property_name_and_value_list ::= */ yytestcase(yyruleno==110); + /* (111) property_name_and_value_list ::= property_name_and_value_list COMMA property_name_and_value */ yytestcase(yyruleno==111); + /* (112) property_name_and_value ::= property_name COLON assignment_expression */ yytestcase(yyruleno==112); + /* (113) property_name ::= IDENTIFIER|STRING|DECIMAL */ yytestcase(yyruleno==113); + /* (115) member_expression_part ::= DOT IDENTIFIER */ yytestcase(yyruleno==115); + /* (126) adjuster ::= */ yytestcase(yyruleno==126); + /* (127) adjuster ::= adjust_expression */ yytestcase(yyruleno==127); + /* (129) adjust_expression ::= adjust_match_expression */ yytestcase(yyruleno==129); break; }; yygoto = yyRuleInfo[yyruleno].lhs; @@ -2055,7 +2072,7 @@ static void yy_syntax_error( GRN_OBJ_FIN(ctx, &message); } } -#line 2059 "grn_ecmascript.c" +#line 2076 "grn_ecmascript.c" grn_expr_parserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } diff --git a/storage/mroonga/vendor/groonga/lib/grn_ecmascript.h b/storage/mroonga/vendor/groonga/lib/grn_ecmascript.h index c8e74b2fb8f..6280690482b 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_ecmascript.h +++ b/storage/mroonga/vendor/groonga/lib/grn_ecmascript.h @@ -44,29 +44,30 @@ #define GRN_EXPR_TOKEN_LCP 44 #define GRN_EXPR_TOKEN_PREFIX 45 #define GRN_EXPR_TOKEN_SUFFIX 46 -#define GRN_EXPR_TOKEN_SHIFTL 47 -#define GRN_EXPR_TOKEN_SHIFTR 48 -#define GRN_EXPR_TOKEN_SHIFTRR 49 -#define GRN_EXPR_TOKEN_PLUS 50 -#define GRN_EXPR_TOKEN_MINUS 51 -#define GRN_EXPR_TOKEN_STAR 52 -#define GRN_EXPR_TOKEN_SLASH 53 -#define GRN_EXPR_TOKEN_MOD 54 -#define GRN_EXPR_TOKEN_DELETE 55 -#define GRN_EXPR_TOKEN_INCR 56 -#define GRN_EXPR_TOKEN_DECR 57 -#define GRN_EXPR_TOKEN_NOT 58 -#define GRN_EXPR_TOKEN_BITWISE_NOT 59 -#define GRN_EXPR_TOKEN_ADJUST 60 -#define GRN_EXPR_TOKEN_EXACT 61 -#define GRN_EXPR_TOKEN_PARTIAL 62 -#define GRN_EXPR_TOKEN_UNSPLIT 63 -#define GRN_EXPR_TOKEN_DECIMAL 64 -#define GRN_EXPR_TOKEN_HEX_INTEGER 65 -#define GRN_EXPR_TOKEN_STRING 66 -#define GRN_EXPR_TOKEN_BOOLEAN 67 -#define GRN_EXPR_TOKEN_NULL 68 -#define GRN_EXPR_TOKEN_BRACKETL 69 -#define GRN_EXPR_TOKEN_BRACKETR 70 -#define GRN_EXPR_TOKEN_DOT 71 -#define GRN_EXPR_TOKEN_NONEXISTENT_COLUMN 72 +#define GRN_EXPR_TOKEN_REGEXP 47 +#define GRN_EXPR_TOKEN_SHIFTL 48 +#define GRN_EXPR_TOKEN_SHIFTR 49 +#define GRN_EXPR_TOKEN_SHIFTRR 50 +#define GRN_EXPR_TOKEN_PLUS 51 +#define GRN_EXPR_TOKEN_MINUS 52 +#define GRN_EXPR_TOKEN_STAR 53 +#define GRN_EXPR_TOKEN_SLASH 54 +#define GRN_EXPR_TOKEN_MOD 55 +#define GRN_EXPR_TOKEN_DELETE 56 +#define GRN_EXPR_TOKEN_INCR 57 +#define GRN_EXPR_TOKEN_DECR 58 +#define GRN_EXPR_TOKEN_NOT 59 +#define GRN_EXPR_TOKEN_BITWISE_NOT 60 +#define GRN_EXPR_TOKEN_ADJUST 61 +#define GRN_EXPR_TOKEN_EXACT 62 +#define GRN_EXPR_TOKEN_PARTIAL 63 +#define GRN_EXPR_TOKEN_UNSPLIT 64 +#define GRN_EXPR_TOKEN_DECIMAL 65 +#define GRN_EXPR_TOKEN_HEX_INTEGER 66 +#define GRN_EXPR_TOKEN_STRING 67 +#define GRN_EXPR_TOKEN_BOOLEAN 68 +#define GRN_EXPR_TOKEN_NULL 69 +#define GRN_EXPR_TOKEN_BRACKETL 70 +#define GRN_EXPR_TOKEN_BRACKETR 71 +#define GRN_EXPR_TOKEN_DOT 72 +#define GRN_EXPR_TOKEN_NONEXISTENT_COLUMN 73 diff --git a/storage/mroonga/vendor/groonga/lib/grn_ecmascript.lemon b/storage/mroonga/vendor/groonga/lib/grn_ecmascript.lemon index 322d7ac8264..1d812655d70 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_ecmascript.lemon +++ b/storage/mroonga/vendor/groonga/lib/grn_ecmascript.lemon @@ -219,6 +219,9 @@ relational_expression ::= relational_expression PREFIX shift_expression. { relational_expression ::= relational_expression SUFFIX shift_expression. { grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_SUFFIX, 2); } +relational_expression ::= relational_expression REGEXP shift_expression. { + grn_expr_append_op(efsi->ctx, efsi->e, GRN_OP_REGEXP, 2); +} shift_expression ::= additive_expression. shift_expression ::= shift_expression SHIFTL additive_expression. { diff --git a/storage/mroonga/vendor/groonga/lib/grn_expr.h b/storage/mroonga/vendor/groonga/lib/grn_expr.h index 0a21f72bb92..75871491d7c 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_expr.h +++ b/storage/mroonga/vendor/groonga/lib/grn_expr.h @@ -44,7 +44,10 @@ typedef grn_bool (*grn_scan_info_each_arg_callback)(grn_ctx *ctx, grn_obj *obj, scan_info *grn_scan_info_open(grn_ctx *ctx, int start); void grn_scan_info_close(grn_ctx *ctx, scan_info *si); void grn_scan_info_put_index(grn_ctx *ctx, scan_info *si, grn_obj *index, - uint32_t sid, int32_t weight); + uint32_t sid, int32_t weight, + grn_obj *scorer, + grn_obj *scorer_args_expr, + uint32_t scorer_args_expr_offset); scan_info **grn_scan_info_put_logical_op(grn_ctx *ctx, scan_info **sis, int *ip, grn_operator op, int start); int grn_scan_info_get_flags(scan_info *si); @@ -59,12 +62,16 @@ int grn_scan_info_get_max_interval(scan_info *si); void grn_scan_info_set_max_interval(scan_info *si, int max_interval); int grn_scan_info_get_similarity_threshold(scan_info *si); void grn_scan_info_set_similarity_threshold(scan_info *si, int similarity_threshold); -grn_obj *grn_scan_info_get_scorer(scan_info *si); -void grn_scan_info_set_scorer(scan_info *si, grn_obj *scorer); grn_bool grn_scan_info_push_arg(scan_info *si, grn_obj *arg); grn_obj *grn_scan_info_get_arg(grn_ctx *ctx, scan_info *si, int i); -int32_t grn_expr_code_get_weight(grn_ctx *ctx, grn_expr_code *ec); +int32_t grn_expr_code_get_weight(grn_ctx *ctx, grn_expr_code *ec, uint32_t *offset); +grn_rc grn_expr_code_inspect_indented(grn_ctx *ctx, + grn_obj *buffer, + grn_expr_code *code, + const char *indent); +void grn_p_expr_code(grn_ctx *ctx, grn_expr_code *code); + void grn_expr_take_obj(grn_ctx *ctx, grn_obj *expr, grn_obj *obj); grn_obj *grn_expr_alloc_const(grn_ctx *ctx, grn_obj *expr); @@ -73,3 +80,4 @@ grn_obj *grn_expr_alloc_const(grn_ctx *ctx, grn_obj *expr); #endif #endif /* GRN_EXPR_H */ + diff --git a/storage/mroonga/vendor/groonga/lib/grn_hash.h b/storage/mroonga/vendor/groonga/lib/grn_hash.h index 44f5bcba0ed..d50836d80f1 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_hash.h +++ b/storage/mroonga/vendor/groonga/lib/grn_hash.h @@ -1,5 +1,5 @@ /* -*- c-basic-offset: 2 -*- */ -/* Copyright(C) 2009-2012 Brazil +/* Copyright(C) 2009-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -187,7 +187,15 @@ GRN_API grn_id grn_table_queue_tail(grn_table_queue *queue); /**** grn_hash ****/ #define GRN_HASH_TINY (0x01<<6) -#define GRN_HASH_MAX_KEY_SIZE GRN_TABLE_MAX_KEY_SIZE +#define GRN_HASH_MAX_KEY_SIZE_NORMAL GRN_TABLE_MAX_KEY_SIZE +#define GRN_HASH_MAX_KEY_SIZE_LARGE (0xffff) + +#define GRN_HASH_IS_LARGE_KEY(hash)\ + ((hash)->key_size > GRN_HASH_MAX_KEY_SIZE_NORMAL) + +typedef struct _grn_hash_header_common grn_hash_header_common; +typedef struct _grn_hash_header_normal grn_hash_header_normal; +typedef struct _grn_hash_header_large grn_hash_header_large; struct _grn_hash { grn_db_obj obj; @@ -205,7 +213,11 @@ struct _grn_hash { /* For grn_io_hash. */ grn_io *io; - struct grn_hash_header *header; + union { + grn_hash_header_common *common; + grn_hash_header_normal *normal; + grn_hash_header_large *large; + } header; uint32_t *lock; // uint32_t nref; // unsigned int max_n_subrecs; @@ -230,24 +242,36 @@ struct _grn_hash { grn_tiny_bitmap bitmap; }; -/* Header of grn_io_hash. */ -struct grn_hash_header { - uint32_t flags; - grn_encoding encoding; - uint32_t key_size; - uint32_t value_size; - grn_id tokenizer; - uint32_t curr_rec; - int32_t curr_key; - uint32_t idx_offset; - uint32_t entry_size; - uint32_t max_offset; - uint32_t n_entries; - uint32_t n_garbages; - uint32_t lock; - grn_id normalizer; - uint32_t reserved[15]; - grn_id garbages[GRN_HASH_MAX_KEY_SIZE]; +#define GRN_HASH_HEADER_COMMON_FIELDS\ + uint32_t flags;\ + grn_encoding encoding;\ + uint32_t key_size;\ + uint32_t value_size;\ + grn_id tokenizer;\ + uint32_t curr_rec;\ + int32_t curr_key;\ + uint32_t idx_offset;\ + uint32_t entry_size;\ + uint32_t max_offset;\ + uint32_t n_entries;\ + uint32_t n_garbages;\ + uint32_t lock;\ + grn_id normalizer;\ + uint32_t reserved[15] + +struct _grn_hash_header_common { + GRN_HASH_HEADER_COMMON_FIELDS; +}; + +struct _grn_hash_header_normal { + GRN_HASH_HEADER_COMMON_FIELDS; + grn_id garbages[GRN_HASH_MAX_KEY_SIZE_NORMAL]; + grn_table_queue queue; +}; + +struct _grn_hash_header_large { + GRN_HASH_HEADER_COMMON_FIELDS; + grn_id garbages[GRN_HASH_MAX_KEY_SIZE_LARGE]; grn_table_queue queue; }; diff --git a/storage/mroonga/vendor/groonga/lib/grn_ii.h b/storage/mroonga/vendor/groonga/lib/grn_ii.h index 0d44079a38c..fe9b1ec26a4 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_ii.h +++ b/storage/mroonga/vendor/groonga/lib/grn_ii.h @@ -152,6 +152,8 @@ struct _grn_select_optarg { void *func_arg; int max_size; grn_obj *scorer; + grn_obj *scorer_args_expr; + unsigned int scorer_args_expr_offset; }; GRN_API grn_rc grn_ii_column_update(grn_ctx *ctx, grn_ii *ii, grn_id id, diff --git a/storage/mroonga/vendor/groonga/lib/grn_io.h b/storage/mroonga/vendor/groonga/lib/grn_io.h index 00a1c3a0936..14f5e496f8b 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_io.h +++ b/storage/mroonga/vendor/groonga/lib/grn_io.h @@ -380,7 +380,7 @@ uint32_t grn_expire(grn_ctx *ctx, int count_thresh, uint32_t limit); *_p++ = _v & 0xff; \ } else { \ *_p++ = 0x8f; \ - memcpy(_p, &_v, sizeof(uint32_t));\ + grn_memcpy(_p, &_v, sizeof(uint32_t));\ _p += sizeof(uint32_t); \ } \ p = _p; \ @@ -395,7 +395,7 @@ uint32_t grn_expire(grn_ctx *ctx, int count_thresh, uint32_t limit); switch (_v >> 4) { \ case 0x08 : \ if (_v == 0x8f) { \ - memcpy(&_v, _p, sizeof(uint32_t));\ + grn_memcpy(&_v, _p, sizeof(uint32_t));\ _p += sizeof(uint32_t); \ } \ break; \ diff --git a/storage/mroonga/vendor/groonga/lib/grn_logger.h b/storage/mroonga/vendor/groonga/lib/grn_logger.h new file mode 100644 index 00000000000..95186ca107a --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/grn_logger.h @@ -0,0 +1,37 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2009-2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ +#ifndef GRN_LOGGER_H +#define GRN_LOGGER_H + +#include "grn.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void grn_logger_init(void); +void grn_logger_fin(grn_ctx *ctx); + +void grn_query_logger_init(void); +void grn_query_logger_fin(grn_ctx *ctx); + +#ifdef __cplusplus +} +#endif + +#endif /* GRN_LOGGER_H */ diff --git a/storage/mroonga/vendor/groonga/lib/grn_msgpack.h b/storage/mroonga/vendor/groonga/lib/grn_msgpack.h new file mode 100644 index 00000000000..db86987c08a --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/grn_msgpack.h @@ -0,0 +1,48 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2009-2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ +#ifndef GRN_MSGPACK_H +#define GRN_MSGPACK_H + +#ifdef GRN_WITH_MESSAGE_PACK +# include <msgpack.h> + +# if MSGPACK_VERSION_MAJOR < 1 +typedef unsigned int msgpack_size_t; + +# define msgpack_pack_str(packer, size) msgpack_pack_raw(packer, size) +# define msgpack_pack_str_body(packer, value, size) \ + msgpack_pack_raw_body(packer, value, size) + +# define MSGPACK_OBJECT_STR MSGPACK_OBJECT_RAW +# define MSGPACK_OBJECT_FLOAT MSGPACK_OBJECT_DOUBLE + +# define MSGPACK_OBJECT_STR_PTR(object) (object)->via.raw.ptr +# define MSGPACK_OBJECT_STR_SIZE(object) (object)->via.raw.size + +# define MSGPACK_OBJECT_FLOAT_VALUE(object) (object)->via.dec +# else /* MSGPACK_VERSION_MAJOR < 1 */ +typedef size_t msgpack_size_t; + +# define MSGPACK_OBJECT_STR_PTR(object) (object)->via.str.ptr +# define MSGPACK_OBJECT_STR_SIZE(object) (object)->via.str.size + +# define MSGPACK_OBJECT_FLOAT_VALUE(object) (object)->via.f64 +# endif /* MSGPACK_VERSION_MAJOR < 1 */ +#endif /* GRN_WITH_MESSAGE_PACK */ + +#endif /* GRN_MSGPACK_H */ diff --git a/storage/mroonga/vendor/groonga/lib/grn_plugin.h b/storage/mroonga/vendor/groonga/lib/grn_plugin.h index 121c3143faf..75b29f56cc9 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_plugin.h +++ b/storage/mroonga/vendor/groonga/lib/grn_plugin.h @@ -37,6 +37,7 @@ typedef void * grn_dl_symbol; typedef struct _grn_plugin grn_plugin; struct _grn_plugin { + char path[PATH_MAX]; grn_dl dl; grn_plugin_func init_func; grn_plugin_func register_func; diff --git a/storage/mroonga/vendor/groonga/lib/grn_rset.h b/storage/mroonga/vendor/groonga/lib/grn_rset.h index 4fa0d08355a..b92e21162df 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_rset.h +++ b/storage/mroonga/vendor/groonga/lib/grn_rset.h @@ -53,7 +53,7 @@ typedef struct { #define GRN_RSET_SUBRECS_NTH(subrecs,size,n) \ ((double *)((byte *)subrecs + n * GRN_RSET_SUBREC_SIZE(size))) #define GRN_RSET_SUBRECS_COPY(subrecs,size,n,src) \ - (memcpy(GRN_RSET_SUBRECS_NTH(subrecs, size, n), src, GRN_RSET_SUBREC_SIZE(size))) + (grn_memcpy(GRN_RSET_SUBRECS_NTH(subrecs, size, n), src, GRN_RSET_SUBREC_SIZE(size))) #define GRN_RSET_SUBRECS_SIZE(subrec_size,n) \ (GRN_RSET_SUBREC_SIZE(subrec_size) * n) diff --git a/storage/mroonga/vendor/groonga/lib/grn_scorer.h b/storage/mroonga/vendor/groonga/lib/grn_scorer.h index 898a83366b0..05f982180db 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_scorer.h +++ b/storage/mroonga/vendor/groonga/lib/grn_scorer.h @@ -39,6 +39,8 @@ struct _grn_scorer_matched_record { uint64_t n_candidates; uint32_t n_tokens; int weight; + grn_obj *args_expr; + unsigned int args_expr_offset; }; diff --git a/storage/mroonga/vendor/groonga/lib/grn_store.h b/storage/mroonga/vendor/groonga/lib/grn_store.h index 742d11341fc..9e1223e0685 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_store.h +++ b/storage/mroonga/vendor/groonga/lib/grn_store.h @@ -71,8 +71,6 @@ void *grn_ra_ref_cache(grn_ctx *ctx, grn_ra *ra, grn_id id, grn_ra_cache *cache) /**** variable sized elements ****/ -extern grn_bool grn_ja_skip_same_value_put; - typedef struct _grn_ja grn_ja; struct _grn_ja { diff --git a/storage/mroonga/vendor/groonga/lib/grn_str.h b/storage/mroonga/vendor/groonga/lib/grn_str.h index ade8c04053e..e6fab611ccd 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_str.h +++ b/storage/mroonga/vendor/groonga/lib/grn_str.h @@ -54,6 +54,7 @@ GRN_API int grn_atoi(const char *nptr, const char *end, const char **rest); GRN_API unsigned int grn_atoui(const char *nptr, const char *end, const char **rest); unsigned int grn_htoui(const char *nptr, const char *end, const char **rest); GRN_API int64_t grn_atoll(const char *nptr, const char *end, const char **rest); +GRN_API uint64_t grn_atoull(const char *nptr, const char *end, const char **rest); grn_rc grn_itoa(int i, char *p, char *end, char **rest); grn_rc grn_lltoa(int64_t i, char *p, char *end, char **rest); grn_rc grn_ulltoa(uint64_t i, char *p, char *end, char **rest); diff --git a/storage/mroonga/vendor/groonga/lib/grn_token_cursor.h b/storage/mroonga/vendor/groonga/lib/grn_token_cursor.h index 81175bc432f..fec62224418 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_token_cursor.h +++ b/storage/mroonga/vendor/groonga/lib/grn_token_cursor.h @@ -26,6 +26,11 @@ extern "C" { #endif +#define GRN_TOKENIZER_BEGIN_MARK_UTF8 "\xEF\xBF\xAF" +#define GRN_TOKENIZER_BEGIN_MARK_UTF8_LEN 3 +#define GRN_TOKENIZER_END_MARK_UTF8 "\xEF\xBF\xB0" +#define GRN_TOKENIZER_END_MARK_UTF8_LEN 3 + typedef enum { GRN_TOKEN_CURSOR_DOING = 0, GRN_TOKEN_CURSOR_DONE, diff --git a/storage/mroonga/vendor/groonga/lib/grn_util.h b/storage/mroonga/vendor/groonga/lib/grn_util.h index 84aa357e4e3..f94140dcb66 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_util.h +++ b/storage/mroonga/vendor/groonga/lib/grn_util.h @@ -1,5 +1,5 @@ /* -*- c-basic-offset: 2 -*- */ -/* Copyright(C) 2010-2011 Brazil +/* Copyright(C) 2010-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -29,6 +29,8 @@ GRN_API grn_rc grn_normalize_offset_and_limit(grn_ctx *ctx, int size, int *offse GRN_API const char *grn_win32_base_dir(void); GRN_API char *grn_path_separator_to_system(char *dest, char *groonga_path); +int grn_mkstemp(char *path_template); + #ifdef __cplusplus } #endif diff --git a/storage/mroonga/vendor/groonga/lib/hash.c b/storage/mroonga/vendor/groonga/lib/hash.c index 22491c5fc8b..2c44f25cc20 100644 --- a/storage/mroonga/vendor/groonga/lib/hash.c +++ b/storage/mroonga/vendor/groonga/lib/hash.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2009-2012 Brazil + Copyright(C) 2009-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -700,7 +700,7 @@ grn_array_get_value(grn_ctx *ctx, grn_array *array, grn_id id, void *valuebuf) void * const value = grn_array_get_value_inline(ctx, array, id); if (value) { if (valuebuf) { - memcpy(valuebuf, value, array->value_size); + grn_memcpy(valuebuf, value, array->value_size); } return array->value_size; } @@ -724,7 +724,7 @@ grn_array_set_value_inline(grn_ctx *ctx, grn_array *array, grn_id id, switch ((flags & GRN_OBJ_SET_MASK)) { case GRN_OBJ_SET : - memcpy(entry, value, array->value_size); + grn_memcpy(entry, value, array->value_size); return GRN_SUCCESS; case GRN_OBJ_INCR : switch (array->value_size) { @@ -863,7 +863,7 @@ grn_array_copy_sort_key(grn_ctx *ctx, grn_array *array, if (!array->keys) { return ctx->rc; } - memcpy(array->keys, keys, sizeof(grn_table_sort_key) * n_keys); + grn_memcpy(array->keys, keys, sizeof(grn_table_sort_key) * n_keys); array->n_keys = n_keys; return GRN_SUCCESS; } @@ -1167,7 +1167,11 @@ grn_array_unblock(grn_ctx *ctx, grn_array *array) /* grn_hash : hash table */ #define GRN_HASH_MAX_SEGMENT 0x400 -#define GRN_HASH_HEADER_SIZE 0x9000 +#define GRN_HASH_HEADER_SIZE_NORMAL 0x9000 +#define GRN_HASH_HEADER_SIZE_LARGE\ + (GRN_HASH_HEADER_SIZE_NORMAL +\ + (sizeof(grn_id) *\ + (GRN_HASH_MAX_KEY_SIZE_LARGE - GRN_HASH_MAX_KEY_SIZE_NORMAL))) #define GRN_HASH_SEGMENT_SIZE 0x400000 #define W_OF_KEY_IN_A_SEGMENT 22 #define IDX_MASK_IN_A_SEGMENT 0xfffff @@ -1298,7 +1302,7 @@ inline static grn_id * grn_hash_idx_at(grn_ctx *ctx, grn_hash *hash, grn_id id) { if (grn_hash_is_io_hash(hash)) { - id = (id & *hash->max_offset) + hash->header->idx_offset; + id = (id & *hash->max_offset) + hash->header.common->idx_offset; return grn_io_hash_idx_at(ctx, hash, id); } else { return hash->index + (id & *hash->max_offset); @@ -1380,15 +1384,18 @@ grn_io_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, key_offset = entry->key.offset; } else { uint32_t segment_id; + grn_hash_header_common *header; + + header = hash->header.common; if (key_size >= GRN_HASH_SEGMENT_SIZE) { return GRN_INVALID_ARGUMENT; } - key_offset = hash->header->curr_key; + key_offset = header->curr_key; segment_id = (key_offset + key_size) >> W_OF_KEY_IN_A_SEGMENT; if ((key_offset >> W_OF_KEY_IN_A_SEGMENT) != segment_id) { - key_offset = hash->header->curr_key = segment_id << W_OF_KEY_IN_A_SEGMENT; + key_offset = header->curr_key = segment_id << W_OF_KEY_IN_A_SEGMENT; } - hash->header->curr_key += key_size; + header->curr_key += key_size; entry->key.offset = key_offset; } @@ -1397,7 +1404,7 @@ grn_io_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, if (!key_ptr) { return GRN_NO_MEMORY_AVAILABLE; } - memcpy(key_ptr, key, key_size); + grn_memcpy(key_ptr, key, key_size); } return GRN_SUCCESS; } @@ -1410,7 +1417,7 @@ grn_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, if (hash->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) { if (grn_hash_is_io_hash(hash)) { if (key_size <= sizeof(entry->io_entry.key.buf)) { - memcpy(entry->io_entry.key.buf, key, key_size); + grn_memcpy(entry->io_entry.key.buf, key, key_size); entry->io_entry.flag = HASH_IMMEDIATE; } else { const grn_rc rc = @@ -1425,7 +1432,7 @@ grn_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, entry->io_entry.key_size = key_size; } else { if (key_size <= sizeof(entry->tiny_entry.key.buf)) { - memcpy(entry->tiny_entry.key.buf, key, key_size); + grn_memcpy(entry->tiny_entry.key.buf, key, key_size); entry->tiny_entry.flag = HASH_IMMEDIATE; } else { grn_ctx * const ctx = hash->ctx; @@ -1433,7 +1440,7 @@ grn_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, if (!entry->tiny_entry.key.ptr) { return GRN_NO_MEMORY_AVAILABLE; } - memcpy(entry->tiny_entry.key.ptr, key, key_size); + grn_memcpy(entry->tiny_entry.key.ptr, key, key_size); entry->tiny_entry.flag = 0; } entry->tiny_entry.hash_value = hash_value; @@ -1444,7 +1451,7 @@ grn_hash_entry_put_key(grn_ctx *ctx, grn_hash *hash, *(uint32_t *)entry->plain_entry.key = hash_value; } else { entry->rich_entry.hash_value = hash_value; - memcpy(entry->rich_entry.key_and_value, key, key_size); + grn_memcpy(entry->rich_entry.key_and_value, key, key_size); } } return GRN_SUCCESS; @@ -1537,7 +1544,8 @@ grn_io_hash_calculate_entry_size(uint32_t key_size, uint32_t value_size, } static grn_io * -grn_io_hash_create_io(grn_ctx *ctx, const char *path, uint32_t entry_size) +grn_io_hash_create_io(grn_ctx *ctx, const char *path, + uint32_t header_size, uint32_t entry_size) { uint32_t w_of_element = 0; grn_io_array_spec array_spec[4]; @@ -1555,7 +1563,7 @@ grn_io_hash_create_io(grn_ctx *ctx, const char *path, uint32_t entry_size) array_spec[GRN_HASH_INDEX_SEGMENT].max_n_segments = 1U << (30 - (22 - 2)); array_spec[GRN_HASH_BITMAP_SEGMENT].w_of_element = 0; array_spec[GRN_HASH_BITMAP_SEGMENT].max_n_segments = 1U << (30 - (22 + 3)); - return grn_io_create_with_array(ctx, path, GRN_HASH_HEADER_SIZE, + return grn_io_create_with_array(ctx, path, header_size, GRN_HASH_SEGMENT_SIZE, grn_io_auto, 4, array_spec); } @@ -1566,12 +1574,17 @@ grn_io_hash_init(grn_ctx *ctx, grn_hash *hash, const char *path, grn_encoding encoding, uint32_t init_size) { grn_io *io; - struct grn_hash_header *header; - uint32_t entry_size, max_offset; + grn_hash_header_common *header; + uint32_t header_size, entry_size, max_offset; + if (key_size <= GRN_HASH_MAX_KEY_SIZE_NORMAL) { + header_size = GRN_HASH_HEADER_SIZE_NORMAL; + } else { + header_size = GRN_HASH_HEADER_SIZE_LARGE; + } entry_size = grn_io_hash_calculate_entry_size(key_size, value_size, flags); - io = grn_io_hash_create_io(ctx, path, entry_size); + io = grn_io_hash_create_io(ctx, path, header_size, entry_size); if (!io) { return GRN_NO_MEMORY_AVAILABLE; } @@ -1587,6 +1600,8 @@ grn_io_hash_init(grn_ctx *ctx, grn_hash *hash, const char *path, encoding = ctx->encoding; } + hash->key_size = key_size; + header = grn_io_header(io); header->flags = flags; header->encoding = encoding; @@ -1610,11 +1625,18 @@ grn_io_hash_init(grn_ctx *ctx, grn_hash *hash, const char *path, header->normalizer = GRN_ID_NIL; } GRN_PTR_INIT(&(hash->token_filters), GRN_OBJ_VECTOR, GRN_ID_NIL); - grn_table_queue_init(ctx, &header->queue); + { + grn_table_queue *queue; + if (GRN_HASH_IS_LARGE_KEY(hash)) { + queue = &(((grn_hash_header_large *)(header))->queue); + } else { + queue = &(((grn_hash_header_normal *)(header))->queue); + } + grn_table_queue_init(ctx, queue); + } hash->obj.header.flags = header->flags; hash->ctx = ctx; - hash->key_size = key_size; hash->encoding = encoding; hash->value_size = value_size; hash->entry_size = entry_size; @@ -1622,7 +1644,7 @@ grn_io_hash_init(grn_ctx *ctx, grn_hash *hash, const char *path, hash->n_entries = &header->n_entries; hash->max_offset = &header->max_offset; hash->io = io; - hash->header = header; + hash->header.common = header; hash->lock = &header->lock; hash->tokenizer = NULL; return GRN_SUCCESS; @@ -1711,7 +1733,7 @@ grn_hash_create(grn_ctx *ctx, const char *path, uint32_t key_size, uint32_t valu if (!ctx) { return NULL; } - if (key_size > GRN_HASH_MAX_KEY_SIZE) { + if (key_size > GRN_HASH_MAX_KEY_SIZE_LARGE) { return NULL; } hash = (grn_hash *)GRN_MALLOC(sizeof(grn_hash)); @@ -1732,7 +1754,7 @@ grn_hash_open(grn_ctx *ctx, const char *path) if (ctx) { grn_io * const io = grn_io_open(ctx, path, grn_io_auto); if (io) { - struct grn_hash_header * const header = grn_io_header(io); + grn_hash_header_common * const header = grn_io_header(io); if (grn_io_get_type(io) == GRN_TABLE_HASH_KEY) { grn_hash * const hash = (grn_hash *)GRN_MALLOC(sizeof(grn_hash)); if (hash) { @@ -1747,7 +1769,7 @@ grn_hash_open(grn_ctx *ctx, const char *path) hash->n_entries = &header->n_entries; hash->max_offset = &header->max_offset; hash->io = io; - hash->header = header; + hash->header.common = header; hash->lock = &header->lock; hash->tokenizer = grn_ctx_at(ctx, header->tokenizer); if (header->flags & GRN_OBJ_KEY_NORMALIZE) { @@ -1911,7 +1933,7 @@ grn_hash_reset(grn_ctx *ctx, grn_hash *hash, uint32_t expected_n_entries) if (grn_hash_is_io_hash(hash)) { uint32_t i; - src_offset = hash->header->idx_offset; + src_offset = hash->header.common->idx_offset; dest_offset = MAX_INDEX_SIZE - src_offset; for (i = 0; i < new_index_size; i += (IDX_MASK_IN_A_SEGMENT + 1)) { /* @@ -1979,7 +2001,7 @@ grn_hash_reset(grn_ctx *ctx, grn_hash *hash, uint32_t expected_n_entries) } if (grn_hash_is_io_hash(hash)) { - hash->header->idx_offset = dest_offset; + hash->header.common->idx_offset = dest_offset; } else { grn_id * const old_index = hash->index; hash->index = new_index; @@ -2038,15 +2060,22 @@ grn_io_hash_add(grn_ctx *ctx, grn_hash *hash, uint32_t hash_value, { grn_id entry_id; grn_hash_entry *entry; - struct grn_hash_header * const header = hash->header; + grn_hash_header_common * const header = hash->header.common; + grn_id *garbages; - entry_id = header->garbages[key_size - 1]; + if (GRN_HASH_IS_LARGE_KEY(hash)) { + garbages = hash->header.large->garbages; + } else { + garbages = hash->header.normal->garbages; + } + + entry_id = garbages[key_size - 1]; if (entry_id) { entry = grn_io_hash_entry_at(ctx, hash, entry_id, GRN_TABLE_ADD); if (!entry) { return GRN_ID_NIL; } - header->garbages[key_size - 1] = *(grn_id *)entry; + garbages[key_size - 1] = *(grn_id *)entry; if (hash->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) { /* keep entry->io_entry's hash_value, flag, key_size and key. */ memset(entry->io_entry.value, 0, header->value_size); @@ -2282,7 +2311,7 @@ grn_hash_get_key(grn_ctx *ctx, grn_hash *hash, grn_id id, void *keybuf, int bufs } key_size = grn_hash_entry_get_key_size(hash, entry); if (bufsize >= key_size) { - memcpy(keybuf, grn_hash_entry_get_key(ctx, hash, entry), key_size); + grn_memcpy(keybuf, grn_hash_entry_get_key(ctx, hash, entry), key_size); } return key_size; } @@ -2320,7 +2349,7 @@ grn_hash_get_value(grn_ctx *ctx, grn_hash *hash, grn_id id, void *valuebuf) return 0; } if (valuebuf) { - memcpy(valuebuf, value, hash->value_size); + grn_memcpy(valuebuf, value, hash->value_size); } return hash->value_size; } @@ -2353,14 +2382,14 @@ grn_hash_get_key_value(grn_ctx *ctx, grn_hash *hash, grn_id id, } key_size = grn_hash_entry_get_key_size(hash, entry); if (bufsize >= key_size) { - memcpy(keybuf, grn_hash_entry_get_key(ctx, hash, entry), key_size); + grn_memcpy(keybuf, grn_hash_entry_get_key(ctx, hash, entry), key_size); } value = grn_hash_entry_get_value(hash, entry); if (!value) { return 0; } if (valuebuf) { - memcpy(valuebuf, value, hash->value_size); + grn_memcpy(valuebuf, value, hash->value_size); } return key_size; } @@ -2400,7 +2429,7 @@ grn_hash_set_value(grn_ctx *ctx, grn_hash *hash, grn_id id, switch (flags & GRN_OBJ_SET_MASK) { case GRN_OBJ_SET : - memcpy(entry_value, value, hash->value_size); + grn_memcpy(entry_value, value, hash->value_size); return GRN_SUCCESS; case GRN_OBJ_INCR : switch (hash->value_size) { @@ -2436,9 +2465,14 @@ grn_hash_set_value(grn_ctx *ctx, grn_hash *hash, grn_id id, *ep = GARBAGE;\ if (grn_hash_is_io_hash(hash)) {\ uint32_t size = key_size - 1;\ - struct grn_hash_header *hh = hash->header;\ - ee->key = hh->garbages[size];\ - hh->garbages[size] = e;\ + grn_id *garbages;\ + if (GRN_HASH_IS_LARGE_KEY(hash)) {\ + garbages = hash->header.large->garbages;\ + } else {\ + garbages = hash->header.normal->garbages;\ + }\ + ee->key = garbages[size];\ + garbages[size] = e;\ grn_io_array_bit_off(ctx, hash->io, GRN_HASH_BITMAP_SEGMENT, e);\ } else {\ ee->key = hash->garbages;\ @@ -2537,7 +2571,7 @@ grn_hash_cursor_close(grn_ctx *ctx, grn_hash_cursor *c) } #define HASH_CURR_MAX(hash) \ - ((grn_hash_is_io_hash(hash)) ? (hash)->header->curr_rec : (hash)->a.max) + ((grn_hash_is_io_hash(hash)) ? (hash)->header.common->curr_rec : (hash)->a.max) grn_hash_cursor * grn_hash_cursor_open(grn_ctx *ctx, grn_hash *hash, @@ -3061,7 +3095,7 @@ void grn_hash_check(grn_ctx *ctx, grn_hash *hash) { char buf[8]; - struct grn_hash_header *h = hash->header; + grn_hash_header_common *h = hash->header.common; GRN_OUTPUT_ARRAY_OPEN("RESULT", 1); GRN_OUTPUT_MAP_OPEN("SUMMARY", 25); GRN_OUTPUT_CSTR("flags"); @@ -3158,7 +3192,7 @@ subrecs_push(byte *subrecs, int size, int n_subrecs, int score, void *body, int } v = subrecs + n * (size + GRN_RSET_SCORE_SIZE); *((int *)v) = score; - memcpy(v + GRN_RSET_SCORE_SIZE, body, size); + grn_memcpy(v + GRN_RSET_SCORE_SIZE, body, size); } inline static void @@ -3191,8 +3225,8 @@ subrecs_replace_min(byte *subrecs, int size, int n_subrecs, int score, void *bod } } v = subrecs + n * (size + GRN_RSET_SCORE_SIZE); - memcpy(v, &score, GRN_RSET_SCORE_SIZE); - memcpy(v + GRN_RSET_SCORE_SIZE, body, size); + grn_memcpy(v, &score, GRN_RSET_SCORE_SIZE); + grn_memcpy(v + GRN_RSET_SCORE_SIZE, body, size); } void @@ -3252,7 +3286,7 @@ grn_rhash_group(grn_hash *s, int limit, grn_group_optarg *optarg) if (gkey) { GRN_FREE(gkey); } return NULL; } - memcpy(&h, s, sizeof(grn_hash)); + grn_memcpy(&h, s, sizeof(grn_hash)); g = s; s = &h; if (grn_rhash_init(ctx, g, unit, rsize, s->record_unit, s->key_size, limit)) { diff --git a/storage/mroonga/vendor/groonga/lib/ii.c b/storage/mroonga/vendor/groonga/lib/ii.c index 3b5478f9ea7..2d32df2fd4d 100644 --- a/storage/mroonga/vendor/groonga/lib/ii.c +++ b/storage/mroonga/vendor/groonga/lib/ii.c @@ -19,6 +19,12 @@ #include <fcntl.h> #include <string.h> #include <sys/stat.h> +#include <math.h> + +#ifdef WIN32 +# include <io.h> +# include <share.h> +#endif /* WIN32 */ #include "grn_ii.h" #include "grn_ctx_impl.h" @@ -42,8 +48,8 @@ #define S_CHUNK (1 << GRN_II_W_CHUNK) #define W_SEGMENT 18 #define S_SEGMENT (1 << W_SEGMENT) -#define W_ARRAY_ELEMENT 3 -#define S_ARRAY_ELEMENT (1 << W_ARRAY_ELEMENT) +#define W_ARRAY_ELEMENT 3 +#define S_ARRAY_ELEMENT (1 << W_ARRAY_ELEMENT) #define W_ARRAY (W_SEGMENT - W_ARRAY_ELEMENT) #define ARRAY_MASK_IN_A_SEGMENT ((1 << W_ARRAY) - 1) #define NOT_ASSIGNED 0xffffffff @@ -1408,7 +1414,7 @@ pack(uint32_t *p, uint32_t i, uint8_t *freq, uint8_t *rp) } } rp = pack_(p - i, i, w, rp); - memcpy(rp, ebuf, ep - ebuf); + grn_memcpy(rp, ebuf, ep - ebuf); return rp + (ep - ebuf); } @@ -1560,7 +1566,7 @@ grn_p_encv(grn_ctx *ctx, datavec *dv, uint32_t dvlen, uint8_t *res) case 0x08 : \ if (_v == 0x8f) { \ if (_p + sizeof(uint32_t) > pe) { return 0; } \ - memcpy(&_v, _p, sizeof(uint32_t)); \ + grn_memcpy(&_v, _p, sizeof(uint32_t)); \ _p += sizeof(uint32_t); \ } \ break; \ @@ -2004,7 +2010,7 @@ buffer_put(grn_ctx *ctx, grn_ii *ii, buffer *b, buffer_term *bt, buffer_rec *r_curr, *r_start = NULL; uint16_t last = 0, *lastp = &bt->pos_in_buffer, pos = BUFFER_REC_POS(b, rnew); int vdelta = 0, delta, delta0 = 0, vhops = 0, nhops = 0, reset = 1; - memcpy(NEXT_ADDR(rnew), bs, size - sizeof(buffer_rec)); + grn_memcpy(NEXT_ADDR(rnew), bs, size - sizeof(buffer_rec)); for (;;) { if (!*lastp) { rnew->step = 0; @@ -2468,7 +2474,7 @@ chunk_flush(grn_ctx *ctx, grn_ii *ii, chunk_info *cinfo, uint8_t *enc, uint32_t if (encsize) { if (!(rc = chunk_new(ctx, ii, &dcn, encsize))) { if ((dc = WIN_MAP(ii->chunk, ctx, &dw, dcn, 0, encsize, grn_io_wronly))) { - memcpy(dc, enc, encsize); + grn_memcpy(dc, enc, encsize); grn_io_win_unmap(&dw); cinfo->segno = dcn; cinfo->size = encsize; @@ -2612,7 +2618,7 @@ buffer_merge(grn_ctx *ctx, grn_ii *ii, uint32_t seg, grn_hash *h, if (!bt->pos_in_buffer) { GRN_ASSERT(!bt->size_in_buffer); if (bt->size_in_chunk) { - memcpy(dcp, sc + bt->pos_in_chunk, bt->size_in_chunk); + grn_memcpy(dcp, sc + bt->pos_in_chunk, bt->size_in_chunk); bt->pos_in_chunk = (uint32_t)(dcp - dc); dcp += bt->size_in_chunk; } @@ -2779,7 +2785,10 @@ buffer_merge(grn_ctx *ctx, grn_ii *ii, uint32_t seg, grn_hash *h, if (sb->header.chunk_size + S_SEGMENT <= (dcp - dc) + encsize) { int i; - char buf[255], *bufp; +#define BUF_SIZE 255 + char buf[BUF_SIZE], *bufp, *buf_end; + buf_end = buf + BUF_SIZE; +#undef BUF_SIZE GRN_LOG(ctx, GRN_LOG_NOTICE, "cs(%d)+(%d)=(%d)<=(%" GRN_FMT_LLD ")+(%d)=(%" GRN_FMT_LLD ")", sb->header.chunk_size, S_SEGMENT, sb->header.chunk_size + S_SEGMENT, @@ -2788,7 +2797,10 @@ buffer_merge(grn_ctx *ctx, grn_ii *ii, uint32_t seg, grn_hash *h, GRN_LOG(ctx, GRN_LOG_NOTICE, "rdv[%d] data_size=%d, flags=%d", j, rdv[j].data_size, rdv[j].flags); for (i = 0, bufp = buf; i < rdv[j].data_size;) { - bufp += sprintf(bufp, " %d", rdv[j].data[i]); + bufp += grn_snprintf(bufp, + buf_end - bufp, + buf_end - bufp, + " %d", rdv[j].data[i]); i++; if (!(i % 32) || i == rdv[j].data_size) { GRN_LOG(ctx, GRN_LOG_NOTICE, "rdv[%d].data[%d]%s", j, i, buf); @@ -2801,7 +2813,10 @@ buffer_merge(grn_ctx *ctx, grn_ii *ii, uint32_t seg, grn_hash *h, GRN_LOG(ctx, GRN_LOG_NOTICE, "dv[%d] data_size=%d, flags=%d", j, dv[j].data_size, dv[j].flags); for (i = 0, bufp = buf; i < dv[j].data_size;) { - bufp += sprintf(bufp, " %d", dv[j].data[i]); + bufp += grn_snprintf(bufp, + buf_end - bufp, + buf_end - bufp, + " %d", dv[j].data[i]); i++; if (!(i % 32) || i == dv[j].data_size) { GRN_LOG(ctx, GRN_LOG_NOTICE, "dv[%d].data[%d]%s", j, i, buf); @@ -2890,7 +2905,7 @@ buffer_flush(grn_ctx *ctx, grn_ii *ii, uint32_t seg, grn_hash *h) sb->header.chunk_size, grn_io_rdonly))) { uint16_t n = sb->header.nterms; memset(db, 0, S_SEGMENT); - memcpy(db->terms, sb->terms, n * sizeof(buffer_term)); + grn_memcpy(db->terms, sb->terms, n * sizeof(buffer_term)); db->header.nterms = n; if (!(rc = buffer_merge(ctx, ii, seg, h, sb, sc, db, dc))) { actual_chunk_size = db->header.chunk_size; @@ -3168,7 +3183,7 @@ term_split(grn_ctx *ctx, grn_obj *lexicon, buffer *sb, buffer *db0, buffer *db1) bt = db0->terms; nt = &db0->header.nterms; for (s = 0; n + 1 < i && s <= th; n++, bt++) { - memcpy(bt, ts[n].bt, sizeof(buffer_term)); + grn_memcpy(bt, ts[n].bt, sizeof(buffer_term)); (*nt)++; s += ts[n].bt->size_in_chunk + 1; } @@ -3176,7 +3191,7 @@ term_split(grn_ctx *ctx, grn_obj *lexicon, buffer *sb, buffer *db0, buffer *db1) bt = db1->terms; nt = &db1->header.nterms; for (; n < i; n++, bt++) { - memcpy(bt, ts[n].bt, sizeof(buffer_term)); + grn_memcpy(bt, ts[n].bt, sizeof(buffer_term)); (*nt)++; } GRN_FREE(ts); @@ -3468,8 +3483,8 @@ _grn_ii_create(grn_ctx *ctx, grn_ii *ii, const char *path, grn_obj *lexicon, uin S_SEGMENT, MAX_PSEG, grn_io_auto, GRN_IO_EXPIRE_SEGMENT); if (!seg) { return NULL; } if (path) { - strcpy(path2, path); - strcat(path2, ".c"); + grn_strcpy(path2, PATH_MAX, path); + grn_strcat(path2, PATH_MAX, ".c"); chunk = grn_io_create(ctx, path2, 0, S_CHUNK, GRN_II_MAX_CHUNK, grn_io_auto, GRN_IO_EXPIRE_SEGMENT); } else { @@ -3525,7 +3540,8 @@ grn_ii_remove(grn_ctx *ctx, const char *path) char buffer[PATH_MAX]; if (!path || strlen(path) > PATH_MAX - 4) { return GRN_INVALID_ARGUMENT; } if ((rc = grn_io_remove(ctx, path))) { goto exit; } - snprintf(buffer, PATH_MAX, "%s.c", path); + grn_snprintf(buffer, PATH_MAX, PATH_MAX, + "%s.c", path); rc = grn_io_remove(ctx, buffer); exit : return rc; @@ -3587,8 +3603,8 @@ grn_ii_open(grn_ctx *ctx, const char *path, grn_obj *lexicon) return NULL; } if (strlen(path) + 6 >= PATH_MAX) { return NULL; } - strcpy(path2, path); - strcat(path2, ".c"); + grn_strcpy(path2, PATH_MAX, path); + grn_strcat(path2, PATH_MAX, ".c"); seg = grn_io_open(ctx, path, grn_io_auto); if (!seg) { return NULL; } chunk = grn_io_open(ctx, path2, grn_io_auto); @@ -4093,6 +4109,41 @@ exit : return c; } +static inline void +grn_ii_cursor_set_min(grn_ctx *ctx, grn_ii_cursor *c, grn_id min) +{ + char grn_ii_cursor_set_min_enable_env[GRN_ENV_BUFFER_SIZE]; + + if (c->min >= min) { + return; + } + + grn_getenv("GRN_II_CURSOR_SET_MIN_ENABLE", + grn_ii_cursor_set_min_enable_env, + GRN_ENV_BUFFER_SIZE); + if (grn_ii_cursor_set_min_enable_env[0]) { + c->min = min; + if (c->buf && c->pc.rid < c->min && c->curr_chunk < c->nchunks) { + uint32_t i, skip_chunk = 0; + grn_id rid; + for (i = 0, rid = GRN_ID_NIL; i < c->nchunks; i++) { + rid += c->cinfo[i].dgap; + if (rid < c->min) { + skip_chunk = i + 1; + } else { + rid -= c->cinfo[i].dgap; + break; + } + } + if (skip_chunk > c->curr_chunk) { + c->pc.rid = rid; + c->curr_chunk = skip_chunk; + c->crp = c->cdp + c->cdf; + } + } + } +} + grn_ii_posting * grn_ii_cursor_next(grn_ctx *ctx, grn_ii_cursor *c) { @@ -4198,34 +4249,55 @@ grn_ii_cursor_next(grn_ctx *ctx, grn_ii_cursor *c) } } if (c->stat & BUFFER_USED) { - if (c->nextb) { - uint32_t lrid = c->pb.rid, lsid = c->pb.sid; /* for check */ - buffer_rec *br = BUFFER_REC_AT(c->buf, c->nextb); - if (buffer_is_reused(ctx, c->ii, c)) { - GRN_LOG(ctx, GRN_LOG_NOTICE, "buffer reused(%d,%d)", c->buffer_pseg, *c->ppseg); - // todo : rewind; - } - c->bp = NEXT_ADDR(br); - GRN_B_DEC(c->pb.rid, c->bp); - if ((c->ii->header->flags & GRN_OBJ_WITH_SECTION)) { - GRN_B_DEC(c->pb.sid, c->bp); - } else { - c->pb.sid = 1; - } - if (lrid > c->pb.rid || (lrid == c->pb.rid && lsid >= c->pb.sid)) { - ERR(GRN_FILE_CORRUPT, "brokend!! (%d:%d) -> (%d:%d) (%d->%d)", lrid, lsid, c->pb.rid, c->pb.sid, c->buffer_pseg, *c->ppseg); - } - c->nextb = br->step; - GRN_B_DEC(c->pb.tf, c->bp); - if ((c->ii->header->flags & GRN_OBJ_WITH_WEIGHT)) { - GRN_B_DEC(c->pb.weight, c->bp); + for (;;) { + if (c->nextb) { + uint32_t lrid = c->pb.rid, lsid = c->pb.sid; /* for check */ + buffer_rec *br = BUFFER_REC_AT(c->buf, c->nextb); + if (buffer_is_reused(ctx, c->ii, c)) { + GRN_LOG(ctx, GRN_LOG_NOTICE, "buffer reused(%d,%d)", c->buffer_pseg, *c->ppseg); + // todo : rewind; + } + c->bp = NEXT_ADDR(br); + GRN_B_DEC(c->pb.rid, c->bp); + if ((c->ii->header->flags & GRN_OBJ_WITH_SECTION)) { + GRN_B_DEC(c->pb.sid, c->bp); + } else { + c->pb.sid = 1; + } + if (lrid > c->pb.rid || (lrid == c->pb.rid && lsid >= c->pb.sid)) { + ERR(GRN_FILE_CORRUPT, "brokend!! (%d:%d) -> (%d:%d) (%d->%d)", lrid, lsid, c->pb.rid, c->pb.sid, c->buffer_pseg, *c->ppseg); + } + if (c->pb.rid < c->min) { + c->pb.rid = 0; + if (br->jump > 0) { + buffer_rec *jump_br = BUFFER_REC_AT(c->buf, br->jump); + uint8_t *jump_bp; + uint32_t jump_rid; + jump_bp = NEXT_ADDR(jump_br); + GRN_B_DEC(jump_rid, jump_bp); + if (jump_rid < c->min) { + c->nextb = br->jump; + } else { + c->nextb = br->step; + } + } else { + c->nextb = br->step; + } + continue; + } + c->nextb = br->step; + GRN_B_DEC(c->pb.tf, c->bp); + if ((c->ii->header->flags & GRN_OBJ_WITH_WEIGHT)) { + GRN_B_DEC(c->pb.weight, c->bp); + } else { + c->pb.weight = 0; + } + c->pb.rest = c->pb.tf; + c->pb.pos = 0; } else { - c->pb.weight = 0; + c->pb.rid = 0; } - c->pb.rest = c->pb.tf; - c->pb.pos = 0; - } else { - c->pb.rid = 0; + break; } } if (c->pb.rid) { @@ -4273,6 +4345,10 @@ grn_ii_cursor_next(grn_ctx *ctx, grn_ii_cursor *c) } else { c->post = &c->pb; c->stat |= SOLE_DOC_USED; + if (c->post->rid < c->min) { + c->post = NULL; + return NULL; + } } } return c->post; @@ -4552,10 +4628,11 @@ cursor_heap_recalc_min(cursor_heap *h) } static inline void -cursor_heap_pop(grn_ctx *ctx, cursor_heap *h) +cursor_heap_pop(grn_ctx *ctx, cursor_heap *h, grn_id min) { if (h->n_entries) { grn_ii_cursor *c = h->bins[0]; + grn_ii_cursor_set_min(ctx, c, min); if (!grn_ii_cursor_next(ctx, c)) { grn_ii_cursor_close(ctx, c); h->bins[0] = h->bins[--h->n_entries]; @@ -5402,7 +5479,7 @@ token_info_skip(grn_ctx *ctx, token_info *ti, uint32_t rid, uint32_t sid) if (!(c = cursor_heap_min(ti->cursors))) { return GRN_END_OF_DATA; } p = c->post; if (p->rid > rid || (p->rid == rid && p->sid >= sid)) { break; } - cursor_heap_pop(ctx, ti->cursors); + cursor_heap_pop(ctx, ti->cursors, rid); } ti->pos = p->pos - ti->offset; ti->p = p; @@ -5521,7 +5598,9 @@ token_info_build(grn_ctx *ctx, grn_obj *lexicon, grn_ii *ii, const char *string, token_cursor->curr_size, token_cursor->pos, ef & EX_PREFIX); break; } - if (!ti) { goto exit; } + if (!ti) { + goto exit; + } tis[(*n)++] = ti; } rc = GRN_SUCCESS; @@ -6079,9 +6158,11 @@ grn_ii_select_sequential_search(grn_ctx *ctx, { /* Disabled by default. */ double too_many_index_match_ratio = -1; - const char *too_many_index_match_ratio_env = - getenv("GRN_II_SELECT_TOO_MANY_INDEX_MATCH_RATIO"); - if (too_many_index_match_ratio_env) { + char too_many_index_match_ratio_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_II_SELECT_TOO_MANY_INDEX_MATCH_RATIO", + too_many_index_match_ratio_env, + GRN_ENV_BUFFER_SIZE); + if (too_many_index_match_ratio_env[0]) { too_many_index_match_ratio = atof(too_many_index_match_ratio_env); } @@ -6161,7 +6242,6 @@ grn_ii_select(grn_ctx *ctx, grn_ii *ii, const char *string, unsigned int string_ grn_wv_mode wvm = grn_wv_none; grn_obj *lexicon = ii->lexicon; grn_scorer_score_func *score_func = NULL; - void *score_func_user_data = NULL; grn_scorer_matched_record record; if (!lexicon || !ii || !s) { return GRN_INVALID_ARGUMENT; } @@ -6240,7 +6320,6 @@ grn_ii_select(grn_ctx *ctx, grn_ii *ii, const char *string, unsigned int string_ if (optarg && optarg->scorer) { grn_proc *scorer = (grn_proc *)(optarg->scorer); score_func = scorer->callbacks.scorer.score; - score_func_user_data = scorer->user_data; record.table = grn_ctx_at(ctx, s->obj.header.domain); record.lexicon = lexicon; record.id = GRN_ID_NIL; @@ -6252,6 +6331,8 @@ grn_ii_select(grn_ctx *ctx, grn_ii *ii, const char *string, unsigned int string_ record.n_candidates = 0; record.n_tokens = 0; record.weight = 0; + record.args_expr = optarg->scorer_args_expr; + record.args_expr_offset = optarg->scorer_args_expr_offset; } for (;;) { @@ -6397,6 +6478,93 @@ exit : return rc; } +uint32_t +grn_ii_estimate_size_for_query(grn_ctx *ctx, grn_ii *ii, + const char *query, unsigned int query_len, + grn_search_optarg *optarg) +{ + grn_rc rc; + grn_obj *lexicon = ii->lexicon; + token_info **tis = NULL; + uint32_t i; + uint32_t n_tis = 0; + grn_bool only_skip_token = GRN_FALSE; + grn_operator mode = GRN_OP_EXACT; + double estimated_size = 0; + + if (query_len == 0) { + return 0; + } + + tis = GRN_MALLOC(sizeof(token_info *) * query_len * 2); + if (!tis) { + return 0; + } + + if (optarg) { + switch (optarg->mode) { + case GRN_OP_NEAR : + case GRN_OP_NEAR2 : + mode = optarg->mode; + break; + case GRN_OP_SIMILAR : + mode = optarg->mode; + break; + case GRN_OP_REGEXP : + mode = optarg->mode; + break; + default : + break; + } + } + + rc = token_info_build(ctx, lexicon, ii, query, query_len, + tis, &n_tis, &only_skip_token, mode); + if (rc != GRN_SUCCESS) { + goto exit; + } + + for (i = 0; i < n_tis; i++) { + token_info *ti = tis[i]; + double term_estimated_size; + term_estimated_size = ((double)ti->size / ti->ntoken); + if (i == 0) { + estimated_size = term_estimated_size; + } else { + estimated_size = fmin(estimated_size, term_estimated_size); + } + } + +exit : + for (i = 0; i < n_tis; i++) { + token_info *ti = tis[i]; + if (ti) { + token_info_close(ctx, ti); + } + } + if (tis) { + GRN_FREE(tis); + } + + return estimated_size; +} + +uint32_t +grn_ii_estimate_size_for_lexicon_cursor(grn_ctx *ctx, grn_ii *ii, + grn_table_cursor *lexicon_cursor) +{ + grn_id term_id; + uint32_t estimated_size = 0; + + while ((term_id = grn_table_cursor_next(ctx, lexicon_cursor)) != GRN_ID_NIL) { + uint32_t term_estimated_size; + term_estimated_size = grn_ii_estimate_size(ctx, ii, term_id); + estimated_size += term_estimated_size; + } + + return estimated_size; +} + grn_rc grn_ii_sel(grn_ctx *ctx, grn_ii *ii, const char *string, unsigned int string_len, grn_hash *s, grn_operator op, grn_search_optarg *optarg) @@ -6417,6 +6585,9 @@ grn_ii_sel(grn_ctx *ctx, grn_ii *ii, const char *string, unsigned int string_len arg.mode = optarg->mode; arg.similarity_threshold = optarg->similarity_threshold; break; + case GRN_OP_REGEXP : + arg.mode = optarg->mode; + break; default : break; } @@ -6425,6 +6596,8 @@ grn_ii_sel(grn_ctx *ctx, grn_ii *ii, const char *string, unsigned int string_len arg.vector_size = optarg->vector_size; } arg.scorer = optarg->scorer; + arg.scorer_args_expr = optarg->scorer_args_expr; + arg.scorer_args_expr_offset = optarg->scorer_args_expr_offset; } /* todo : support subrec grn_rset_init(ctx, s, grn_rec_document, 0, grn_rec_none, 0, 0); @@ -6755,8 +6928,12 @@ grn_ii_inspect_values(grn_ctx *ctx, grn_ii *ii, grn_obj *buf) /********************** buffered index builder ***********************/ -const grn_id II_BUFFER_RID_FLAG = 0x80000000; -const grn_id II_BUFFER_WEIGHT_FLAG = 0x40000000; +const grn_id II_BUFFER_TYPE_MASK = 0xc0000000; +#define II_BUFFER_TYPE_RID 0x80000000 +#define II_BUFFER_TYPE_WEIGHT 0x40000000 +#define II_BUFFER_TYPE(id) (((id) & II_BUFFER_TYPE_MASK)) +#define II_BUFFER_PACK(value, type) ((value) | (type)) +#define II_BUFFER_UNPACK(id, type) ((id) & ~(type)) #ifdef II_BUFFER_ORDER_BY_ID const int II_BUFFER_ORDER = GRN_CURSOR_BY_ID; #else /* II_BUFFER_ORDER_BY_ID */ @@ -6932,7 +7109,7 @@ encode_terms(grn_ctx *ctx, grn_ii_buffer *ii_buffer, } if (outbufp_ + II_BUFFER_BLOCK_READ_UNIT_SIZE < outbufp) { uint32_t size = outbufp - outbufp_ + sizeof(uint32_t); - memcpy(pnext, &size, sizeof(uint32_t)); + grn_memcpy(pnext, &size, sizeof(uint32_t)); pnext = outbufp; outbufp += sizeof(uint32_t); outbufp_ = outbufp; @@ -6941,7 +7118,7 @@ encode_terms(grn_ctx *ctx, grn_ii_buffer *ii_buffer, grn_table_cursor_close(ctx, tc); if (outbufp_ < outbufp) { uint32_t size = outbufp - outbufp_; - memcpy(pnext, &size, sizeof(uint32_t)); + grn_memcpy(pnext, &size, sizeof(uint32_t)); } return outbufp - outbuf; } @@ -6958,59 +7135,66 @@ encode_postings(grn_ctx *ctx, grn_ii_buffer *ii_buffer, uint8_t *outbuf) uint32_t flags = ii_buffer->ii->header->flags; for (rest = ii_buffer->block_pos; rest; bp++, rest--) { grn_id id = *bp; - if (id & II_BUFFER_RID_FLAG) { - rid = id - II_BUFFER_RID_FLAG; + switch (II_BUFFER_TYPE(id)) { + case II_BUFFER_TYPE_RID : + rid = II_BUFFER_UNPACK(id, II_BUFFER_TYPE_RID); if ((flags & GRN_OBJ_WITH_SECTION) && rest) { sid = *++bp; rest--; } weight = 0; pos = 0; - } else if (id & II_BUFFER_WEIGHT_FLAG) { - weight = id - II_BUFFER_WEIGHT_FLAG; - } else { - ii_buffer_counter *counter = &ii_buffer->counters[id - 1]; - if (counter->last_rid == rid && counter->last_sid == sid) { - counter->last_tf++; - counter->last_weight += weight; - } else { - if (counter->last_tf) { - uint8_t *p = outbuf + counter->offset_tf; - GRN_B_ENC(counter->last_tf - 1, p); - counter->offset_tf = p - outbuf; - if (flags & GRN_OBJ_WITH_WEIGHT) { - p = outbuf + counter->offset_weight; - GRN_B_ENC(counter->last_weight, p); - counter->offset_weight = p - outbuf; + break; + case II_BUFFER_TYPE_WEIGHT : + weight = II_BUFFER_UNPACK(id, II_BUFFER_TYPE_WEIGHT); + break; + default : + { + ii_buffer_counter *counter = &ii_buffer->counters[id - 1]; + if (counter->last_rid == rid && counter->last_sid == sid) { + counter->last_tf++; + counter->last_weight += weight; + } else { + if (counter->last_tf) { + uint8_t *p = outbuf + counter->offset_tf; + GRN_B_ENC(counter->last_tf - 1, p); + counter->offset_tf = p - outbuf; + if (flags & GRN_OBJ_WITH_WEIGHT) { + p = outbuf + counter->offset_weight; + GRN_B_ENC(counter->last_weight, p); + counter->offset_weight = p - outbuf; + } } - } - { - uint8_t *p = outbuf + counter->offset_rid; - GRN_B_ENC(rid - counter->last_rid, p); - counter->offset_rid = p - outbuf; - } - if (flags & GRN_OBJ_WITH_SECTION) { - uint8_t *p = outbuf + counter->offset_sid; - if (counter->last_rid != rid) { - GRN_B_ENC(sid - 1, p); - } else { - GRN_B_ENC(sid - counter->last_sid - 1, p); + { + uint8_t *p = outbuf + counter->offset_rid; + GRN_B_ENC(rid - counter->last_rid, p); + counter->offset_rid = p - outbuf; } - counter->offset_sid = p - outbuf; + if (flags & GRN_OBJ_WITH_SECTION) { + uint8_t *p = outbuf + counter->offset_sid; + if (counter->last_rid != rid) { + GRN_B_ENC(sid - 1, p); + } else { + GRN_B_ENC(sid - counter->last_sid - 1, p); + } + counter->offset_sid = p - outbuf; + } + counter->last_rid = rid; + counter->last_sid = sid; + counter->last_tf = 1; + counter->last_weight = weight; + counter->last_pos = 0; + } + if ((flags & GRN_OBJ_WITH_POSITION) && rest) { + uint8_t *p = outbuf + counter->offset_pos; + pos = *++bp; + rest--; + GRN_B_ENC(pos - counter->last_pos, p); + counter->offset_pos = p - outbuf; + counter->last_pos = pos; } - counter->last_rid = rid; - counter->last_sid = sid; - counter->last_tf = 1; - counter->last_weight = weight; - counter->last_pos = 0; - } - if (flags & GRN_OBJ_WITH_POSITION) { - uint8_t *p = outbuf + counter->offset_pos; - GRN_B_ENC(pos - counter->last_pos, p); - counter->offset_pos = p - outbuf; - counter->last_pos = pos; } - pos++; + break; } } } @@ -7046,7 +7230,7 @@ grn_ii_buffer_flush(grn_ctx *ctx, grn_ii_buffer *ii_buffer) encode_postings(ctx, ii_buffer, outbuf); encode_last_tf(ctx, ii_buffer, outbuf); { - ssize_t r = GRN_WRITE(ii_buffer->tmpfd, outbuf, encsize); + ssize_t r = grn_write(ii_buffer->tmpfd, outbuf, encsize); if (r != encsize) { ERR(GRN_INPUT_OUTPUT_ERROR, "write returned %" GRN_FMT_LLD " != %" GRN_FMT_LLU, (long long int)r, (unsigned long long int)encsize); @@ -7126,7 +7310,7 @@ grn_ii_buffer_tokenize(grn_ctx *ctx, grn_ii_buffer *ii_buffer, grn_id rid, { if (value_len) { grn_obj *tmp_lexicon; - uint32_t est_len = value_len + 2; + uint32_t est_len = value_len * 2 + 2; if (ii_buffer->block_buf_size < ii_buffer->block_pos + est_len) { grn_ii_buffer_flush(ctx, ii_buffer); } @@ -7142,24 +7326,27 @@ grn_ii_buffer_tokenize(grn_ctx *ctx, grn_ii_buffer *ii_buffer, grn_id rid, grn_token_cursor *token_cursor; grn_id *buffer = ii_buffer->block_buf; uint32_t block_pos = ii_buffer->block_pos; - buffer[block_pos++] = rid + II_BUFFER_RID_FLAG; - if ((ii_buffer->ii->header->flags & GRN_OBJ_WITH_SECTION)) { + uint32_t ii_flags = ii_buffer->ii->header->flags; + buffer[block_pos++] = II_BUFFER_PACK(rid, II_BUFFER_TYPE_RID); + if (ii_flags & GRN_OBJ_WITH_SECTION) { buffer[block_pos++] = sid; } if (weight) { - buffer[block_pos++] = weight + II_BUFFER_WEIGHT_FLAG; + buffer[block_pos++] = II_BUFFER_PACK(weight, II_BUFFER_TYPE_WEIGHT); } if ((token_cursor = grn_token_cursor_open(ctx, tmp_lexicon, value, value_len, GRN_TOKEN_ADD, token_flags))) { - uint32_t pos; - for (pos = 0; !token_cursor->status; pos++) { + while (!token_cursor->status) { grn_id tid; if ((tid = grn_token_cursor_next(ctx, token_cursor))) { ii_buffer_counter *counter; counter = get_buffer_counter(ctx, ii_buffer, tmp_lexicon, tid); if (!counter) { return; } buffer[block_pos++] = tid; + if (ii_flags & GRN_OBJ_WITH_POSITION) { + buffer[block_pos++] = token_cursor->pos; + } if (counter->last_rid != rid) { counter->offset_rid += GRN_B_ENC_SIZE(rid - counter->last_rid); counter->last_rid = rid; @@ -7187,8 +7374,9 @@ grn_ii_buffer_tokenize(grn_ctx *ctx, grn_ii_buffer *ii_buffer, grn_id rid, counter->last_pos = 0; counter->nrecs++; } - counter->offset_pos += GRN_B_ENC_SIZE(pos - counter->last_pos); - counter->last_pos = pos; + counter->offset_pos += + GRN_B_ENC_SIZE(token_cursor->pos - counter->last_pos); + counter->last_pos = token_cursor->pos; counter->last_tf++; counter->last_weight += weight; counter->nposts++; @@ -7219,11 +7407,20 @@ grn_ii_buffer_fetch(grn_ctx *ctx, grn_ii_buffer *ii_buffer, return; } } - if (grn_lseek(ii_buffer->tmpfd, block->head, SEEK_SET) != block->head) { - SERR("grn_lseek"); - return; + { + off64_t seeked_position; + seeked_position = grn_lseek(ii_buffer->tmpfd, block->head, SEEK_SET); + if (seeked_position != block->head) { + ERRNO_ERR("grn_lseek"); + GRN_LOG(ctx, GRN_LOG_ERROR, + "failed to " + "grn_lseek(%" GRN_FMT_OFF64_T ") -> %" GRN_FMT_OFF64_T, + block->head, + seeked_position); + return; + } } - if (read(ii_buffer->tmpfd, block->buffer, bytesize) != bytesize) { + if (grn_read(ii_buffer->tmpfd, block->buffer, bytesize) != bytesize) { SERR("read"); return; } @@ -7239,8 +7436,8 @@ grn_ii_buffer_fetch(grn_ctx *ctx, grn_ii_buffer *ii_buffer, block->nextsize = 0; } else { block->rest = block->nextsize - sizeof(uint32_t); - memcpy(&block->nextsize, - &block->buffer[block->rest], sizeof(uint32_t)); + grn_memcpy(&block->nextsize, + &block->buffer[block->rest], sizeof(uint32_t)); } } } @@ -7528,16 +7725,10 @@ grn_ii_buffer_open(grn_ctx *ctx, grn_ii *ii, if (ii_buffer->counters) { ii_buffer->block_buf = GRN_MALLOCN(grn_id, II_BUFFER_BLOCK_SIZE); if (ii_buffer->block_buf) { - int open_flags = 0; -#ifdef WIN32 - open_flags |= O_BINARY; -#endif - snprintf(ii_buffer->tmpfpath, PATH_MAX, - "%sXXXXXX", grn_io_path(ii->seg)); + grn_snprintf(ii_buffer->tmpfpath, PATH_MAX, PATH_MAX, + "%sXXXXXX", grn_io_path(ii->seg)); ii_buffer->block_buf_size = II_BUFFER_BLOCK_SIZE; - ii_buffer->tmpfd = GRN_MKOSTEMP(ii_buffer->tmpfpath, - open_flags, - S_IRUSR|S_IWUSR); + ii_buffer->tmpfd = grn_mkstemp(ii_buffer->tmpfpath); if (ii_buffer->tmpfd != -1) { grn_obj_flags flags; grn_table_get_info(ctx, ii->lexicon, &flags, NULL, NULL, NULL, NULL); @@ -7577,7 +7768,7 @@ grn_ii_buffer_commit(grn_ctx *ctx, grn_ii_buffer *ii_buffer) grn_ii_buffer_flush(ctx, ii_buffer); } if (ii_buffer->tmpfd != -1) { - GRN_CLOSE(ii_buffer->tmpfd); + grn_close(ii_buffer->tmpfd); } if (ii_buffer->block_buf) { GRN_FREE(ii_buffer->block_buf); @@ -7604,13 +7795,11 @@ grn_ii_buffer_commit(grn_ctx *ctx, grn_ii_buffer *ii_buffer) ii_buffer->nblocks, ii_buffer->update_buffer_size); datavec_init(ctx, ii_buffer->data_vectors, ii_buffer->ii->n_elements, 0, 0); -#ifdef WIN32 - ii_buffer->tmpfd = GRN_OPEN(ii_buffer->tmpfpath, O_RDONLY|O_BINARY); -#else /* WIN32 */ - ii_buffer->tmpfd = GRN_OPEN(ii_buffer->tmpfpath, O_RDONLY); -#endif /* WIN32 */ + grn_open(ii_buffer->tmpfd, + ii_buffer->tmpfpath, + O_RDONLY | GRN_OPEN_FLAG_BINARY); if (ii_buffer->tmpfd == -1) { - SERR("oepn"); + ERRNO_ERR("oepn"); return ctx->rc; } { @@ -7652,8 +7841,8 @@ grn_ii_buffer_commit(grn_ctx *ctx, grn_ii_buffer *ii_buffer) GRN_LOG(ctx, GRN_LOG_NOTICE, "tmpfile_size:%" GRN_FMT_INT64D " > total_chunk_size:%" GRN_FMT_SIZE, ii_buffer->filepos, ii_buffer->total_chunk_size); - GRN_CLOSE(ii_buffer->tmpfd); - unlink(ii_buffer->tmpfpath); + grn_close(ii_buffer->tmpfd); + grn_unlink(ii_buffer->tmpfpath); ii_buffer->tmpfd = -1; return ctx->rc; } @@ -7671,8 +7860,8 @@ grn_ii_buffer_close(grn_ctx *ctx, grn_ii_buffer *ii_buffer) grn_obj_close(ctx, ii_buffer->tmp_lexicon); } if (ii_buffer->tmpfd != -1) { - GRN_CLOSE(ii_buffer->tmpfd); - unlink(ii_buffer->tmpfpath); + grn_close(ii_buffer->tmpfd); + grn_unlink(ii_buffer->tmpfpath); } if (ii_buffer->block_buf) { GRN_FREE(ii_buffer->block_buf); @@ -7749,7 +7938,21 @@ grn_ii_buffer_parse(grn_ctx *ctx, grn_ii_buffer *ii_buffer, grn_rc grn_ii_build(grn_ctx *ctx, grn_ii *ii, uint64_t sparsity) { - grn_ii_buffer *ii_buffer = grn_ii_buffer_open(ctx, ii, sparsity); + grn_ii_buffer *ii_buffer; + + { + grn_obj *data_table; + + data_table = grn_ctx_at(ctx, DB_OBJ(ii)->range); + if (!data_table) { + return ctx->rc; + } + if (grn_table_size(ctx, data_table) == 0) { + return ctx->rc; + } + } + + ii_buffer = grn_ii_buffer_open(ctx, ii, sparsity); if (ii_buffer) { grn_id *s = ii->obj.source; if ((ii->obj.source_size) && s) { diff --git a/storage/mroonga/vendor/groonga/lib/io.c b/storage/mroonga/vendor/groonga/lib/io.c index 2ea581b8db7..a78c7f242c9 100644 --- a/storage/mroonga/vendor/groonga/lib/io.c +++ b/storage/mroonga/vendor/groonga/lib/io.c @@ -29,9 +29,13 @@ #include "grn_hash.h" #include "grn_ctx_impl.h" +#ifdef WIN32 +# include <share.h> +#endif /* WIN32 */ + #define GRN_IO_IDSTR "GROONGA:IO:00001" -#define GRN_IO_VERSION_DEFAULT 0 +#define GRN_IO_VERSION_DEFAULT 1 #define GRN_IO_FILE_SIZE_V1 1073741824UL @@ -41,14 +45,6 @@ # define GRN_IO_FILE_SIZE_V0 GRN_IO_FILE_SIZE_V1 #endif /* WIN32 */ -#ifndef O_BINARY -# ifdef _O_BINARY -# define O_BINARY _O_BINARY -# else -# define O_BINARY 0 -# endif -#endif - typedef struct _grn_io_fileinfo { #ifdef WIN32 HANDLE fh; @@ -65,10 +61,10 @@ typedef struct _grn_io_fileinfo { static uint32_t grn_io_version_default = GRN_IO_VERSION_DEFAULT; -inline static grn_rc grn_open(grn_ctx *ctx, fileinfo *fi, const char *path, int flags); +inline static grn_rc grn_fileinfo_open(grn_ctx *ctx, fileinfo *fi, const char *path, int flags); inline static void grn_fileinfo_init(fileinfo *fis, int nfis); -inline static int grn_opened(fileinfo *fi); -inline static grn_rc grn_close(grn_ctx *ctx, fileinfo *fi); +inline static int grn_fileinfo_opened(fileinfo *fi); +inline static grn_rc grn_fileinfo_close(grn_ctx *ctx, fileinfo *fi); #ifdef WIN32 inline static void * grn_mmap(grn_ctx *ctx, grn_io *io, HANDLE *fmo, fileinfo *fi, @@ -106,10 +102,12 @@ inline static grn_rc grn_pwrite(grn_ctx *ctx, fileinfo *fi, void *buf, size_t co grn_rc grn_io_init(void) { - const char *version_env; + char version_env[GRN_ENV_BUFFER_SIZE]; - version_env = getenv("GRN_IO_VERSION"); - if (version_env) { + grn_getenv("GRN_IO_VERSION", + version_env, + GRN_ENV_BUFFER_SIZE); + if (version_env[0]) { grn_io_version_default = atoi(version_env); } @@ -194,7 +192,7 @@ grn_io_create_tmp(uint32_t header_size, uint32_t segment_size, header->n_arrays = 0; header->flags = flags; header->lock = 0; - memcpy(header->idstr, GRN_IO_IDSTR, 16); + grn_memcpy(header->idstr, GRN_IO_IDSTR, 16); if ((io = GRN_GMALLOCN(grn_io, 1))) { grn_io_mapinfo *maps = NULL; if ((maps = GRN_GCALLOC(sizeof(grn_io_mapinfo) * max_segment))) { @@ -283,7 +281,7 @@ grn_io_create(grn_ctx *ctx, const char *path, uint32_t header_size, uint32_t seg bs, file_size); if ((fis = GRN_GMALLOCN(fileinfo, max_nfiles))) { grn_fileinfo_init(fis, max_nfiles); - if (!grn_open(ctx, fis, path, O_RDWR|O_CREAT|O_EXCL)) { + if (!grn_fileinfo_open(ctx, fis, path, O_RDWR|O_CREAT|O_EXCL)) { header = (struct _grn_io_header *)GRN_MMAP(&grn_gctx, NULL, &fis->fmo, fis, 0, b); if (header) { @@ -294,12 +292,12 @@ grn_io_create(grn_ctx *ctx, const char *path, uint32_t header_size, uint32_t seg header->n_arrays = 0; header->flags = flags; header->lock = 0; - memcpy(header->idstr, GRN_IO_IDSTR, 16); + grn_memcpy(header->idstr, GRN_IO_IDSTR, 16); grn_msync(ctx, header, b); if ((io = GRN_GMALLOCN(grn_io, 1))) { grn_io_mapinfo *maps = NULL; if ((maps = GRN_GCALLOC(sizeof(grn_io_mapinfo) * max_segment))) { - strncpy(io->path, path, PATH_MAX); + grn_strncpy(io->path, PATH_MAX, path, PATH_MAX); io->header = header; io->user_header = (((byte *) header) + IO_HEADER_SIZE); io->maps = maps; @@ -321,7 +319,8 @@ grn_io_create(grn_ctx *ctx, const char *path, uint32_t header_size, uint32_t seg } GRN_MUNMAP(&grn_gctx, NULL, &fis->fmo, fis, header, b); } - grn_close(ctx, fis); + grn_fileinfo_close(ctx, fis); + grn_unlink(path); } GRN_GFREE(fis); } @@ -395,7 +394,7 @@ grn_io_create_with_array(grn_ctx *ctx, const char *path, if ((io = grn_io_create(ctx, path, header_size + hsize, segment_size, nsegs, mode, GRN_IO_EXPIRE_GTICK))) { hp = io->user_header; - memcpy(hp, array_specs, sizeof(grn_io_array_spec) * n_arrays); + grn_memcpy(hp, array_specs, sizeof(grn_io_array_spec) * n_arrays); io->header->n_arrays = n_arrays; io->header->segment_tail = 1; if (!array_init_(io, n_arrays, hsize, msize)) { @@ -474,11 +473,12 @@ grn_io_detect_type(grn_ctx *ctx, const char *path) { struct _grn_io_header h; uint32_t res = 0; - int fd = GRN_OPEN(path, O_RDWR | O_BINARY); + int fd; + grn_open(fd, path, O_RDWR | GRN_OPEN_FLAG_BINARY); if (fd != -1) { struct stat s; if (fstat(fd, &s) != -1 && s.st_size >= sizeof(struct _grn_io_header)) { - if (read(fd, &h, sizeof(struct _grn_io_header)) == sizeof(struct _grn_io_header)) { + if (grn_read(fd, &h, sizeof(struct _grn_io_header)) == sizeof(struct _grn_io_header)) { if (!memcmp(h.idstr, GRN_IO_IDSTR, 16)) { res = h.type; } else { @@ -490,9 +490,9 @@ grn_io_detect_type(grn_ctx *ctx, const char *path) } else { ERR(GRN_INVALID_FORMAT, "grn_io_detect_type failed"); } - GRN_CLOSE(fd); + grn_close(fd); } else { - SERR(path); + ERRNO_ERR(path); } return res; } @@ -509,10 +509,14 @@ grn_io_open(grn_ctx *ctx, const char *path, grn_io_mode mode) if (!path || !*path || (strlen(path) > PATH_MAX - 4)) { return NULL; } { struct _grn_io_header h; - int fd = GRN_OPEN(path, O_RDWR | O_BINARY); - if (fd == -1) { SERR(path); return NULL; } + int fd; + grn_open(fd, path, O_RDWR | GRN_OPEN_FLAG_BINARY); + if (fd == -1) { + ERRNO_ERR(path); + return NULL; + } if (fstat(fd, &s) != -1 && s.st_size >= sizeof(struct _grn_io_header)) { - if (read(fd, &h, sizeof(struct _grn_io_header)) == sizeof(struct _grn_io_header)) { + if (grn_read(fd, &h, sizeof(struct _grn_io_header)) == sizeof(struct _grn_io_header)) { if (!memcmp(h.idstr, GRN_IO_IDSTR, 16)) { header_size = h.header_size; segment_size = h.segment_size; @@ -523,13 +527,13 @@ grn_io_open(grn_ctx *ctx, const char *path, grn_io_mode mode) } } } - GRN_CLOSE(fd); + grn_close(fd); if (!segment_size) { return NULL; } } b = grn_io_compute_base(header_size); bs = grn_io_compute_base_segment(b, segment_size); grn_fileinfo_init(&fi, 1); - if (!grn_open(ctx, &fi, path, O_RDWR)) { + if (!grn_fileinfo_open(ctx, &fi, path, O_RDWR)) { struct _grn_io_header *header; header = GRN_MMAP(&grn_gctx, NULL, &(fi.fmo), &fi, 0, b); if (header) { @@ -543,15 +547,15 @@ grn_io_open(grn_ctx *ctx, const char *path, grn_io_mode mode) fis = GRN_GMALLOCN(fileinfo, max_nfiles); if (!fis) { GRN_MUNMAP(&grn_gctx, NULL, &(fi.fmo), &fi, header, b); - grn_close(ctx, &fi); + grn_fileinfo_close(ctx, &fi); return NULL; } grn_fileinfo_init(fis, max_nfiles); - memcpy(fis, &fi, sizeof(fileinfo)); + grn_memcpy(fis, &fi, sizeof(fileinfo)); if ((io = GRN_GMALLOC(sizeof(grn_io)))) { grn_io_mapinfo *maps = NULL; if ((maps = GRN_GCALLOC(sizeof(grn_io_mapinfo) * max_segment))) { - strncpy(io->path, path, PATH_MAX); + grn_strncpy(io->path, PATH_MAX, path, PATH_MAX); io->header = header; io->user_header = (((byte *) header) + IO_HEADER_SIZE); { @@ -578,7 +582,7 @@ grn_io_open(grn_ctx *ctx, const char *path, grn_io_mode mode) GRN_GFREE(fis); GRN_MUNMAP(&grn_gctx, NULL, &(fi.fmo), &fi, header, b); } - grn_close(ctx, &fi); + grn_fileinfo_close(ctx, &fi); } return NULL; } @@ -624,7 +628,7 @@ grn_io_close(grn_ctx *ctx, grn_io *io) int i; for (i = 0; i < max_nfiles; i++) { fileinfo *fi = &(io->fis[i]); - grn_close(ctx, fi); + grn_fileinfo_close(ctx, fi); } GRN_GFREE(io->fis); } @@ -671,7 +675,7 @@ inline static void gen_pathname(const char *path, char *buffer, int fno) { size_t len = strlen(path); - memcpy(buffer, path, len); + grn_memcpy(buffer, path, len); if (fno) { buffer[len] = '.'; grn_itoh(fno, buffer + len + 1, 3); @@ -716,8 +720,8 @@ grn_io_remove(grn_ctx *ctx, const char *path) if (stat(path, &s)) { SERR("stat"); return ctx->rc; - } else if (unlink(path)) { - SERR(path); + } else if (grn_unlink(path)) { + ERRNO_ERR(path); return ctx->rc; } else { int fno; @@ -725,7 +729,9 @@ grn_io_remove(grn_ctx *ctx, const char *path) for (fno = 1; ; fno++) { gen_pathname(path, buffer, fno); if (!stat(buffer, &s)) { - if (unlink(buffer)) { SERR(buffer); } + if (grn_unlink(buffer)) { + ERRNO_ERR(buffer); + } } else { break; } @@ -790,10 +796,10 @@ grn_io_read_ja(grn_io *io, grn_ctx *ctx, grn_io_ja_einfo *einfo, uint32_t epos, rest = pos + size - file_size; size = file_size - pos; } - if (!grn_opened(fi)) { + if (!grn_fileinfo_opened(fi)) { char path[PATH_MAX]; gen_pathname(io->path, path, fno); - if (grn_open(ctx, fi, path, O_RDWR|O_CREAT)) { + if (grn_fileinfo_open(ctx, fi, path, O_RDWR|O_CREAT)) { *value = NULL; *value_len = 0; GRN_FREE(v); @@ -838,10 +844,10 @@ grn_io_read_ja(grn_io *io, grn_ctx *ctx, grn_io_ja_einfo *einfo, uint32_t epos, byte *vr = (byte *)v + size; do { fi = &io->fis[++fno]; - if (!grn_opened(fi)) { + if (!grn_fileinfo_opened(fi)) { char path[PATH_MAX]; gen_pathname(io->path, path, fno); - if (grn_open(ctx, fi, path, O_RDWR|O_CREAT)) { + if (grn_fileinfo_open(ctx, fi, path, O_RDWR|O_CREAT)) { *value = NULL; *value_len = 0; GRN_FREE(v); @@ -881,16 +887,16 @@ grn_io_write_ja(grn_io *io, grn_ctx *ctx, uint32_t key, rest = pos + size - file_size; size = file_size - pos; } - if (!grn_opened(fi)) { + if (!grn_fileinfo_opened(fi)) { char path[PATH_MAX]; gen_pathname(io->path, path, fno); - if ((rc = grn_open(ctx, fi, path, O_RDWR|O_CREAT))) { return rc; } + if ((rc = grn_fileinfo_open(ctx, fi, path, O_RDWR|O_CREAT))) { return rc; } } if (value_len <= 256) { ja_element je; je.head.size = value_len; je.head.key = key; - memcpy(je.body, value, value_len); + grn_memcpy(je.body, value, value_len); rc = grn_pwrite(ctx, fi, &je, size, pos); } else { grn_io_ja_ehead eh; @@ -905,10 +911,10 @@ grn_io_write_ja(grn_io *io, grn_ctx *ctx, uint32_t key, byte *vr = (byte *)value + size - sizeof(grn_io_ja_ehead); do { fi = &io->fis[++fno]; - if (!grn_opened(fi)) { + if (!grn_fileinfo_opened(fi)) { char path[PATH_MAX]; gen_pathname(io->path, path, fno); - if ((rc = grn_open(ctx, fi, path, O_RDWR|O_CREAT))) { return rc; } + if ((rc = grn_fileinfo_open(ctx, fi, path, O_RDWR|O_CREAT))) { return rc; } } size = rest > file_size ? file_size : rest; if ((rc = grn_pwrite(ctx, fi, vr, size, 0))) { return rc; } @@ -932,10 +938,10 @@ grn_io_write_ja_ehead(grn_io *io, grn_ctx *ctx, uint32_t key, fileinfo *fi = &io->fis[fno]; off_t base = fno ? 0 : io->base - (uint64_t)segment_size + io->base_seg; off_t pos = (uint64_t)segment_size * (bseg % segments_per_file) + offset + base; - if (!grn_opened(fi)) { + if (!grn_fileinfo_opened(fi)) { char path[PATH_MAX]; gen_pathname(io->path, path, fno); - if ((rc = grn_open(ctx, fi, path, O_RDWR|O_CREAT))) { return rc; } + if ((rc = grn_fileinfo_open(ctx, fi, path, O_RDWR|O_CREAT))) { return rc; } } { grn_io_ja_ehead eh; @@ -987,7 +993,7 @@ grn_io_win_map(grn_io *io, grn_ctx *ctx, grn_io_win *iw, uint32_t segment, return NULL; } s = (offset + r > segment_size) ? segment_size - offset : r; - memcpy(p, q + offset, s); + grn_memcpy(p, q + offset, s); GRN_IO_SEG_UNREF(io, segment); } } @@ -1027,7 +1033,7 @@ grn_io_win_unmap(grn_io_win *iw) GRN_IO_SEG_REF(io, segment, q); if (!q) { return GRN_NO_MEMORY_AVAILABLE; } s = (offset + r > segment_size) ? segment_size - offset : r; - memcpy(q + offset, p, s); + grn_memcpy(q + offset, p, s); GRN_IO_SEG_UNREF(io, segment); } } @@ -1063,10 +1069,10 @@ grn_io_win_unmap(grn_io_win *iw) off_t base = fno ? 0 : io->base - (uint64_t)segment_size * io->base_seg;\ off_t pos = (uint64_t)segment_size * (bseg % segments_per_file) + base;\ fileinfo *fi = &io->fis[fno];\ - if (!grn_opened(fi)) {\ + if (!grn_fileinfo_opened(fi)) {\ char path[PATH_MAX];\ gen_pathname(io->path, path, fno);\ - if (!grn_open(ctx, fi, path, O_RDWR|O_CREAT)) { \ + if (!grn_fileinfo_open(ctx, fi, path, O_RDWR|O_CREAT)) { \ DO_MAP(io, &info->fmo, fi, pos, segment_size, segno, info->map);\ }\ } else {\ @@ -1312,7 +1318,7 @@ static size_t mmap_size = 0; #ifdef WIN32 inline static grn_rc -grn_open_v1(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) +grn_fileinfo_open_v1(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) { CRITICAL_SECTION_INIT(fi->cs); return GRN_SUCCESS; @@ -1390,7 +1396,7 @@ grn_munmap_v1(grn_ctx *ctx, HANDLE *fmo, fileinfo *fi, } inline static grn_rc -grn_open_v0(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) +grn_fileinfo_open_v0(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) { /* signature may be wrong.. */ fi->fmo = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, NULL); @@ -1475,7 +1481,7 @@ grn_munmap_v0(grn_ctx *ctx, fileinfo *fi, void *start, size_t length) } inline static grn_rc -grn_open_common(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) +grn_fileinfo_open_common(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) { /* may be wrong if flags is just only O_RDWR */ if ((flags & O_CREAT)) { @@ -1520,7 +1526,7 @@ exit : } inline static grn_rc -grn_open(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) +grn_fileinfo_open(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) { grn_rc rc; struct _grn_io_header io_header; @@ -1528,7 +1534,7 @@ grn_open(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) DWORD read_bytes; int version = grn_io_version_default; - rc = grn_open_common(ctx, fi, path, flags); + rc = grn_fileinfo_open_common(ctx, fi, path, flags); if (rc != GRN_SUCCESS) { return rc; } @@ -1543,9 +1549,9 @@ grn_open(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) } if (version == 0) { - return grn_open_v0(ctx, fi, path, flags); + return grn_fileinfo_open_v0(ctx, fi, path, flags); } else { - return grn_open_v1(ctx, fi, path, flags); + return grn_fileinfo_open_v1(ctx, fi, path, flags); } } @@ -1598,7 +1604,7 @@ grn_munmap(grn_ctx *ctx, grn_io *io, } inline static grn_rc -grn_close(grn_ctx *ctx, fileinfo *fi) +grn_fileinfo_close(grn_ctx *ctx, fileinfo *fi) { if (fi->fmo != NULL) { CloseHandle(fi->fmo); @@ -1622,7 +1628,7 @@ grn_fileinfo_init(fileinfo *fis, int nfis) } inline static int -grn_opened(fileinfo *fi) +grn_fileinfo_opened(fileinfo *fi) { return fi->fh != INVALID_HANDLE_VALUE; } @@ -1681,15 +1687,16 @@ grn_pwrite(grn_ctx *ctx, fileinfo *fi, void *buf, size_t count, off_t offset) #else /* WIN32 */ inline static grn_rc -grn_open(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) +grn_fileinfo_open(grn_ctx *ctx, fileinfo *fi, const char *path, int flags) { struct stat st; - if ((fi->fd = GRN_OPEN(path, flags, GRN_IO_FILE_CREATE_MODE)) == -1) { - SERR(path); + grn_open(fi->fd, path, flags); + if (fi->fd == -1) { + ERRNO_ERR(path); return ctx->rc; } if (fstat(fi->fd, &st) == -1) { - SERR(path); + ERRNO_ERR(path); return ctx->rc; } fi->dev = st.st_dev; @@ -1704,16 +1711,16 @@ grn_fileinfo_init(fileinfo *fis, int nfis) } inline static int -grn_opened(fileinfo *fi) +grn_fileinfo_opened(fileinfo *fi) { return fi->fd != -1; } inline static grn_rc -grn_close(grn_ctx *ctx, fileinfo *fi) +grn_fileinfo_close(grn_ctx *ctx, fileinfo *fi) { if (fi->fd != -1) { - if (GRN_CLOSE(fi->fd) == -1) { + if (grn_close(fi->fd) == -1) { SERR("close"); return ctx->rc; } @@ -1764,11 +1771,17 @@ grn_fail_mmap(grn_ctx *ctx, grn_io *io, fileinfo *fi, const char* file, int line, const char *func) { if (grn_fail_malloc_check(length, file, line, func)) { - return grn_mmap(ctx, fi, offset, length); + return grn_mmap(ctx, io, fi, offset, length); } else { MERR("fail_mmap(%" GRN_FMT_SIZE ",%d,%" GRN_FMT_LLU ") " "(%s:%d@%s) <%" GRN_FMT_SIZE ">", - length, fi ? fi->fd : 0, offset, file, line, func, mmap_size); + length, + fi ? fi->fd : 0, + (long long unsigned int)offset, + file, + line, + func, + mmap_size); return NULL; } } diff --git a/storage/mroonga/vendor/groonga/lib/logger.c b/storage/mroonga/vendor/groonga/lib/logger.c new file mode 100644 index 00000000000..fc9fe71767e --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/logger.c @@ -0,0 +1,584 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2009-2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "grn_logger.h" +#include "grn_ctx.h" +#include "grn_ctx_impl.h" + +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> + +#ifdef WIN32 +# include <share.h> +#endif /* WIN32 */ + +#ifdef WIN32 +# define fileno(file) _fileno(file) +#endif + +static void +rotate_log_file(grn_ctx *ctx, const char *current_path) +{ + char rotated_path[PATH_MAX]; + grn_timeval now; + struct tm tm_buffer; + struct tm *tm; + + grn_timeval_now(ctx, &now); + tm = grn_timeval2tm(ctx, &now, &tm_buffer); + grn_snprintf(rotated_path, PATH_MAX, PATH_MAX, + "%s.%04d-%02d-%02d-%02d-%02d-%02d-%06d", + current_path, + tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, + tm->tm_hour, tm->tm_min, tm->tm_sec, + (int)(GRN_TIME_NSEC_TO_USEC(now.tv_nsec))); + rename(current_path, rotated_path); +} + +static char *default_logger_path = NULL; +static FILE *default_logger_file = NULL; +static grn_critical_section default_logger_lock; +static off_t default_logger_size = 0; +static off_t default_logger_rotate_threshold_size = 0; + +#define LOGGER_NEED_ROTATE(size, threshold) \ + ((threshold) > 0 && (size) >= (threshold)) + +static void +default_logger_log(grn_ctx *ctx, grn_log_level level, + const char *timestamp, const char *title, + const char *message, const char *location, void *user_data) +{ + const char slev[] = " EACewnid-"; + if (default_logger_path) { + CRITICAL_SECTION_ENTER(default_logger_lock); + if (!default_logger_file) { + default_logger_file = grn_fopen(default_logger_path, "a"); + default_logger_size = 0; + if (default_logger_file) { + struct stat stat; + if (fstat(fileno(default_logger_file), &stat) != -1) { + default_logger_size = stat.st_size; + } + } + } + if (default_logger_file) { + int written; + if (location && *location) { + written = fprintf(default_logger_file, "%s|%c|%s %s %s\n", + timestamp, *(slev + level), title, message, location); + } else { + written = fprintf(default_logger_file, "%s|%c|%s %s\n", timestamp, + *(slev + level), title, message); + } + if (written > 0) { + default_logger_size += written; + if (LOGGER_NEED_ROTATE(default_logger_size, + default_logger_rotate_threshold_size)) { + fclose(default_logger_file); + default_logger_file = NULL; + rotate_log_file(ctx, default_logger_path); + } else { + fflush(default_logger_file); + } + } + } + CRITICAL_SECTION_LEAVE(default_logger_lock); + } +} + +static void +default_logger_reopen(grn_ctx *ctx, void *user_data) +{ + GRN_LOG(ctx, GRN_LOG_NOTICE, "log will be closed."); + CRITICAL_SECTION_ENTER(default_logger_lock); + if (default_logger_file) { + fclose(default_logger_file); + default_logger_file = NULL; + } + CRITICAL_SECTION_LEAVE(default_logger_lock); + GRN_LOG(ctx, GRN_LOG_NOTICE, "log opened."); +} + +static void +default_logger_fin(grn_ctx *ctx, void *user_data) +{ + CRITICAL_SECTION_ENTER(default_logger_lock); + if (default_logger_file) { + fclose(default_logger_file); + default_logger_file = NULL; + } + CRITICAL_SECTION_LEAVE(default_logger_lock); +} + +static grn_logger default_logger = { + GRN_LOG_DEFAULT_LEVEL, + GRN_LOG_TIME|GRN_LOG_MESSAGE, + NULL, + default_logger_log, + default_logger_reopen, + default_logger_fin +}; + +static grn_logger current_logger = { + GRN_LOG_DEFAULT_LEVEL, + GRN_LOG_TIME|GRN_LOG_MESSAGE, + NULL, + NULL, + NULL, + NULL +}; + +void +grn_default_logger_set_max_level(grn_log_level max_level) +{ + default_logger.max_level = max_level; + if (current_logger.log == default_logger_log) { + current_logger.max_level = max_level; + } +} + +grn_log_level +grn_default_logger_get_max_level(void) +{ + return default_logger.max_level; +} + +void +grn_default_logger_set_path(const char *path) +{ + if (default_logger_path) { + free(default_logger_path); + } + + if (path) { + default_logger_path = grn_strdup_raw(path); + } else { + default_logger_path = NULL; + } +} + +const char * +grn_default_logger_get_path(void) +{ + return default_logger_path; +} + +void +grn_default_logger_set_rotate_threshold_size(off_t threshold) +{ + default_logger_rotate_threshold_size = threshold; +} + +off_t +grn_default_logger_get_rotate_threshold_size(void) +{ + return default_logger_rotate_threshold_size; +} + +void +grn_logger_reopen(grn_ctx *ctx) +{ + if (current_logger.reopen) { + current_logger.reopen(ctx, current_logger.user_data); + } +} + +static void +current_logger_fin(grn_ctx *ctx) +{ + if (current_logger.fin) { + current_logger.fin(ctx, current_logger.user_data); + } +} + +static void +logger_info_func_wrapper(grn_ctx *ctx, grn_log_level level, + const char *timestamp, const char *title, + const char *message, const char *location, + void *user_data) +{ + grn_logger_info *info = user_data; + info->func(level, timestamp, title, message, location, info->func_arg); +} + +/* Deprecated since 2.1.2. */ +grn_rc +grn_logger_info_set(grn_ctx *ctx, const grn_logger_info *info) +{ + if (info) { + grn_logger logger; + + memset(&logger, 0, sizeof(grn_logger)); + logger.max_level = info->max_level; + logger.flags = info->flags; + if (info->func) { + logger.log = logger_info_func_wrapper; + logger.user_data = (grn_logger_info *)info; + } else { + logger.log = default_logger_log; + logger.reopen = default_logger_reopen; + logger.fin = default_logger_fin; + } + return grn_logger_set(ctx, &logger); + } else { + return grn_logger_set(ctx, NULL); + } +} + +grn_rc +grn_logger_set(grn_ctx *ctx, const grn_logger *logger) +{ + current_logger_fin(ctx); + if (logger) { + current_logger = *logger; + } else { + current_logger = default_logger; + } + return GRN_SUCCESS; +} + +void +grn_logger_set_max_level(grn_ctx *ctx, grn_log_level max_level) +{ + current_logger.max_level = max_level; +} + +grn_log_level +grn_logger_get_max_level(grn_ctx *ctx) +{ + return current_logger.max_level; +} + +grn_bool +grn_logger_pass(grn_ctx *ctx, grn_log_level level) +{ + return level <= current_logger.max_level; +} + +#define TBUFSIZE GRN_TIMEVAL_STR_SIZE +#define MBUFSIZE 0x1000 +#define LBUFSIZE 0x400 + +void +grn_logger_put(grn_ctx *ctx, grn_log_level level, + const char *file, int line, const char *func, const char *fmt, ...) +{ + if (level <= current_logger.max_level && current_logger.log) { + char tbuf[TBUFSIZE]; + char mbuf[MBUFSIZE]; + char lbuf[LBUFSIZE]; + tbuf[0] = '\0'; + if (current_logger.flags & GRN_LOG_TIME) { + grn_timeval tv; + grn_timeval_now(ctx, &tv); + grn_timeval2str(ctx, &tv, tbuf, TBUFSIZE); + } + if (current_logger.flags & GRN_LOG_MESSAGE) { + va_list argp; + va_start(argp, fmt); + vsnprintf(mbuf, MBUFSIZE - 1, fmt, argp); + va_end(argp); + mbuf[MBUFSIZE - 1] = '\0'; + } else { + mbuf[0] = '\0'; + } + if (current_logger.flags & GRN_LOG_LOCATION) { + grn_snprintf(lbuf, LBUFSIZE, LBUFSIZE, + "%d %s:%d %s()", getpid(), file, line, func); + lbuf[LBUFSIZE - 1] = '\0'; + } else { + lbuf[0] = '\0'; + } + current_logger.log(ctx, level, tbuf, "", mbuf, lbuf, + current_logger.user_data); + } +} + +void +grn_logger_init(void) +{ + if (!default_logger_path) { + default_logger_path = grn_strdup_raw(GRN_LOG_PATH); + } + grn_memcpy(¤t_logger, &default_logger, sizeof(grn_logger)); + CRITICAL_SECTION_INIT(default_logger_lock); +} + +void +grn_logger_fin(grn_ctx *ctx) +{ + current_logger_fin(ctx); + if (default_logger_path) { + free(default_logger_path); + default_logger_path = NULL; + } + CRITICAL_SECTION_FIN(default_logger_lock); +} + + +static char *default_query_logger_path = NULL; +static FILE *default_query_logger_file = NULL; +static grn_critical_section default_query_logger_lock; +static off_t default_query_logger_size = 0; +static off_t default_query_logger_rotate_threshold_size = 0; + +static void +default_query_logger_log(grn_ctx *ctx, unsigned int flag, + const char *timestamp, const char *info, + const char *message, void *user_data) +{ + if (default_query_logger_path) { + CRITICAL_SECTION_ENTER(default_query_logger_lock); + if (!default_query_logger_file) { + default_query_logger_file = grn_fopen(default_query_logger_path, "a"); + default_query_logger_size = 0; + if (default_query_logger_file) { + struct stat stat; + if (fstat(fileno(default_query_logger_file), &stat) != -1) { + default_query_logger_size = stat.st_size; + } + } + } + if (default_query_logger_file) { + int written; + written = fprintf(default_query_logger_file, "%s|%s%s\n", + timestamp, info, message); + if (written > 0) { + default_query_logger_size += written; + if (LOGGER_NEED_ROTATE(default_query_logger_size, + default_query_logger_rotate_threshold_size)) { + fclose(default_query_logger_file); + default_query_logger_file = NULL; + rotate_log_file(ctx, default_query_logger_path); + } else { + fflush(default_query_logger_file); + } + } + } + CRITICAL_SECTION_LEAVE(default_query_logger_lock); + } +} + +static void +default_query_logger_close(grn_ctx *ctx, void *user_data) +{ + GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_DESTINATION, " ", + "query log will be closed: <%s>", default_query_logger_path); + CRITICAL_SECTION_ENTER(default_query_logger_lock); + if (default_query_logger_file) { + fclose(default_query_logger_file); + default_query_logger_file = NULL; + } + CRITICAL_SECTION_LEAVE(default_query_logger_lock); +} + +static void +default_query_logger_reopen(grn_ctx *ctx, void *user_data) +{ + default_query_logger_close(ctx, user_data); + if (default_query_logger_path) { + GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_DESTINATION, " ", + "query log is opened: <%s>", default_query_logger_path); + } +} + +static void +default_query_logger_fin(grn_ctx *ctx, void *user_data) +{ + if (default_query_logger_file) { + default_query_logger_close(ctx, user_data); + } +} + +static grn_query_logger default_query_logger = { + GRN_QUERY_LOG_DEFAULT, + NULL, + default_query_logger_log, + default_query_logger_reopen, + default_query_logger_fin +}; + +static grn_query_logger current_query_logger = { + GRN_QUERY_LOG_DEFAULT, + NULL, + NULL, + NULL, + NULL +}; + +void +grn_default_query_logger_set_flags(unsigned int flags) +{ + default_query_logger.flags = flags; + if (current_query_logger.log == default_query_logger_log) { + current_query_logger.flags = flags; + } +} + +unsigned int +grn_default_query_logger_get_flags(void) +{ + return default_query_logger.flags; +} + +void +grn_default_query_logger_set_path(const char *path) +{ + if (default_query_logger_path) { + free(default_query_logger_path); + } + + if (path) { + default_query_logger_path = grn_strdup_raw(path); + } else { + default_query_logger_path = NULL; + } +} + +const char * +grn_default_query_logger_get_path(void) +{ + return default_query_logger_path; +} + +void +grn_default_query_logger_set_rotate_threshold_size(off_t threshold) +{ + default_query_logger_rotate_threshold_size = threshold; +} + +off_t +grn_default_query_logger_get_rotate_threshold_size(void) +{ + return default_query_logger_rotate_threshold_size; +} + +void +grn_query_logger_reopen(grn_ctx *ctx) +{ + if (current_query_logger.reopen) { + current_query_logger.reopen(ctx, current_query_logger.user_data); + } +} + +static void +current_query_logger_fin(grn_ctx *ctx) +{ + if (current_query_logger.fin) { + current_query_logger.fin(ctx, current_query_logger.user_data); + } +} + +grn_rc +grn_query_logger_set(grn_ctx *ctx, const grn_query_logger *logger) +{ + current_query_logger_fin(ctx); + if (logger) { + current_query_logger = *logger; + } else { + current_query_logger = default_query_logger; + } + return GRN_SUCCESS; +} + +grn_bool +grn_query_logger_pass(grn_ctx *ctx, unsigned int flag) +{ + return current_query_logger.flags & flag; +} + +#define TIMESTAMP_BUFFER_SIZE TBUFSIZE +/* 8+a(%p) + 1(|) + 1(mark) + 15(elapsed time) = 25+a */ +#define INFO_BUFFER_SIZE 40 + +void +grn_query_logger_put(grn_ctx *ctx, unsigned int flag, const char *mark, + const char *format, ...) +{ + char timestamp[TIMESTAMP_BUFFER_SIZE]; + char info[INFO_BUFFER_SIZE]; + grn_obj *message = &ctx->impl->query_log_buf; + + if (!current_query_logger.log) { + return; + } + + { + grn_timeval tv; + timestamp[0] = '\0'; + grn_timeval_now(ctx, &tv); + grn_timeval2str(ctx, &tv, timestamp, TIMESTAMP_BUFFER_SIZE); + } + + if (flag & (GRN_QUERY_LOG_COMMAND | GRN_QUERY_LOG_DESTINATION)) { + grn_snprintf(info, INFO_BUFFER_SIZE, INFO_BUFFER_SIZE, + "%p|%s", ctx, mark); + info[INFO_BUFFER_SIZE - 1] = '\0'; + } else { + grn_timeval tv; + uint64_t elapsed_time; + grn_timeval_now(ctx, &tv); + elapsed_time = + (uint64_t)(tv.tv_sec - ctx->impl->tv.tv_sec) * GRN_TIME_NSEC_PER_SEC + + (tv.tv_nsec - ctx->impl->tv.tv_nsec); + + grn_snprintf(info, INFO_BUFFER_SIZE, INFO_BUFFER_SIZE, + "%p|%s%015" GRN_FMT_INT64U " ", ctx, mark, elapsed_time); + info[INFO_BUFFER_SIZE - 1] = '\0'; + } + + { + va_list args; + + va_start(args, format); + GRN_BULK_REWIND(message); + grn_text_vprintf(ctx, message, format, args); + va_end(args); + GRN_TEXT_PUTC(ctx, message, '\0'); + } + + current_query_logger.log(ctx, flag, timestamp, info, GRN_TEXT_VALUE(message), + current_query_logger.user_data); +} + +void +grn_query_logger_init(void) +{ + grn_memcpy(¤t_query_logger, + &default_query_logger, sizeof(grn_query_logger)); + CRITICAL_SECTION_INIT(default_query_logger_lock); +} + +void +grn_query_logger_fin(grn_ctx *ctx) +{ + current_query_logger_fin(ctx); + if (default_query_logger_path) { + free(default_query_logger_path); + } + CRITICAL_SECTION_FIN(default_query_logger_lock); +} + +void +grn_log_reopen(grn_ctx *ctx) +{ + grn_logger_reopen(ctx); + grn_query_logger_reopen(ctx); +} diff --git a/storage/mroonga/vendor/groonga/lib/mrb.c b/storage/mroonga/vendor/groonga/lib/mrb.c index 4d1e4b5a218..60d2172d520 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb.c +++ b/storage/mroonga/vendor/groonga/lib/mrb.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2013 Brazil + Copyright(C) 2013-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -28,6 +28,10 @@ #include <ctype.h> +#ifdef WIN32 +# include <share.h> +#endif /* WIN32 */ + #define BUFFER_SIZE 2048 #define E_LOAD_ERROR (mrb_class_get(mrb, "LoadError")) @@ -45,9 +49,9 @@ grn_mrb_get_default_system_ruby_scripts_dir(void) base_dir = grn_win32_base_dir(); base_dir_length = strlen(base_dir); - strcpy(win32_ruby_scripts_dir_buffer, base_dir); - strcat(win32_ruby_scripts_dir_buffer, "/"); - strcat(win32_ruby_scripts_dir_buffer, relative_path); + grn_strcpy(win32_ruby_scripts_dir_buffer, PATH_MAX, base_dir); + grn_strcat(win32_ruby_scripts_dir_buffer, PATH_MAX, "/"); + grn_strcat(win32_ruby_scripts_dir_buffer, PATH_MAX, relative_path); win32_ruby_scripts_dir = win32_ruby_scripts_dir_buffer; } return win32_ruby_scripts_dir; @@ -64,14 +68,16 @@ grn_mrb_get_default_system_ruby_scripts_dir(void) const char * grn_mrb_get_system_ruby_scripts_dir(grn_ctx *ctx) { - const char *ruby_scripts_dir; + static char ruby_scripts_dir[GRN_ENV_BUFFER_SIZE]; - ruby_scripts_dir = getenv("GRN_RUBY_SCRIPTS_DIR"); - if (!ruby_scripts_dir) { - ruby_scripts_dir = grn_mrb_get_default_system_ruby_scripts_dir(); + grn_getenv("GRN_RUBY_SCRIPTS_DIR", + ruby_scripts_dir, + GRN_ENV_BUFFER_SIZE); + if (ruby_scripts_dir[0]) { + return ruby_scripts_dir; + } else { + return grn_mrb_get_default_system_ruby_scripts_dir(); } - - return ruby_scripts_dir; } static grn_bool @@ -89,7 +95,8 @@ grn_mrb_is_absolute_path(const char *path) } static grn_bool -grn_mrb_expand_script_path(grn_ctx *ctx, const char *path, char *expanded_path) +grn_mrb_expand_script_path(grn_ctx *ctx, const char *path, + char *expanded_path, size_t expanded_path_size) { const char *ruby_scripts_dir; char dir_last_char; @@ -98,15 +105,15 @@ grn_mrb_expand_script_path(grn_ctx *ctx, const char *path, char *expanded_path) if (grn_mrb_is_absolute_path(path)) { expanded_path[0] = '\0'; } else if (path[0] == '.' && path[1] == '/') { - strcpy(expanded_path, ctx->impl->mrb.base_directory); - strcat(expanded_path, "/"); + grn_strcpy(expanded_path, expanded_path_size, ctx->impl->mrb.base_directory); + grn_strcat(expanded_path, expanded_path_size, "/"); } else { ruby_scripts_dir = grn_mrb_get_system_ruby_scripts_dir(ctx); - strcpy(expanded_path, ruby_scripts_dir); + grn_strcpy(expanded_path, expanded_path_size, ruby_scripts_dir); dir_last_char = ruby_scripts_dir[strlen(expanded_path) - 1]; if (dir_last_char != '/') { - strcat(expanded_path, "/"); + grn_strcat(expanded_path, expanded_path_size, "/"); } } @@ -120,7 +127,7 @@ grn_mrb_expand_script_path(grn_ctx *ctx, const char *path, char *expanded_path) return GRN_FALSE; } - strcat(expanded_path, path); + grn_strcat(expanded_path, expanded_path_size, path); return GRN_TRUE; } @@ -139,16 +146,17 @@ grn_mrb_load(grn_ctx *ctx, const char *path) return mrb_nil_value(); } - if (!grn_mrb_expand_script_path(ctx, path, expanded_path)) { + if (!grn_mrb_expand_script_path(ctx, path, expanded_path, PATH_MAX)) { return mrb_nil_value(); } - file = fopen(expanded_path, "r"); + file = grn_fopen(expanded_path, "r"); if (!file) { char message[BUFFER_SIZE]; mrb_value exception; - snprintf(message, BUFFER_SIZE - 1, - "fopen: failed to open mruby script file: <%s>", expanded_path); + grn_snprintf(message, BUFFER_SIZE, BUFFER_SIZE, + "fopen: failed to open mruby script file: <%s>", + expanded_path); SERR(message); exception = mrb_exc_new(mrb, E_LOAD_ERROR, ctx->errbuf, strlen(ctx->errbuf)); @@ -160,8 +168,8 @@ grn_mrb_load(grn_ctx *ctx, const char *path) char current_base_directory[PATH_MAX]; char *last_directory; - strcpy(current_base_directory, data->base_directory); - strcpy(data->base_directory, expanded_path); + grn_strcpy(current_base_directory, PATH_MAX, data->base_directory); + grn_strcpy(data->base_directory, PATH_MAX, expanded_path); last_directory = strrchr(data->base_directory, '/'); if (last_directory) { last_directory[0] = '\0'; @@ -181,7 +189,7 @@ grn_mrb_load(grn_ctx *ctx, const char *path) } mrb_parser_free(parser); - strcpy(data->base_directory, current_base_directory); + grn_strcpy(data->base_directory, PATH_MAX, current_base_directory); } return result; diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_accessor.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_accessor.c index bd288af8491..daa0caada06 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_accessor.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_accessor.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2013-2014 Brazil + Copyright(C) 2013-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -26,6 +26,7 @@ #include "../grn_db.h" #include "mrb_accessor.h" +#include "mrb_converter.h" static struct mrb_data_type mrb_grn_accessor_type = { "Groonga::Accessor", @@ -49,8 +50,25 @@ mrb_grn_accessor_next(mrb_state *mrb, mrb_value self) grn_accessor *accessor; accessor = DATA_PTR(self); - if (!accessor->next) { return mrb_nil_value(); } - return mrb_cptr_value(mrb, accessor->next); + return grn_mrb_value_from_grn_obj(mrb, (grn_obj *)(accessor->next)); +} + +static mrb_value +mrb_grn_accessor_have_next_p(mrb_state *mrb, mrb_value self) +{ + grn_accessor *accessor; + + accessor = DATA_PTR(self); + return mrb_bool_value(accessor->next != NULL); +} + +static mrb_value +mrb_grn_accessor_object(mrb_state *mrb, mrb_value self) +{ + grn_accessor *accessor; + + accessor = DATA_PTR(self); + return grn_mrb_value_from_grn_obj(mrb, accessor->obj); } void @@ -67,5 +85,9 @@ grn_mrb_accessor_init(grn_ctx *ctx) mrb_grn_accessor_initialize, MRB_ARGS_REQ(1)); mrb_define_method(mrb, klass, "next", mrb_grn_accessor_next, MRB_ARGS_NONE()); + mrb_define_method(mrb, klass, "have_next?", + mrb_grn_accessor_have_next_p, MRB_ARGS_NONE()); + mrb_define_method(mrb, klass, "object", + mrb_grn_accessor_object, MRB_ARGS_NONE()); } #endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_bulk.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_bulk.c index 2a304b68781..eb57a016148 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_bulk.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_bulk.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2014 Brazil + Copyright(C) 2014-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -127,6 +127,13 @@ grn_mrb_value_from_bulk(mrb_state *mrb, grn_obj *bulk) mrb_fixnum_value(usec)); } break; + case GRN_DB_SHORT_TEXT : + case GRN_DB_TEXT : + case GRN_DB_LONG_TEXT : + mrb_value_ = mrb_str_new_static(mrb, + GRN_TEXT_VALUE(bulk), + GRN_TEXT_LEN(bulk)); + break; default : { #define MESSAGE_SIZE 4096 @@ -141,14 +148,14 @@ grn_mrb_value_from_bulk(mrb_state *mrb, grn_obj *bulk) domain_name, GRN_TABLE_MAX_KEY_SIZE); grn_obj_unlink(ctx, domain); } else { - strcpy(domain_name, "unknown"); + grn_strcpy(domain_name, GRN_TABLE_MAX_KEY_SIZE, "unknown"); domain_name_size = strlen(domain_name); } - snprintf(message, MESSAGE_SIZE, - "unsupported bulk value type: <%d>(%.*s)", - bulk->header.domain, - domain_name_size, - domain_name); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "unsupported bulk value type: <%d>(%.*s)", + bulk->header.domain, + domain_name_size, + domain_name); mrb_raise(mrb, E_RANGE_ERROR, message); #undef MESSAGE_SIZE } @@ -158,6 +165,17 @@ grn_mrb_value_from_bulk(mrb_state *mrb, grn_obj *bulk) return mrb_value_; } +grn_bool +grn_mrb_bulk_cast(mrb_state *mrb, grn_obj *from, grn_obj *to, grn_id domain_id) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + grn_rc rc; + + grn_obj_reinit(ctx, to, domain_id, 0); + rc = grn_obj_cast(ctx, from, to, GRN_FALSE); + return rc == GRN_SUCCESS; +} + static mrb_value mrb_grn_bulk_initialize(mrb_state *mrb, mrb_value self) { diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_bulk.h b/storage/mroonga/vendor/groonga/lib/mrb/mrb_bulk.h index e85b81568da..dd1f46fb84e 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_bulk.h +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_bulk.h @@ -32,6 +32,10 @@ mrb_value grn_mrb_value_from_bulk(mrb_state *mrb, grn_obj *bulk); grn_obj *grn_mrb_value_to_bulk(mrb_state *mrb, mrb_value mrb_value_, grn_obj *bulk); +grn_bool grn_mrb_bulk_cast(mrb_state *mrb, + grn_obj *from, + grn_obj *to, + grn_id domain_id); #ifdef __cplusplus } diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_content_type.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_content_type.c new file mode 100644 index 00000000000..7f4ff139de1 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_content_type.c @@ -0,0 +1,49 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "../grn_ctx_impl.h" + +#ifdef GRN_WITH_MRUBY +#include <mruby.h> + +#include "mrb_ctx.h" +#include "mrb_content_type.h" + +void +grn_mrb_content_type_init(grn_ctx *ctx) +{ + grn_mrb_data *data = &(ctx->impl->mrb); + mrb_state *mrb = data->state; + struct RClass *module; + + module = mrb_define_module_under(mrb, data->module, "ContentType"); + + mrb_define_const(mrb, module, "NONE", + mrb_fixnum_value(GRN_CONTENT_NONE)); + mrb_define_const(mrb, module, "TSV", + mrb_fixnum_value(GRN_CONTENT_TSV)); + mrb_define_const(mrb, module, "JSON", + mrb_fixnum_value(GRN_CONTENT_JSON)); + mrb_define_const(mrb, module, "XML", + mrb_fixnum_value(GRN_CONTENT_XML)); + mrb_define_const(mrb, module, "MSGPACK", + mrb_fixnum_value(GRN_CONTENT_MSGPACK)); + mrb_define_const(mrb, module, "GROONGA_COMMAND_LIST", + mrb_fixnum_value(GRN_CONTENT_GROONGA_COMMAND_LIST)); +} +#endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_content_type.h b/storage/mroonga/vendor/groonga/lib/mrb/mrb_content_type.h new file mode 100644 index 00000000000..46103ff1a62 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_content_type.h @@ -0,0 +1,34 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef GRN_MRB_CONTENT_TYPE_H +#define GRN_MRB_CONTENT_TYPE_H + +#include "../grn_ctx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void grn_mrb_content_type_init(grn_ctx *ctx); + +#ifdef __cplusplus +} +#endif + +#endif /* GRN_MRB_CONTENT_TYPE_H */ diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_converter.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_converter.c index 9686feb8bc9..e4c3aceb435 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_converter.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_converter.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2013-2014 Brazil + Copyright(C) 2013-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -23,10 +23,119 @@ #include <mruby.h> #include <mruby/class.h> #include <mruby/data.h> +#include <mruby/string.h> #include "mrb_converter.h" #include "mrb_bulk.h" +void +grn_mrb_value_to_raw_data_buffer_init(mrb_state *mrb, + grn_mrb_value_to_raw_data_buffer *buffer) +{ + GRN_VOID_INIT(&(buffer->from)); + GRN_VOID_INIT(&(buffer->to)); +} + +void +grn_mrb_value_to_raw_data_buffer_fin(mrb_state *mrb, + grn_mrb_value_to_raw_data_buffer *buffer) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + + GRN_OBJ_FIN(ctx, &(buffer->from)); + GRN_OBJ_FIN(ctx, &(buffer->to)); +} + +void +grn_mrb_value_to_raw_data(mrb_state *mrb, + const char *context, + mrb_value mrb_value_, + grn_id domain_id, + grn_mrb_value_to_raw_data_buffer *buffer, + void **raw_value, + unsigned int *raw_value_size) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + enum mrb_vtype mrb_value_type; + grn_bool try_cast = GRN_FALSE; + grn_obj *from_bulk = NULL; + + if (mrb_nil_p(mrb_value_)) { + *raw_value = NULL; + *raw_value_size = 0; + return; + } + + mrb_value_type = mrb_type(mrb_value_); + + switch (mrb_value_type) { + case MRB_TT_STRING : + switch (domain_id) { + case GRN_DB_SHORT_TEXT : + case GRN_DB_TEXT : + case GRN_DB_LONG_TEXT : + *raw_value = RSTRING_PTR(mrb_value_); + *raw_value_size = RSTRING_LEN(mrb_value_); + break; + default : + try_cast = GRN_TRUE; + break; + } + break; + default : + { + struct RClass *klass; + grn_mrb_data *data = &(ctx->impl->mrb); + + klass = mrb_class(mrb, mrb_value_); + if (domain_id == GRN_DB_TIME && + klass == data->builtin.time_class) { + mrb_value mrb_sec; + mrb_value mrb_usec; + + mrb_sec = mrb_funcall(mrb, mrb_value_, "to_i", 0); + mrb_usec = mrb_funcall(mrb, mrb_value_, "usec", 0); + buffer->value.time_value = GRN_TIME_PACK(mrb_fixnum(mrb_sec), + mrb_fixnum(mrb_usec)); + *raw_value = &(buffer->value.time_value); + *raw_value_size = sizeof(buffer->value.time_value); + } else { + try_cast = GRN_TRUE; + if (mrb_value_type == MRB_TT_DATA && + klass == mrb_class_get_under(mrb, data->module, "Bulk")) { + from_bulk = DATA_PTR(mrb_value_); + } + } + } + break; + } + + if (!try_cast) { + return; + } + + if (!from_bulk) { + from_bulk = &(buffer->from); + grn_mrb_value_to_bulk(mrb, mrb_value_, from_bulk); + } + if (!grn_mrb_bulk_cast(mrb, from_bulk, &(buffer->to), domain_id)) { + grn_obj *domain; + char domain_name[GRN_TABLE_MAX_KEY_SIZE]; + int domain_name_size; + + domain = grn_ctx_at(ctx, domain_id); + domain_name_size = grn_obj_name(ctx, domain, domain_name, + GRN_TABLE_MAX_KEY_SIZE); + mrb_raisef(mrb, E_ARGUMENT_ERROR, + "%S: failed to convert to %S: %S", + mrb_str_new_static(mrb, context, strlen(context)), + mrb_str_new_static(mrb, domain_name, domain_name_size), + mrb_value_); + } + *raw_value = GRN_BULK_HEAD(&(buffer->to)); + *raw_value_size = GRN_BULK_VSIZE(&(buffer->to)); +} + struct RClass * grn_mrb_class_from_grn_obj(mrb_state *mrb, grn_obj *object) { @@ -85,8 +194,8 @@ grn_mrb_class_from_grn_obj(mrb_state *mrb, grn_obj *object) if (!klass) { #define BUFFER_SIZE 1024 char buffer[BUFFER_SIZE]; - snprintf(buffer, BUFFER_SIZE - 1, - "can't find class for object type: %#x", object->header.type); + grn_snprintf(buffer, BUFFER_SIZE, BUFFER_SIZE, + "can't find class for object type: %#x", object->header.type); mrb_raise(mrb, E_ARGUMENT_ERROR, buffer); #undef BUFFER_SIZE } @@ -154,16 +263,11 @@ mrb_grn_converter_singleton_convert(mrb_state *mrb, mrb_value klass) grn_mrb_value_to_bulk(mrb, mrb_from, from); to_type = grn_mrb_class_to_type(mrb, mrb_class_ptr(mrb_to_class)); - grn_obj_reinit(ctx, to, to_type, 0); - { - grn_rc rc; - rc = grn_obj_cast(ctx, from, to, GRN_FALSE); - if (rc != GRN_SUCCESS) { - mrb_raisef(mrb, E_ARGUMENT_ERROR, - "failed to convert to %S: %S", - mrb_to_class, - from); - } + if (!grn_mrb_bulk_cast(mrb, from, to, to_type)) { + mrb_raisef(mrb, E_ARGUMENT_ERROR, + "failed to convert to %S: %S", + mrb_to_class, + mrb_from); } return grn_mrb_value_from_bulk(mrb, to); diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_converter.h b/storage/mroonga/vendor/groonga/lib/mrb/mrb_converter.h index 3dfe5e5a01a..c2cb58c0e6d 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_converter.h +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_converter.h @@ -27,6 +27,26 @@ extern "C" { void grn_mrb_converter_init(grn_ctx *ctx); +typedef struct { + grn_obj from; + grn_obj to; + union { + int64_t time_value; + } value; +} grn_mrb_value_to_raw_data_buffer; + +void grn_mrb_value_to_raw_data_buffer_init(mrb_state *mrb, + grn_mrb_value_to_raw_data_buffer *buffer); +void grn_mrb_value_to_raw_data_buffer_fin(mrb_state *mrb, + grn_mrb_value_to_raw_data_buffer *buffer); +void grn_mrb_value_to_raw_data(mrb_state *mrb, + const char *context, + mrb_value mrb_value_, + grn_id domain_id, + grn_mrb_value_to_raw_data_buffer *buffer, + void **raw_value, + unsigned int *raw_value_size); + struct RClass *grn_mrb_class_from_grn_obj(mrb_state *mrb, grn_obj *object); mrb_value grn_mrb_value_from_grn_obj(mrb_state *mrb, grn_obj *object); diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_ctx.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_ctx.c index f52342ae18e..3814313f3d0 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_ctx.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_ctx.c @@ -58,6 +58,10 @@ ctx_array_reference(mrb_state *mrb, mrb_value self) mrb_get_args(mrb, "o", &mrb_id_or_name); + if (mrb_nil_p(mrb_id_or_name)) { + return mrb_nil_value(); + } + if (mrb_fixnum_p(mrb_id_or_name)) { grn_id id = mrb_fixnum(mrb_id_or_name); object = grn_ctx_at(ctx, id); @@ -220,449 +224,449 @@ grn_mrb_ctx_check(mrb_state *mrb) return; case GRN_END_OF_DATA: error_class = mrb_class_get_under(mrb, module, "EndOfData"); - snprintf(message, MESSAGE_SIZE, - "end of data: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "end of data: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_UNKNOWN_ERROR: error_class = mrb_class_get_under(mrb, module, "UnknownError"); - snprintf(message, MESSAGE_SIZE, - "unknown error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "unknown error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_OPERATION_NOT_PERMITTED: error_class = mrb_class_get_under(mrb, module, "OperationNotPermitted"); - snprintf(message, MESSAGE_SIZE, - "operation not permitted: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "operation not permitted: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NO_SUCH_FILE_OR_DIRECTORY: error_class = mrb_class_get_under(mrb, module, "NoSuchFileOrDirectory"); - snprintf(message, MESSAGE_SIZE, - "no such file or directory: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "no such file or directory: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NO_SUCH_PROCESS: error_class = mrb_class_get_under(mrb, module, "NoSuchProcess"); - snprintf(message, MESSAGE_SIZE, - "no such process: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "no such process: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_INTERRUPTED_FUNCTION_CALL: error_class = mrb_class_get_under(mrb, module, "InterruptedFunctionCall"); - snprintf(message, MESSAGE_SIZE, - "interrupted function call: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "interrupted function call: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_INPUT_OUTPUT_ERROR: error_class = mrb_class_get_under(mrb, module, "InputOutputError"); - snprintf(message, MESSAGE_SIZE, - "input output error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "input output error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NO_SUCH_DEVICE_OR_ADDRESS: error_class = mrb_class_get_under(mrb, module, "NoSuchDeviceOrAddress"); - snprintf(message, MESSAGE_SIZE, - "no such device or address: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "no such device or address: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_ARG_LIST_TOO_LONG: error_class = mrb_class_get_under(mrb, module, "ArgListTooLong"); - snprintf(message, MESSAGE_SIZE, - "arg list too long: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "arg list too long: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_EXEC_FORMAT_ERROR: error_class = mrb_class_get_under(mrb, module, "ExecFormatError"); - snprintf(message, MESSAGE_SIZE, - "exec format error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "exec format error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_BAD_FILE_DESCRIPTOR: error_class = mrb_class_get_under(mrb, module, "BadFileDescriptor"); - snprintf(message, MESSAGE_SIZE, - "bad file descriptor: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "bad file descriptor: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NO_CHILD_PROCESSES: error_class = mrb_class_get_under(mrb, module, "NoChildProcesses"); - snprintf(message, MESSAGE_SIZE, - "no child processes: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "no child processes: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_RESOURCE_TEMPORARILY_UNAVAILABLE: error_class = mrb_class_get_under(mrb, module, "ResourceTemporarilyUnavailable"); - snprintf(message, MESSAGE_SIZE, - "resource temporarily unavailable: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "resource temporarily unavailable: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NOT_ENOUGH_SPACE: error_class = mrb_class_get_under(mrb, module, "NotEnoughSpace"); - snprintf(message, MESSAGE_SIZE, - "not enough space: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "not enough space: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_PERMISSION_DENIED: error_class = mrb_class_get_under(mrb, module, "PermissionDenied"); - snprintf(message, MESSAGE_SIZE, - "permission denied: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "permission denied: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_BAD_ADDRESS: error_class = mrb_class_get_under(mrb, module, "BadAddress"); - snprintf(message, MESSAGE_SIZE, - "bad address: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "bad address: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_RESOURCE_BUSY: error_class = mrb_class_get_under(mrb, module, "ResourceBusy"); - snprintf(message, MESSAGE_SIZE, - "resource busy: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "resource busy: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_FILE_EXISTS: error_class = mrb_class_get_under(mrb, module, "FileExists"); - snprintf(message, MESSAGE_SIZE, - "file exists: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "file exists: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_IMPROPER_LINK: error_class = mrb_class_get_under(mrb, module, "ImproperLink"); - snprintf(message, MESSAGE_SIZE, - "improper link: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "improper link: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NO_SUCH_DEVICE: error_class = mrb_class_get_under(mrb, module, "NoSuchDevice"); - snprintf(message, MESSAGE_SIZE, - "no such device: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "no such device: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NOT_A_DIRECTORY: error_class = mrb_class_get_under(mrb, module, "NotDirectory"); - snprintf(message, MESSAGE_SIZE, - "not directory: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "not directory: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_IS_A_DIRECTORY: error_class = mrb_class_get_under(mrb, module, "IsDirectory"); - snprintf(message, MESSAGE_SIZE, - "is directory: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "is directory: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_INVALID_ARGUMENT: error_class = mrb_class_get_under(mrb, module, "InvalidArgument"); - snprintf(message, MESSAGE_SIZE, - "invalid argument: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "invalid argument: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_TOO_MANY_OPEN_FILES_IN_SYSTEM: error_class = mrb_class_get_under(mrb, module, "TooManyOpenFilesInSystem"); - snprintf(message, MESSAGE_SIZE, - "too many open files in system: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "too many open files in system: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_TOO_MANY_OPEN_FILES: error_class = mrb_class_get_under(mrb, module, "TooManyOpenFiles"); - snprintf(message, MESSAGE_SIZE, - "too many open files: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "too many open files: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_INAPPROPRIATE_I_O_CONTROL_OPERATION: error_class = mrb_class_get_under(mrb, module, "InappropriateIOControlOperation"); - snprintf(message, MESSAGE_SIZE, - "inappropriate IO control operation: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "inappropriate IO control operation: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_FILE_TOO_LARGE: error_class = mrb_class_get_under(mrb, module, "FileTooLarge"); - snprintf(message, MESSAGE_SIZE, - "file too large: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "file too large: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NO_SPACE_LEFT_ON_DEVICE: error_class = mrb_class_get_under(mrb, module, "NoSpaceLeftOnDevice"); - snprintf(message, MESSAGE_SIZE, - "no space left on device: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "no space left on device: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_INVALID_SEEK: error_class = mrb_class_get_under(mrb, module, "InvalidSeek"); - snprintf(message, MESSAGE_SIZE, - "invalid seek: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "invalid seek: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_READ_ONLY_FILE_SYSTEM: error_class = mrb_class_get_under(mrb, module, "ReadOnlyFileSystem"); - snprintf(message, MESSAGE_SIZE, - "read only file system: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "read only file system: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_TOO_MANY_LINKS: error_class = mrb_class_get_under(mrb, module, "TooManyLinks"); - snprintf(message, MESSAGE_SIZE, - "too many links: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "too many links: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_BROKEN_PIPE: error_class = mrb_class_get_under(mrb, module, "BrokenPipe"); - snprintf(message, MESSAGE_SIZE, - "broken pipe: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "broken pipe: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_DOMAIN_ERROR: error_class = mrb_class_get_under(mrb, module, "DomainError"); - snprintf(message, MESSAGE_SIZE, - "domain error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "domain error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_RESULT_TOO_LARGE: error_class = mrb_class_get_under(mrb, module, "ResultTooLarge"); - snprintf(message, MESSAGE_SIZE, - "result too large: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "result too large: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_RESOURCE_DEADLOCK_AVOIDED: error_class = mrb_class_get_under(mrb, module, "ResourceDeadlockAvoided"); - snprintf(message, MESSAGE_SIZE, - "resource deadlock avoided: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "resource deadlock avoided: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NO_MEMORY_AVAILABLE: error_class = mrb_class_get_under(mrb, module, "NoMemoryAvailable"); - snprintf(message, MESSAGE_SIZE, - "no memory available: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "no memory available: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_FILENAME_TOO_LONG: error_class = mrb_class_get_under(mrb, module, "FilenameTooLong"); - snprintf(message, MESSAGE_SIZE, - "filename too long: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "filename too long: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NO_LOCKS_AVAILABLE: error_class = mrb_class_get_under(mrb, module, "NoLocksAvailable"); - snprintf(message, MESSAGE_SIZE, - "no locks available: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "no locks available: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_FUNCTION_NOT_IMPLEMENTED: error_class = mrb_class_get_under(mrb, module, "FunctionNotImplemented"); - snprintf(message, MESSAGE_SIZE, - "function not implemented: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "function not implemented: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_DIRECTORY_NOT_EMPTY: error_class = mrb_class_get_under(mrb, module, "DirectoryNotEmpty"); - snprintf(message, MESSAGE_SIZE, - "directory not empty: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "directory not empty: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_ILLEGAL_BYTE_SEQUENCE: error_class = mrb_class_get_under(mrb, module, "IllegalByteSequence"); - snprintf(message, MESSAGE_SIZE, - "illegal byte sequence: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "illegal byte sequence: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_SOCKET_NOT_INITIALIZED: error_class = mrb_class_get_under(mrb, module, "SocketNotInitialized"); - snprintf(message, MESSAGE_SIZE, - "socket not initialized: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "socket not initialized: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_OPERATION_WOULD_BLOCK: error_class = mrb_class_get_under(mrb, module, "OperationWouldBlock"); - snprintf(message, MESSAGE_SIZE, - "operation would block: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "operation would block: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_ADDRESS_IS_NOT_AVAILABLE: error_class = mrb_class_get_under(mrb, module, "AddressIsNotAvailable"); - snprintf(message, MESSAGE_SIZE, - "address is not available: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "address is not available: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NETWORK_IS_DOWN: error_class = mrb_class_get_under(mrb, module, "NetworkIsDown"); - snprintf(message, MESSAGE_SIZE, - "network is down: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "network is down: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NO_BUFFER: error_class = mrb_class_get_under(mrb, module, "NoBuffer"); - snprintf(message, MESSAGE_SIZE, - "no buffer: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "no buffer: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_SOCKET_IS_ALREADY_CONNECTED: error_class = mrb_class_get_under(mrb, module, "SocketIsAlreadyConnected"); - snprintf(message, MESSAGE_SIZE, - "socket is already connected: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "socket is already connected: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_SOCKET_IS_NOT_CONNECTED: error_class = mrb_class_get_under(mrb, module, "SocketIsNotConnected"); - snprintf(message, MESSAGE_SIZE, - "socket is not connected: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "socket is not connected: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_SOCKET_IS_ALREADY_SHUTDOWNED: error_class = mrb_class_get_under(mrb, module, "SocketIsAlreadyShutdowned"); - snprintf(message, MESSAGE_SIZE, - "socket is already shutdowned: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "socket is already shutdowned: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_OPERATION_TIMEOUT: error_class = mrb_class_get_under(mrb, module, "OperationTimeout"); - snprintf(message, MESSAGE_SIZE, - "operation timeout: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "operation timeout: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_CONNECTION_REFUSED: error_class = mrb_class_get_under(mrb, module, "ConnectionRefused"); - snprintf(message, MESSAGE_SIZE, - "connection refused: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "connection refused: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_RANGE_ERROR: error_class = mrb_class_get_under(mrb, module, "RangeError"); - snprintf(message, MESSAGE_SIZE, - "range error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "range error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_TOKENIZER_ERROR: error_class = mrb_class_get_under(mrb, module, "TokenizerError"); - snprintf(message, MESSAGE_SIZE, - "tokenizer error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "tokenizer error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_FILE_CORRUPT: error_class = mrb_class_get_under(mrb, module, "FileCorrupt"); - snprintf(message, MESSAGE_SIZE, - "file corrupt: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "file corrupt: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_INVALID_FORMAT: error_class = mrb_class_get_under(mrb, module, "InvalidFormat"); - snprintf(message, MESSAGE_SIZE, - "invalid format: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "invalid format: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_OBJECT_CORRUPT: error_class = mrb_class_get_under(mrb, module, "ObjectCorrupt"); - snprintf(message, MESSAGE_SIZE, - "object corrupt: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "object corrupt: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_TOO_MANY_SYMBOLIC_LINKS: error_class = mrb_class_get_under(mrb, module, "TooManySymbolicLinks"); - snprintf(message, MESSAGE_SIZE, - "too many symbolic links: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "too many symbolic links: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NOT_SOCKET: error_class = mrb_class_get_under(mrb, module, "NotSocket"); - snprintf(message, MESSAGE_SIZE, - "not socket: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "not socket: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_OPERATION_NOT_SUPPORTED: error_class = mrb_class_get_under(mrb, module, "OperationNotSupported"); - snprintf(message, MESSAGE_SIZE, - "operation not supported: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "operation not supported: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_ADDRESS_IS_IN_USE: error_class = mrb_class_get_under(mrb, module, "AddressIsInUse"); - snprintf(message, MESSAGE_SIZE, - "address is in use: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "address is in use: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_ZLIB_ERROR: error_class = mrb_class_get_under(mrb, module, "ZlibError"); - snprintf(message, MESSAGE_SIZE, - "zlib error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "zlib error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_LZ4_ERROR: error_class = mrb_class_get_under(mrb, module, "LZ4Error"); - snprintf(message, MESSAGE_SIZE, - "LZ4 error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "LZ4 error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_STACK_OVER_FLOW: error_class = mrb_class_get_under(mrb, module, "StackOverFlow"); - snprintf(message, MESSAGE_SIZE, - "stack over flow: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "stack over flow: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_SYNTAX_ERROR: error_class = mrb_class_get_under(mrb, module, "SyntaxError"); - snprintf(message, MESSAGE_SIZE, - "syntax error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "syntax error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_RETRY_MAX: error_class = mrb_class_get_under(mrb, module, "RetryMax"); - snprintf(message, MESSAGE_SIZE, - "retry max: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "retry max: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_INCOMPATIBLE_FILE_FORMAT: error_class = mrb_class_get_under(mrb, module, "IncompatibleFileFormat"); - snprintf(message, MESSAGE_SIZE, - "incompatible file format: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "incompatible file format: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_UPDATE_NOT_ALLOWED: error_class = mrb_class_get_under(mrb, module, "UpdateNotAllowed"); - snprintf(message, MESSAGE_SIZE, - "update not allowed: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "update not allowed: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_TOO_SMALL_OFFSET: error_class = mrb_class_get_under(mrb, module, "TooSmallOffset"); - snprintf(message, MESSAGE_SIZE, - "too small offset: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "too small offset: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_TOO_LARGE_OFFSET: error_class = mrb_class_get_under(mrb, module, "TooLargeOffset"); - snprintf(message, MESSAGE_SIZE, - "too large offset: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "too large offset: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_TOO_SMALL_LIMIT: error_class = mrb_class_get_under(mrb, module, "TooSmallLimit"); - snprintf(message, MESSAGE_SIZE, - "too small limit: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "too small limit: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_CAS_ERROR: error_class = mrb_class_get_under(mrb, module, "CASError"); - snprintf(message, MESSAGE_SIZE, - "CAS error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "CAS error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_UNSUPPORTED_COMMAND_VERSION: error_class = mrb_class_get_under(mrb, module, "UnsupportedCommandVersion"); - snprintf(message, MESSAGE_SIZE, - "unsupported command version: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "unsupported command version: <%s>(%d)", + ctx->errbuf, ctx->rc); break; case GRN_NORMALIZER_ERROR: error_class = mrb_class_get_under(mrb, module, "NormalizerError"); - snprintf(message, MESSAGE_SIZE, - "normalizer error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "normalizer error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; default: error_class = mrb_class_get_under(mrb, module, "Error"); - snprintf(message, MESSAGE_SIZE, - "unsupported error: <%s>(%d)", - ctx->errbuf, ctx->rc); + grn_snprintf(message, MESSAGE_SIZE, MESSAGE_SIZE, + "unsupported error: <%s>(%d)", + ctx->errbuf, ctx->rc); break; } #undef MESSAGE_SIZE diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_expr.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_expr.c index df86d5579e4..dcae12a131a 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_expr.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_expr.c @@ -33,7 +33,9 @@ #include "mrb_accessor.h" #include "mrb_ctx.h" #include "mrb_expr.h" +#include "mrb_operator.h" #include "mrb_converter.h" +#include "mrb_options.h" static struct mrb_data_type mrb_grn_scan_info_type = { "Groonga::ScanInfo", @@ -99,17 +101,35 @@ mrb_grn_expr_code_initialize(mrb_state *mrb, mrb_value self) static mrb_value mrb_grn_scan_info_put_index(mrb_state *mrb, mrb_value self) { + grn_ctx *ctx = (grn_ctx *)mrb->ud; + scan_info *si; + mrb_value mrb_index; int sid; int32_t weight; - scan_info *si; - grn_ctx *ctx = (grn_ctx *)mrb->ud; + mrb_value mrb_scorer; + mrb_value mrb_scorer_args_expr; + int32_t scorer_args_expr_offset; grn_obj *index; - mrb_value mrb_index; - - mrb_get_args(mrb, "oii", &mrb_index, &sid, &weight); + grn_obj *scorer = NULL; + grn_obj *scorer_args_expr = NULL; + + mrb_get_args(mrb, "oiiooi", + &mrb_index, &sid, &weight, + &mrb_scorer, + &mrb_scorer_args_expr, + &scorer_args_expr_offset); si = DATA_PTR(self); index = DATA_PTR(mrb_index); - grn_scan_info_put_index(ctx, si, index, sid, weight); + if (!mrb_nil_p(mrb_scorer)) { + scorer = DATA_PTR(mrb_scorer); + } + if (!mrb_nil_p(mrb_scorer_args_expr)) { + scorer_args_expr = DATA_PTR(mrb_scorer_args_expr); + } + grn_scan_info_put_index(ctx, si, index, sid, weight, + scorer, + scorer_args_expr, + scorer_args_expr_offset); return self; } @@ -121,17 +141,19 @@ mrb_grn_scan_info_get_op(mrb_state *mrb, mrb_value self) si = DATA_PTR(self); op = grn_scan_info_get_op(si); - return mrb_fixnum_value(op); + return grn_mrb_value_from_operator(mrb, op); } static mrb_value mrb_grn_scan_info_set_op(mrb_state *mrb, mrb_value self) { scan_info *si; + mrb_value mrb_op; grn_operator op; - mrb_get_args(mrb, "i", &op); + mrb_get_args(mrb, "o", &mrb_op, &op); si = DATA_PTR(self); + op = grn_mrb_value_to_operator(mrb, mrb_op); grn_scan_info_set_op(si, op); return self; } @@ -191,10 +213,12 @@ static mrb_value mrb_grn_scan_info_set_logical_op(mrb_state *mrb, mrb_value self) { scan_info *si; + mrb_value mrb_logical_op; grn_operator logical_op; - mrb_get_args(mrb, "i", &logical_op); + mrb_get_args(mrb, "o", &mrb_logical_op); si = DATA_PTR(self); + logical_op = grn_mrb_value_to_operator(mrb, mrb_logical_op); grn_scan_info_set_logical_op(si, logical_op); return self; } @@ -207,7 +231,7 @@ mrb_grn_scan_info_get_logical_op(mrb_state *mrb, mrb_value self) si = DATA_PTR(self); logical_op = grn_scan_info_get_logical_op(si); - return mrb_fixnum_value(logical_op); + return grn_mrb_value_from_operator(mrb, logical_op); } static mrb_value @@ -257,33 +281,6 @@ mrb_grn_scan_info_get_similarity_threshold(mrb_state *mrb, mrb_value self) } static mrb_value -mrb_grn_scan_info_set_scorer(mrb_state *mrb, mrb_value self) -{ - scan_info *si; - mrb_value mrb_scorer; - - mrb_get_args(mrb, "o", &mrb_scorer); - si = DATA_PTR(self); - if (mrb_nil_p(mrb_scorer)) { - grn_scan_info_set_scorer(si, NULL); - } else { - grn_scan_info_set_scorer(si, DATA_PTR(mrb_scorer)); - } - return self; -} - -static mrb_value -mrb_grn_scan_info_get_scorer(mrb_state *mrb, mrb_value self) -{ - scan_info *si; - grn_obj *scorer; - - si = DATA_PTR(self); - scorer = grn_scan_info_get_scorer(si); - return grn_mrb_value_from_grn_obj(mrb, scorer); -} - -static mrb_value mrb_grn_scan_info_get_arg(mrb_state *mrb, mrb_value self) { grn_ctx *ctx = (grn_ctx *)mrb->ud; @@ -315,11 +312,86 @@ mrb_grn_scan_info_push_arg(mrb_state *mrb, mrb_value self) } static mrb_value +mrb_grn_expr_code_inspect(mrb_state *mrb, mrb_value self) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + grn_expr_code *code; + mrb_value inspected; + + code = DATA_PTR(self); + + inspected = mrb_str_buf_new(mrb, 48); + + mrb_str_cat_lit(mrb, inspected, "#<"); + mrb_str_cat_cstr(mrb, inspected, mrb_obj_classname(mrb, self)); + mrb_str_cat_lit(mrb, inspected, ":"); + mrb_str_concat(mrb, inspected, mrb_ptr_to_str(mrb, mrb_cptr(self))); + + { + int32_t weight; + uint32_t offset; + + weight = grn_expr_code_get_weight(ctx, DATA_PTR(self), &offset); + + mrb_str_cat_lit(mrb, inspected, " weight="); + mrb_str_concat(mrb, inspected, + mrb_funcall(mrb, + mrb_fixnum_value(weight), + "inspect", + 0)); + mrb_str_cat_lit(mrb, inspected, ", offset="); + mrb_str_concat(mrb, inspected, + mrb_funcall(mrb, + mrb_fixnum_value(offset), + "inspect", + 0)); + } + + mrb_str_cat_lit(mrb, inspected, ", modify="); + mrb_str_concat(mrb, inspected, + mrb_funcall(mrb, + mrb_fixnum_value(code->modify), + "inspect", + 0)); + + mrb_str_cat_lit(mrb, inspected, ", op="); + mrb_str_concat(mrb, inspected, + mrb_funcall(mrb, + grn_mrb_value_from_operator(mrb, code->op), + "inspect", + 0)); + + mrb_str_cat_lit(mrb, inspected, ", flags="); + mrb_str_concat(mrb, inspected, + mrb_funcall(mrb, + mrb_fixnum_value(code->flags), + "inspect", + 0)); + + mrb_str_cat_lit(mrb, inspected, ", value="); + mrb_str_concat(mrb, inspected, + mrb_funcall(mrb, + grn_mrb_value_from_grn_obj(mrb, code->value), + "inspect", + 0)); + + mrb_str_cat_lit(mrb, inspected, ">"); + + return inspected; +} + +static mrb_value mrb_grn_expr_code_get_weight(mrb_state *mrb, mrb_value self) { grn_ctx *ctx = (grn_ctx *)mrb->ud; + int32_t weight; + uint32_t offset; + mrb_value mrb_values[2]; - return mrb_fixnum_value(grn_expr_code_get_weight(ctx, DATA_PTR(self))); + weight = grn_expr_code_get_weight(ctx, DATA_PTR(self), &offset); + mrb_values[0] = mrb_fixnum_value(weight); + mrb_values[1] = mrb_fixnum_value(offset); + return mrb_ary_new_from_values(mrb, 2, mrb_values); } static mrb_value @@ -337,7 +409,7 @@ mrb_grn_expr_code_get_op(mrb_state *mrb, mrb_value self) grn_expr_code *expr_code; expr_code = DATA_PTR(self); - return mrb_fixnum_value(expr_code->op); + return grn_mrb_value_from_operator(mrb, expr_code->op); } static mrb_value @@ -491,8 +563,7 @@ mrb_grn_expression_parse(mrb_state *mrb, mrb_value self) if (!mrb_nil_p(mrb_options)) { mrb_value mrb_flags; - mrb_flags = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "flags"))); + mrb_flags = grn_mrb_options_get_lit(mrb, mrb_options, "flags"); if (!mrb_nil_p(mrb_flags)) { flags = mrb_fixnum(mrb_flags); } @@ -512,13 +583,15 @@ mrb_grn_expression_append_object(mrb_state *mrb, mrb_value self) grn_obj *expr; mrb_value mrb_object; grn_obj *object; + mrb_value mrb_op; grn_operator op; int n_args; expr = DATA_PTR(self); - mrb_get_args(mrb, "oii", &mrb_object, &op, &n_args); + mrb_get_args(mrb, "ooi", &mrb_object, &mrb_op, &n_args); object = DATA_PTR(mrb_object); + op = grn_mrb_value_to_operator(mrb, mrb_op); grn_expr_append_obj(ctx, expr, object, op, n_args); grn_mrb_ctx_check(mrb); @@ -531,12 +604,14 @@ mrb_grn_expression_append_constant(mrb_state *mrb, mrb_value self) grn_ctx *ctx = (grn_ctx *)mrb->ud; grn_obj *expr; mrb_value mrb_constant; + mrb_value mrb_op; grn_operator op; int n_args; expr = DATA_PTR(self); - mrb_get_args(mrb, "oii", &mrb_constant, &op, &n_args); + mrb_get_args(mrb, "ooi", &mrb_constant, &mrb_op, &n_args); + op = grn_mrb_value_to_operator(mrb, mrb_op); switch (mrb_type(mrb_constant)) { case MRB_TT_FALSE : if (mrb_nil_p(mrb_constant)) { @@ -624,12 +699,14 @@ mrb_grn_expression_append_operator(mrb_state *mrb, mrb_value self) { grn_ctx *ctx = (grn_ctx *)mrb->ud; grn_obj *expr; - grn_operator op; + mrb_value mrb_op; int n_args; + grn_operator op; expr = DATA_PTR(self); - mrb_get_args(mrb, "ii", &op, &n_args); + mrb_get_args(mrb, "oi", &mrb_op, &n_args); + op = grn_mrb_value_to_operator(mrb, mrb_op); grn_expr_append_op(ctx, expr, op, n_args); grn_mrb_ctx_check(mrb); @@ -649,7 +726,7 @@ grn_mrb_expr_init(grn_ctx *ctx) mrb_define_method(mrb, klass, "initialize", mrb_grn_scan_info_initialize, MRB_ARGS_REQ(1)); mrb_define_method(mrb, klass, "put_index", - mrb_grn_scan_info_put_index, MRB_ARGS_REQ(3)); + mrb_grn_scan_info_put_index, MRB_ARGS_REQ(6)); mrb_define_method(mrb, klass, "op", mrb_grn_scan_info_get_op, MRB_ARGS_NONE()); mrb_define_method(mrb, klass, "op=", @@ -674,10 +751,6 @@ grn_mrb_expr_init(grn_ctx *ctx) mrb_grn_scan_info_get_similarity_threshold, MRB_ARGS_NONE()); mrb_define_method(mrb, klass, "similarity_threshold=", mrb_grn_scan_info_set_similarity_threshold, MRB_ARGS_REQ(1)); - mrb_define_method(mrb, klass, "scorer", - mrb_grn_scan_info_get_scorer, MRB_ARGS_NONE()); - mrb_define_method(mrb, klass, "scorer=", - mrb_grn_scan_info_set_scorer, MRB_ARGS_REQ(1)); mrb_define_method(mrb, klass, "get_arg", mrb_grn_scan_info_get_arg, MRB_ARGS_REQ(1)); mrb_define_method(mrb, klass, "push_arg", @@ -688,6 +761,8 @@ grn_mrb_expr_init(grn_ctx *ctx) MRB_SET_INSTANCE_TT(klass, MRB_TT_DATA); mrb_define_method(mrb, klass, "initialize", mrb_grn_expr_code_initialize, MRB_ARGS_REQ(1)); + mrb_define_method(mrb, klass, "inspect", + mrb_grn_expr_code_inspect, MRB_ARGS_NONE()); mrb_define_method(mrb, klass, "weight", mrb_grn_expr_code_get_weight, MRB_ARGS_NONE()); mrb_define_method(mrb, klass, "value", @@ -697,6 +772,14 @@ grn_mrb_expr_init(grn_ctx *ctx) mrb_define_method(mrb, klass, "flags", mrb_grn_expr_code_get_flags, MRB_ARGS_NONE()); + { + struct RClass *expression_code_class = klass; + struct RClass *flags_module; + flags_module = mrb_define_module_under(mrb, expression_code_class, "Flags"); + mrb_define_const(mrb, flags_module, "RELATIONAL_EXPRESSION", + mrb_fixnum_value(GRN_EXPR_CODE_RELATIONAL_EXPRESSION)); + } + klass = mrb_define_class_under(mrb, module, "Expression", object_class); MRB_SET_INSTANCE_TT(klass, MRB_TT_DATA); @@ -724,11 +807,6 @@ grn_mrb_expr_init(grn_ctx *ctx) mrb_grn_expression_append_constant, MRB_ARGS_REQ(3)); mrb_define_method(mrb, klass, "append_operator", mrb_grn_expression_append_operator, MRB_ARGS_REQ(2)); - - grn_mrb_load(ctx, "expression.rb"); - grn_mrb_load(ctx, "scan_info.rb"); - grn_mrb_load(ctx, "scan_info_data.rb"); - grn_mrb_load(ctx, "scan_info_builder.rb"); } scan_info ** @@ -747,7 +825,7 @@ grn_mrb_scan_info_build(grn_ctx *ctx, grn_obj *expr, int *n, mrb_expression = grn_mrb_value_from_grn_obj(mrb, expr); mrb_sis = mrb_funcall(mrb, mrb_expression, "build_scan_info", 2, - mrb_fixnum_value(op), + grn_mrb_value_from_operator(mrb, op), mrb_fixnum_value(size)); if (mrb_nil_p(mrb_sis)) { @@ -781,4 +859,31 @@ exit: return sis; } + +unsigned int +grn_mrb_expr_estimate_size(grn_ctx *ctx, grn_obj *expr, grn_obj *table) +{ + grn_mrb_data *data = &(ctx->impl->mrb); + mrb_state *mrb = data->state; + mrb_value mrb_expression; + mrb_value mrb_table; + mrb_value mrb_size; + unsigned int size; + int arena_index; + + arena_index = mrb_gc_arena_save(mrb); + + mrb_expression = grn_mrb_value_from_grn_obj(mrb, expr); + mrb_table = grn_mrb_value_from_grn_obj(mrb, table); + mrb_size = mrb_funcall(mrb, mrb_expression, "estimate_size", 1, mrb_table); + if (mrb->exc) { + size = grn_table_size(ctx, table); + } else { + size = mrb_fixnum(mrb_size); + } + + mrb_gc_arena_restore(mrb, arena_index); + + return size; +} #endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_expr.h b/storage/mroonga/vendor/groonga/lib/mrb/mrb_expr.h index 687565dd6d9..0564401dac9 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_expr.h +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_expr.h @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2013 Brazil + Copyright(C) 2013-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -28,6 +28,9 @@ extern "C" { void grn_mrb_expr_init(grn_ctx *ctx); scan_info **grn_mrb_scan_info_build(grn_ctx *ctx, grn_obj *expr, int *n, grn_operator op, uint32_t size); +unsigned int grn_mrb_expr_estimate_size(grn_ctx *ctx, + grn_obj *expr, + grn_obj *table); #ifdef __cplusplus } diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_hash_table.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_hash_table.c index a5a4e797de2..31d00956137 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_hash_table.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_hash_table.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2014 Brazil + Copyright(C) 2014-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -22,8 +22,11 @@ #include <mruby.h> #include <mruby/class.h> #include <mruby/data.h> +#include <mruby/string.h> +#include "mrb_ctx.h" #include "mrb_hash_table.h" +#include "mrb_options.h" static struct mrb_data_type mrb_grn_hash_table_type = { "Groonga::HashTable", @@ -31,6 +34,56 @@ static struct mrb_data_type mrb_grn_hash_table_type = { }; static mrb_value +mrb_grn_hash_table_singleton_create(mrb_state *mrb, mrb_value klass) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + mrb_value mrb_options = mrb_nil_value(); + const char *name = NULL; + unsigned int name_size = 0; + const char *path = NULL; + grn_obj_flags flags = GRN_OBJ_TABLE_HASH_KEY; + grn_obj *key_type = NULL; + grn_obj *value_type = NULL; + grn_obj *table; + + mrb_get_args(mrb, "|H", &mrb_options); + + if (!mrb_nil_p(mrb_options)) { + mrb_value mrb_name; + mrb_value mrb_flags; + mrb_value mrb_key_type; + mrb_value mrb_value_type; + + mrb_name = grn_mrb_options_get_lit(mrb, mrb_options, "name"); + if (!mrb_nil_p(mrb_name)) { + name = RSTRING_PTR(mrb_name); + name_size = RSTRING_LEN(mrb_name); + } + + mrb_flags = grn_mrb_options_get_lit(mrb, mrb_options, "flags"); + if (!mrb_nil_p(mrb_flags)) { + flags |= mrb_fixnum(mrb_flags); + } + + mrb_key_type = grn_mrb_options_get_lit(mrb, mrb_options, "key_type"); + if (!mrb_nil_p(mrb_key_type)) { + key_type = DATA_PTR(mrb_key_type); + } + + mrb_value_type = grn_mrb_options_get_lit(mrb, mrb_options, "value_type"); + if (!mrb_nil_p(mrb_value_type)) { + key_type = DATA_PTR(mrb_value_type); + } + } + + table = grn_table_create(ctx, name, name_size, path, flags, + key_type, value_type); + grn_mrb_ctx_check(mrb); + + return mrb_funcall(mrb, klass, "new", 1, mrb_cptr_value(mrb, table)); +} + +static mrb_value mrb_grn_hash_table_initialize(mrb_state *mrb, mrb_value self) { mrb_value mrb_hash_table_ptr; @@ -54,6 +107,10 @@ grn_mrb_hash_table_init(grn_ctx *ctx) klass = mrb_define_class_under(mrb, module, "HashTable", table_class); MRB_SET_INSTANCE_TT(klass, MRB_TT_DATA); + mrb_define_singleton_method(mrb, (struct RObject *)klass, "create", + mrb_grn_hash_table_singleton_create, + MRB_ARGS_OPT(1)); + mrb_define_method(mrb, klass, "initialize", mrb_grn_hash_table_initialize, MRB_ARGS_REQ(1)); } diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_index_column.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_index_column.c index d91aa576e8e..eb85ff955fa 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_index_column.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_index_column.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2014 Brazil + Copyright(C) 2014-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -17,13 +17,17 @@ */ #include "../grn_ctx_impl.h" +#include "../grn_ii.h" #ifdef GRN_WITH_MRUBY #include <mruby.h> #include <mruby/class.h> #include <mruby/data.h> +#include "mrb_converter.h" #include "mrb_index_column.h" +#include "mrb_operator.h" +#include "mrb_options.h" static struct mrb_data_type mrb_grn_index_column_type = { "Groonga::IndexColumn", @@ -41,6 +45,84 @@ mrb_grn_index_column_initialize(mrb_state *mrb, mrb_value self) return self; } +static mrb_value +mrb_grn_index_column_get_lexicon(mrb_state *mrb, mrb_value self) +{ + grn_obj *index_column; + grn_obj *lexicon; + + index_column = DATA_PTR(self); + lexicon = ((grn_ii *)index_column)->lexicon; + + return grn_mrb_value_from_grn_obj(mrb, lexicon); +} + +static mrb_value +mrb_grn_index_column_estimate_size_for_term_id(mrb_state *mrb, mrb_value self) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + grn_obj *index_column; + mrb_int term_id; + unsigned int size; + + index_column = DATA_PTR(self); + mrb_get_args(mrb, "i", &term_id); + + size = grn_ii_estimate_size(ctx, (grn_ii *)index_column, term_id); + return mrb_fixnum_value(size); +} + +static mrb_value +mrb_grn_index_column_estimate_size_for_query(mrb_state *mrb, mrb_value self) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + grn_obj *index_column; + char *query; + mrb_int query_len; + mrb_value mrb_options = mrb_nil_value(); + grn_search_optarg optarg; + unsigned int size; + + index_column = DATA_PTR(self); + mrb_get_args(mrb, "s|H", &query, &query_len, &mrb_options); + + memset(&optarg, 0, sizeof(grn_search_optarg)); + optarg.mode = GRN_OP_EXACT; + + if (!mrb_nil_p(mrb_options)) { + mrb_value mrb_mode; + + mrb_mode = grn_mrb_options_get_lit(mrb, mrb_options, "mode"); + if (!mrb_nil_p(mrb_mode)) { + optarg.mode = grn_mrb_value_to_operator(mrb, mrb_mode); + } + } + + size = grn_ii_estimate_size_for_query(ctx, (grn_ii *)index_column, + query, query_len, &optarg); + return mrb_fixnum_value(size); +} + +static mrb_value +mrb_grn_index_column_estimate_size_for_lexicon_cursor(mrb_state *mrb, + mrb_value self) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + grn_obj *index_column; + mrb_value mrb_lexicon_cursor; + grn_table_cursor *lexicon_cursor; + unsigned int size; + + index_column = DATA_PTR(self); + mrb_get_args(mrb, "o", &mrb_lexicon_cursor); + + lexicon_cursor = DATA_PTR(mrb_lexicon_cursor); + size = grn_ii_estimate_size_for_lexicon_cursor(ctx, + (grn_ii *)index_column, + lexicon_cursor); + return mrb_fixnum_value(size); +} + void grn_mrb_index_column_init(grn_ctx *ctx) { @@ -55,5 +137,19 @@ grn_mrb_index_column_init(grn_ctx *ctx) MRB_SET_INSTANCE_TT(klass, MRB_TT_DATA); mrb_define_method(mrb, klass, "initialize", mrb_grn_index_column_initialize, MRB_ARGS_REQ(1)); + + mrb_define_method(mrb, klass, "lexicon", + mrb_grn_index_column_get_lexicon, + MRB_ARGS_NONE()); + + mrb_define_method(mrb, klass, "estimate_size_for_term_id", + mrb_grn_index_column_estimate_size_for_term_id, + MRB_ARGS_REQ(1)); + mrb_define_method(mrb, klass, "estimate_size_for_query", + mrb_grn_index_column_estimate_size_for_query, + MRB_ARGS_ARG(1, 1)); + mrb_define_method(mrb, klass, "estimate_size_for_lexicon_cursor", + mrb_grn_index_column_estimate_size_for_lexicon_cursor, + MRB_ARGS_REQ(1)); } #endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_index_cursor.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_index_cursor.c index f57d59d727c..34133ff7ae0 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_index_cursor.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_index_cursor.c @@ -17,6 +17,8 @@ */ #include "../grn_ctx_impl.h" +#include "../grn_ii.h" +#include "../grn_db.h" #ifdef GRN_WITH_MRUBY #include <mruby.h> @@ -24,9 +26,12 @@ #include <mruby/data.h> #include <mruby/string.h> #include <mruby/hash.h> +#include <mruby/variable.h> #include "mrb_ctx.h" #include "mrb_index_cursor.h" +#include "mrb_converter.h" +#include "mrb_options.h" static struct mrb_data_type mrb_grn_index_cursor_type = { "Groonga::IndexCursor", @@ -46,6 +51,7 @@ mrb_grn_index_cursor_singleton_open_raw(mrb_state *mrb, mrb_value klass) grn_id rid_min = GRN_ID_NIL; grn_id rid_max = GRN_ID_MAX; int flags = 0; + mrb_value mrb_index_cursor; mrb_get_args(mrb, "oo|H", &mrb_table_cursor, &mrb_index, &mrb_options); @@ -58,7 +64,10 @@ mrb_grn_index_cursor_singleton_open_raw(mrb_state *mrb, mrb_value klass) rid_min, rid_max, flags); grn_mrb_ctx_check(mrb); - return mrb_funcall(mrb, klass, "new", 1, mrb_cptr_value(mrb, index_cursor)); + mrb_index_cursor = mrb_funcall(mrb, klass, "new", 1, + mrb_cptr_value(mrb, index_cursor)); + mrb_iv_set(mrb, mrb_index_cursor, mrb_intern_lit(mrb, "@index"), mrb_index); + return mrb_index_cursor; } static mrb_value @@ -103,6 +112,91 @@ mrb_grn_index_cursor_count(mrb_state *mrb, mrb_value self) return mrb_fixnum_value(n_records); } +static mrb_value +mrb_grn_index_cursor_select(mrb_state *mrb, mrb_value self) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + mrb_value mrb_result_set; + mrb_value mrb_options; + grn_obj *index_cursor; + grn_obj *expr = NULL; + grn_obj *expr_variable = NULL; + int offset = 0; + int limit = 10; + int n_matched_records = 0; + mrb_value mrb_index; + grn_obj *index; + grn_obj *lexicon; + grn_obj *data_table; + grn_hash *result_set; + grn_posting *posting; + grn_id term_id; + grn_operator op = GRN_OP_OR; + + mrb_get_args(mrb, "o|H", &mrb_result_set, &mrb_options); + + index_cursor = DATA_PTR(self); + result_set = DATA_PTR(mrb_result_set); + + if (!mrb_nil_p(mrb_options)) { + mrb_value mrb_expr; + mrb_value mrb_offset; + mrb_value mrb_limit; + + mrb_expr = grn_mrb_options_get_lit(mrb, mrb_options, "expression"); + if (!mrb_nil_p(mrb_expr)) { + expr = DATA_PTR(mrb_expr); + expr_variable = grn_expr_get_var_by_offset(ctx, expr, 0); + } + + mrb_offset = grn_mrb_options_get_lit(mrb, mrb_options, "offset"); + if (!mrb_nil_p(mrb_offset)) { + offset = mrb_fixnum(mrb_offset); + } + + mrb_limit = grn_mrb_options_get_lit(mrb, mrb_options, "limit"); + if (!mrb_nil_p(mrb_limit)) { + limit = mrb_fixnum(mrb_limit); + } + } + + if (limit <= 0) { + return mrb_fixnum_value(n_matched_records); + } + + mrb_index = mrb_iv_get(mrb, self, mrb_intern_lit(mrb, "@index")); + index = DATA_PTR(mrb_index); + lexicon = ((grn_ii *)index)->lexicon; + data_table = grn_ctx_at(ctx, grn_obj_get_range(ctx, index)); + + while ((posting = grn_index_cursor_next(ctx, index_cursor, &term_id))) { + if (expr) { + grn_bool matched_raw; + grn_obj *matched; + + GRN_RECORD_SET(ctx, expr_variable, posting->rid); + matched = grn_expr_exec(ctx, expr, 0); + GRN_TRUEP(ctx, matched, matched_raw); + if (!matched_raw) { + continue; + } + } + n_matched_records++; + if (offset > 0) { + offset--; + continue; + } + grn_ii_posting_add(ctx, (grn_ii_posting *)posting, result_set, op); + limit--; + if (limit == 0) { + break; + } + } + grn_ii_resolve_sel_and(ctx, result_set, op); + + return mrb_fixnum_value(n_matched_records); +} + void grn_mrb_index_cursor_init(grn_ctx *ctx) { @@ -124,5 +218,7 @@ grn_mrb_index_cursor_init(grn_ctx *ctx) mrb_grn_index_cursor_close, MRB_ARGS_NONE()); mrb_define_method(mrb, klass, "count", mrb_grn_index_cursor_count, MRB_ARGS_NONE()); + mrb_define_method(mrb, klass, "select", + mrb_grn_index_cursor_select, MRB_ARGS_ARG(1, 1)); } #endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_object.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_object.c index 60398b91829..8efaa41e50d 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_object.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_object.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2013-2014 Brazil + Copyright(C) 2013-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -27,9 +27,39 @@ #include "../grn_mrb.h" #include "mrb_object.h" +#include "mrb_operator.h" #include "mrb_converter.h" static mrb_value +object_inspect(mrb_state *mrb, mrb_value self) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + grn_obj *object; + mrb_value inspected; + + object = DATA_PTR(self); + inspected = mrb_str_buf_new(mrb, 48); + + mrb_str_cat_lit(mrb, inspected, "#<"); + mrb_str_cat_cstr(mrb, inspected, mrb_obj_classname(mrb, self)); + mrb_str_cat_lit(mrb, inspected, ":"); + mrb_str_concat(mrb, inspected, mrb_ptr_to_str(mrb, mrb_cptr(self))); + if (object) { + grn_obj buffer; + GRN_TEXT_INIT(&buffer, 0); + grn_inspect(ctx, &buffer, object); + mrb_str_cat_lit(mrb, inspected, " "); + mrb_str_cat(mrb, inspected, GRN_TEXT_VALUE(&buffer), GRN_TEXT_LEN(&buffer)); + GRN_OBJ_FIN(ctx, &buffer); + } else { + mrb_str_cat_lit(mrb, inspected, " (closed)"); + } + mrb_str_cat_lit(mrb, inspected, ">"); + + return inspected; +} + +static mrb_value object_get_id(mrb_state *mrb, mrb_value self) { grn_ctx *ctx = (grn_ctx *)mrb->ud; @@ -60,19 +90,19 @@ object_find_index(mrb_state *mrb, mrb_value self) grn_ctx *ctx = (grn_ctx *)mrb->ud; grn_obj *object; mrb_value mrb_operator; - grn_obj *index; - int n_indexes; - int section_id; + grn_operator operator; + grn_index_datum index_datum; + int n_index_data; mrb_get_args(mrb, "o", &mrb_operator); object = DATA_PTR(self); - n_indexes = grn_column_index(ctx, - object, - mrb_fixnum(mrb_operator), - &index, - 1, - §ion_id); - if (n_indexes == 0) { + operator = grn_mrb_value_to_operator(mrb, mrb_operator); + n_index_data = grn_column_find_index_data(ctx, + object, + operator, + &index_datum, + 1); + if (n_index_data == 0) { return mrb_nil_value(); } else { grn_mrb_data *data; @@ -81,8 +111,8 @@ object_find_index(mrb_state *mrb, mrb_value self) data = &(ctx->impl->mrb); klass = mrb_class_get_under(mrb, data->module, "IndexInfo"); - args[0] = grn_mrb_value_from_grn_obj(mrb, index); - args[1] = mrb_fixnum_value(section_id); + args[0] = grn_mrb_value_from_grn_obj(mrb, index_datum.index); + args[1] = mrb_fixnum_value(index_datum.section); return mrb_obj_new(mrb, klass, 2, args); } } @@ -140,6 +170,39 @@ object_close(mrb_state *mrb, mrb_value self) } static mrb_value +object_get_domain_id(mrb_state *mrb, mrb_value self) +{ + grn_obj *object; + grn_id domain_id; + + object = DATA_PTR(self); + domain_id = object->header.domain; + + if (domain_id == GRN_ID_NIL) { + return mrb_nil_value(); + } else { + return mrb_fixnum_value(domain_id); + } +} + +static mrb_value +object_get_range_id(mrb_state *mrb, mrb_value self) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + grn_obj *object; + grn_id range_id; + + object = DATA_PTR(self); + range_id = grn_obj_get_range(ctx, object); + + if (range_id == GRN_ID_NIL) { + return mrb_nil_value(); + } else { + return mrb_fixnum_value(range_id); + } +} + +static mrb_value object_is_temporary(mrb_state *mrb, mrb_value self) { grn_obj *object; @@ -173,6 +236,9 @@ grn_mrb_object_init(grn_ctx *ctx) MRB_SET_INSTANCE_TT(klass, MRB_TT_DATA); data->object_class = klass; + mrb_define_method(mrb, klass, "inspect", + object_inspect, MRB_ARGS_NONE()); + mrb_define_method(mrb, klass, "id", object_get_id, MRB_ARGS_NONE()); mrb_define_method(mrb, klass, "name", object_get_name, MRB_ARGS_NONE()); mrb_define_method(mrb, klass, "find_index", @@ -182,6 +248,11 @@ grn_mrb_object_init(grn_ctx *ctx) mrb_define_method(mrb, klass, "==", object_equal, MRB_ARGS_REQ(1)); mrb_define_method(mrb, klass, "close", object_close, MRB_ARGS_NONE()); + mrb_define_method(mrb, klass, "domain_id", object_get_domain_id, + MRB_ARGS_NONE()); + mrb_define_method(mrb, klass, "range_id", object_get_range_id, + MRB_ARGS_NONE()); + mrb_define_method(mrb, klass, "temporary?", object_is_temporary, MRB_ARGS_NONE()); mrb_define_method(mrb, klass, "persistent?", object_is_persistent, diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_object_flags.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_object_flags.c new file mode 100644 index 00000000000..c8accada692 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_object_flags.c @@ -0,0 +1,95 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "../grn_ctx_impl.h" + +#ifdef GRN_WITH_MRUBY +# include <mruby.h> +# include <mruby/string.h> +# include <mruby/class.h> +# include <mruby/data.h> + +# include "../grn_mrb.h" +# include "mrb_object.h" +# include "mrb_operator.h" +# include "mrb_converter.h" + +void +grn_mrb_object_flags_init(grn_ctx *ctx) +{ + grn_mrb_data *data = &(ctx->impl->mrb); + mrb_state *mrb = data->state; + struct RClass *module = data->module; + struct RClass *flags_module; + + flags_module = mrb_define_module_under(mrb, module, "ObjectFlags"); + +#define MRB_DEFINE_FLAG(name) \ + mrb_define_const(mrb, flags_module, #name, \ + mrb_fixnum_value(GRN_OBJ_ ## name)) + + MRB_DEFINE_FLAG(TABLE_TYPE_MASK); + MRB_DEFINE_FLAG(TABLE_HASH_KEY); + MRB_DEFINE_FLAG(TABLE_PAT_KEY); + MRB_DEFINE_FLAG(TABLE_DAT_KEY); + MRB_DEFINE_FLAG(TABLE_NO_KEY); + + MRB_DEFINE_FLAG(KEY_MASK); + MRB_DEFINE_FLAG(KEY_UINT); + MRB_DEFINE_FLAG(KEY_INT); + MRB_DEFINE_FLAG(KEY_FLOAT); + MRB_DEFINE_FLAG(KEY_GEO_POINT); + + MRB_DEFINE_FLAG(KEY_WITH_SIS); + MRB_DEFINE_FLAG(KEY_NORMALIZE); + + MRB_DEFINE_FLAG(COLUMN_TYPE_MASK); + MRB_DEFINE_FLAG(COLUMN_SCALAR); + MRB_DEFINE_FLAG(COLUMN_VECTOR); + MRB_DEFINE_FLAG(COLUMN_INDEX); + + MRB_DEFINE_FLAG(COMPRESS_MASK); + MRB_DEFINE_FLAG(COMPRESS_NONE); + MRB_DEFINE_FLAG(COMPRESS_ZLIB); + MRB_DEFINE_FLAG(COMPRESS_LZ4); + + MRB_DEFINE_FLAG(WITH_SECTION); + MRB_DEFINE_FLAG(WITH_WEIGHT); + MRB_DEFINE_FLAG(WITH_POSITION); + MRB_DEFINE_FLAG(RING_BUFFER); + + MRB_DEFINE_FLAG(UNIT_MASK); + MRB_DEFINE_FLAG(UNIT_DOCUMENT_NONE); + MRB_DEFINE_FLAG(UNIT_DOCUMENT_SECTION); + MRB_DEFINE_FLAG(UNIT_DOCUMENT_POSITION); + MRB_DEFINE_FLAG(UNIT_SECTION_NONE); + MRB_DEFINE_FLAG(UNIT_SECTION_POSITION); + MRB_DEFINE_FLAG(UNIT_POSITION_NONE); + MRB_DEFINE_FLAG(UNIT_USERDEF_DOCUMENT); + MRB_DEFINE_FLAG(UNIT_USERDEF_SECTION); + MRB_DEFINE_FLAG(UNIT_USERDEF_POSITION); + + MRB_DEFINE_FLAG(NO_SUBREC); + MRB_DEFINE_FLAG(WITH_SUBREC); + + MRB_DEFINE_FLAG(KEY_VAR_SIZE); + + MRB_DEFINE_FLAG(TEMPORARY); + MRB_DEFINE_FLAG(PERSISTENT); +} +#endif /* GRN_WITH_MRUBY */ diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_object_flags.h b/storage/mroonga/vendor/groonga/lib/mrb/mrb_object_flags.h new file mode 100644 index 00000000000..60c82227078 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_object_flags.h @@ -0,0 +1,34 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef GRN_MRB_OBJECT_FLAGS_H +#define GRN_MRB_OBJECT_FLAGS_H + +#include "../grn_ctx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void grn_mrb_object_flags_init(grn_ctx *ctx); + +#ifdef __cplusplus +} +#endif + +#endif /* GRN_MRB_OBJECT_FLAGS_H */ diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_operator.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_operator.c index ca2fc74a38a..2e0cb481722 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_operator.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_operator.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2014 Brazil + Copyright(C) 2014-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -23,168 +23,132 @@ #include "mrb_operator.h" +mrb_value +grn_mrb_value_from_operator(mrb_state *mrb, grn_operator op) +{ + grn_ctx *ctx = (grn_ctx *)(mrb->ud); + grn_mrb_data *data = &(ctx->impl->mrb); + mrb_value mrb_op_raw; + mrb_value mrb_op; + + mrb_op_raw = mrb_fixnum_value(op); + mrb_op = mrb_funcall(mrb, mrb_obj_value(data->groonga.operator_class), + "find", 1, mrb_op_raw); + if (mrb_nil_p(mrb_op)) { + return mrb_op_raw; + } else { + return mrb_op; + } +} + +grn_operator +grn_mrb_value_to_operator(mrb_state *mrb, mrb_value mrb_op) +{ + if (!mrb_fixnum_p(mrb_op)) { + mrb_op = mrb_funcall(mrb, mrb_op, "value", 0); + } + + return mrb_fixnum(mrb_op); +} + void grn_mrb_operator_init(grn_ctx *ctx) { - mrb_state *mrb = ctx->impl->mrb.state; + grn_mrb_data *data = &(ctx->impl->mrb); + mrb_state *mrb = data->state; struct RClass *module = ctx->impl->mrb.module; - struct RClass *operator_module; + struct RClass *klass; + mrb_value klass_obj; + + klass = mrb_class_get_under(mrb, module, "Operator"); + data->groonga.operator_class = klass; + + klass_obj = mrb_obj_value(klass); +#define DEFINE_OPERATOR(name) \ + mrb_funcall(mrb, klass_obj, "register", 1, \ + mrb_funcall(mrb, klass_obj, "new", 2, \ + mrb_str_new_lit(mrb, #name), \ + mrb_fixnum_value(GRN_OP_ ## name))) - operator_module = mrb_define_module_under(mrb, module, "Operator"); + DEFINE_OPERATOR(PUSH); + DEFINE_OPERATOR(POP); + DEFINE_OPERATOR(NOP); + DEFINE_OPERATOR(CALL); + DEFINE_OPERATOR(INTERN); + DEFINE_OPERATOR(GET_REF); + DEFINE_OPERATOR(GET_VALUE); + DEFINE_OPERATOR(AND); + DEFINE_OPERATOR(AND_NOT); + DEFINE_OPERATOR(OR); + DEFINE_OPERATOR(ASSIGN); + DEFINE_OPERATOR(STAR_ASSIGN); + DEFINE_OPERATOR(SLASH_ASSIGN); + DEFINE_OPERATOR(MOD_ASSIGN); + DEFINE_OPERATOR(PLUS_ASSIGN); + DEFINE_OPERATOR(MINUS_ASSIGN); + DEFINE_OPERATOR(SHIFTL_ASSIGN); + DEFINE_OPERATOR(SHIFTR_ASSIGN); + DEFINE_OPERATOR(SHIFTRR_ASSIGN); + DEFINE_OPERATOR(AND_ASSIGN); + DEFINE_OPERATOR(XOR_ASSIGN); + DEFINE_OPERATOR(OR_ASSIGN); + DEFINE_OPERATOR(JUMP); + DEFINE_OPERATOR(CJUMP); + DEFINE_OPERATOR(COMMA); + DEFINE_OPERATOR(BITWISE_OR); + DEFINE_OPERATOR(BITWISE_XOR); + DEFINE_OPERATOR(BITWISE_AND); + DEFINE_OPERATOR(BITWISE_NOT); + DEFINE_OPERATOR(EQUAL); + DEFINE_OPERATOR(NOT_EQUAL); + DEFINE_OPERATOR(LESS); + DEFINE_OPERATOR(GREATER); + DEFINE_OPERATOR(LESS_EQUAL); + DEFINE_OPERATOR(GREATER_EQUAL); + DEFINE_OPERATOR(IN); + DEFINE_OPERATOR(MATCH); + DEFINE_OPERATOR(NEAR); + DEFINE_OPERATOR(NEAR2); + DEFINE_OPERATOR(SIMILAR); + DEFINE_OPERATOR(TERM_EXTRACT); + DEFINE_OPERATOR(SHIFTL); + DEFINE_OPERATOR(SHIFTR); + DEFINE_OPERATOR(SHIFTRR); + DEFINE_OPERATOR(PLUS); + DEFINE_OPERATOR(MINUS); + DEFINE_OPERATOR(STAR); + DEFINE_OPERATOR(SLASH); + DEFINE_OPERATOR(MOD); + DEFINE_OPERATOR(DELETE); + DEFINE_OPERATOR(INCR); + DEFINE_OPERATOR(DECR); + DEFINE_OPERATOR(INCR_POST); + DEFINE_OPERATOR(DECR_POST); + DEFINE_OPERATOR(NOT); + DEFINE_OPERATOR(ADJUST); + DEFINE_OPERATOR(EXACT); + DEFINE_OPERATOR(LCP); + DEFINE_OPERATOR(PARTIAL); + DEFINE_OPERATOR(UNSPLIT); + DEFINE_OPERATOR(PREFIX); + DEFINE_OPERATOR(SUFFIX); + DEFINE_OPERATOR(GEO_DISTANCE1); + DEFINE_OPERATOR(GEO_DISTANCE2); + DEFINE_OPERATOR(GEO_DISTANCE3); + DEFINE_OPERATOR(GEO_DISTANCE4); + DEFINE_OPERATOR(GEO_WITHINP5); + DEFINE_OPERATOR(GEO_WITHINP6); + DEFINE_OPERATOR(GEO_WITHINP8); + DEFINE_OPERATOR(OBJ_SEARCH); + DEFINE_OPERATOR(EXPR_GET_VAR); + DEFINE_OPERATOR(TABLE_CREATE); + DEFINE_OPERATOR(TABLE_SELECT); + DEFINE_OPERATOR(TABLE_SORT); + DEFINE_OPERATOR(TABLE_GROUP); + DEFINE_OPERATOR(JSON_PUT); + DEFINE_OPERATOR(GET_MEMBER); + DEFINE_OPERATOR(REGEXP); - mrb_define_const(mrb, operator_module, "PUSH", - mrb_fixnum_value(GRN_OP_PUSH)); - mrb_define_const(mrb, operator_module, "POP", - mrb_fixnum_value(GRN_OP_POP)); - mrb_define_const(mrb, operator_module, "NOP", - mrb_fixnum_value(GRN_OP_NOP)); - mrb_define_const(mrb, operator_module, "CALL", - mrb_fixnum_value(GRN_OP_CALL)); - mrb_define_const(mrb, operator_module, "INTERN", - mrb_fixnum_value(GRN_OP_INTERN)); - mrb_define_const(mrb, operator_module, "GET_REF", - mrb_fixnum_value(GRN_OP_GET_REF)); - mrb_define_const(mrb, operator_module, "GET_VALUE", - mrb_fixnum_value(GRN_OP_GET_VALUE)); - mrb_define_const(mrb, operator_module, "AND", - mrb_fixnum_value(GRN_OP_AND)); - mrb_define_const(mrb, operator_module, "AND_NOT", - mrb_fixnum_value(GRN_OP_AND_NOT)); - mrb_define_const(mrb, operator_module, "OR", - mrb_fixnum_value(GRN_OP_OR)); - mrb_define_const(mrb, operator_module, "ASSIGN", - mrb_fixnum_value(GRN_OP_ASSIGN)); - mrb_define_const(mrb, operator_module, "STAR_ASSIGN", - mrb_fixnum_value(GRN_OP_STAR_ASSIGN)); - mrb_define_const(mrb, operator_module, "SLASH_ASSIGN", - mrb_fixnum_value(GRN_OP_SLASH_ASSIGN)); - mrb_define_const(mrb, operator_module, "MOD_ASSIGN", - mrb_fixnum_value(GRN_OP_MOD_ASSIGN)); - mrb_define_const(mrb, operator_module, "PLUS_ASSIGN", - mrb_fixnum_value(GRN_OP_PLUS_ASSIGN)); - mrb_define_const(mrb, operator_module, "MINUS_ASSIGN", - mrb_fixnum_value(GRN_OP_MINUS_ASSIGN)); - mrb_define_const(mrb, operator_module, "SHIFTL_ASSIGN", - mrb_fixnum_value(GRN_OP_SHIFTL_ASSIGN)); - mrb_define_const(mrb, operator_module, "SHIFTR_ASSIGN", - mrb_fixnum_value(GRN_OP_SHIFTR_ASSIGN)); - mrb_define_const(mrb, operator_module, "SHIFTRR_ASSIGN", - mrb_fixnum_value(GRN_OP_SHIFTRR_ASSIGN)); - mrb_define_const(mrb, operator_module, "AND_ASSIGN", - mrb_fixnum_value(GRN_OP_AND_ASSIGN)); - mrb_define_const(mrb, operator_module, "XOR_ASSIGN", - mrb_fixnum_value(GRN_OP_XOR_ASSIGN)); - mrb_define_const(mrb, operator_module, "OR_ASSIGN", - mrb_fixnum_value(GRN_OP_OR_ASSIGN)); - mrb_define_const(mrb, operator_module, "JUMP", - mrb_fixnum_value(GRN_OP_JUMP)); - mrb_define_const(mrb, operator_module, "CJUMP", - mrb_fixnum_value(GRN_OP_CJUMP)); - mrb_define_const(mrb, operator_module, "COMMA", - mrb_fixnum_value(GRN_OP_COMMA)); - mrb_define_const(mrb, operator_module, "BITWISE_OR", - mrb_fixnum_value(GRN_OP_BITWISE_OR)); - mrb_define_const(mrb, operator_module, "BITWISE_XOR", - mrb_fixnum_value(GRN_OP_BITWISE_XOR)); - mrb_define_const(mrb, operator_module, "BITWISE_AND", - mrb_fixnum_value(GRN_OP_BITWISE_AND)); - mrb_define_const(mrb, operator_module, "BITWISE_NOT", - mrb_fixnum_value(GRN_OP_BITWISE_NOT)); - mrb_define_const(mrb, operator_module, "EQUAL", - mrb_fixnum_value(GRN_OP_EQUAL)); - mrb_define_const(mrb, operator_module, "NOT_EQUAL", - mrb_fixnum_value(GRN_OP_NOT_EQUAL)); - mrb_define_const(mrb, operator_module, "LESS", - mrb_fixnum_value(GRN_OP_LESS)); - mrb_define_const(mrb, operator_module, "GREATER", - mrb_fixnum_value(GRN_OP_GREATER)); - mrb_define_const(mrb, operator_module, "LESS_EQUAL", - mrb_fixnum_value(GRN_OP_LESS_EQUAL)); - mrb_define_const(mrb, operator_module, "GREATER_EQUAL", - mrb_fixnum_value(GRN_OP_GREATER_EQUAL)); - mrb_define_const(mrb, operator_module, "IN", - mrb_fixnum_value(GRN_OP_IN)); - mrb_define_const(mrb, operator_module, "MATCH", - mrb_fixnum_value(GRN_OP_MATCH)); - mrb_define_const(mrb, operator_module, "NEAR", - mrb_fixnum_value(GRN_OP_NEAR)); - mrb_define_const(mrb, operator_module, "NEAR2", - mrb_fixnum_value(GRN_OP_NEAR2)); - mrb_define_const(mrb, operator_module, "SIMILAR", - mrb_fixnum_value(GRN_OP_SIMILAR)); - mrb_define_const(mrb, operator_module, "TERM_EXTRACT", - mrb_fixnum_value(GRN_OP_TERM_EXTRACT)); - mrb_define_const(mrb, operator_module, "SHIFTL", - mrb_fixnum_value(GRN_OP_SHIFTL)); - mrb_define_const(mrb, operator_module, "SHIFTR", - mrb_fixnum_value(GRN_OP_SHIFTR)); - mrb_define_const(mrb, operator_module, "SHIFTRR", - mrb_fixnum_value(GRN_OP_SHIFTRR)); - mrb_define_const(mrb, operator_module, "PLUS", - mrb_fixnum_value(GRN_OP_PLUS)); - mrb_define_const(mrb, operator_module, "MINUS", - mrb_fixnum_value(GRN_OP_MINUS)); - mrb_define_const(mrb, operator_module, "STAR", - mrb_fixnum_value(GRN_OP_STAR)); - mrb_define_const(mrb, operator_module, "SLASH", - mrb_fixnum_value(GRN_OP_SLASH)); - mrb_define_const(mrb, operator_module, "MOD", - mrb_fixnum_value(GRN_OP_MOD)); - mrb_define_const(mrb, operator_module, "DELETE", - mrb_fixnum_value(GRN_OP_DELETE)); - mrb_define_const(mrb, operator_module, "INCR", - mrb_fixnum_value(GRN_OP_INCR)); - mrb_define_const(mrb, operator_module, "DECR", - mrb_fixnum_value(GRN_OP_DECR)); - mrb_define_const(mrb, operator_module, "INCR_POST", - mrb_fixnum_value(GRN_OP_INCR_POST)); - mrb_define_const(mrb, operator_module, "DECR_POST", - mrb_fixnum_value(GRN_OP_DECR_POST)); - mrb_define_const(mrb, operator_module, "NOT", - mrb_fixnum_value(GRN_OP_NOT)); - mrb_define_const(mrb, operator_module, "ADJUST", - mrb_fixnum_value(GRN_OP_ADJUST)); - mrb_define_const(mrb, operator_module, "EXACT", - mrb_fixnum_value(GRN_OP_EXACT)); - mrb_define_const(mrb, operator_module, "LCP", - mrb_fixnum_value(GRN_OP_LCP)); - mrb_define_const(mrb, operator_module, "PARTIAL", - mrb_fixnum_value(GRN_OP_PARTIAL)); - mrb_define_const(mrb, operator_module, "UNSPLIT", - mrb_fixnum_value(GRN_OP_UNSPLIT)); - mrb_define_const(mrb, operator_module, "PREFIX", - mrb_fixnum_value(GRN_OP_PREFIX)); - mrb_define_const(mrb, operator_module, "SUFFIX", - mrb_fixnum_value(GRN_OP_SUFFIX)); - mrb_define_const(mrb, operator_module, "GEO_DISTANCE1", - mrb_fixnum_value(GRN_OP_GEO_DISTANCE1)); - mrb_define_const(mrb, operator_module, "GEO_DISTANCE2", - mrb_fixnum_value(GRN_OP_GEO_DISTANCE2)); - mrb_define_const(mrb, operator_module, "GEO_DISTANCE3", - mrb_fixnum_value(GRN_OP_GEO_DISTANCE3)); - mrb_define_const(mrb, operator_module, "GEO_DISTANCE4", - mrb_fixnum_value(GRN_OP_GEO_DISTANCE4)); - mrb_define_const(mrb, operator_module, "GEO_WITHINP5", - mrb_fixnum_value(GRN_OP_GEO_WITHINP5)); - mrb_define_const(mrb, operator_module, "GEO_WITHINP6", - mrb_fixnum_value(GRN_OP_GEO_WITHINP6)); - mrb_define_const(mrb, operator_module, "GEO_WITHINP8", - mrb_fixnum_value(GRN_OP_GEO_WITHINP8)); - mrb_define_const(mrb, operator_module, "OBJ_SEARCH", - mrb_fixnum_value(GRN_OP_OBJ_SEARCH)); - mrb_define_const(mrb, operator_module, "EXPR_GET_VAR", - mrb_fixnum_value(GRN_OP_EXPR_GET_VAR)); - mrb_define_const(mrb, operator_module, "TABLE_CREATE", - mrb_fixnum_value(GRN_OP_TABLE_CREATE)); - mrb_define_const(mrb, operator_module, "TABLE_SELECT", - mrb_fixnum_value(GRN_OP_TABLE_SELECT)); - mrb_define_const(mrb, operator_module, "TABLE_SORT", - mrb_fixnum_value(GRN_OP_TABLE_SORT)); - mrb_define_const(mrb, operator_module, "TABLE_GROUP", - mrb_fixnum_value(GRN_OP_TABLE_GROUP)); - mrb_define_const(mrb, operator_module, "JSON_PUT", - mrb_fixnum_value(GRN_OP_JSON_PUT)); - mrb_define_const(mrb, operator_module, "GET_MEMBER", - mrb_fixnum_value(GRN_OP_GET_MEMBER)); +#undef DEFINE_OPERATOR } #endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_operator.h b/storage/mroonga/vendor/groonga/lib/mrb/mrb_operator.h index 683b9dacbd9..b76c4983446 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_operator.h +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_operator.h @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2014 Brazil + Copyright(C) 2014-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -26,6 +26,8 @@ extern "C" { #endif void grn_mrb_operator_init(grn_ctx *ctx); +mrb_value grn_mrb_value_from_operator(mrb_state *mrb, grn_operator op); +grn_operator grn_mrb_value_to_operator(mrb_state *mrb, mrb_value mrb_op); #ifdef __cplusplus } diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_options.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_options.c new file mode 100644 index 00000000000..ff3a1c0e731 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_options.c @@ -0,0 +1,39 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "../grn_ctx_impl.h" +#include "../grn_db.h" + +#ifdef GRN_WITH_MRUBY +#include <mruby.h> +#include <mruby/hash.h> + +#include "mrb_options.h" + +mrb_value +grn_mrb_options_get_static(mrb_state *mrb, + mrb_value mrb_options, + const char *key, + size_t key_size) +{ + mrb_sym mrb_key; + + mrb_key = mrb_intern_static(mrb, key, key_size); + return mrb_hash_get(mrb, mrb_options, mrb_symbol_value(mrb_key)); +} +#endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_options.h b/storage/mroonga/vendor/groonga/lib/mrb/mrb_options.h new file mode 100644 index 00000000000..1aa547d3be9 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_options.h @@ -0,0 +1,40 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef GRN_MRB_OPTIONS_H +#define GRN_MRB_OPTIONS_H + +#include "../grn_ctx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +mrb_value grn_mrb_options_get_static(mrb_state *mrb, + mrb_value mrb_options, + const char *key, + size_t key_size); +#define grn_mrb_options_get_lit(mrb, mrb_options, literal) \ + grn_mrb_options_get_static(mrb, mrb_options, \ + (literal), mrb_strlen_lit(literal)) + +#ifdef __cplusplus +} +#endif + +#endif /* GRN_MRB_OPTIONS_H */ diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_table.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_table.c index 2b86fdb482b..651a77e4549 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_table.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_table.c @@ -29,6 +29,41 @@ #include "mrb_ctx.h" #include "mrb_table.h" #include "mrb_converter.h" +#include "mrb_options.h" + +static mrb_value +mrb_grn_table_array_reference(mrb_state *mrb, mrb_value self) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + grn_obj *table; + grn_id key_domain_id; + mrb_value mrb_key; + grn_id record_id; + grn_mrb_value_to_raw_data_buffer buffer; + void *key; + unsigned int key_size; + + mrb_get_args(mrb, "o", &mrb_key); + + table = DATA_PTR(self); + if (table->header.type == GRN_DB) { + key_domain_id = GRN_DB_SHORT_TEXT; + } else { + key_domain_id = table->header.domain; + } + + grn_mrb_value_to_raw_data_buffer_init(mrb, &buffer); + grn_mrb_value_to_raw_data(mrb, "key", mrb_key, key_domain_id, + &buffer, &key, &key_size); + record_id = grn_table_get(ctx, table, key, key_size); + grn_mrb_value_to_raw_data_buffer_fin(mrb, &buffer); + + if (record_id == GRN_ID_NIL) { + return mrb_nil_value(); + } else { + return mrb_fixnum_value(record_id); + } +} static mrb_value mrb_grn_table_is_locked(mrb_state *mrb, mrb_value self) @@ -86,14 +121,12 @@ mrb_grn_table_select(mrb_state *mrb, mrb_value self) mrb_value mrb_result; mrb_value mrb_operator; - mrb_result = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "result"))); + mrb_result = grn_mrb_options_get_lit(mrb, mrb_options, "result"); if (!mrb_nil_p(mrb_result)) { result = DATA_PTR(mrb_result); } - mrb_operator = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "operator"))); + mrb_operator = grn_mrb_options_get_lit(mrb, mrb_options, "operator"); if (!mrb_nil_p(mrb_operator)) { operator = mrb_fixnum(mrb_operator); } @@ -133,8 +166,7 @@ mrb_grn_table_sort(mrb_state *mrb, mrb_value self) mrb_value mrb_sort_order; mrb_sort_options = RARRAY_PTR(mrb_keys)[i]; - mrb_sort_key = mrb_hash_get(mrb, mrb_sort_options, - mrb_symbol_value(mrb_intern_lit(mrb, "key"))); + mrb_sort_key = grn_mrb_options_get_lit(mrb, mrb_sort_options, "key"); switch (mrb_type(mrb_sort_key)) { case MRB_TT_STRING : keys[i].key = grn_obj_column(ctx, table, @@ -158,9 +190,7 @@ mrb_grn_table_sort(mrb_state *mrb, mrb_value self) } keys[i].flags = 0; - mrb_sort_order = - mrb_hash_get(mrb, mrb_sort_options, - mrb_symbol_value(mrb_intern_lit(mrb, "order"))); + mrb_sort_order = grn_mrb_options_get_lit(mrb, mrb_sort_options, "order"); if (mrb_nil_p(mrb_sort_order) || (mrb_symbol(mrb_sort_order) == mrb_intern_lit(mrb, "ascending"))) { keys[i].flags |= GRN_TABLE_SORT_ASC; @@ -173,14 +203,12 @@ mrb_grn_table_sort(mrb_state *mrb, mrb_value self) mrb_value mrb_offset; mrb_value mrb_limit; - mrb_offset = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "offset"))); + mrb_offset = grn_mrb_options_get_lit(mrb, mrb_options, "offset"); if (!mrb_nil_p(mrb_offset)) { offset = mrb_fixnum(mrb_offset); } - mrb_limit = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "limit"))); + mrb_limit = grn_mrb_options_get_lit(mrb, mrb_options, "limit"); if (!mrb_nil_p(mrb_limit)) { limit = mrb_fixnum(mrb_limit); } @@ -210,6 +238,9 @@ grn_mrb_table_init(grn_ctx *ctx) klass = mrb_define_class_under(mrb, module, "Table", object_class); MRB_SET_INSTANCE_TT(klass, MRB_TT_DATA); + mrb_define_method(mrb, klass, "[]", + mrb_grn_table_array_reference, MRB_ARGS_REQ(1)); + mrb_define_method(mrb, klass, "locked?", mrb_grn_table_is_locked, MRB_ARGS_NONE()); diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_table_cursor.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_table_cursor.c index 660596070d3..76d4429d24f 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_table_cursor.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_table_cursor.c @@ -26,62 +26,17 @@ #include <mruby/hash.h> #include "mrb_ctx.h" +#include "mrb_bulk.h" #include "mrb_table_cursor.h" +#include "mrb_converter.h" +#include "mrb_options.h" + static struct mrb_data_type mrb_grn_table_cursor_type = { "Groonga::TableCursor", NULL }; -typedef union { - int64_t time_value; -} border_value_buffer; - -static void -mrb_value_to_border_value(mrb_state *mrb, - const char *type, - mrb_value mrb_border_value, - border_value_buffer *buffer, - void **border_value, - unsigned int *border_value_size) -{ - grn_ctx *ctx = (grn_ctx *)mrb->ud; - - if (mrb_nil_p(mrb_border_value)) { - return; - } - - switch (mrb_type(mrb_border_value)) { - case MRB_TT_STRING : - *border_value = RSTRING_PTR(mrb_border_value); - *border_value_size = RSTRING_LEN(mrb_border_value); - break; - default : - { - struct RClass *klass; - - klass = mrb_class(mrb, mrb_border_value); - if (klass == ctx->impl->mrb.builtin.time_class) { - mrb_value mrb_sec; - mrb_value mrb_usec; - - mrb_sec = mrb_funcall(mrb, mrb_border_value, "to_i", 0); - mrb_usec = mrb_funcall(mrb, mrb_border_value, "usec", 0); - buffer->time_value = GRN_TIME_PACK(mrb_fixnum(mrb_sec), - mrb_fixnum(mrb_usec)); - *border_value = &(buffer->time_value); - *border_value_size = sizeof(buffer->time_value); - } else { - mrb_raisef(mrb, E_NOTIMP_ERROR, - "%s: only String and Time is supported for now: %S", - type, - mrb_border_value); - } - } - break; - } -} - static mrb_value mrb_grn_table_cursor_singleton_open_raw(mrb_state *mrb, mrb_value klass) { @@ -92,10 +47,10 @@ mrb_grn_table_cursor_singleton_open_raw(mrb_state *mrb, mrb_value klass) grn_obj *table; void *min = NULL; unsigned int min_size = 0; - border_value_buffer min_buffer; + grn_mrb_value_to_raw_data_buffer min_buffer; void *max = NULL; unsigned int max_size = 0; - border_value_buffer max_buffer; + grn_mrb_value_to_raw_data_buffer max_buffer; int offset = 0; int limit = -1; int flags = 0; @@ -103,21 +58,29 @@ mrb_grn_table_cursor_singleton_open_raw(mrb_state *mrb, mrb_value klass) mrb_get_args(mrb, "o|H", &mrb_table, &mrb_options); table = DATA_PTR(mrb_table); + grn_mrb_value_to_raw_data_buffer_init(mrb, &min_buffer); + grn_mrb_value_to_raw_data_buffer_init(mrb, &max_buffer); if (!mrb_nil_p(mrb_options)) { + grn_id key_domain_id; mrb_value mrb_min; mrb_value mrb_max; mrb_value mrb_flags; - mrb_min = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "min"))); - mrb_value_to_border_value(mrb, "min", mrb_min, &min_buffer, &min, &min_size); + if (table->header.type == GRN_DB) { + key_domain_id = GRN_DB_SHORT_TEXT; + } else { + key_domain_id = table->header.domain; + } + + mrb_min = grn_mrb_options_get_lit(mrb, mrb_options, "min"); + grn_mrb_value_to_raw_data(mrb, "min", mrb_min, + key_domain_id, &min_buffer, &min, &min_size); - mrb_max = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "max"))); - mrb_value_to_border_value(mrb, "max", mrb_max, &max_buffer, &max, &max_size); + mrb_max = grn_mrb_options_get_lit(mrb, mrb_options, "max"); + grn_mrb_value_to_raw_data(mrb, "max", mrb_max, + key_domain_id, &max_buffer, &max, &max_size); - mrb_flags = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "flags"))); + mrb_flags = grn_mrb_options_get_lit(mrb, mrb_options, "flags"); if (!mrb_nil_p(mrb_flags)) { flags = mrb_fixnum(mrb_flags); } @@ -126,6 +89,8 @@ mrb_grn_table_cursor_singleton_open_raw(mrb_state *mrb, mrb_value klass) min, min_size, max, max_size, offset, limit, flags); + grn_mrb_value_to_raw_data_buffer_fin(mrb, &min_buffer); + grn_mrb_value_to_raw_data_buffer_fin(mrb, &max_buffer); grn_mrb_ctx_check(mrb); return mrb_funcall(mrb, klass, "new", 1, mrb_cptr_value(mrb, table_cursor)); diff --git a/storage/mroonga/vendor/groonga/lib/mrb/mrb_writer.c b/storage/mroonga/vendor/groonga/lib/mrb/mrb_writer.c index e14073e5fe2..e35a066e560 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/mrb_writer.c +++ b/storage/mroonga/vendor/groonga/lib/mrb/mrb_writer.c @@ -28,6 +28,7 @@ #include "../grn_output.h" #include "mrb_ctx.h" #include "mrb_writer.h" +#include "mrb_options.h" static mrb_value writer_write(mrb_state *mrb, mrb_value self) @@ -164,14 +165,12 @@ writer_write_table_records(mrb_state *mrb, mrb_value self) mrb_value mrb_offset; mrb_value mrb_limit; - mrb_offset = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "offset"))); + mrb_offset = grn_mrb_options_get_lit(mrb, mrb_options, "offset"); if (!mrb_nil_p(mrb_offset)) { offset = mrb_fixnum(mrb_offset); } - mrb_limit = mrb_hash_get(mrb, mrb_options, - mrb_symbol_value(mrb_intern_lit(mrb, "limit"))); + mrb_limit = grn_mrb_options_get_lit(mrb, mrb_options, "limit"); if (!mrb_nil_p(mrb_limit)) { limit = mrb_fixnum(mrb_limit); } @@ -195,6 +194,19 @@ writer_write_table_records(mrb_state *mrb, mrb_value self) return mrb_nil_value(); } +static mrb_value +writer_set_content_type(mrb_state *mrb, mrb_value self) +{ + grn_ctx *ctx = (grn_ctx *)mrb->ud; + grn_content_type content_type; + + mrb_get_args(mrb, "i", &content_type); + + grn_ctx_set_output_type(ctx, content_type); + + return mrb_nil_value(); +} + void grn_mrb_writer_init(grn_ctx *ctx) { @@ -219,5 +231,8 @@ grn_mrb_writer_init(grn_ctx *ctx) writer_write_table_columns, MRB_ARGS_REQ(2)); mrb_define_method(mrb, klass, "write_table_records", writer_write_table_records, MRB_ARGS_ARG(2, 1)); + + mrb_define_method(mrb, klass, "content_type=", + writer_set_content_type, MRB_ARGS_REQ(1)); } #endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/Makefile.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/Makefile.am index 631923e2e06..9b6acf8bbda 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/Makefile.am +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/Makefile.am @@ -1,3 +1,9 @@ +SUBDIRS = \ + command_line \ + context \ + initialize \ + logger + include sources.am EXTRA_DIST = \ @@ -5,5 +11,5 @@ EXTRA_DIST = \ $(RUBY_SCRIPT_FILES) if WITH_MRUBY -nobase_ruby_scripts_DATA = $(RUBY_SCRIPT_FILES) +ruby_scripts_DATA = $(RUBY_SCRIPT_FILES) endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/backtrace_entry.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/backtrace_entry.rb index 68ea9e4b6f2..34f95e968f5 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/backtrace_entry.rb +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/backtrace_entry.rb @@ -5,16 +5,24 @@ module Groonga match_data = /:(\d+):?/.match(entry) file = match_data.pre_match line = match_data[1].to_i - method = match_data.post_match.gsub(/\Ain /, "") - new(file, line, method) + detail_match_data = /\A(in )?(\S+)\s*/.match(match_data.post_match) + if detail_match_data[1] + method = detail_match_data[2] + message = detail_match_data.post_match + else + method = "" + message = match_data.post_match + end + new(file, line, method, message) end end - attr_reader :file, :line, :method - def initialize(file, line, method) + attr_reader :file, :line, :method, :message + def initialize(file, line, method, message) @file = file @line = line @method = method + @message = message end end end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/command_line/Makefile.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/command_line/Makefile.am new file mode 100644 index 00000000000..8d580810674 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/command_line/Makefile.am @@ -0,0 +1,9 @@ +include sources.am + +EXTRA_DIST = \ + $(RUBY_SCRIPT_FILES) + +if WITH_MRUBY +ruby_scripts_command_linedir = $(ruby_scriptsdir)/command_line +ruby_scripts_command_line_DATA = $(RUBY_SCRIPT_FILES) +endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/command_line/sources.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/command_line/sources.am new file mode 100644 index 00000000000..759948eecd6 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/command_line/sources.am @@ -0,0 +1,2 @@ +RUBY_SCRIPT_FILES = \ + grndb.rb diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/context/Makefile.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/context/Makefile.am new file mode 100644 index 00000000000..8d862082cce --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/context/Makefile.am @@ -0,0 +1,9 @@ +include sources.am + +EXTRA_DIST = \ + $(RUBY_SCRIPT_FILES) + +if WITH_MRUBY +ruby_scripts_contextdir = $(ruby_scriptsdir)/context +ruby_scripts_context_DATA = $(RUBY_SCRIPT_FILES) +endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/context/sources.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/context/sources.am new file mode 100644 index 00000000000..1f0d1624ce5 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/context/sources.am @@ -0,0 +1,3 @@ +RUBY_SCRIPT_FILES = \ + error_level.rb \ + rc.rb diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/expression.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/expression.rb index 29290e207f2..7b1199b7795 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/expression.rb +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/expression.rb @@ -1,3 +1,8 @@ +require "scan_info" +require "scan_info_builder" + +require "expression_size_estimator" + module Groonga class Expression def build_scan_info(op, size) @@ -9,5 +14,23 @@ module Groonga nil end end + + def estimate_size(table) + begin + estimator = ExpressionSizeEstimator.new(self, table) + estimator.estimate + rescue GroongaError => groonga_error + context.set_groonga_error(groonga_error) + table.size + rescue => error + context.record_error(:unknown_error, error) + table.size + end + end + + private + def context + @context ||= Context.instance + end end end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/expression_size_estimator.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/expression_size_estimator.rb new file mode 100644 index 00000000000..73b44cae3dd --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/expression_size_estimator.rb @@ -0,0 +1,155 @@ +module Groonga + class ExpressionSizeEstimator + def initialize(expression, table) + @expression = expression + @table = table + @table_size = @table.size + end + + def estimate + builder = ScanInfoBuilder.new(@expression, Operator::OR, 0) + data_list = builder.build + return @table_size if data_list.nil? + + or_data_list = group_data_list(data_list) + or_sizes = or_data_list.collect do |and_data_list| + and_sizes = and_data_list.collect do |data| + size = estimate_data(data) + if data.logical_op == Operator::AND_NOT + size = @table_size - size + size = 0 if size < 0 + end + size + end + and_sizes.min + end + or_sizes.max + end + + private + def group_data_list(data_list) + or_data_list = [[]] + data_list.each do |data| + next if data.op == Operator::NOP + + and_data_list = or_data_list.last + if and_data_list.empty? + and_data_list << data + else + case data.logical_op + when Operator::AND, Operator::AND_NOT + and_data_list << data + else + and_data_list = [data] + or_data_list << and_data_list + end + end + end + or_data_list + end + + def estimate_data(data) + search_index = data.search_indexes.first + return @table_size if search_index.nil? + + index_column = resolve_index_column(search_index.index_column, + data.op) + return @table_size if index_column.nil? + + size = nil + case data.op + when Operator::MATCH + size = estimate_match(data, index_column) + when Operator::REGEXP + size = estimate_regexp(data, index_column) + when Operator::EQUAL + size = estimate_equal(data, index_column) + when Operator::LESS, + Operator::LESS_EQUAL, + Operator::GREATER, + Operator::GREATER_EQUAL + size = estimate_range(data, index_column) + when Operator::CALL + procedure = data.args.first + if procedure.is_a?(Procedure) and procedure.name == "between" + size = estimate_between(data, index_column) + end + end + size || @table_size + end + + def resolve_index_column(index_column, operator) + while index_column.is_a?(Accessor) + index_info = index_column.find_index(operator) + return nil if index_info.nil? + index_column = index_info.index + end + + index_column + end + + def estimate_match(data, index_column) + index_column.estimate_size(:query => data.query.value) + end + + def estimate_regexp(data, index_column) + index_column.estimate_size(:query => data.query.value, + :mode => data.op) + end + + def estimate_equal(data, index_column) + lexicon = index_column.lexicon + term_id = lexicon[data.query] + return 0 if term_id.nil? + + index_column.estimate_size(:term_id => term_id) + end + + def estimate_range(data, index_column) + lexicon = index_column.lexicon + value = data.query.value + options = {} + case data.op + when Operator::LESS + options[:max] = value + options[:flags] = TableCursorFlags::LT + when Operator::LESS_EQUAL + options[:max] = value + options[:flags] = TableCursorFlags::LE + when Operator::GREATER + options[:min] = value + options[:flags] = TableCursorFlags::GT + when Operator::GREATER_EQUAL + options[:min] = value + options[:flags] = TableCursorFlags::GE + end + TableCursor.open(lexicon, options) do |cursor| + index_column.estimate_size(:lexicon_cursor => cursor) + end + end + + def estimate_between(data, index_column) + lexicon = index_column.lexicon + _, _, min, min_border, max, max_border = data.args + options = { + :min => min, + :max => max, + :flags => 0, + } + if min_border == "include" + options[:flags] |= TableCursorFlags::LT + else + options[:flags] |= TableCursorFlags::LE + end + if max_border == "include" + options[:flags] |= TableCursorFlags::GT + else + options[:flags] |= TableCursorFlags::GE + end + + TableCursor.open(lexicon, options) do |cursor| + index_column.estimate_size(:lexicon_cursor => cursor) + end + end + end +end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/index_column.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/index_column.rb new file mode 100644 index 00000000000..25ebc149367 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/index_column.rb @@ -0,0 +1,39 @@ +module Groonga + class IndexColumn + private :estimate_size_for_term_id + private :estimate_size_for_query + private :estimate_size_for_lexicon_cursor + + # Estimate the number of matched records for term ID or query. + # + # @overload estimate_size(:term_id => term_id) + # @return [Integer] the number of matched records for the term ID. + # + # @overload estimate_size(:query => query) + # @return [Integer] the number of matched records for the query. + # + # @overload estimate_size(:lexicon_cursor => lexicon_cursor) + # @return [Integer] the number of matched records for the lexicon cursor. + # + def estimate_size(parameters) + term_id = parameters[:term_id] + if term_id + return estimate_size_for_term_id(term_id) + end + + query = parameters[:query] + if query + return estimate_size_for_query(query, parameters) + end + + lexicon_cursor = parameters[:lexicon_cursor] + if lexicon_cursor + return estimate_size_for_lexicon_cursor(lexicon_cursor) + end + + message = + "must specify :term_id, :query, :lexicon_cursor: #{parameters.inspect}" + raise InvalidArgument, message + end + end +end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/Makefile.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/Makefile.am new file mode 100644 index 00000000000..e7531fdc029 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/Makefile.am @@ -0,0 +1,9 @@ +include sources.am + +EXTRA_DIST = \ + $(RUBY_SCRIPT_FILES) + +if WITH_MRUBY +ruby_scripts_initializedir = $(ruby_scriptsdir)/initialize +ruby_scripts_initialize_DATA = $(RUBY_SCRIPT_FILES) +endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/post.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/post.rb index 7110437c200..ea26a031e0a 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/post.rb +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/post.rb @@ -4,11 +4,15 @@ require "context" require "writer" +require "object" require "database" +require "index_column" require "command" require "table_cursor" require "index_cursor" +require "expression" + require "plugin_loader" require "eval_context" diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/pre.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/pre.rb index c9a74a80dd9..99300d11a92 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/pre.rb +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/pre.rb @@ -1 +1,3 @@ require "backtrace_entry" + +require "operator" diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/sources.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/sources.am new file mode 100644 index 00000000000..3c26e19b2ae --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/initialize/sources.am @@ -0,0 +1,3 @@ +RUBY_SCRIPT_FILES = \ + pre.rb \ + post.rb diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger.rb index 33021355824..cb747a418d6 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger.rb +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger.rb @@ -9,12 +9,13 @@ module Groonga message = "#{error.class}: #{error.message}" end backtrace = error.backtrace - first_raw_entry = backtrace.first - if first_raw_entry - first_entry = BacktraceEntry.parse(first_raw_entry) - file = first_entry.file - line = first_entry.line - method = first_entry.method + last_raw_entry = backtrace.last + if last_raw_entry + last_entry = BacktraceEntry.parse(last_raw_entry) + file = last_entry.file + line = last_entry.line + method = last_entry.method + # message = "#{file}:#{line}:#{method}: #{message}" else file = "" line = 0 @@ -22,8 +23,11 @@ module Groonga end log(log_level, file, line, method, message) - backtrace.each do |raw_entry| + backtrace.reverse_each.with_index do |raw_entry, i| + next if i == 0 entry = BacktraceEntry.parse(raw_entry) + message = entry.message + message = raw_entry if message.empty? log(log_level, entry.file, entry.line, entry.method, raw_entry) end end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger/Makefile.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger/Makefile.am new file mode 100644 index 00000000000..448e72ca5bc --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger/Makefile.am @@ -0,0 +1,9 @@ +include sources.am + +EXTRA_DIST = \ + $(RUBY_SCRIPT_FILES) + +if WITH_MRUBY +ruby_scripts_loggerdir = $(ruby_scriptsdir)/logger +ruby_scripts_logger_DATA = $(RUBY_SCRIPT_FILES) +endif diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger/sources.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger/sources.am new file mode 100644 index 00000000000..7231ee4eecb --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/logger/sources.am @@ -0,0 +1,2 @@ +RUBY_SCRIPT_FILES = \ + level.rb diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/object.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/object.rb index f1cc1a6aa79..d98b5069e7f 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/object.rb +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/object.rb @@ -1,7 +1,11 @@ module Groonga class Object - def inspect - super[0..-2] + ": #{grn_inspect}>" + def domain + Context.instance[domain_id] + end + + def range + Context.instance[range_id] end end end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/operator.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/operator.rb new file mode 100644 index 00000000000..349695e1a53 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/operator.rb @@ -0,0 +1,22 @@ +module Groonga + class Operator + @values = {} + class << self + def register(operator) + const_set(operator.name, operator) + @values[operator.value] = operator + end + + def find(value) + @values[value] + end + end + + attr_reader :name + attr_reader :value + def initialize(name, value) + @name = name + @value = value + end + end +end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info.rb index 64262f8f975..a98cf792a9c 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info.rb +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info.rb @@ -19,14 +19,16 @@ module Groonga if data.similarity_threshold self.similarity_threshold = data.similarity_threshold end - if data.scorer - self.scorer = data.scorer - end data.args.each do |arg| push_arg(arg) end - data.indexes.each do |index, section_id, weight| - put_index(index, section_id, weight) + data.search_indexes.each do |search_index| + put_index(search_index.index_column, + search_index.section_id, + search_index.weight, + search_index.scorer, + search_index.scorer_args_expr, + search_index.scorer_args_expr_offset || 0) end end end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_builder.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_builder.rb index b002eaa14c0..dc003f88948 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_builder.rb +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_builder.rb @@ -1,11 +1,6 @@ -module Groonga - # TODO: Move me - class ExpressionCode - module Flags - RELATIONAL_EXPRESSION = 0x01 - end - end +require "scan_info_data" +module Groonga class ScanInfoBuilder module Status START = 0 @@ -39,6 +34,7 @@ module Groonga Operator::GEO_WITHINP6, Operator::GEO_WITHINP8, Operator::TERM_EXTRACT, + Operator::REGEXP, ] ARITHMETIC_OPERATORS = [ @@ -294,10 +290,10 @@ module Groonga return false if data.args[0] != next_data.args[0] - data_indexes = data.indexes - return false if data_indexes.empty? + data_search_indexes = data.search_indexes + return false if data_search_indexes.empty? - data_indexes == next_data.indexes + data_search_indexes == next_data.search_indexes end def lower_condition?(operator) @@ -325,7 +321,7 @@ module Groonga between_data.op = Operator::CALL between_data.logical_op = data.logical_op between_data.args = create_between_data_args(data, next_data) - between_data.indexes = data.indexes + between_data.search_indexes = data.search_indexes between_data end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_data.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_data.rb index b0ebfea3cee..4a6e58a951a 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_data.rb +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_data.rb @@ -1,3 +1,5 @@ +require "scan_info_search_index" + module Groonga class ScanInfoData attr_accessor :start @@ -6,23 +8,21 @@ module Groonga attr_accessor :logical_op attr_accessor :query attr_accessor :args - attr_accessor :indexes + attr_accessor :search_indexes attr_accessor :flags attr_accessor :max_interval attr_accessor :similarity_threshold - attr_accessor :scorer def initialize(start) @start = start @end = 0 - @op = 0 + @op = Operator::NOP @logical_op = Operator::OR @query = nil @args = [] - @indexes = [] + @search_indexes = [] @flags = ScanInfo::Flags::PUSH @max_interval = nil @similarity_threshold = nil - @scorer = nil end def match_resolve_index @@ -102,6 +102,41 @@ module Groonga self.query = arg end end + if @op == Operator::REGEXP and not index_searchable_regexp?(@query) + @search_indexes.clear + end + end + + def index_searchable_regexp?(pattern) + return false if pattern.nil? + + previous_char = nil + pattern.value.each_char do |char| + if previous_char == "\\" + case char + when "Z" + return false + when "b", "B" + return false + when "d", "D", "h", "H", "p", "s", "S", "w", "W" + return false + when "X" + return false + when "k", "g", "1", "2", "3", "4", "5", "6", "7", "8", "9" + return false + when "\\" + previous_char = nil + next + end + else + case char + when ".", "[", "]", "|", "?", "+", "*", "{", "}", "^", "$", "(", ")" + return false + end + end + previous_char = char + end + true end def match_resolve_index_expression(expression) @@ -109,19 +144,82 @@ module Groonga n_codes = codes.size i = 0 while i < n_codes - i = match_resolve_index_expression_codes(codes, i, n_codes) + i = match_resolve_index_expression_codes(expression, codes, i, n_codes) + end + end + + def match_resolve_index_expression_codes(expression, codes, i, n_codes) + code = codes[i] + value = code.value + return i + 1 if value.nil? + + case value + when Accessor, Column + index_info, offset = + match_resolve_index_expression_find_index(expression, + codes, i, n_codes) + i += offset - 1 + if index_info + if value.is_a?(Accessor) + self.flags |= ScanInfo::Flags::ACCESSOR + end + weight, offset = codes[i].weight + i += offset + put_search_index(index_info.index, index_info.section_id, weight) + end + when Procedure + unless value.scorer? + message = "procedure must be scorer: #{scorer.name}>" + raise ErrorMessage, message + end + scorer = value + i += 1 + index_info, offset = + match_resolve_index_expression_find_index(expression, + codes, i, n_codes) + i += offset + if index_info + scorer_args_expr_offset = 0 + if codes[i].op != Operator::CALL + scorer_args_expr_offset = i + end + while i < n_codes and codes[i].op != Operator::CALL + i += 1 + end + weight, offset = codes[i].weight + i += offset + search_index = ScanInfoSearchIndex.new(index_info.index, + index_info.section_id, + weight, + scorer, + expression, + scorer_args_expr_offset) + @search_indexes << search_index + end + when Table + raise ErrorMessage, "invalid match target: <#{value.name}>" end + i + 1 end - def match_resolve_index_expression_codes(codes, i, n_codes) + def match_resolve_index_expression_find_index(expression, codes, i, n_codes) code = codes[i] value = code.value + index_info = nil + offset = 1 case value when Accessor - match_resolve_index_expression_accessor(code) + accessor = value + index_info = accessor.find_index(@op) + if index_info + if accessor.have_next? and index_info.index != accessor.object + index_info = IndexInfo.new(accessor, index_info.section_id) + end + end when FixedSizeColumn, VariableSizeColumn - match_resolve_index_expression_data_column(code) + index_info = value.find_index(@op) when IndexColumn + index = value section_id = 0 rest_n_codes = n_codes - i if rest_n_codes >= 2 and @@ -130,26 +228,12 @@ module Groonga codes[i + 1].value.domain == ID::INT32) and codes[i + 2].op == Operator::GET_MEMBER section_id = codes[i + 1].value.value + 1 - code = codes[i + 2] - i += 2 + offset += 2 end - put_index(value, section_id, code.weight) - when Procedure - unless value.scorer? - message = "procedure must be scorer: #{scorer.name}>" - raise ErrorMessage, message - end - @scorer = value - rest_n_codes = n_codes - i - if rest_n_codes == 0 - message = "match target is required as an argument: <#{scorer.name}>" - raise ErrorMessage, message - end - i = match_resolve_index_expression_codes(codes, i + 1, n_codes) - when Table - raise ErrorMessage, "invalid match target: <#{value.name}>" + index_info = IndexInfo.new(index, section_id) end - i + 1 + + [index_info, offset] end def match_resolve_index_expression_accessor(expr_code) @@ -157,10 +241,13 @@ module Groonga self.flags |= ScanInfo::Flags::ACCESSOR index_info = accessor.find_index(op) return if index_info.nil? + + section_id = index_info.section_id + weight = expr_code.weight if accessor.next - put_index(accessor, index_info.section_id, expr_code.weight) + put_search_index(accessor, section_id, weight) else - put_index(index_info.index, index_info.section_id, expr_code.weight) + put_search_index(index_info.index, section_id, weight) end end @@ -168,13 +255,13 @@ module Groonga column = expr_code.value index_info = column.find_index(op) return if index_info.nil? - put_index(index_info.index, index_info.section_id, expr_code.weight) + put_search_index(index_info.index, index_info.section_id, expr_code.weight) end def match_resolve_index_db_obj(db_obj) index_info = db_obj.find_index(op) return if index_info.nil? - put_index(index_info.index, index_info.section_id, 1) + put_search_index(index_info.index, index_info.section_id, 1) end def match_resolve_index_accessor(accessor) @@ -182,9 +269,9 @@ module Groonga index_info = accessor.find_index(op) return if index_info.nil? if accessor.next - put_index(accessor, index_info.section_id, 1) + put_search_index(accessor, index_info.section_id, 1) else - put_index(index_info.index, index_info.section_id, 1) + put_search_index(index_info.index, index_info.section_id, 1) end end @@ -202,18 +289,19 @@ module Groonga def call_relational_resolve_index_db_obj(db_obj) index_info = db_obj.find_index(op) return if index_info.nil? - put_index(index_info.index, index_info.section_id, 1) + put_search_index(index_info.index, index_info.section_id, 1) end def call_relational_resolve_index_accessor(accessor) self.flags |= ScanInfo::Flags::ACCESSOR index_info = accessor.find_index(op) return if index_info.nil? - put_index(index_info.index, index_info.section_id, 1) + put_search_index(index_info.index, index_info.section_id, 1) end - def put_index(index, section_id, weight) - @indexes << [index, section_id, weight] + def put_search_index(index, section_id, weight) + search_index = ScanInfoSearchIndex.new(index, section_id, weight) + @search_indexes << search_index end end end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_search_index.rb b/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_search_index.rb new file mode 100644 index 00000000000..a2818160310 --- /dev/null +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/scan_info_search_index.rb @@ -0,0 +1,9 @@ +module Groonga + class ScanInfoSearchIndex < Struct.new(:index_column, + :section_id, + :weight, + :scorer, + :scorer_args_expr, + :scorer_args_expr_offset) + end +end diff --git a/storage/mroonga/vendor/groonga/lib/mrb/scripts/sources.am b/storage/mroonga/vendor/groonga/lib/mrb/scripts/sources.am index fa90c59ce23..5ddcba18d4c 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/scripts/sources.am +++ b/storage/mroonga/vendor/groonga/lib/mrb/scripts/sources.am @@ -1,24 +1,23 @@ RUBY_SCRIPT_FILES = \ backtrace_entry.rb \ command.rb \ - command_line/grndb.rb \ context.rb \ - context/error_level.rb \ - context/rc.rb \ database.rb \ error.rb \ eval_context.rb \ expression.rb \ + expression_size_estimator.rb \ + index_column.rb \ index_cursor.rb \ index_info.rb \ - initialize/pre.rb \ - initialize/post.rb \ logger.rb \ - logger/level.rb \ + object.rb \ + operator.rb \ plugin_loader.rb \ require.rb \ scan_info.rb \ scan_info_builder.rb \ scan_info_data.rb \ + scan_info_search_index.rb \ table_cursor.rb \ writer.rb diff --git a/storage/mroonga/vendor/groonga/lib/mrb/sources.am b/storage/mroonga/vendor/groonga/lib/mrb/sources.am index c4f02fca639..5cda570bc37 100644 --- a/storage/mroonga/vendor/groonga/lib/mrb/sources.am +++ b/storage/mroonga/vendor/groonga/lib/mrb/sources.am @@ -11,6 +11,8 @@ libgrnmrb_la_SOURCES = \ mrb_command.h \ mrb_command_input.c \ mrb_command_input.h \ + mrb_content_type.c \ + mrb_content_type.h \ mrb_converter.c \ mrb_converter.h \ mrb_ctx.c \ @@ -37,8 +39,12 @@ libgrnmrb_la_SOURCES = \ mrb_logger.h \ mrb_object.c \ mrb_object.h \ + mrb_object_flags.c \ + mrb_object_flags.h \ mrb_operator.c \ mrb_operator.h \ + mrb_options.c \ + mrb_options.h \ mrb_patricia_trie.c \ mrb_patricia_trie.h \ mrb_procedure.c \ diff --git a/storage/mroonga/vendor/groonga/lib/normalizer.c b/storage/mroonga/vendor/groonga/lib/normalizer.c index 27d34a4850b..5999bf64317 100644 --- a/storage/mroonga/vendor/groonga/lib/normalizer.c +++ b/storage/mroonga/vendor/groonga/lib/normalizer.c @@ -728,7 +728,7 @@ utf8_normalize(grn_ctx *ctx, grn_string *nstr) nstr->ctypes = ctypes; } } - memcpy(d, p, lp); + grn_memcpy(d, p, lp); d_ = d; d += lp; length++; diff --git a/storage/mroonga/vendor/groonga/lib/operator.c b/storage/mroonga/vendor/groonga/lib/operator.c index 27aa5df5819..14a870cd885 100644 --- a/storage/mroonga/vendor/groonga/lib/operator.c +++ b/storage/mroonga/vendor/groonga/lib/operator.c @@ -19,9 +19,18 @@ #include "grn.h" #include "grn_db.h" #include "grn_str.h" +#include "grn_normalizer.h" #include <string.h> +#ifdef GRN_WITH_ONIGMO +# define GRN_SUPPORT_REGEXP +#endif + +#ifdef GRN_SUPPORT_REGEXP +# include <oniguruma.h> +#endif + static const char *operator_names[] = { "push", "pop", @@ -99,13 +108,16 @@ static const char *operator_names[] = { "table_sort", "table_group", "json_put", - "get_member" + "get_member", + "regexp" }; +#define GRN_OP_LAST GRN_OP_REGEXP + const char * grn_operator_to_string(grn_operator op) { - if (GRN_OP_PUSH <= op && op <= GRN_OP_GET_MEMBER) { + if (op <= GRN_OP_LAST) { return operator_names[op]; } else { return "unknown"; @@ -310,12 +322,16 @@ grn_operator_to_string(grn_operator op) if (!grn_obj_cast(ctx, x, &dest, GRN_FALSE)) {\ r = (GRN_BULK_VSIZE(&dest) == GRN_BULK_VSIZE(y) &&\ !memcmp(GRN_BULK_HEAD(&dest), GRN_BULK_HEAD(y), GRN_BULK_VSIZE(y))); \ + } else {\ + r = GRN_FALSE;\ }\ } else {\ GRN_OBJ_INIT(&dest, GRN_BULK, 0, x->header.domain);\ if (!grn_obj_cast(ctx, y, &dest, GRN_FALSE)) {\ r = (GRN_BULK_VSIZE(&dest) == GRN_BULK_VSIZE(x) &&\ !memcmp(GRN_BULK_HEAD(&dest), GRN_BULK_HEAD(x), GRN_BULK_VSIZE(x))); \ + } else {\ + r = GRN_FALSE;\ }\ }\ GRN_OBJ_FIN(ctx, &dest);\ @@ -594,3 +610,319 @@ grn_operator_exec_greater_equal(grn_ctx *ctx, grn_obj *x, grn_obj *y) DO_COMPARE(x, y, r, >=); GRN_API_RETURN(r); } + +static grn_bool +string_have_sub_text(grn_ctx *ctx, + const char *text, unsigned int text_len, + const char *sub_text, unsigned int sub_text_len) +{ + /* TODO: Use more fast algorithm such as Boyer-Moore algorithm that + * is used in snip.c. */ + const char *text_end = text + text_len; + unsigned int sub_text_current = 0; + + for (; text < text_end; text++) { + if (text[0] == sub_text[sub_text_current]) { + sub_text_current++; + if (sub_text_current == sub_text_len) { + return GRN_TRUE; + } + } else { + sub_text_current = 0; + } + } + + return GRN_FALSE; +} + +static grn_bool +string_have_prefix(grn_ctx *ctx, + const char *target, unsigned int target_len, + const char *prefix, unsigned int prefix_len) +{ + return (target_len >= prefix_len && + strncmp(target, prefix, prefix_len) == 0); +} + +static grn_bool +string_match_regexp(grn_ctx *ctx, + const char *target, unsigned int target_len, + const char *pattern, unsigned int pattern_len) +{ +#ifdef GRN_SUPPORT_REGEXP + OnigRegex regex; + OnigEncoding onig_encoding; + int onig_result; + OnigErrorInfo onig_error_info; + + if (ctx->encoding == GRN_ENC_NONE) { + return GRN_FALSE; + } + + switch (ctx->encoding) { + case GRN_ENC_EUC_JP : + onig_encoding = ONIG_ENCODING_EUC_JP; + break; + case GRN_ENC_UTF8 : + onig_encoding = ONIG_ENCODING_UTF8; + break; + case GRN_ENC_SJIS : + onig_encoding = ONIG_ENCODING_CP932; + break; + case GRN_ENC_LATIN1 : + onig_encoding = ONIG_ENCODING_ISO_8859_1; + break; + case GRN_ENC_KOI8R : + onig_encoding = ONIG_ENCODING_KOI8_R; + break; + default : + return GRN_FALSE; + } + + onig_result = onig_new(®ex, + pattern, + pattern + pattern_len, + ONIG_OPTION_ASCII_RANGE, + onig_encoding, + ONIG_SYNTAX_RUBY, + &onig_error_info); + if (onig_result != ONIG_NORMAL) { + char message[ONIG_MAX_ERROR_MESSAGE_LEN]; + onig_error_code_to_str(message, onig_result, onig_error_info); + ERR(GRN_INVALID_ARGUMENT, + "[operator][regexp] " + "failed to create regular expression object: <%.*s>: %s", + pattern_len, pattern, + message); + return GRN_FALSE; + } + + { + OnigPosition position; + position = onig_search(regex, + target, + target + target_len, + target, + target + target_len, + NULL, + ONIG_OPTION_NONE); + onig_free(regex); + return position != ONIG_MISMATCH; + } +#else + return GRN_FALSE; +#endif +} + +static grn_bool +exec_text_operator(grn_ctx *ctx, + grn_operator op, + const char *target, + unsigned int target_len, + const char *query, + unsigned int query_len) +{ + grn_bool matched = GRN_FALSE; + + switch (op) { + case GRN_OP_MATCH : + matched = string_have_sub_text(ctx, target, target_len, query, query_len); + break; + case GRN_OP_PREFIX : + matched = string_have_prefix(ctx, target, target_len, query, query_len); + break; + case GRN_OP_REGEXP : + matched = string_match_regexp(ctx, target, target_len, query, query_len); + break; + default : + matched = GRN_FALSE; + break; + } + + return matched; +} + +static grn_bool +exec_text_operator_raw_text_raw_text(grn_ctx *ctx, + grn_operator op, + const char *target, + unsigned int target_len, + const char *query, + unsigned int query_len) +{ + grn_obj *normalizer; + grn_obj *norm_target; + grn_obj *norm_query; + const char *norm_target_raw; + const char *norm_query_raw; + unsigned int norm_target_raw_length_in_bytes; + unsigned int norm_query_raw_length_in_bytes; + grn_bool matched = GRN_FALSE; + + if (target_len == 0 || query_len == 0) { + return GRN_FALSE; + } + + if (op == GRN_OP_REGEXP) { + return exec_text_operator(ctx, op, + target, target_len, + query, query_len); + } + + normalizer = grn_ctx_get(ctx, GRN_NORMALIZER_AUTO_NAME, -1); + norm_target = grn_string_open(ctx, target, target_len, normalizer, 0); + norm_query = grn_string_open(ctx, query, query_len, normalizer, 0); + grn_string_get_normalized(ctx, norm_target, + &norm_target_raw, + &norm_target_raw_length_in_bytes, + NULL); + grn_string_get_normalized(ctx, norm_query, + &norm_query_raw, + &norm_query_raw_length_in_bytes, + NULL); + + matched = exec_text_operator(ctx, op, + norm_target_raw, + norm_target_raw_length_in_bytes, + norm_query_raw, + norm_query_raw_length_in_bytes); + + grn_obj_close(ctx, norm_target); + grn_obj_close(ctx, norm_query); + grn_obj_unlink(ctx, normalizer); + + return matched; +} + +static grn_bool +exec_text_operator_record_text(grn_ctx *ctx, + grn_operator op, + grn_obj *record, grn_obj *table, + grn_obj *query) +{ + grn_obj *normalizer; + char record_key[GRN_TABLE_MAX_KEY_SIZE]; + int record_key_len; + grn_bool matched = GRN_FALSE; + + if (table->header.domain != GRN_DB_SHORT_TEXT) { + return GRN_FALSE; + } + + if (GRN_TEXT_LEN(query) == 0) { + return GRN_FALSE; + } + + record_key_len = grn_table_get_key(ctx, table, GRN_RECORD_VALUE(record), + record_key, GRN_TABLE_MAX_KEY_SIZE); + grn_table_get_info(ctx, table, NULL, NULL, NULL, &normalizer, NULL); + if (normalizer && (op != GRN_OP_REGEXP)) { + grn_obj *norm_query; + const char *norm_query_raw; + unsigned int norm_query_raw_length_in_bytes; + norm_query = grn_string_open(ctx, + GRN_TEXT_VALUE(query), + GRN_TEXT_LEN(query), + normalizer, + 0); + grn_string_get_normalized(ctx, norm_query, + &norm_query_raw, + &norm_query_raw_length_in_bytes, + NULL); + matched = exec_text_operator(ctx, + op, + record_key, + record_key_len, + norm_query_raw, + norm_query_raw_length_in_bytes); + grn_obj_close(ctx, norm_query); + } else { + matched = exec_text_operator_raw_text_raw_text(ctx, + op, + record_key, + record_key_len, + GRN_TEXT_VALUE(query), + GRN_TEXT_LEN(query)); + } + + return matched; +} + +static grn_bool +exec_text_operator_text_text(grn_ctx *ctx, + grn_operator op, + grn_obj *target, + grn_obj *query) +{ + return exec_text_operator_raw_text_raw_text(ctx, + op, + GRN_TEXT_VALUE(target), + GRN_TEXT_LEN(target), + GRN_TEXT_VALUE(query), + GRN_TEXT_LEN(query)); +} + +static grn_bool +exec_text_operator_bulk_bulk(grn_ctx *ctx, + grn_operator op, + grn_obj *target, + grn_obj *query) +{ + switch (target->header.domain) { + case GRN_DB_SHORT_TEXT : + case GRN_DB_TEXT : + case GRN_DB_LONG_TEXT : + switch (query->header.domain) { + case GRN_DB_SHORT_TEXT : + case GRN_DB_TEXT : + case GRN_DB_LONG_TEXT : + return exec_text_operator_text_text(ctx, op, target, query); + default : + break; + } + return GRN_FALSE; + default: + { + grn_obj *domain; + domain = grn_ctx_at(ctx, target->header.domain); + if (GRN_OBJ_TABLEP(domain)) { + switch (query->header.domain) { + case GRN_DB_SHORT_TEXT : + case GRN_DB_TEXT : + case GRN_DB_LONG_TEXT : + return exec_text_operator_record_text(ctx, op, target, domain, query); + default : + break; + } + } + } + return GRN_FALSE; + } +} + +grn_bool +grn_operator_exec_match(grn_ctx *ctx, grn_obj *target, grn_obj *sub_text) +{ + grn_bool matched; + GRN_API_ENTER; + matched = exec_text_operator_bulk_bulk(ctx, GRN_OP_MATCH, target, sub_text); + GRN_API_RETURN(matched); +} + +grn_bool +grn_operator_exec_prefix(grn_ctx *ctx, grn_obj *target, grn_obj *prefix) +{ + grn_bool matched; + GRN_API_ENTER; + matched = exec_text_operator_bulk_bulk(ctx, GRN_OP_PREFIX, target, prefix); + GRN_API_RETURN(matched); +} + +grn_bool +grn_operator_exec_regexp(grn_ctx *ctx, grn_obj *target, grn_obj *pattern) +{ + grn_bool matched; + GRN_API_ENTER; + matched = exec_text_operator_bulk_bulk(ctx, GRN_OP_REGEXP, target, pattern); + GRN_API_RETURN(matched); +} diff --git a/storage/mroonga/vendor/groonga/lib/output.c b/storage/mroonga/vendor/groonga/lib/output.c index 20bd2a8694a..0036e9a8619 100644 --- a/storage/mroonga/vendor/groonga/lib/output.c +++ b/storage/mroonga/vendor/groonga/lib/output.c @@ -54,6 +54,8 @@ put_delimiter(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type) case GRN_CONTENT_MSGPACK : // do nothing break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + break; case GRN_CONTENT_NONE: break; } @@ -88,6 +90,8 @@ grn_output_array_open(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_typ msgpack_pack_array(&ctx->impl->msgpacker, nelements); #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + break; case GRN_CONTENT_NONE: break; } @@ -119,6 +123,8 @@ grn_output_array_close(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_ty case GRN_CONTENT_MSGPACK : // do nothing break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + break; case GRN_CONTENT_NONE: break; } @@ -155,6 +161,8 @@ grn_output_map_open(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type, msgpack_pack_map(&ctx->impl->msgpacker, nelements); #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + break; case GRN_CONTENT_NONE: break; } @@ -186,6 +194,8 @@ grn_output_map_close(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type case GRN_CONTENT_MSGPACK : // do nothing break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + break; case GRN_CONTENT_NONE: break; } @@ -214,6 +224,9 @@ grn_output_int32(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type, in msgpack_pack_int32(&ctx->impl->msgpacker, value); #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + grn_text_itoa(ctx, outbuf, value); + break; case GRN_CONTENT_NONE: break; } @@ -241,6 +254,9 @@ grn_output_int64(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type, in msgpack_pack_int64(&ctx->impl->msgpacker, value); #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + grn_text_lltoa(ctx, outbuf, value); + break; case GRN_CONTENT_NONE: break; } @@ -268,6 +284,9 @@ grn_output_uint64(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type, i msgpack_pack_uint64(&ctx->impl->msgpacker, value); #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + grn_text_ulltoa(ctx, outbuf, value); + break; case GRN_CONTENT_NONE: break; } @@ -295,6 +314,9 @@ grn_output_float(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type, do msgpack_pack_double(&ctx->impl->msgpacker, value); #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + grn_text_ftoa(ctx, outbuf, value); + break; case GRN_CONTENT_NONE: break; } @@ -320,10 +342,13 @@ grn_output_str(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type, break; case GRN_CONTENT_MSGPACK : #ifdef GRN_WITH_MESSAGE_PACK - msgpack_pack_raw(&ctx->impl->msgpacker, value_len); - msgpack_pack_raw_body(&ctx->impl->msgpacker, value, value_len); + msgpack_pack_str(&ctx->impl->msgpacker, value_len); + msgpack_pack_str_body(&ctx->impl->msgpacker, value, value_len); #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + GRN_TEXT_PUT(ctx, outbuf, value, value_len); + break; case GRN_CONTENT_NONE: break; } @@ -362,6 +387,9 @@ grn_output_bool(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type, grn } #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + GRN_TEXT_PUTS(ctx, outbuf, value ? "true" : "false"); + break; case GRN_CONTENT_NONE: break; } @@ -386,6 +414,8 @@ grn_output_null(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type) msgpack_pack_nil(&ctx->impl->msgpacker); #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + break; case GRN_CONTENT_NONE: break; } @@ -426,6 +456,9 @@ grn_output_time(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type, int msgpack_pack_double(&ctx->impl->msgpacker, dv); #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + grn_text_ftoa(ctx, outbuf, dv); + break; case GRN_CONTENT_NONE: break; } @@ -477,8 +510,8 @@ grn_output_geo_point(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type grn_text_itoa(ctx, &buf, value->latitude); GRN_TEXT_PUTC(ctx, &buf, 'x'); grn_text_itoa(ctx, &buf, value->longitude); - msgpack_pack_raw(&ctx->impl->msgpacker, GRN_TEXT_LEN(&buf)); - msgpack_pack_raw_body(&ctx->impl->msgpacker, + msgpack_pack_str(&ctx->impl->msgpacker, GRN_TEXT_LEN(&buf)); + msgpack_pack_str_body(&ctx->impl->msgpacker, GRN_TEXT_VALUE(&buf), GRN_TEXT_LEN(&buf)); grn_obj_close(ctx, &buf); @@ -487,6 +520,17 @@ grn_output_geo_point(grn_ctx *ctx, grn_obj *outbuf, grn_content_type output_type } #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + if (value) { + GRN_TEXT_PUTC(ctx, outbuf, '"'); + grn_text_itoa(ctx, outbuf, value->latitude); + GRN_TEXT_PUTC(ctx, outbuf, 'x'); + grn_text_itoa(ctx, outbuf, value->longitude); + GRN_TEXT_PUTC(ctx, outbuf, '"'); + } else { + GRN_TEXT_PUTS(ctx, outbuf, "\"\""); + } + break; case GRN_CONTENT_NONE: break; } @@ -1802,7 +1846,7 @@ typedef struct { } msgpack_writer_ctx; static inline int -msgpack_buffer_writer(void* data, const char* buf, unsigned int len) +msgpack_buffer_writer(void* data, const char* buf, msgpack_size_t len) { msgpack_writer_ctx *writer_ctx = (msgpack_writer_ctx *)data; return grn_bulk_write(writer_ctx->ctx, writer_ctx->buffer, buf, len); @@ -1980,8 +2024,8 @@ grn_output_envelope(grn_ctx *ctx, msgpack_pack_double(&header_packer, elapsed); if (rc != GRN_SUCCESS) { - msgpack_pack_raw(&header_packer, strlen(ctx->errbuf)); - msgpack_pack_raw_body(&header_packer, ctx->errbuf, strlen(ctx->errbuf)); + msgpack_pack_str(&header_packer, strlen(ctx->errbuf)); + msgpack_pack_str_body(&header_packer, ctx->errbuf, strlen(ctx->errbuf)); if (ctx->errfunc && ctx->errfile) { grn_obj *command = GRN_CTX_USER_DATA(ctx)->ptr; int error_detail_size; @@ -1996,33 +2040,35 @@ grn_output_envelope(grn_ctx *ctx, } msgpack_pack_array(&header_packer, error_detail_size); - msgpack_pack_raw(&header_packer, strlen(ctx->errfunc)); - msgpack_pack_raw_body(&header_packer, ctx->errfunc, strlen(ctx->errfunc)); + msgpack_pack_str(&header_packer, strlen(ctx->errfunc)); + msgpack_pack_str_body(&header_packer, ctx->errfunc, strlen(ctx->errfunc)); - msgpack_pack_raw(&header_packer, strlen(ctx->errfile)); - msgpack_pack_raw_body(&header_packer, ctx->errfile, strlen(ctx->errfile)); + msgpack_pack_str(&header_packer, strlen(ctx->errfile)); + msgpack_pack_str_body(&header_packer, ctx->errfile, strlen(ctx->errfile)); msgpack_pack_int(&header_packer, ctx->errline); if (command) { if (file) { - msgpack_pack_raw(&header_packer, strlen(file)); - msgpack_pack_raw_body(&header_packer, file, strlen(file)); + msgpack_pack_str(&header_packer, strlen(file)); + msgpack_pack_str_body(&header_packer, file, strlen(file)); } else { - msgpack_pack_raw(&header_packer, 7); - msgpack_pack_raw_body(&header_packer, "(stdin)", 7); + msgpack_pack_str(&header_packer, 7); + msgpack_pack_str_body(&header_packer, "(stdin)", 7); } msgpack_pack_int(&header_packer, line); - msgpack_pack_raw(&header_packer, GRN_TEXT_LEN(command)); - msgpack_pack_raw_body(&header_packer, GRN_TEXT_VALUE(command), GRN_TEXT_LEN(command)); + msgpack_pack_str(&header_packer, GRN_TEXT_LEN(command)); + msgpack_pack_str_body(&header_packer, GRN_TEXT_VALUE(command), GRN_TEXT_LEN(command)); } } } } #endif break; + case GRN_CONTENT_GROONGA_COMMAND_LIST : + break; case GRN_CONTENT_NONE: break; } diff --git a/storage/mroonga/vendor/groonga/lib/pat.c b/storage/mroonga/vendor/groonga/lib/pat.c index add8469a009..e24dbe7bbf0 100644 --- a/storage/mroonga/vendor/groonga/lib/pat.c +++ b/storage/mroonga/vendor/groonga/lib/pat.c @@ -202,7 +202,7 @@ key_put(grn_ctx *ctx, grn_pat *pat, const uint8_t *key, int len) uint8_t *dest; KEY_AT(pat, res, dest, GRN_TABLE_ADD); if (!dest) { return 0; } - memcpy(dest, key, len); + grn_memcpy(dest, key, len); } pat->header->curr_key += len; return res; @@ -227,7 +227,7 @@ pat_node_set_key(grn_ctx *ctx, grn_pat *pat, pat_node *n, const uint8_t *key, un PAT_LEN_SET(n, len); if (len <= sizeof(uint32_t)) { PAT_IMD_ON(n); - memcpy(&n->key, key, len); + grn_memcpy(&n->key, key, len); } else { PAT_IMD_OFF(n); n->key = key_put(ctx, pat, key, len); @@ -725,7 +725,7 @@ _grn_pat_add(grn_ctx *ctx, grn_pat *pat, const uint8_t *key, uint32_t size, uint pat->header->garbages[size2] = rn->lr[0]; if (!(keybuf = pat_node_get_key(ctx, pat, rn))) { return 0; } PAT_LEN_SET(rn, size); - memcpy(keybuf, key, size); + grn_memcpy(keybuf, key, size); } else { if (!(rn = pat_node_new(ctx, pat, &r))) { return 0; } pat_node_set_key(ctx, pat, rn, key, size); @@ -1137,14 +1137,18 @@ _grn_pat_del(grn_ctx *ctx, grn_pat *pat, const char *key, uint32_t key_size, int di->stat = DL_PHASE2; di->d = r; if (otherside) { - PAT_AT(pat, otherside, rno); - if (rno && c0 < PAT_CHK(rno) && PAT_CHK(rno) <= c) { - if (!delinfo_search(pat, otherside)) { - GRN_LOG(ctx, GRN_LOG_ERROR, "no delinfo found %d", otherside); + if (otherside == r) { + otherside = 0; + } else { + PAT_AT(pat, otherside, rno); + if (rno && c0 < PAT_CHK(rno) && PAT_CHK(rno) <= c) { + if (!delinfo_search(pat, otherside)) { + GRN_LOG(ctx, GRN_LOG_DEBUG, "no delinfo found %d", otherside); + } + PAT_CHK_SET(rno, 0); } - PAT_CHK_SET(rno, 0); + if (proot == p0 && !rno->check) { rno->lr[0] = rno->lr[1] = otherside; } } - if (proot == p0 && !rno->check) { rno->lr[0] = rno->lr[1] = otherside; } } *p0 = otherside; } else { @@ -1201,14 +1205,18 @@ _grn_pat_del(grn_ctx *ctx, grn_pat *pat, const char *key, uint32_t key_size, int if (proot == p0 && !rn0->check) { rn0->lr[0] = rn0->lr[1] = otherside; } } else { if (otherside) { - PAT_AT(pat, otherside, rno); - if (rno && c0 < PAT_CHK(rno) && PAT_CHK(rno) <= c) { - if (!delinfo_search(pat, otherside)) { - GRN_LOG(ctx, GRN_LOG_ERROR, "no delinfo found %d", otherside); + if (otherside == r) { + otherside = 0; + } else { + PAT_AT(pat, otherside, rno); + if (rno && c0 < PAT_CHK(rno) && PAT_CHK(rno) <= c) { + if (!delinfo_search(pat, otherside)) { + GRN_LOG(ctx, GRN_LOG_ERROR, "no delinfo found %d", otherside); + } + PAT_CHK_SET(rno, 0); } - PAT_CHK_SET(rno, 0); + if (proot == p0 && !rno->check) { rno->lr[0] = rno->lr[1] = otherside; } } - if (proot == p0 && !rno->check) { rno->lr[0] = rno->lr[1] = otherside; } } *p0 = otherside; } @@ -1295,7 +1303,7 @@ grn_pat_get_key(grn_ctx *ctx, grn_pat *pat, grn_id id, void *keybuf, int bufsize if (KEY_NEEDS_CONVERT(pat, len)) { KEY_DEC(pat, keybuf, key, len); } else { - memcpy(keybuf, key, len); + grn_memcpy(keybuf, key, len); } } return len; @@ -1341,9 +1349,9 @@ grn_pat_get_value(grn_ctx *ctx, grn_pat *pat, grn_id id, void *valuebuf) if (v) { if (valuebuf) { if (pat->obj.header.flags & GRN_OBJ_KEY_WITH_SIS) { - memcpy(valuebuf, v + sizeof(sis_node), value_size); + grn_memcpy(valuebuf, v + sizeof(sis_node), value_size); } else { - memcpy(valuebuf, v, value_size); + grn_memcpy(valuebuf, v, value_size); } } return value_size; @@ -1377,7 +1385,7 @@ grn_pat_set_value(grn_ctx *ctx, grn_pat *pat, grn_id id, if (pat->obj.header.flags & GRN_OBJ_KEY_WITH_SIS) { v += sizeof(sis_node); } switch ((flags & GRN_OBJ_SET_MASK)) { case GRN_OBJ_SET : - memcpy(v, value, value_size); + grn_memcpy(v, value, value_size); return GRN_SUCCESS; case GRN_OBJ_INCR : switch (value_size) { @@ -2773,7 +2781,7 @@ rk_emit(rk_tree_node *rn, char **str) #define RK_OUTPUT(e,l) do {\ if (oc < oe) {\ uint32_t l_ = (oc + (l) < oe) ? (l) : (oe - oc);\ - memcpy(oc, (e), l_);\ + grn_memcpy(oc, (e), l_);\ oc += l_;\ ic_ = ic;\ }\ @@ -2863,7 +2871,7 @@ search_push(grn_ctx *ctx, grn_pat *pat, grn_pat_cursor *c, if (l + key_len <= GRN_TABLE_MAX_KEY_SIZE) { int ch = c0; grn_id i; - memcpy(key + key_len, e, l); + grn_memcpy(key + key_len, e, l); if ((i = sub_search(ctx, pat, id, &ch, key, key_len + l))) { search_push(ctx, pat, c, key, key_len + l, rn->next, i, ch, flags); } diff --git a/storage/mroonga/vendor/groonga/lib/plugin.c b/storage/mroonga/vendor/groonga/lib/plugin.c index 7fd0fbd1971..48ccded0b92 100644 --- a/storage/mroonga/vendor/groonga/lib/plugin.c +++ b/storage/mroonga/vendor/groonga/lib/plugin.c @@ -45,10 +45,6 @@ static grn_hash *grn_plugins = NULL; static grn_critical_section grn_plugins_lock; -#ifdef GRN_WITH_MRUBY -static const char *grn_plugin_mrb_suffix = ".rb"; -#endif /* GRN_WITH_MRUBY */ - #ifdef HAVE_DLFCN_H # include <dlfcn.h> # define grn_dl_open(filename) dlopen(filename, RTLD_LAZY | RTLD_LOCAL) @@ -68,6 +64,8 @@ static const char *grn_plugin_mrb_suffix = ".rb"; # define grn_dl_clear_error() #endif +#define GRN_PLUGIN_KEY_SIZE(filename) (strlen((filename)) + 1) + static int compute_name_size(const char *name, int name_size) { @@ -88,7 +86,8 @@ grn_plugin_reference(grn_ctx *ctx, const char *filename) grn_plugin **plugin = NULL; CRITICAL_SECTION_ENTER(grn_plugins_lock); - id = grn_hash_get(&grn_gctx, grn_plugins, filename, strlen(filename), + id = grn_hash_get(&grn_gctx, grn_plugins, + filename, GRN_PLUGIN_KEY_SIZE(filename), (void **)&plugin); if (plugin) { (*plugin)->refcount++; @@ -102,7 +101,8 @@ const char * grn_plugin_path(grn_ctx *ctx, grn_id id) { const char *path; - uint32_t key_size; + grn_plugin *plugin; + int value_size; const char *system_plugins_dir; size_t system_plugins_dir_size; @@ -111,13 +111,14 @@ grn_plugin_path(grn_ctx *ctx, grn_id id) } CRITICAL_SECTION_ENTER(grn_plugins_lock); - path = _grn_hash_key(&grn_gctx, grn_plugins, id, &key_size); + value_size = grn_hash_get_value(&grn_gctx, grn_plugins, id, &plugin); CRITICAL_SECTION_LEAVE(grn_plugins_lock); - if (!path) { + if (!plugin) { return NULL; } + path = plugin->path; system_plugins_dir = grn_plugin_get_system_plugins_dir(); system_plugins_dir_size = strlen(system_plugins_dir); if (strncmp(system_plugins_dir, path, system_plugins_dir_size) == 0) { @@ -254,6 +255,11 @@ grn_plugin_open_mrb(grn_ctx *ctx, const char *filename, size_t filename_size) grn_id id = GRN_ID_NIL; grn_plugin **plugin = NULL; + if (!ctx->impl->mrb.state) { + ERR(GRN_FUNCTION_NOT_IMPLEMENTED, "mruby support isn't enabled"); + return GRN_ID_NIL; + } + id = grn_hash_add(&grn_gctx, grn_plugins, filename, filename_size, (void **)&plugin, NULL); if (!id) { @@ -266,6 +272,7 @@ grn_plugin_open_mrb(grn_ctx *ctx, const char *filename, size_t filename_size) return GRN_ID_NIL; } + grn_memcpy((*plugin)->path, filename, filename_size); (*plugin)->dl = NULL; (*plugin)->init_func = NULL; (*plugin)->register_func = NULL; @@ -284,7 +291,7 @@ grn_plugin_open(grn_ctx *ctx, const char *filename) grn_plugin **plugin = NULL; size_t filename_size; - filename_size = strlen(filename); + filename_size = GRN_PLUGIN_KEY_SIZE(filename); CRITICAL_SECTION_ENTER(grn_plugins_lock); if ((id = grn_hash_get(&grn_gctx, grn_plugins, filename, filename_size, @@ -294,11 +301,15 @@ grn_plugin_open(grn_ctx *ctx, const char *filename) } #ifdef GRN_WITH_MRUBY - if (filename_size > strlen(grn_plugin_mrb_suffix) && - strcmp(filename + (filename_size - strlen(grn_plugin_mrb_suffix)), - grn_plugin_mrb_suffix) == 0) { - id = grn_plugin_open_mrb(ctx, filename, filename_size); - goto exit; + { + const char *mrb_suffix; + mrb_suffix = grn_plugin_get_ruby_suffix(); + if (filename_size > strlen(mrb_suffix) && + strcmp(filename + (strlen(filename) - strlen(mrb_suffix)), + mrb_suffix) == 0) { + id = grn_plugin_open_mrb(ctx, filename, filename_size); + goto exit; + } } #endif /* GRN_WITH_MRUBY */ @@ -307,6 +318,7 @@ grn_plugin_open(grn_ctx *ctx, const char *filename) (void **)&plugin, NULL))) { *plugin = GRN_GMALLOCN(grn_plugin, 1); if (*plugin) { + grn_memcpy((*plugin)->path, filename, filename_size); if (grn_plugin_initialize(ctx, *plugin, dl, id, filename)) { GRN_GFREE(*plugin); *plugin = NULL; @@ -439,6 +451,12 @@ grn_plugin_get_suffix(void) return GRN_PLUGIN_SUFFIX; } +const char * +grn_plugin_get_ruby_suffix(void) +{ + return ".rb"; +} + grn_rc grn_plugin_register_by_path(grn_ctx *ctx, const char *path) { @@ -476,9 +494,9 @@ grn_plugin_get_default_system_plugins_dir(void) base_dir = grn_win32_base_dir(); base_dir_length = strlen(base_dir); - strcpy(win32_plugins_dir_buffer, base_dir); - strcat(win32_plugins_dir_buffer, "/"); - strcat(win32_plugins_dir_buffer, relative_path); + grn_strcpy(win32_plugins_dir_buffer, PATH_MAX, base_dir); + grn_strcat(win32_plugins_dir_buffer, PATH_MAX, "/"); + grn_strcat(win32_plugins_dir_buffer, PATH_MAX, relative_path); win32_plugins_dir = win32_plugins_dir_buffer; } return win32_plugins_dir; @@ -495,14 +513,16 @@ grn_plugin_get_default_system_plugins_dir(void) const char * grn_plugin_get_system_plugins_dir(void) { - const char *plugins_dir; + static char plugins_dir[GRN_ENV_BUFFER_SIZE]; - plugins_dir = getenv("GRN_PLUGINS_DIR"); - if (!plugins_dir) { - plugins_dir = grn_plugin_get_default_system_plugins_dir(); + grn_getenv("GRN_PLUGINS_DIR", + plugins_dir, + GRN_ENV_BUFFER_SIZE); + if (plugins_dir[0]) { + return plugins_dir; + } else { + return grn_plugin_get_default_system_plugins_dir(); } - - return plugins_dir; } static char * @@ -521,14 +541,19 @@ grn_plugin_find_path_raw(grn_ctx *ctx, const char *path) return GRN_STRDUP(path); } -#if GRN_WITH_MRUBY +#ifdef GRN_WITH_MRUBY static char * grn_plugin_find_path_mrb(grn_ctx *ctx, const char *path, size_t path_len) { char mrb_path[PATH_MAX]; - const char *mrb_suffix = grn_plugin_mrb_suffix; + const char *mrb_suffix; size_t mrb_path_len; + mrb_suffix = grn_plugin_get_ruby_suffix(); + if (!ctx->impl->mrb.state) { + return NULL; + } + mrb_path_len = path_len + strlen(mrb_suffix); if (mrb_path_len >= PATH_MAX) { ERR(GRN_FILENAME_TOO_LONG, @@ -537,8 +562,8 @@ grn_plugin_find_path_mrb(grn_ctx *ctx, const char *path, size_t path_len) return NULL; } - strcpy(mrb_path, path); - strcat(mrb_path, mrb_suffix); + grn_strcpy(mrb_path, PATH_MAX, path); + grn_strcat(mrb_path, PATH_MAX, mrb_suffix); return grn_plugin_find_path_raw(ctx, mrb_path); } #else /* GRN_WITH_MRUBY */ @@ -565,8 +590,8 @@ grn_plugin_find_path_so(grn_ctx *ctx, const char *path, size_t path_len) return NULL; } - strcpy(so_path, path); - strcat(so_path, so_suffix); + grn_strcpy(so_path, PATH_MAX, path); + grn_strcat(so_path, PATH_MAX, so_suffix); return grn_plugin_find_path_raw(ctx, so_path); } @@ -598,10 +623,10 @@ grn_plugin_find_path_libs_so(grn_ctx *ctx, const char *path, size_t path_len) } libs_so_path[0] = '\0'; - strncat(libs_so_path, path, base_name - path); - strcat(libs_so_path, libs_path); - strcat(libs_so_path, base_name); - strcat(libs_so_path, so_suffix); + grn_strncat(libs_so_path, PATH_MAX, path, base_name - path); + grn_strcat(libs_so_path, PATH_MAX, libs_path); + grn_strcat(libs_so_path, PATH_MAX, base_name); + grn_strcat(libs_so_path, PATH_MAX, so_suffix); return grn_plugin_find_path_raw(ctx, libs_so_path); } @@ -620,11 +645,11 @@ grn_plugin_find_path(grn_ctx *ctx, const char *name) path[0] = '\0'; } else { plugins_dir = grn_plugin_get_system_plugins_dir(); - strcpy(path, plugins_dir); + grn_strcpy(path, PATH_MAX, plugins_dir); dir_last_char = plugins_dir[strlen(path) - 1]; if (dir_last_char != '/') { - strcat(path, "/"); + grn_strcat(path, PATH_MAX, "/"); } } @@ -637,7 +662,7 @@ grn_plugin_find_path(grn_ctx *ctx, const char *name) path, name); goto exit; } - strcat(path, name); + grn_strcat(path, PATH_MAX, name); found_path = grn_plugin_find_path_raw(ctx, path); if (found_path) { @@ -673,6 +698,30 @@ exit : GRN_API_RETURN(found_path); } +static void +grn_plugin_set_name_resolve_error(grn_ctx *ctx, const char *name, + const char *tag) +{ + const char *prefix, *prefix_separator, *suffix; + + if (name[0] == '/') { + prefix = ""; + prefix_separator = ""; + suffix = ""; + } else { + prefix = grn_plugin_get_system_plugins_dir(); + if (prefix[strlen(prefix) - 1] != '/') { + prefix_separator = "/"; + } else { + prefix_separator = ""; + } + suffix = grn_plugin_get_suffix(); + } + ERR(GRN_NO_SUCH_FILE_OR_DIRECTORY, + "%s cannot find plugin file: <%s%s%s%s>", + tag, prefix, prefix_separator, name, suffix); +} + grn_rc grn_plugin_register(grn_ctx *ctx, const char *name) { @@ -686,23 +735,86 @@ grn_plugin_register(grn_ctx *ctx, const char *name) GRN_FREE(path); } else { if (ctx->rc == GRN_SUCCESS) { - const char *prefix, *prefix_separator, *suffix; - if (name[0] == '/') { - prefix = ""; - prefix_separator = ""; - suffix = ""; + grn_plugin_set_name_resolve_error(ctx, name, "[plugin][register]"); + } + rc = ctx->rc; + } + GRN_API_RETURN(rc); +} + +grn_rc +grn_plugin_unregister_by_path(grn_ctx *ctx, const char *path) +{ + grn_obj *db; + grn_id plugin_id; + + if (!ctx || !ctx->impl) { + ERR(GRN_INVALID_ARGUMENT, "[plugin][unregister] ctx isn't initialized"); + return ctx->rc; + } + + db = ctx->impl->db; + if (!db) { + ERR(GRN_INVALID_ARGUMENT, "[plugin][unregister] DB isn't initialized"); + return ctx->rc; + } + + GRN_API_ENTER; + + CRITICAL_SECTION_ENTER(grn_plugins_lock); + plugin_id = grn_hash_get(&grn_gctx, grn_plugins, + path, GRN_PLUGIN_KEY_SIZE(path), + NULL); + CRITICAL_SECTION_LEAVE(grn_plugins_lock); + + if (plugin_id == GRN_ID_NIL) { + GRN_API_RETURN(ctx->rc); + } + + { + grn_table_cursor *cursor; + grn_id id; + + cursor = grn_table_cursor_open(ctx, db, + NULL, 0, + NULL, 0, + 0, -1, GRN_CURSOR_BY_ID); + if (!cursor) { + GRN_API_RETURN(ctx->rc); + } + + while ((id = grn_table_cursor_next(ctx, cursor))) { + grn_obj *obj; + obj = grn_ctx_at(ctx, id); + if (!obj) { + continue; + } + if (obj->header.type == GRN_PROC && DB_OBJ(obj)->range == plugin_id) { + grn_obj_remove(ctx, obj); } else { - prefix = grn_plugin_get_system_plugins_dir(); - if (prefix[strlen(prefix) - 1] != '/') { - prefix_separator = "/"; - } else { - prefix_separator = ""; - } - suffix = grn_plugin_get_suffix(); + grn_obj_unlink(ctx, obj); } - ERR(GRN_NO_SUCH_FILE_OR_DIRECTORY, - "cannot find plugin file: <%s%s%s%s>", - prefix, prefix_separator, name, suffix); + } + grn_table_cursor_close(ctx, cursor); + } + + GRN_API_RETURN(ctx->rc); +} + +grn_rc +grn_plugin_unregister(grn_ctx *ctx, const char *name) +{ + grn_rc rc; + char *path; + + GRN_API_ENTER; + path = grn_plugin_find_path(ctx, name); + if (path) { + rc = grn_plugin_unregister_by_path(ctx, path); + GRN_FREE(path); + } else { + if (ctx->rc == GRN_SUCCESS) { + grn_plugin_set_name_resolve_error(ctx, name, "[plugin][unregister]"); } rc = ctx->rc; } diff --git a/storage/mroonga/vendor/groonga/lib/proc.c b/storage/mroonga/vendor/groonga/lib/proc.c index 03deb0c4b70..fc6aa690e74 100644 --- a/storage/mroonga/vendor/groonga/lib/proc.c +++ b/storage/mroonga/vendor/groonga/lib/proc.c @@ -27,19 +27,18 @@ #include "grn_expr.h" #include <string.h> -#include <float.h> #include <stdlib.h> #include <fcntl.h> #include <sys/stat.h> +#ifdef WIN32 +# include <share.h> +#endif /* WIN32 */ + #ifndef O_NOFOLLOW #define O_NOFOLLOW 0 #endif -#ifndef O_BINARY -#define O_BINARY 0 -#endif - typedef grn_rc (*grn_substitute_term_func) (grn_ctx *ctx, const char *term, unsigned int term_len, @@ -65,7 +64,8 @@ grn_bulk_put_from_file(grn_ctx *ctx, grn_obj *bulk, const char *path) /* FIXME: implement more smartly with grn_bulk */ int fd, ret = 0; struct stat stat; - if ((fd = GRN_OPEN(path, O_RDONLY|O_NOFOLLOW|O_BINARY)) == -1) { + grn_open(fd, path, O_RDONLY|O_NOFOLLOW|GRN_OPEN_FLAG_BINARY); + if (fd == -1) { switch (errno) { case EACCES : ERR(GRN_OPERATION_NOT_PERMITTED, "request is not allowed: <%s>", path); @@ -91,7 +91,7 @@ grn_bulk_put_from_file(grn_ctx *ctx, grn_obj *bulk, const char *path) if ((buf = GRN_MALLOC(rest))) { ssize_t ss; for (bp = buf; rest; rest -= ss, bp += ss) { - if ((ss = GRN_READ(fd, bp, rest)) == -1) { goto exit; } + if ((ss = grn_read(fd, bp, rest)) == -1) { goto exit; } } GRN_TEXT_PUT(ctx, bulk, buf, stat.st_size); ret = 1; @@ -101,7 +101,7 @@ grn_bulk_put_from_file(grn_ctx *ctx, grn_obj *bulk, const char *path) ERR(GRN_INVALID_ARGUMENT, "cannot stat file: <%s>", path); } exit : - GRN_CLOSE(fd); + grn_close(fd); return ret; } @@ -866,7 +866,7 @@ grn_select(grn_ctx *ctx, const char *table, unsigned int table_len, grn_obj *outbuf = ctx->impl->outbuf; grn_content_type output_type = ctx->impl->output_type; grn_obj *table_, *match_columns_ = NULL, *cond = NULL, *scorer_, *res = NULL, *sorted; - char cache_key[GRN_TABLE_MAX_KEY_SIZE]; + char cache_key[GRN_CACHE_MAX_KEY_SIZE]; uint32_t cache_key_size; long long int threshold, original_threshold = 0; grn_cache *cache_obj = grn_cache_current_get(ctx); @@ -887,52 +887,57 @@ grn_select(grn_ctx *ctx, const char *table, unsigned int table_len, sizeof(int) * 2; } } - if (cache_key_size <= GRN_TABLE_MAX_KEY_SIZE) { + if (cache_key_size <= GRN_CACHE_MAX_KEY_SIZE) { grn_obj *cache_value; char *cp = cache_key; - memcpy(cp, table, table_len); + grn_memcpy(cp, table, table_len); cp += table_len; *cp++ = '\0'; - memcpy(cp, match_columns, match_columns_len); + grn_memcpy(cp, match_columns, match_columns_len); cp += match_columns_len; *cp++ = '\0'; - memcpy(cp, query, query_len); + grn_memcpy(cp, query, query_len); cp += query_len; *cp++ = '\0'; - memcpy(cp, filter, filter_len); + grn_memcpy(cp, filter, filter_len); cp += filter_len; *cp++ = '\0'; - memcpy(cp, scorer, scorer_len); + grn_memcpy(cp, scorer, scorer_len); cp += scorer_len; *cp++ = '\0'; - memcpy(cp, sortby, sortby_len); + grn_memcpy(cp, sortby, sortby_len); cp += sortby_len; *cp++ = '\0'; - memcpy(cp, output_columns, output_columns_len); + grn_memcpy(cp, output_columns, output_columns_len); cp += output_columns_len; *cp++ = '\0'; { unsigned int i; for (i = 0; i < n_drilldowns; i++) { drilldown_info *drilldown = &(drilldowns[i]); - memcpy(cp, drilldown->keys, drilldown->keys_len); + grn_memcpy(cp, drilldown->keys, drilldown->keys_len); cp += drilldown->keys_len; *cp++ = '\0'; - memcpy(cp, drilldown->sortby, drilldown->sortby_len); + grn_memcpy(cp, drilldown->sortby, drilldown->sortby_len); cp += drilldown->sortby_len; *cp++ = '\0'; - memcpy(cp, drilldown->output_columns, drilldown->output_columns_len); + grn_memcpy(cp, drilldown->output_columns, drilldown->output_columns_len); cp += drilldown->output_columns_len; *cp++ = '\0'; } } - memcpy(cp, match_escalation_threshold, match_escalation_threshold_len); + grn_memcpy(cp, match_escalation_threshold, match_escalation_threshold_len); cp += match_escalation_threshold_len; *cp++ = '\0'; - memcpy(cp, query_expander, query_expander_len); + grn_memcpy(cp, query_expander, query_expander_len); cp += query_expander_len; *cp++ = '\0'; - memcpy(cp, query_flags, query_flags_len); + grn_memcpy(cp, query_flags, query_flags_len); cp += query_flags_len; *cp++ = '\0'; - memcpy(cp, adjuster, adjuster_len); + grn_memcpy(cp, adjuster, adjuster_len); cp += adjuster_len; *cp++ = '\0'; - memcpy(cp, &output_type, sizeof(grn_content_type)); cp += sizeof(grn_content_type); - memcpy(cp, &offset, sizeof(int)); cp += sizeof(int); - memcpy(cp, &limit, sizeof(int)); cp += sizeof(int); + grn_memcpy(cp, &output_type, sizeof(grn_content_type)); + cp += sizeof(grn_content_type); + grn_memcpy(cp, &offset, sizeof(int)); + cp += sizeof(int); + grn_memcpy(cp, &limit, sizeof(int)); + cp += sizeof(int); { unsigned int i; for (i = 0; i < n_drilldowns; i++) { drilldown_info *drilldown = &(drilldowns[i]); - memcpy(cp, &(drilldown->offset), sizeof(int)); cp += sizeof(int); - memcpy(cp, &(drilldown->limit), sizeof(int)); cp += sizeof(int); + grn_memcpy(cp, &(drilldown->offset), sizeof(int)); + cp += sizeof(int); + grn_memcpy(cp, &(drilldown->limit), sizeof(int)); + cp += sizeof(int); } } cache_value = grn_cache_fetch(ctx, cache_obj, cache_key, cache_key_size); @@ -1138,7 +1143,7 @@ grn_select(grn_ctx *ctx, const char *table, unsigned int table_len, GRN_OUTPUT_ARRAY_OPEN("RESULT", 0); } GRN_OUTPUT_ARRAY_CLOSE(); - if (!ctx->rc && cacheable && cache_key_size <= GRN_TABLE_MAX_KEY_SIZE + if (!ctx->rc && cacheable && cache_key_size <= GRN_CACHE_MAX_KEY_SIZE && (!cache || cache_len != 2 || *cache != 'n' || *(cache + 1) != 'o')) { grn_cache_update(ctx, cache_obj, cache_key, cache_key_size, outbuf); } @@ -1257,8 +1262,10 @@ proc_select(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) drilldown->label_len = label_len; #define GET_VAR(name)\ - snprintf(key_name, GRN_TABLE_MAX_KEY_SIZE,\ - "drilldown[%.*s]." # name, label_len, label);\ + grn_snprintf(key_name, \ + GRN_TABLE_MAX_KEY_SIZE, \ + GRN_TABLE_MAX_KEY_SIZE, \ + "drilldown[%.*s]." # name, label_len, label); \ name = GRN_PROC_GET_VAR(key_name); GET_VAR(keys); @@ -1943,7 +1950,7 @@ proc_column_remove(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_ if ((fullname_len = grn_obj_name(ctx, table, fullname, GRN_TABLE_MAX_KEY_SIZE))) { fullname[fullname_len] = GRN_DB_DELIMITER; - memcpy((fullname + fullname_len + 1), colname, colname_len); + grn_memcpy((fullname + fullname_len + 1), colname, colname_len); fullname_len += colname_len + 1; //TODO:check fullname_len < GRN_TABLE_MAX_KEY_SIZE col = grn_ctx_get(ctx, fullname, fullname_len); @@ -2336,7 +2343,7 @@ proc_missing(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) } if ((plen = GRN_TEXT_LEN(VAR(0))) + grn_document_root_len < PATH_MAX) { char path[PATH_MAX]; - memcpy(path, grn_document_root, grn_document_root_len); + grn_memcpy(path, grn_document_root, grn_document_root_len); path[grn_document_root_len] = '/'; grn_str_url_path_normalize(ctx, GRN_TEXT_VALUE(VAR(0)), @@ -2605,7 +2612,7 @@ proc_delete(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) GRN_EXPR_SYNTAX_SCRIPT); if (ctx->rc) { char original_error_message[GRN_CTX_MSGSIZE]; - strcpy(original_error_message, ctx->errbuf); + grn_strcpy(original_error_message, GRN_CTX_MSGSIZE, ctx->errbuf); rc = ctx->rc; ERR(rc, "[table][record][delete] failed to parse filter: " @@ -2644,6 +2651,110 @@ exit : static const size_t DUMP_FLUSH_THRESHOLD_SIZE = 256 * 1024; static void +dump_plugins(grn_ctx *ctx, grn_obj *outbuf) +{ + grn_obj *db = ctx->impl->db; + grn_table_cursor *cursor; + grn_id id; + grn_hash *processed_paths; + const char *system_plugins_dir; + const char *native_plugin_suffix; + const char *ruby_plugin_suffix; + + cursor = grn_table_cursor_open(ctx, db, NULL, 0, NULL, 0, 0, -1, + GRN_CURSOR_BY_ID); + if (!cursor) { + return; + } + + processed_paths = grn_hash_create(ctx, NULL, GRN_TABLE_MAX_KEY_SIZE, 0, + GRN_OBJ_TABLE_HASH_KEY | + GRN_OBJ_KEY_VAR_SIZE); + if (!processed_paths) { + grn_table_cursor_close(ctx, cursor); + return; + } + + system_plugins_dir = grn_plugin_get_system_plugins_dir(); + native_plugin_suffix = grn_plugin_get_suffix(); + ruby_plugin_suffix = grn_plugin_get_ruby_suffix(); + while ((id = grn_table_cursor_next(ctx, cursor)) != GRN_ID_NIL) { + grn_obj *object; + const char *path; + grn_id processed_path_id; + + object = grn_ctx_at(ctx, id); + if (!object) { + ERRCLR(ctx); + continue; + } + + if (!grn_obj_is_proc(ctx, object)) { + grn_obj_unlink(ctx, object); + continue; + } + + if (grn_obj_is_builtin(ctx, object)) { + grn_obj_unlink(ctx, object); + continue; + } + + path = grn_obj_path(ctx, object); + if (!path) { + grn_obj_unlink(ctx, object); + continue; + } + + processed_path_id = grn_hash_get(ctx, processed_paths, + path, strlen(path), + NULL); + if (processed_path_id != GRN_ID_NIL) { + grn_obj_unlink(ctx, object); + continue; + } + + grn_hash_add(ctx, processed_paths, + path, strlen(path), + NULL, NULL); + + { + const char *relative_path; + const char *libs_path = "/.libs/"; + const char *start_libs; + char name[PATH_MAX]; + + name[0] = '\0'; + if (strncmp(path, system_plugins_dir, strlen(system_plugins_dir)) == 0) { + relative_path = path + strlen(system_plugins_dir); + } else { + relative_path = path; + } + start_libs = strstr(relative_path, libs_path); + if (start_libs) { + grn_strncat(name, PATH_MAX, relative_path, start_libs - relative_path); + grn_strcat(name, PATH_MAX, "/"); + grn_strcat(name, PATH_MAX, start_libs + strlen(libs_path)); + } else { + grn_strcat(name, PATH_MAX, relative_path); + } + if (strlen(name) > strlen(native_plugin_suffix) && + strcmp(name + strlen(name) - strlen(native_plugin_suffix), + native_plugin_suffix) == 0) { + name[strlen(name) - strlen(native_plugin_suffix)] = '\0'; + } else if (strlen(name) > strlen(ruby_plugin_suffix) && + strcmp(name + strlen(name) - strlen(ruby_plugin_suffix), + ruby_plugin_suffix) == 0) { + name[strlen(name) - strlen(ruby_plugin_suffix)] = '\0'; + } + grn_text_printf(ctx, outbuf, "plugin_register %s\n", name); + } + } + grn_table_cursor_close(ctx, cursor); + + grn_hash_close(ctx, processed_paths); +} + +static void dump_name(grn_ctx *ctx, grn_obj *outbuf, const char *name, int name_len) { grn_obj escaped_name; @@ -2771,7 +2882,7 @@ reference_column_p(grn_ctx *ctx, grn_obj *column) static void dump_columns(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table, - grn_obj *pending_columns) + grn_obj *pending_reference_columns) { grn_hash *columns; columns = grn_hash_create(ctx, NULL, sizeof(grn_id), 0, @@ -2787,8 +2898,10 @@ dump_columns(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table, GRN_HASH_EACH(ctx, columns, id, &key, NULL, NULL, { grn_obj *column; if ((column = grn_ctx_at(ctx, *key))) { - if (reference_column_p(ctx, column)) { - GRN_PTR_PUT(ctx, pending_columns, column); + if (GRN_OBJ_INDEX_COLUMNP(column)) { + /* do nothing */ + } else if (reference_column_p(ctx, column)) { + GRN_PTR_PUT(ctx, pending_reference_columns, column); } else { dump_column(ctx, outbuf, table, column); grn_obj_unlink(ctx, column); @@ -2834,6 +2947,8 @@ dump_records(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table) grn_table_cursor *cursor; int i, ncolumns, n_use_columns; grn_obj columnbuf, delete_commands, use_columns, column_name; + grn_bool have_index_column = GRN_FALSE; + grn_bool have_data_column = GRN_FALSE; switch (table->header.type) { case GRN_TABLE_HASH_KEY: @@ -2849,12 +2964,6 @@ dump_records(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table) return; } - GRN_TEXT_INIT(&delete_commands, 0); - - GRN_TEXT_PUTS(ctx, outbuf, "load --table "); - dump_obj_name(ctx, outbuf, table); - GRN_TEXT_PUTS(ctx, outbuf, "\n[\n"); - GRN_PTR_INIT(&columnbuf, GRN_OBJ_VECTOR, GRN_ID_NIL); grn_obj_columns(ctx, table, DUMP_COLUMNS, strlen(DUMP_COLUMNS), &columnbuf); columns = (grn_obj **)GRN_BULK_HEAD(&columnbuf); @@ -2864,27 +2973,47 @@ dump_records(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table) GRN_TEXT_INIT(&column_name, 0); for (i = 0; i < ncolumns; i++) { if (GRN_OBJ_INDEX_COLUMNP(columns[i])) { + have_index_column = GRN_TRUE; continue; } + + if (columns[i]->header.type != GRN_ACCESSOR) { + have_data_column = GRN_TRUE; + } + GRN_BULK_REWIND(&column_name); grn_column_name_(ctx, columns[i], &column_name); - if (((table->header.type == GRN_TABLE_HASH_KEY || - table->header.type == GRN_TABLE_PAT_KEY || - table->header.type == GRN_TABLE_DAT_KEY) && - GRN_TEXT_LEN(&column_name) == GRN_COLUMN_NAME_ID_LEN && - !memcmp(GRN_TEXT_VALUE(&column_name), - GRN_COLUMN_NAME_ID, - GRN_COLUMN_NAME_ID_LEN)) || - (table->header.type == GRN_TABLE_NO_KEY && - GRN_TEXT_LEN(&column_name) == GRN_COLUMN_NAME_KEY_LEN && - !memcmp(GRN_TEXT_VALUE(&column_name), - GRN_COLUMN_NAME_KEY, - GRN_COLUMN_NAME_KEY_LEN))) { + if (table->header.type != GRN_TABLE_NO_KEY && + GRN_TEXT_LEN(&column_name) == GRN_COLUMN_NAME_ID_LEN && + memcmp(GRN_TEXT_VALUE(&column_name), + GRN_COLUMN_NAME_ID, + GRN_COLUMN_NAME_ID_LEN) == 0) { + continue; + } + + if (table->header.type == GRN_TABLE_NO_KEY && + GRN_TEXT_LEN(&column_name) == GRN_COLUMN_NAME_KEY_LEN && + memcmp(GRN_TEXT_VALUE(&column_name), + GRN_COLUMN_NAME_KEY, + GRN_COLUMN_NAME_KEY_LEN) == 0) { continue; } + GRN_PTR_PUT(ctx, &use_columns, columns[i]); } + if (have_index_column && !have_data_column) { + goto exit; + } + + if (GRN_TEXT_LEN(outbuf) > 0) { + GRN_TEXT_PUTC(ctx, outbuf, '\n'); + } + + GRN_TEXT_PUTS(ctx, outbuf, "load --table "); + dump_obj_name(ctx, outbuf, table); + GRN_TEXT_PUTS(ctx, outbuf, "\n[\n"); + n_use_columns = GRN_BULK_VSIZE(&use_columns) / sizeof(grn_obj *); GRN_TEXT_PUTC(ctx, outbuf, '['); for (i = 0; i < n_use_columns; i++) { @@ -2897,6 +3026,7 @@ dump_records(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table) } GRN_TEXT_PUTS(ctx, outbuf, "],\n"); + GRN_TEXT_INIT(&delete_commands, 0); cursor = grn_table_cursor_open(ctx, table, NULL, 0, NULL, 0, 0, -1, GRN_CURSOR_BY_KEY); for (i = 0; (id = grn_table_cursor_next(ctx, cursor)) != GRN_ID_NIL; @@ -2984,14 +3114,16 @@ dump_records(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table) grn_ctx_output_flush(ctx, 0); } } + grn_table_cursor_close(ctx, cursor); GRN_TEXT_PUTS(ctx, outbuf, "\n]\n"); GRN_TEXT_PUT(ctx, outbuf, GRN_TEXT_VALUE(&delete_commands), GRN_TEXT_LEN(&delete_commands)); grn_obj_unlink(ctx, &delete_commands); + +exit : grn_obj_unlink(ctx, &column_name); grn_obj_unlink(ctx, &use_columns); - grn_table_cursor_close(ctx, cursor); for (i = 0; i < ncolumns; i++) { grn_obj_unlink(ctx, columns[i]); } @@ -3000,7 +3132,7 @@ dump_records(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table) static void dump_table(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table, - grn_obj *pending_columns) + grn_obj *pending_reference_columns) { grn_obj *domain = NULL, *range = NULL; grn_obj_flags default_flags = GRN_OBJ_PERSISTENT; @@ -3018,6 +3150,11 @@ dump_table(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table, break; } + if (GRN_TEXT_LEN(outbuf) > 0) { + GRN_TEXT_PUTC(ctx, outbuf, '\n'); + grn_ctx_output_flush(ctx, 0); + } + GRN_TEXT_PUTS(ctx, outbuf, "table_create "); dump_obj_name(ctx, outbuf, table); GRN_TEXT_PUTC(ctx, outbuf, ' '); @@ -3081,68 +3218,77 @@ dump_table(grn_ctx *ctx, grn_obj *outbuf, grn_obj *table, grn_obj_unlink(ctx, domain); } - dump_columns(ctx, outbuf, table, pending_columns); + dump_columns(ctx, outbuf, table, pending_reference_columns); } -/* can we move this to groonga.h? */ -#define GRN_PTR_POP(obj,value) do {\ - if (GRN_BULK_VSIZE(obj) >= sizeof(grn_obj *)) {\ - GRN_BULK_INCR_LEN((obj), -(sizeof(grn_obj *)));\ - value = *(grn_obj **)(GRN_BULK_CURR(obj));\ - } else {\ - value = NULL;\ - }\ -} while (0) +static void +dump_pending_columns(grn_ctx *ctx, grn_obj *outbuf, grn_obj *pending_columns) +{ + size_t i, n_columns; + + n_columns = GRN_BULK_VSIZE(pending_columns) / sizeof(grn_obj *); + if (n_columns == 0) { + return; + } + + if (GRN_TEXT_LEN(outbuf) > 0) { + GRN_TEXT_PUTC(ctx, outbuf, '\n'); + grn_ctx_output_flush(ctx, 0); + } + + for (i = 0; i < n_columns; i++) { + grn_obj *table, *column; + + column = GRN_PTR_VALUE_AT(pending_columns, i); + table = grn_ctx_at(ctx, column->header.domain); + dump_column(ctx, outbuf, table, column); + grn_obj_unlink(ctx, column); + grn_obj_unlink(ctx, table); + } +} static void dump_schema(grn_ctx *ctx, grn_obj *outbuf) { grn_obj *db = ctx->impl->db; grn_table_cursor *cur; - if ((cur = grn_table_cursor_open(ctx, db, NULL, 0, NULL, 0, 0, -1, - GRN_CURSOR_BY_ID))) { - grn_id id; - grn_obj pending_columns; + grn_id id; + grn_obj pending_reference_columns; - GRN_PTR_INIT(&pending_columns, GRN_OBJ_VECTOR, GRN_ID_NIL); - while ((id = grn_table_cursor_next(ctx, cur)) != GRN_ID_NIL) { - grn_obj *object; - - if ((object = grn_ctx_at(ctx, id))) { - switch (object->header.type) { - case GRN_TABLE_HASH_KEY: - case GRN_TABLE_PAT_KEY: - case GRN_TABLE_DAT_KEY: - case GRN_TABLE_NO_KEY: - dump_table(ctx, outbuf, object, &pending_columns); - break; - default: - break; - } - grn_obj_unlink(ctx, object); - } else { - /* XXX: this clause is executed when MeCab tokenizer is enabled in - database but the groonga isn't supported MeCab. - We should return error mesage about it and error exit status - but it's too difficult for this architecture. :< */ - ERRCLR(ctx); - } - } - grn_table_cursor_close(ctx, cur); + cur = grn_table_cursor_open(ctx, db, NULL, 0, NULL, 0, 0, -1, + GRN_CURSOR_BY_ID); + if (!cur) { + return; + } + + GRN_PTR_INIT(&pending_reference_columns, GRN_OBJ_VECTOR, GRN_ID_NIL); + while ((id = grn_table_cursor_next(ctx, cur)) != GRN_ID_NIL) { + grn_obj *object; - while (GRN_TRUE) { - grn_obj *table, *column; - GRN_PTR_POP(&pending_columns, column); - if (!column) { + if ((object = grn_ctx_at(ctx, id))) { + switch (object->header.type) { + case GRN_TABLE_HASH_KEY: + case GRN_TABLE_PAT_KEY: + case GRN_TABLE_DAT_KEY: + case GRN_TABLE_NO_KEY: + dump_table(ctx, outbuf, object, &pending_reference_columns); + break; + default: break; } - table = grn_ctx_at(ctx, column->header.domain); - dump_column(ctx, outbuf, table, column); - grn_obj_unlink(ctx, column); - grn_obj_unlink(ctx, table); + grn_obj_unlink(ctx, object); + } else { + /* XXX: this clause is executed when MeCab tokenizer is enabled in + database but the groonga isn't supported MeCab. + We should return error mesage about it and error exit status + but it's too difficult for this architecture. :< */ + ERRCLR(ctx); } - grn_obj_close(ctx, &pending_columns); } + grn_table_cursor_close(ctx, cur); + + dump_pending_columns(ctx, outbuf, &pending_reference_columns); + grn_obj_close(ctx, &pending_reference_columns); } static void @@ -3227,21 +3373,114 @@ dump_all_records(grn_ctx *ctx, grn_obj *outbuf) } } +static void +dump_indexes(grn_ctx *ctx, grn_obj *outbuf) +{ + grn_obj *db = ctx->impl->db; + grn_table_cursor *cursor; + grn_id id; + grn_bool is_first_index_column = GRN_TRUE; + + cursor = grn_table_cursor_open(ctx, db, NULL, 0, NULL, 0, 0, -1, + GRN_CURSOR_BY_ID); + if (!cursor) { + return; + } + + while ((id = grn_table_cursor_next(ctx, cursor)) != GRN_ID_NIL) { + grn_obj *object; + + object = grn_ctx_at(ctx, id); + if (!object) { + /* XXX: this clause is executed when MeCab tokenizer is enabled in + database but the groonga isn't supported MeCab. + We should return error mesage about it and error exit status + but it's too difficult for this architecture. :< */ + ERRCLR(ctx); + continue; + } + + if (object->header.type == GRN_COLUMN_INDEX) { + grn_obj *table; + grn_obj *column = object; + + if (is_first_index_column && GRN_TEXT_LEN(outbuf) > 0) { + GRN_TEXT_PUTC(ctx, outbuf, '\n'); + } + is_first_index_column = GRN_FALSE; + + table = grn_ctx_at(ctx, column->header.domain); + dump_column(ctx, outbuf, table, column); + grn_obj_unlink(ctx, table); + } + grn_obj_unlink(ctx, object); + } + grn_table_cursor_close(ctx, cursor); +} + +static grn_bool +bool_option_value(grn_obj *option, grn_bool default_value) +{ + const char *value; + size_t value_length; + + value = GRN_TEXT_VALUE(option); + value_length = GRN_TEXT_LEN(option); + + if (value_length == 0) { + return default_value; + } + + if (value_length == strlen("yes") && + strncmp(value, "yes", value_length) == 0) { + return GRN_TRUE; + } else if (value_length == strlen("no") && + strncmp(value, "no", value_length) == 0) { + return GRN_FALSE; + } else { + return default_value; + } +} + static grn_obj * proc_dump(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) { grn_obj *outbuf = ctx->impl->outbuf; - ctx->impl->output_type = GRN_CONTENT_NONE; - ctx->impl->mime_type = "text/x-groonga-command-list"; - dump_schema(ctx, outbuf); - grn_ctx_output_flush(ctx, 0); - /* To update index columns correctly, we first create the whole schema, then - load non-derivative records, while skipping records of index columns. That - way, groonga will silently do the job of updating index columns for us. */ - if (GRN_TEXT_LEN(VAR(0)) > 0) { - dump_selected_tables_records(ctx, outbuf, VAR(0)); - } else { - dump_all_records(ctx, outbuf); + grn_obj *tables = VAR(0); + grn_obj *dump_plugins_raw = VAR(1); + grn_obj *dump_schema_raw = VAR(2); + grn_obj *dump_records_raw = VAR(3); + grn_obj *dump_indexes_raw = VAR(4); + grn_bool is_dump_plugins; + grn_bool is_dump_schema; + grn_bool is_dump_records; + grn_bool is_dump_indexes; + + grn_ctx_set_output_type(ctx, GRN_CONTENT_GROONGA_COMMAND_LIST); + + is_dump_plugins = bool_option_value(dump_plugins_raw, GRN_TRUE); + is_dump_schema = bool_option_value(dump_schema_raw, GRN_TRUE); + is_dump_records = bool_option_value(dump_records_raw, GRN_TRUE); + is_dump_indexes = bool_option_value(dump_indexes_raw, GRN_TRUE); + + if (is_dump_plugins) { + dump_plugins(ctx, outbuf); + } + if (is_dump_schema) { + dump_schema(ctx, outbuf); + } + if (is_dump_records) { + /* To update index columns correctly, we first create the whole schema, then + load non-derivative records, while skipping records of index columns. That + way, groonga will silently do the job of updating index columns for us. */ + if (GRN_TEXT_LEN(tables) > 0) { + dump_selected_tables_records(ctx, outbuf, tables); + } else { + dump_all_records(ctx, outbuf); + } + } + if (is_dump_indexes) { + dump_indexes(ctx, outbuf); } /* remove the last newline because another one will be added by the caller. @@ -5476,9 +5715,11 @@ selector_between_sequential_search(grn_ctx *ctx, double too_many_index_match_ratio = 0.01; { - const char *too_many_index_match_ratio_env = - getenv("GRN_BETWEEN_TOO_MANY_INDEX_MATCH_RATIO"); - if (too_many_index_match_ratio_env) { + char too_many_index_match_ratio_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_BETWEEN_TOO_MANY_INDEX_MATCH_RATIO", + too_many_index_match_ratio_env, + GRN_ENV_BUFFER_SIZE); + if (too_many_index_match_ratio_env[0]) { too_many_index_match_ratio = atof(too_many_index_match_ratio_env); } } @@ -5721,14 +5962,15 @@ func_highlight_html(grn_ctx *ctx, int nargs, grn_obj **args, } if (condition) { + size_t i, n_keywords; grn_obj current_keywords; GRN_PTR_INIT(¤t_keywords, GRN_OBJ_VECTOR, GRN_ID_NIL); grn_expr_get_keywords(ctx, condition, ¤t_keywords); - for (;;) { + n_keywords = GRN_BULK_VSIZE(¤t_keywords) / sizeof(grn_obj *); + for (i = 0; i < n_keywords; i++) { grn_obj *keyword; - GRN_PTR_POP(¤t_keywords, keyword); - if (!keyword) { break; } + keyword = GRN_PTR_VALUE_AT(¤t_keywords, i); grn_table_add(ctx, keywords, GRN_TEXT_VALUE(keyword), GRN_TEXT_LEN(keyword), @@ -5971,9 +6213,11 @@ selector_in_values_sequential_search(grn_ctx *ctx, double too_many_index_match_ratio = 0.01; { - const char *too_many_index_match_ratio_env = - getenv("GRN_IN_VALUES_TOO_MANY_INDEX_MATCH_RATIO"); - if (too_many_index_match_ratio_env) { + char too_many_index_match_ratio_env[GRN_ENV_BUFFER_SIZE]; + grn_getenv("GRN_IN_VALUES_TOO_MANY_INDEX_MATCH_RATIO", + too_many_index_match_ratio_env, + GRN_ENV_BUFFER_SIZE); + if (too_many_index_match_ratio_env[0]) { too_many_index_match_ratio = atof(too_many_index_match_ratio_env); } } @@ -6433,6 +6677,38 @@ proc_request_cancel(grn_ctx *ctx, int nargs, grn_obj **args, return NULL; } +static grn_obj * +proc_plugin_register(grn_ctx *ctx, int nargs, grn_obj **args, + grn_user_data *user_data) +{ + if (GRN_TEXT_LEN(VAR(0))) { + const char *name; + GRN_TEXT_PUTC(ctx, VAR(0), '\0'); + name = GRN_TEXT_VALUE(VAR(0)); + grn_plugin_register(ctx, name); + } else { + ERR(GRN_INVALID_ARGUMENT, "[plugin_register] name is missing"); + } + GRN_OUTPUT_BOOL(!ctx->rc); + return NULL; +} + +static grn_obj * +proc_plugin_unregister(grn_ctx *ctx, int nargs, grn_obj **args, + grn_user_data *user_data) +{ + if (GRN_TEXT_LEN(VAR(0))) { + const char *name; + GRN_TEXT_PUTC(ctx, VAR(0), '\0'); + name = GRN_TEXT_VALUE(VAR(0)); + grn_plugin_unregister(ctx, name); + } else { + ERR(GRN_INVALID_ARGUMENT, "[plugin_unregister] name is missing"); + } + GRN_OUTPUT_BOOL(!ctx->rc); + return NULL; +} + #define DEF_VAR(v,name_str) do {\ (v).name = (name_str);\ (v).name_size = GRN_STRLEN(name_str);\ @@ -6559,8 +6835,13 @@ grn_db_init_builtin_query(grn_ctx *ctx) DEF_COMMAND("cache_limit", proc_cache_limit, 1, vars); DEF_VAR(vars[0], "tables"); - DEF_COMMAND("dump", proc_dump, 1, vars); + DEF_VAR(vars[1], "dump_plugins"); + DEF_VAR(vars[2], "dump_schema"); + DEF_VAR(vars[3], "dump_records"); + DEF_VAR(vars[4], "dump_indexes"); + DEF_COMMAND("dump", proc_dump, 5, vars); + /* Deprecated. Use "plugin_register" instead. */ DEF_VAR(vars[0], "path"); DEF_COMMAND("register", proc_register, 1, vars); @@ -6700,4 +6981,10 @@ grn_db_init_builtin_query(grn_ctx *ctx) DEF_VAR(vars[0], "id"); DEF_COMMAND("request_cancel", proc_request_cancel, 1, vars); + + DEF_VAR(vars[0], "name"); + DEF_COMMAND("plugin_register", proc_plugin_register, 1, vars); + + DEF_VAR(vars[0], "name"); + DEF_COMMAND("plugin_unregister", proc_plugin_unregister, 1, vars); } diff --git a/storage/mroonga/vendor/groonga/lib/scorer.c b/storage/mroonga/vendor/groonga/lib/scorer.c index f5bc0435108..2670bb3d4c6 100644 --- a/storage/mroonga/vendor/groonga/lib/scorer.c +++ b/storage/mroonga/vendor/groonga/lib/scorer.c @@ -100,6 +100,61 @@ grn_scorer_matched_record_get_weight(grn_ctx *ctx, return record->weight; } +grn_obj * +grn_scorer_matched_record_get_arg(grn_ctx *ctx, + grn_scorer_matched_record *record, + unsigned int i) +{ + grn_expr *expr; + grn_expr_code *codes_original; + uint32_t codes_curr_original; + grn_obj *arg; + + if (!record->args_expr) { + return NULL; + } + + expr = (grn_expr *)(record->args_expr); + /* TODO: support getting column value */ + codes_original = expr->codes; + codes_curr_original = expr->codes_curr; + expr->codes += record->args_expr_offset; + expr->codes_curr = 1; /* TODO: support 1 or more codes */ + arg = grn_expr_exec(ctx, (grn_obj *)expr, 0); + expr->codes_curr = codes_curr_original; + expr->codes = codes_original; + + return arg; +} + +unsigned int +grn_scorer_matched_record_get_n_args(grn_ctx *ctx, + grn_scorer_matched_record *record) +{ + grn_expr *expr; + grn_expr_code *codes; + unsigned int n_args = 0; + + if (!record->args_expr) { + return 0; + } + + expr = (grn_expr *)(record->args_expr); + codes = expr->codes + record->args_expr_offset; + if (codes[0].op == GRN_OP_CALL) { + return 0; + } + + n_args++; + for (; codes[0].op != GRN_OP_CALL; codes++) { + if (codes[0].op == GRN_OP_COMMA) { + n_args++; + } + } + + return n_args; +} + grn_rc grn_scorer_register(grn_ctx *ctx, const char *plugin_name_ptr, diff --git a/storage/mroonga/vendor/groonga/lib/scorers.c b/storage/mroonga/vendor/groonga/lib/scorers.c index b039e011920..dcec87c8ecc 100644 --- a/storage/mroonga/vendor/groonga/lib/scorers.c +++ b/storage/mroonga/vendor/groonga/lib/scorers.c @@ -16,6 +16,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include "grn_db.h" + #include <groonga/scorer.h> #include <math.h> @@ -49,9 +51,46 @@ scorer_tf_idf(grn_ctx *ctx, grn_scorer_matched_record *record) } } +static double +scorer_tf_at_most(grn_ctx *ctx, grn_scorer_matched_record *record) +{ + double tf; + double max; + grn_obj *max_raw; + + tf = grn_scorer_matched_record_get_n_occurrences(ctx, record) + + grn_scorer_matched_record_get_total_term_weights(ctx, record); + max_raw = grn_scorer_matched_record_get_arg(ctx, record, 0); + + if (!max_raw) { + return tf; + } + + if (max_raw->header.type != GRN_BULK) { + return tf; + } + + if (max_raw->header.domain == GRN_DB_FLOAT) { + max = GRN_FLOAT_VALUE(max_raw); + } else { + grn_obj casted_max_raw; + GRN_FLOAT_INIT(&casted_max_raw, 0); + if (grn_obj_cast(ctx, max_raw, &casted_max_raw, GRN_FALSE) != GRN_SUCCESS) { + GRN_OBJ_FIN(ctx, &casted_max_raw); + return tf; + } else { + max = GRN_FLOAT_VALUE(&casted_max_raw); + } + GRN_OBJ_FIN(ctx, &casted_max_raw); + } + + return fmin(tf, max); +} + grn_rc grn_db_init_builtin_scorers(grn_ctx *ctx) { grn_scorer_register(ctx, "scorer_tf_idf", -1, scorer_tf_idf); + grn_scorer_register(ctx, "scorer_tf_at_most", -1, scorer_tf_at_most); return GRN_SUCCESS; } diff --git a/storage/mroonga/vendor/groonga/lib/snip.c b/storage/mroonga/vendor/groonga/lib/snip.c index 739ee345738..da5ba704c85 100644 --- a/storage/mroonga/vendor/groonga/lib/snip.c +++ b/storage/mroonga/vendor/groonga/lib/snip.c @@ -306,7 +306,7 @@ grn_snip_strndup(grn_ctx *ctx, const char *string, unsigned int string_len) if (!copied_string) { return NULL; } - memcpy(copied_string, string, string_len); + grn_memcpy(copied_string, string, string_len); copied_string[string_len]= '\0'; /* not required, but for ql use */ return copied_string; } @@ -775,7 +775,9 @@ grn_snip_get_result(grn_ctx *ctx, grn_obj *snip, const unsigned int index, char if (snip_->tag_result[j].end_offset > sres->end_offset) { continue; } - memcpy(p, snip_->tag_result[j].cond->opentag, snip_->tag_result[j].cond->opentag_len); + grn_memcpy(p, + snip_->tag_result[j].cond->opentag, + snip_->tag_result[j].cond->opentag_len); p += snip_->tag_result[j].cond->opentag_len; } @@ -820,8 +822,9 @@ grn_snip_get_result(grn_ctx *ctx, grn_obj *snip, const unsigned int index, char snip_->tag_result[k].end_offset <= sres->end_offset; k--) { /* TODO: avoid all loop */ if (snip_->tag_result[k].end_offset == i + 1) { - memcpy(p, snip_->tag_result[k].cond->closetag, - snip_->tag_result[k].cond->closetag_len); + grn_memcpy(p, + snip_->tag_result[k].cond->closetag, + snip_->tag_result[k].cond->closetag_len); p += snip_->tag_result[k].cond->closetag_len; } if (k <= sres->first_tag_result_idx) { diff --git a/storage/mroonga/vendor/groonga/lib/sources.am b/storage/mroonga/vendor/groonga/lib/sources.am index 25e5df5fd4d..459b73792ab 100644 --- a/storage/mroonga/vendor/groonga/lib/sources.am +++ b/storage/mroonga/vendor/groonga/lib/sources.am @@ -24,8 +24,11 @@ libgroonga_la_SOURCES = \ grn_ii.h \ io.c \ grn_io.h \ + logger.c \ + grn_logger.h \ mrb.c \ grn_mrb.h \ + grn_msgpack.h \ nfkc.c \ normalizer.c \ grn_normalizer.h \ diff --git a/storage/mroonga/vendor/groonga/lib/store.c b/storage/mroonga/vendor/groonga/lib/store.c index 6b3b01ce4be..027f86baac8 100644 --- a/storage/mroonga/vendor/groonga/lib/store.c +++ b/storage/mroonga/vendor/groonga/lib/store.c @@ -338,8 +338,6 @@ struct grn_ja_header { #define SEGMENTS_GINFO_ON(ja,seg,width) (SEGMENTS_AT(ja,seg) = SEG_GINFO|(width)) #define SEGMENTS_OFF(ja,seg) (SEGMENTS_AT(ja,seg) = 0) -grn_bool grn_ja_skip_same_value_put = GRN_TRUE; - static grn_ja * _grn_ja_create(grn_ctx *ctx, grn_ja *ja, const char *path, unsigned int max_element_size, uint32_t flags) @@ -881,12 +879,12 @@ set_value(grn_ctx *ctx, grn_ja *ja, grn_id id, void *value, uint32_t value_len, if ((rc = grn_ja_alloc(ctx, ja, id, value_len + sizeof(uint32_t), einfo, &iw))) { return rc; } - memcpy(iw.addr, value, value_len); + grn_memcpy(iw.addr, value, value_len); memset((byte *)iw.addr + value_len, 0, sizeof(uint32_t)); grn_io_win_unmap(&iw); } else { if ((rc = grn_ja_alloc(ctx, ja, id, value_len, einfo, &iw))) { return rc; } - memcpy(iw.addr, value, value_len); + grn_memcpy(iw.addr, value, value_len); grn_io_win_unmap(&iw); } return rc; @@ -901,8 +899,7 @@ grn_ja_put_raw(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_io_win iw; grn_ja_einfo einfo; - if (grn_ja_skip_same_value_put && - (flags & GRN_OBJ_SET_MASK) == GRN_OBJ_SET && + if ((flags & GRN_OBJ_SET_MASK) == GRN_OBJ_SET && value_len > 0) { grn_io_win jw; uint32_t old_len; @@ -935,11 +932,11 @@ grn_ja_put_raw(grn_ctx *ctx, grn_ja *ja, grn_id id, GRN_ASSERT(pos < el); if (el <= pos + value_len) { uint32_t rest = el - pos; - memcpy(b + pos, value, rest); - memcpy(b, (byte *)value + rest, value_len - rest); + grn_memcpy(b + pos, value, rest); + grn_memcpy(b, (byte *)value + rest, value_len - rest); *((uint32_t *)(b + el)) = value_len - rest; } else { - memcpy(b + pos, value, value_len); + grn_memcpy(b + pos, value, value_len); *((uint32_t *)(b + el)) = pos + value_len; } return GRN_SUCCESS; @@ -950,8 +947,8 @@ grn_ja_put_raw(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_ja_unref(ctx, &jw); return rc; } - memcpy(iw.addr, oldvalue, old_len); - memcpy((byte *)iw.addr + old_len, value, value_len); + grn_memcpy(iw.addr, oldvalue, old_len); + grn_memcpy((byte *)iw.addr + old_len, value, value_len); memset((byte *)iw.addr + old_len + value_len, 0, sizeof(uint32_t)); grn_io_win_unmap(&iw); } @@ -960,8 +957,8 @@ grn_ja_put_raw(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_ja_unref(ctx, &jw); return rc; } - memcpy(iw.addr, oldvalue, old_len); - memcpy((byte *)iw.addr + old_len, value, value_len); + grn_memcpy(iw.addr, oldvalue, old_len); + grn_memcpy((byte *)iw.addr + old_len, value, value_len); grn_io_win_unmap(&iw); } grn_ja_unref(ctx, &jw); @@ -985,11 +982,11 @@ grn_ja_put_raw(grn_ctx *ctx, grn_ja *ja, grn_id id, GRN_ASSERT(pos < el); if (pos < value_len) { uint32_t rest = value_len - pos; - memcpy(b, (byte *)value + rest, pos); - memcpy(b + el - rest, value, rest); + grn_memcpy(b, (byte *)value + rest, pos); + grn_memcpy(b + el - rest, value, rest); *((uint32_t *)(b + el)) = el - rest; } else { - memcpy(b + pos - value_len, value, value_len); + grn_memcpy(b + pos - value_len, value, value_len); *((uint32_t *)(b + el)) = pos - value_len; } return GRN_SUCCESS; @@ -1000,8 +997,8 @@ grn_ja_put_raw(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_ja_unref(ctx, &jw); return rc; } - memcpy(iw.addr, value, value_len); - memcpy((byte *)iw.addr + value_len, oldvalue, old_len); + grn_memcpy(iw.addr, value, value_len); + grn_memcpy((byte *)iw.addr + value_len, oldvalue, old_len); memset((byte *)iw.addr + value_len + old_len, 0, sizeof(uint32_t)); grn_io_win_unmap(&iw); } @@ -1010,8 +1007,8 @@ grn_ja_put_raw(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_ja_unref(ctx, &jw); return rc; } - memcpy(iw.addr, value, value_len); - memcpy((byte *)iw.addr + value_len, oldvalue, old_len); + grn_memcpy(iw.addr, value, value_len); + grn_memcpy((byte *)iw.addr + value_len, oldvalue, old_len); grn_io_win_unmap(&iw); } grn_ja_unref(ctx, &jw); @@ -1100,9 +1097,13 @@ grn_ja_putv(grn_ctx *ctx, grn_ja *ja, grn_id id, grn_obj *vector, int flags) size_t sizev = body ? GRN_BULK_VSIZE(body) : 0; size_t sizef = GRN_BULK_VSIZE(&footer); if ((rc = grn_ja_alloc(ctx, ja, id, sizeh + sizev + sizef, &einfo, &iw))) { goto exit; } - memcpy(iw.addr, GRN_BULK_HEAD(&header), sizeh); - if (body) { memcpy((char *)iw.addr + sizeh, GRN_BULK_HEAD(body), sizev); } - if (f) { memcpy((char *)iw.addr + sizeh + sizev, GRN_BULK_HEAD(&footer), sizef); } + grn_memcpy(iw.addr, GRN_BULK_HEAD(&header), sizeh); + if (body) { + grn_memcpy((char *)iw.addr + sizeh, GRN_BULK_HEAD(body), sizev); + } + if (f) { + grn_memcpy((char *)iw.addr + sizeh + sizev, GRN_BULK_HEAD(&footer), sizef); + } grn_io_win_unmap(&iw); rc = grn_ja_replace(ctx, ja, id, &einfo, NULL); } diff --git a/storage/mroonga/vendor/groonga/lib/str.c b/storage/mroonga/vendor/groonga/lib/str.c index f91b89c96c0..1bbe43a591c 100644 --- a/storage/mroonga/vendor/groonga/lib/str.c +++ b/storage/mroonga/vendor/groonga/lib/str.c @@ -27,6 +27,14 @@ #endif /* _ISOC99_SOURCE */ #include <math.h> +#if defined(HAVE__GMTIME64_S) && defined(__GNUC__) +# ifdef _WIN64 +# define gmtime_s(tm, time) _gmtime64_s(tm, time) +# else /* _WIN64 */ +# define gmtime_s(tm, time) _gmtime32_s(tm, time) +# endif /* _WIN64 */ +#endif /* defined(HAVE__GMTIME64_S) && defined(__GNUC__) */ + /* For Visual C++ 2010. Drop the code when we drop Visual C++ 2010 support. */ #if defined(_MSC_VER) && _MSC_VER < 1800 # define va_copy(destination, source) destination = source @@ -530,7 +538,7 @@ normalize_utf8(grn_ctx *ctx, grn_str *nstr) nstr->ctypes = ctypes; } } - memcpy(d, p, lp); + grn_memcpy(d, p, lp); d_ = d; d += lp; length++; @@ -1146,7 +1154,7 @@ grn_fakenstr_open(grn_ctx *ctx, const char *str, size_t str_len, grn_encoding en } nstr->orig = str; nstr->orig_blen = str_len; - memcpy(nstr->norm, str, str_len); + grn_memcpy(nstr->norm, str, str_len); nstr->norm[str_len] = '\0'; nstr->norm_blen = str_len; nstr->ctypes = NULL; @@ -1492,6 +1500,20 @@ grn_atoll(const char *nptr, const char *end, const char **rest) return n ? -v : v; } +uint64_t +grn_atoull(const char *nptr, const char *end, const char **rest) +{ + uint64_t v = 0, t; + while (nptr < end && *nptr >= '0' && *nptr <= '9') { + t = v * 10 + (*nptr - '0'); + if (t < v) { v = 0; break; } + v = t; + nptr++; + } + if (rest) { *rest = nptr; } + return v; +} + unsigned int grn_htoui(const char *nptr, const char *end, const char **rest) { @@ -1916,7 +1938,7 @@ grn_bulk_resize(grn_ctx *ctx, grn_obj *buf, unsigned int newsize) if (rounded_newsize < newsize) { return GRN_NOT_ENOUGH_SPACE; } newsize = rounded_newsize; if (!(head = GRN_MALLOC(newsize))) { return GRN_NO_MEMORY_AVAILABLE; } - memcpy(head, GRN_BULK_HEAD(buf), GRN_BULK_VSIZE(buf)); + grn_memcpy(head, GRN_BULK_HEAD(buf), GRN_BULK_VSIZE(buf)); buf->u.b.curr = head + grn_bulk_margin_size + GRN_BULK_VSIZE(buf); buf->u.b.head = head + grn_bulk_margin_size; buf->u.b.tail = head + newsize; @@ -1942,7 +1964,7 @@ grn_bulk_write(grn_ctx *ctx, grn_obj *buf, const char *str, unsigned int len) if ((rc = grn_bulk_resize(ctx, buf, GRN_BULK_VSIZE(buf) + len))) { return rc; } } curr = GRN_BULK_CURR(buf); - memcpy(curr, str, len); + grn_memcpy(curr, str, len); GRN_BULK_INCR_LEN(buf, len); return rc; } @@ -2074,26 +2096,30 @@ grn_text_ulltoa(grn_ctx *ctx, grn_obj *buf, unsigned long long int i) inline static void ftoa_(grn_ctx *ctx, grn_obj *buf, double d) { - char *curr; + char *start; + size_t before_size; size_t len; #define DIGIT_NUMBER 15 - grn_bulk_reserve(ctx, buf, DIGIT_NUMBER + 1); - curr = GRN_BULK_CURR(buf); - len = sprintf(curr, "%#.*g", DIGIT_NUMBER, d); +#define FIRST_BUFFER_SIZE (DIGIT_NUMBER + 4) + before_size = GRN_BULK_VSIZE(buf); + grn_bulk_reserve(ctx, buf, FIRST_BUFFER_SIZE); + grn_text_printf(ctx, buf, "%#.*g", DIGIT_NUMBER, d); + len = GRN_BULK_VSIZE(buf) - before_size; + start = GRN_BULK_CURR(buf) - len; +#undef FIRST_BUFFER_SIZE #undef DIGIT_NUMBER - if (curr[len - 1] == '.') { - GRN_BULK_INCR_LEN(buf, len); + if (start[len - 1] == '.') { GRN_TEXT_PUTC(ctx, buf, '0'); } else { char *p, *q; - curr[len] = '\0'; - if ((p = strchr(curr, 'e'))) { + start[len] = '\0'; + if ((p = strchr(start, 'e'))) { for (q = p; *(q - 2) != '.' && *(q - 1) == '0'; q--) { len--; } - memmove(q, p, curr + len - q); + grn_memmove(q, p, start + len - q); } else { - for (q = curr + len; *(q - 2) != '.' && *(q - 1) == '0'; q--) { len--; } + for (q = start + len; *(q - 2) != '.' && *(q - 1) == '0'; q--) { len--; } } - GRN_BULK_INCR_LEN(buf, len); + grn_bulk_truncate(ctx, buf, before_size + len); } } @@ -2491,9 +2517,11 @@ grn_text_printf(grn_ctx *ctx, grn_obj *bulk, const char *format, ...) grn_rc grn_text_vprintf(grn_ctx *ctx, grn_obj *bulk, const char *format, va_list args) { - int rest_size, written_size; + grn_bool is_written = GRN_FALSE; + int written_size; { + int rest_size; va_list copied_args; rest_size = GRN_BULK_REST(bulk); @@ -2501,9 +2529,43 @@ grn_text_vprintf(grn_ctx *ctx, grn_obj *bulk, const char *format, va_list args) written_size = vsnprintf(GRN_BULK_CURR(bulk), rest_size, format, copied_args); va_end(copied_args); + + if (written_size < rest_size) { + is_written = GRN_TRUE; + } } - if (written_size >= rest_size) { +#ifdef WIN32 + if (written_size == -1 && errno == ERANGE) { +# define N_NEW_SIZES 3 + int i; + int new_sizes[N_NEW_SIZES]; + + new_sizes[0] = GRN_BULK_REST(bulk) + strlen(format) * 2; + new_sizes[1] = new_sizes[0] + 4096; + new_sizes[2] = new_sizes[0] + 65536; + + for (i = 0; i < N_NEW_SIZES; i++) { + grn_rc rc; + int new_size = new_sizes[i]; + va_list copied_args; + + rc = grn_bulk_reserve(ctx, bulk, GRN_BULK_VSIZE(bulk) + new_size); + if (rc) { + return rc; + } + va_copy(copied_args, args); + written_size = vsnprintf(GRN_BULK_CURR(bulk), new_size, + format, copied_args); + va_end(copied_args); + if (written_size != -1) { + break; + } + } +# undef N_NEW_SIZES + } +#else /* WIN32 */ + if (!is_written) { grn_rc rc; int required_size = written_size + 1; /* "+ 1" for terminate '\0'. */ @@ -2514,6 +2576,7 @@ grn_text_vprintf(grn_ctx *ctx, grn_obj *bulk, const char *format, va_list args) written_size = vsnprintf(GRN_BULK_CURR(bulk), required_size, format, args); } +#endif /* WIN32 */ if (written_size < 0) { return GRN_INVALID_ARGUMENT; @@ -3148,7 +3211,7 @@ grn_str_url_path_normalize(grn_ctx *ctx, const char *path, size_t path_len, } } if (be - b >= pc - p) { - memcpy(b, p, (pc - p)); + grn_memcpy(b, p, (pc - p)); b += pc - p; p = pc; if (p < pe && *pc == '/' && be > b) { diff --git a/storage/mroonga/vendor/groonga/lib/string.c b/storage/mroonga/vendor/groonga/lib/string.c index 132d51257ac..3249865b900 100644 --- a/storage/mroonga/vendor/groonga/lib/string.c +++ b/storage/mroonga/vendor/groonga/lib/string.c @@ -54,7 +54,7 @@ grn_fake_string_open(grn_ctx *ctx, grn_string *string) if (!grn_tokenizer_is_tokenized_delimiter(ctx, source_current, char_length, ctx->encoding)) { - memcpy(destination, source_current, char_length); + grn_memcpy(destination, source_current, char_length); destination += char_length; destination_length += char_length; } @@ -63,7 +63,7 @@ grn_fake_string_open(grn_ctx *ctx, grn_string *string) nstr->normalized[destination_length] = '\0'; nstr->normalized_length_in_bytes = destination_length; } else { - memcpy(nstr->normalized, str, str_len); + grn_memcpy(nstr->normalized, str, str_len); nstr->normalized[str_len] = '\0'; nstr->normalized_length_in_bytes = str_len; } @@ -140,8 +140,21 @@ grn_string_open_(grn_ctx *ctx, const char *str, unsigned int str_len, return NULL; } + is_normalizer_auto = (normalizer == GRN_NORMALIZER_AUTO); + if (is_normalizer_auto) { + normalizer = grn_ctx_get(ctx, GRN_NORMALIZER_AUTO_NAME, -1); + if (!normalizer) { + ERR(GRN_INVALID_ARGUMENT, + "[string][open] NormalizerAuto normalizer isn't available"); + return NULL; + } + } + string = GRN_MALLOCN(grn_string, 1); if (!string) { + if (is_normalizer_auto) { + grn_obj_unlink(ctx, normalizer); + } GRN_LOG(ctx, GRN_LOG_ALERT, "[string][open] failed to allocate memory"); return NULL; @@ -163,12 +176,6 @@ grn_string_open_(grn_ctx *ctx, const char *str, unsigned int str_len, return (grn_obj *)grn_fake_string_open(ctx, string); } - is_normalizer_auto = (normalizer == GRN_NORMALIZER_AUTO); - if (is_normalizer_auto) { - normalizer = grn_ctx_get(ctx, GRN_NORMALIZER_AUTO_NAME, -1); - } - - /* TODO: check rc */ grn_normalizer_normalize(ctx, normalizer, (grn_obj *)string); if (ctx->rc) { grn_obj_close(ctx, obj); diff --git a/storage/mroonga/vendor/groonga/lib/token_cursor.c b/storage/mroonga/vendor/groonga/lib/token_cursor.c index d3e2e2d8523..d98d1d46342 100644 --- a/storage/mroonga/vendor/groonga/lib/token_cursor.c +++ b/storage/mroonga/vendor/groonga/lib/token_cursor.c @@ -216,15 +216,17 @@ grn_token_cursor_next(grn_ctx *ctx, grn_token_cursor *token_cursor) token_cursor->force_prefix = GRN_TRUE; } if (token_cursor->curr_size == 0) { - char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE]; - int tokenizer_name_length; - tokenizer_name_length = - grn_obj_name(ctx, token_cursor->tokenizer, - tokenizer_name, GRN_TABLE_MAX_KEY_SIZE); - GRN_LOG(ctx, GRN_WARN, - "[token_next] ignore an empty token: <%.*s>: <%.*s>", - tokenizer_name_length, tokenizer_name, - token_cursor->orig_blen, token_cursor->orig); + if (token_cursor->status != GRN_TOKEN_CURSOR_DONE) { + char tokenizer_name[GRN_TABLE_MAX_KEY_SIZE]; + int tokenizer_name_length; + tokenizer_name_length = + grn_obj_name(ctx, token_cursor->tokenizer, + tokenizer_name, GRN_TABLE_MAX_KEY_SIZE); + GRN_LOG(ctx, GRN_WARN, + "[token_next] ignore an empty token: <%.*s>: <%.*s>", + tokenizer_name_length, tokenizer_name, + token_cursor->orig_blen, token_cursor->orig); + } continue; } if (token_cursor->curr_size > GRN_TABLE_MAX_KEY_SIZE) { diff --git a/storage/mroonga/vendor/groonga/lib/tokenizer.c b/storage/mroonga/vendor/groonga/lib/tokenizer.c index ef9eb5bf7b5..e72d3b43d2b 100644 --- a/storage/mroonga/vendor/groonga/lib/tokenizer.c +++ b/storage/mroonga/vendor/groonga/lib/tokenizer.c @@ -167,7 +167,7 @@ grn_tokenizer_query_open(grn_ctx *ctx, int num_args, grn_obj **args, return NULL; } query->normalized_query = normalized_query; - memcpy(query_buf, GRN_TEXT_VALUE(query_str), query_length); + grn_memcpy(query_buf, GRN_TEXT_VALUE(query_str), query_length); query_buf[query_length] = '\0'; query->query_buf = query_buf; query->ptr = query_buf; diff --git a/storage/mroonga/vendor/groonga/lib/tokenizers.c b/storage/mroonga/vendor/groonga/lib/tokenizers.c index 3f6df15a690..8ed0b8c37fa 100644 --- a/storage/mroonga/vendor/groonga/lib/tokenizers.c +++ b/storage/mroonga/vendor/groonga/lib/tokenizers.c @@ -1,6 +1,6 @@ /* -*- c-basic-offset: 2 -*- */ /* - Copyright(C) 2009-2014 Brazil + Copyright(C) 2009-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -467,6 +467,260 @@ ngram_fin(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) return NULL; } +/* regexp tokenizer */ + +typedef struct { + grn_tokenizer_token token; + grn_tokenizer_query *query; + struct { + grn_bool have_begin; + grn_bool have_end; + int32_t n_skip_tokens; + } get; + grn_bool is_begin; + grn_bool is_end; + grn_bool is_first_token; + grn_bool is_overlapping; + const char *next; + const char *end; + grn_obj buffer; +} grn_regexp_tokenizer; + +static grn_obj * +regexp_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) +{ + unsigned int normalize_flags = 0; + grn_tokenizer_query *query; + const char *normalized; + unsigned int normalized_length_in_bytes; + grn_regexp_tokenizer *tokenizer; + + query = grn_tokenizer_query_open(ctx, nargs, args, normalize_flags); + if (!query) { + return NULL; + } + + tokenizer = GRN_MALLOC(sizeof(grn_regexp_tokenizer)); + if (!tokenizer) { + grn_tokenizer_query_close(ctx, query); + ERR(GRN_NO_MEMORY_AVAILABLE, + "[tokenizer][regexp] failed to allocate memory"); + return NULL; + } + user_data->ptr = tokenizer; + + grn_tokenizer_token_init(ctx, &(tokenizer->token)); + tokenizer->query = query; + + tokenizer->get.have_begin = GRN_FALSE; + tokenizer->get.have_end = GRN_FALSE; + tokenizer->get.n_skip_tokens = 0; + + tokenizer->is_begin = GRN_TRUE; + tokenizer->is_end = GRN_FALSE; + tokenizer->is_first_token = GRN_TRUE; + tokenizer->is_overlapping = GRN_FALSE; + + grn_string_get_normalized(ctx, tokenizer->query->normalized_query, + &normalized, &normalized_length_in_bytes, + NULL); + tokenizer->next = normalized; + tokenizer->end = tokenizer->next + normalized_length_in_bytes; + + if (tokenizer->query->tokenize_mode == GRN_TOKEN_GET) { + unsigned int query_length = tokenizer->query->length; + if (query_length >= 2) { + const char *query_string = tokenizer->query->ptr; + grn_encoding encoding = tokenizer->query->encoding; + if (query_string[0] == '\\' && query_string[1] == 'A') { + tokenizer->get.have_begin = GRN_TRUE; + /* TODO: It assumes that both "\\" and "A" are normalized to 1 + characters. Normalizer may omit character or expand to + multiple characters. */ + tokenizer->next += grn_charlen_(ctx, tokenizer->next, tokenizer->end, + encoding); + tokenizer->next += grn_charlen_(ctx, tokenizer->next, tokenizer->end, + encoding); + } + if (query_string[query_length - 2] == '\\' && + query_string[query_length - 1] == 'z') { + tokenizer->get.have_end = GRN_TRUE; + /* TODO: It assumes that both "\\" and "z" are normalized to 1 + byte characters. Normalizer may omit character or expand to + multiple characters. */ + tokenizer->end -= grn_charlen_(ctx, + tokenizer->end - 1, + tokenizer->end, + encoding); + tokenizer->end -= grn_charlen_(ctx, + tokenizer->end - 1, + tokenizer->end, + encoding); + } + } + } + + GRN_TEXT_INIT(&(tokenizer->buffer), 0); + + return NULL; +} + +static grn_obj * +regexp_next(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) +{ + int char_len; + grn_token_status status = 0; + grn_regexp_tokenizer *tokenizer = user_data->ptr; + unsigned int n_characters = 0; + int ngram_unit = 2; + grn_obj *buffer = &(tokenizer->buffer); + const char *current = tokenizer->next; + const char *end = tokenizer->end; + grn_tokenize_mode mode = tokenizer->query->tokenize_mode; + grn_bool escaping = GRN_FALSE; + + GRN_BULK_REWIND(buffer); + + if (mode == GRN_TOKEN_GET) { + if (tokenizer->get.have_begin) { + grn_tokenizer_token_push(ctx, + &(tokenizer->token), + GRN_TOKENIZER_BEGIN_MARK_UTF8, + GRN_TOKENIZER_BEGIN_MARK_UTF8_LEN, + status); + tokenizer->get.have_begin = GRN_FALSE; + return NULL; + } + + if (tokenizer->is_end && tokenizer->get.have_end) { + status |= GRN_TOKEN_LAST | GRN_TOKEN_REACH_END; + grn_tokenizer_token_push(ctx, + &(tokenizer->token), + GRN_TOKENIZER_END_MARK_UTF8, + GRN_TOKENIZER_END_MARK_UTF8_LEN, + status); + return NULL; + } + } else { + if (tokenizer->is_begin) { + grn_tokenizer_token_push(ctx, + &(tokenizer->token), + GRN_TOKENIZER_BEGIN_MARK_UTF8, + GRN_TOKENIZER_BEGIN_MARK_UTF8_LEN, + status); + tokenizer->is_begin = GRN_FALSE; + return NULL; + } + + if (tokenizer->is_end) { + status |= GRN_TOKEN_LAST | GRN_TOKEN_REACH_END; + grn_tokenizer_token_push(ctx, + &(tokenizer->token), + GRN_TOKENIZER_END_MARK_UTF8, + GRN_TOKENIZER_END_MARK_UTF8_LEN, + status); + return NULL; + } + } + + char_len = grn_charlen_(ctx, current, end, tokenizer->query->encoding); + if (char_len == 0) { + status |= GRN_TOKEN_LAST | GRN_TOKEN_REACH_END; + grn_tokenizer_token_push(ctx, &(tokenizer->token), "", 0, status); + return NULL; + } + + while (GRN_TRUE) { + if (!escaping && mode == GRN_TOKEN_GET && + char_len == 1 && current[0] == '\\') { + current += char_len; + escaping = GRN_TRUE; + } else { + n_characters++; + GRN_TEXT_PUT(ctx, buffer, current, char_len); + current += char_len; + escaping = GRN_FALSE; + if (n_characters == 1) { + tokenizer->next = current; + } + if (n_characters == ngram_unit) { + break; + } + } + + char_len = grn_charlen_(ctx, (const char *)current, (const char *)end, + tokenizer->query->encoding); + if (char_len == 0) { + break; + } + } + + if (tokenizer->is_overlapping) { + status |= GRN_TOKEN_OVERLAP; + } + if (n_characters < ngram_unit) { + status |= GRN_TOKEN_UNMATURED; + } + tokenizer->is_overlapping = (n_characters > 1); + + if (mode == GRN_TOKEN_GET) { + if ((end - tokenizer->next) < ngram_unit) { + if (tokenizer->get.have_end) { + if (tokenizer->next == end) { + tokenizer->is_end = GRN_TRUE; + } + if (status & GRN_TOKEN_UNMATURED) { + if (tokenizer->is_first_token) { + status |= GRN_TOKEN_FORCE_PREFIX; + } else { + status |= GRN_TOKEN_SKIP; + } + } + } else { + tokenizer->is_end = GRN_TRUE; + status |= GRN_TOKEN_LAST | GRN_TOKEN_REACH_END; + if (status & GRN_TOKEN_UNMATURED) { + status |= GRN_TOKEN_FORCE_PREFIX; + } + } + } else { + if (tokenizer->get.n_skip_tokens > 0) { + tokenizer->get.n_skip_tokens--; + status |= GRN_TOKEN_SKIP; + } else { + tokenizer->get.n_skip_tokens = ngram_unit - 1; + } + } + } else { + if (tokenizer->next == end) { + tokenizer->is_end = GRN_TRUE; + } + } + + grn_tokenizer_token_push(ctx, + &(tokenizer->token), + GRN_TEXT_VALUE(buffer), + GRN_TEXT_LEN(buffer), + status); + tokenizer->is_first_token = GRN_FALSE; + + return NULL; +} + +static grn_obj * +regexp_fin(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) +{ + grn_regexp_tokenizer *tokenizer = user_data->ptr; + if (!tokenizer) { + return NULL; + } + grn_tokenizer_token_fin(ctx, &(tokenizer->token)); + grn_tokenizer_query_close(ctx, tokenizer->query); + GRN_OBJ_FIN(ctx, &(tokenizer->buffer)); + GRN_FREE(tokenizer); + return NULL; +} + /* external */ grn_rc @@ -560,5 +814,7 @@ grn_db_init_builtin_tokenizers(grn_ctx *ctx) bigramisad_init, ngram_next, ngram_fin, vars); DEF_TOKENIZER("TokenDelimitNull", delimit_null_init, delimited_next, delimited_fin, vars); + DEF_TOKENIZER("TokenRegexp", + regexp_init, regexp_next, regexp_fin, vars); return GRN_SUCCESS; } diff --git a/storage/mroonga/vendor/groonga/lib/util.c b/storage/mroonga/vendor/groonga/lib/util.c index dd703b53928..17172d6cfeb 100644 --- a/storage/mroonga/vendor/groonga/lib/util.c +++ b/storage/mroonga/vendor/groonga/lib/util.c @@ -20,9 +20,18 @@ #include "grn_ii.h" #include "grn_util.h" #include "grn_string.h" +#include "grn_expr.h" #include <string.h> #include <stdio.h> +#include <stdlib.h> +#include <fcntl.h> +#include <sys/stat.h> + +#ifdef WIN32 +# include <io.h> +# include <share.h> +#endif /* WIN32 */ grn_rc grn_normalize_offset_and_limit(grn_ctx *ctx, int size, int *p_offset, int *p_limit) @@ -211,8 +220,13 @@ grn_inspect_type(grn_ctx *ctx, grn_obj *buf, unsigned char type) break; default: { - char type_in_hex[5]; /* "0xXX" */ - sprintf(type_in_hex, "%#02x", type); +#define TYPE_IN_HEX_SIZE 5 /* "0xXX" */ + char type_in_hex[TYPE_IN_HEX_SIZE]; + grn_snprintf(type_in_hex, + TYPE_IN_HEX_SIZE, + TYPE_IN_HEX_SIZE, + "%#02x", type); +#undef TYPE_IN_HEX_SIZE GRN_TEXT_PUTS(ctx, buf, "(unknown: "); GRN_TEXT_PUTS(ctx, buf, type_in_hex); GRN_TEXT_PUTS(ctx, buf, ")"); @@ -279,6 +293,36 @@ grn_proc_inspect(grn_ctx *ctx, grn_obj *buf, grn_obj *obj) } grn_rc +grn_expr_code_inspect_indented(grn_ctx *ctx, + grn_obj *buffer, + grn_expr_code *code, + const char *indent) +{ + if (!code) { + GRN_TEXT_PUTS(ctx, buffer, "(NULL)"); + return GRN_SUCCESS; + } + + GRN_TEXT_PUTS(ctx, buffer, "<"); + GRN_TEXT_PUTS(ctx, buffer, grn_operator_to_string(code->op)); + GRN_TEXT_PUTS(ctx, buffer, " "); + GRN_TEXT_PUTS(ctx, buffer, "n_args:"); + grn_text_itoa(ctx, buffer, code->nargs); + GRN_TEXT_PUTS(ctx, buffer, ", "); + GRN_TEXT_PUTS(ctx, buffer, "flags:"); + grn_text_itoh(ctx, buffer, code->flags, 1); + GRN_TEXT_PUTS(ctx, buffer, ", "); + GRN_TEXT_PUTS(ctx, buffer, "modify:"); + grn_text_itoa(ctx, buffer, code->modify); + GRN_TEXT_PUTS(ctx, buffer, ", "); + GRN_TEXT_PUTS(ctx, buffer, "value:"); + grn_inspect_indented(ctx, buffer, code->value, " "); + GRN_TEXT_PUTS(ctx, buffer, ">"); + + return GRN_SUCCESS; +} + +grn_rc grn_expr_inspect(grn_ctx *ctx, grn_obj *buffer, grn_obj *expr) { grn_expr *e = (grn_expr *)expr; @@ -305,33 +349,15 @@ grn_expr_inspect(grn_ctx *ctx, grn_obj *buffer, grn_obj *expr) } { - uint32_t i, j; - grn_expr_var *var; + uint32_t i; grn_expr_code *code; GRN_TEXT_PUTS(ctx, buffer, "\n codes:{"); - for (j = 0, code = e->codes; j < e->codes_curr; j++, code++) { - if (j) { GRN_TEXT_PUTC(ctx, buffer, ','); } + for (i = 0, code = e->codes; i < e->codes_curr; i++, code++) { + if (i) { GRN_TEXT_PUTC(ctx, buffer, ','); } GRN_TEXT_PUTS(ctx, buffer, "\n "); - grn_text_itoa(ctx, buffer, j); - GRN_TEXT_PUTS(ctx, buffer, ":<"); - GRN_TEXT_PUTS(ctx, buffer, grn_operator_to_string(code->op)); - GRN_TEXT_PUTS(ctx, buffer, "("); - for (i = 0, var = e->vars; i < e->nvars; i++, var++) { - if (i) { GRN_TEXT_PUTC(ctx, buffer, ','); } - GRN_TEXT_PUTC(ctx, buffer, '?'); - if (var->name_size) { - GRN_TEXT_PUT(ctx, buffer, var->name, var->name_size); - } else { - grn_text_itoa(ctx, buffer, (int)i); - } - } - GRN_TEXT_PUTS(ctx, buffer, "), "); - GRN_TEXT_PUTS(ctx, buffer, "modify:"); - grn_text_itoa(ctx, buffer, code->modify); - GRN_TEXT_PUTS(ctx, buffer, ", "); - GRN_TEXT_PUTS(ctx, buffer, "value:"); - grn_inspect_indented(ctx, buffer, code->value, " "); - GRN_TEXT_PUTS(ctx, buffer, ">"); + grn_text_itoa(ctx, buffer, i); + GRN_TEXT_PUTS(ctx, buffer, ":"); + grn_expr_code_inspect_indented(ctx, buffer, code, " "); } GRN_TEXT_PUTS(ctx, buffer, "\n }"); } @@ -1256,6 +1282,17 @@ grn_p_ii_values(grn_ctx *ctx, grn_obj *ii) grn_obj_unlink(ctx, &buffer); } +void +grn_p_expr_code(grn_ctx *ctx, grn_expr_code *code) +{ + grn_obj buffer; + + GRN_TEXT_INIT(&buffer, 0); + grn_expr_code_inspect_indented(ctx, &buffer, code, ""); + printf("%.*s\n", (int)GRN_TEXT_LEN(&buffer), GRN_TEXT_VALUE(&buffer)); + grn_obj_unlink(ctx, &buffer); +} + #ifdef WIN32 static char *win32_base_dir = NULL; const char * @@ -1271,7 +1308,7 @@ grn_win32_base_dir(void) absolute_dll_filename, MAX_PATH); if (absolute_dll_filename_size == 0) { - win32_base_dir = strdup("."); + win32_base_dir = grn_strdup_raw("."); } else { DWORD ansi_dll_filename_size; ansi_dll_filename_size = @@ -1279,7 +1316,7 @@ grn_win32_base_dir(void) absolute_dll_filename, absolute_dll_filename_size, NULL, 0, NULL, NULL); if (ansi_dll_filename_size == 0) { - win32_base_dir = strdup("."); + win32_base_dir = grn_strdup_raw("."); } else { char *path; win32_base_dir = malloc(ansi_dll_filename_size + 1); @@ -1310,3 +1347,41 @@ grn_win32_base_dir(void) return win32_base_dir; } #endif + +#ifdef WIN32 +int +grn_mkstemp(char *path_template) +{ + errno_t error; + size_t path_template_size; + int fd; + + path_template_size = strlen(path_template) + 1; + error = _mktemp_s(path_template, path_template_size); + if (error != 0) { + return -1; + } + + error = _sopen_s(&fd, + path_template, + _O_RDWR | _O_CREAT | _O_EXCL | _O_BINARY, + _SH_DENYNO, + _S_IREAD | _S_IWRITE); + if (error != 0) { + return -1; + } + + return fd; +} +#else /* WIN32 */ +int +grn_mkstemp(char *path_template) +{ +# ifdef HAVE_MKSTEMP + return mkstemp(path_template); +# else /* HAVE_MKSTEMP */ + mktemp(path_template); + return open(path_template, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); +# endif /* HAVE_MKSTEMP */ +} +#endif /* WIN32 */ diff --git a/storage/mroonga/vendor/groonga/nginx_version b/storage/mroonga/vendor/groonga/nginx_version index f65dc1e213e..27f9cd322bb 100644 --- a/storage/mroonga/vendor/groonga/nginx_version +++ b/storage/mroonga/vendor/groonga/nginx_version @@ -1 +1 @@ -1.7.9 +1.8.0 diff --git a/storage/mroonga/vendor/groonga/plugins/CMakeLists.txt b/storage/mroonga/vendor/groonga/plugins/CMakeLists.txt index 89a28f50b99..37f6127b0d2 100644 --- a/storage/mroonga/vendor/groonga/plugins/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/plugins/CMakeLists.txt @@ -20,6 +20,7 @@ add_subdirectory(query_expanders) add_subdirectory(ruby) add_subdirectory(token_filters) add_subdirectory(sharding) +add_subdirectory(functions) if(GRN_WITH_MRUBY) read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/ruby_scripts.am RUBY_SCRIPTS) diff --git a/storage/mroonga/vendor/groonga/plugins/Makefile.am b/storage/mroonga/vendor/groonga/plugins/Makefile.am index 92007edd732..dc3967899c2 100644 --- a/storage/mroonga/vendor/groonga/plugins/Makefile.am +++ b/storage/mroonga/vendor/groonga/plugins/Makefile.am @@ -5,7 +5,8 @@ SUBDIRS = \ query_expanders \ ruby \ token_filters \ - sharding + sharding \ + functions EXTRA_DIST = \ CMakeLists.txt diff --git a/storage/mroonga/vendor/groonga/plugins/functions/CMakeLists.txt b/storage/mroonga/vendor/groonga/plugins/functions/CMakeLists.txt new file mode 100644 index 00000000000..28db8b42ef3 --- /dev/null +++ b/storage/mroonga/vendor/groonga/plugins/functions/CMakeLists.txt @@ -0,0 +1,32 @@ +# Copyright(C) 2015 Brazil +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/../../lib + ) + +set(GRN_FUNCTIONS_PLUGIN_DIR "${GRN_RELATIVE_PLUGINS_DIR}/functions") + +read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/vector_sources.am + VECTOR_SOURCES) +add_library(vector_functions MODULE ${VECTOR_SOURCES}) +set_source_files_properties(${VECTOR_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") +set_target_properties(vector_functions PROPERTIES + PREFIX "" + OUTPUT_NAME "vector") +target_link_libraries(vector_functions libgroonga) +install(TARGETS vector_functions DESTINATION "${GRN_FUNCTIONS_PLUGIN_DIR}") diff --git a/storage/mroonga/vendor/groonga/plugins/functions/Makefile.am b/storage/mroonga/vendor/groonga/plugins/functions/Makefile.am new file mode 100644 index 00000000000..5d8d1d9cc84 --- /dev/null +++ b/storage/mroonga/vendor/groonga/plugins/functions/Makefile.am @@ -0,0 +1,20 @@ +EXTRA_DIST = \ + CMakeLists.txt + +AM_CPPFLAGS = \ + -I$(top_builddir) \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/lib + +AM_LDFLAGS = \ + -avoid-version \ + -module \ + -no-undefined + +LIBS = \ + $(top_builddir)/lib/libgroonga.la + +function_plugins_LTLIBRARIES = +function_plugins_LTLIBRARIES += vector.la + +include vector_sources.am diff --git a/storage/mroonga/vendor/groonga/plugins/functions/vector.c b/storage/mroonga/vendor/groonga/plugins/functions/vector.c new file mode 100644 index 00000000000..34f7a98fd27 --- /dev/null +++ b/storage/mroonga/vendor/groonga/plugins/functions/vector.c @@ -0,0 +1,90 @@ +/* -*- c-basic-offset: 2 -*- */ +/* + Copyright(C) 2015 Brazil + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include <groonga/plugin.h> + +static grn_obj * +func_vector_size(grn_ctx *ctx, int n_args, grn_obj **args, + grn_user_data *user_data) +{ + grn_obj *target; + unsigned int size; + grn_obj *grn_size; + + if (n_args != 1) { + GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, + "vector_size(): wrong number of arguments (%d for 1)", + n_args); + return NULL; + } + + target = args[0]; + switch (target->header.type) { + case GRN_VECTOR : + case GRN_PVECTOR : + case GRN_UVECTOR : + size = grn_vector_size(ctx, target); + break; + default : + { + grn_obj inspected; + + GRN_TEXT_INIT(&inspected, 0); + grn_inspect(ctx, target, &inspected); + GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, + "vector_size(): target object must be vector: <%.*s>", + (int)GRN_TEXT_LEN(&inspected), + GRN_TEXT_VALUE(&inspected)); + GRN_OBJ_FIN(ctx, &inspected); + return NULL; + } + break; + } + + grn_size = grn_plugin_proc_alloc(ctx, user_data, GRN_DB_UINT32, 0); + if (!grn_size) { + return NULL; + } + + GRN_UINT32_SET(ctx, grn_size, size); + + return grn_size; +} + +grn_rc +GRN_PLUGIN_INIT(grn_ctx *ctx) +{ + return ctx->rc; +} + +grn_rc +GRN_PLUGIN_REGISTER(grn_ctx *ctx) +{ + grn_rc rc = GRN_SUCCESS; + + grn_proc_create(ctx, "vector_size", -1, GRN_PROC_FUNCTION, func_vector_size, + NULL, NULL, 0, NULL); + + return rc; +} + +grn_rc +GRN_PLUGIN_FIN(grn_ctx *ctx) +{ + return GRN_SUCCESS; +} diff --git a/storage/mroonga/vendor/groonga/plugins/functions/vector_sources.am b/storage/mroonga/vendor/groonga/plugins/functions/vector_sources.am new file mode 100644 index 00000000000..1d98e651845 --- /dev/null +++ b/storage/mroonga/vendor/groonga/plugins/functions/vector_sources.am @@ -0,0 +1,2 @@ +vector_la_SOURCES = \ + vector.c diff --git a/storage/mroonga/vendor/groonga/plugins/query_expanders/tsv.c b/storage/mroonga/vendor/groonga/plugins/query_expanders/tsv.c index 1b9f177711a..06bb322acb2 100644 --- a/storage/mroonga/vendor/groonga/plugins/query_expanders/tsv.c +++ b/storage/mroonga/vendor/groonga/plugins/query_expanders/tsv.c @@ -1,5 +1,5 @@ /* -*- c-basic-offset: 2 -*- */ -/* Copyright(C) 2012 Brazil +/* Copyright(C) 2012-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -25,6 +25,10 @@ #include <stdio.h> #include <string.h> +#ifdef WIN32 +# include <share.h> +#endif /* WIN32 */ + #ifdef HAVE__STRNICMP # ifdef strncasecmp # undef strncasecmp @@ -37,24 +41,20 @@ static grn_hash *synonyms = NULL; #ifdef WIN32 -static char *win32_synonyms_file = NULL; +static char win32_synonyms_file[MAX_PATH] = ""; const char * get_system_synonyms_file(void) { - if (!win32_synonyms_file) { + if (win32_synonyms_file[0] == '\0') { const char *base_dir; const char *relative_path = GRN_QUERY_EXPANDER_TSV_RELATIVE_SYNONYMS_FILE; - char *synonyms_file; size_t base_dir_length; base_dir = grn_plugin_win32_base_dir(); base_dir_length = strlen(base_dir); - synonyms_file = - malloc(base_dir_length + strlen("/") + strlen(relative_path) + 1); - strcpy(synonyms_file, base_dir); - strcat(synonyms_file, "/"); - strcat(synonyms_file, relative_path); - win32_synonyms_file = synonyms_file; + grn_strcpy(win32_synonyms_file, MAX_PATH, base_dir); + grn_strcat(win32_synonyms_file, MAX_PATH, "/"); + grn_strcat(win32_synonyms_file, MAX_PATH, relative_path); } return win32_synonyms_file; } @@ -182,26 +182,35 @@ parse_synonyms_file_line(grn_ctx *ctx, const char *line, int line_length, return; } - grn_bulk_truncate(ctx, value, MAX_SYNONYM_BYTES - 1); - GRN_TEXT_PUTC(ctx, value, '\0'); - memcpy(value_location, GRN_TEXT_VALUE(value), MAX_SYNONYM_BYTES); + if (GRN_TEXT_LEN(value) <= MAX_SYNONYM_BYTES - 1) { + GRN_TEXT_PUTC(ctx, value, '\0'); + } else { + grn_bulk_truncate(ctx, value, MAX_SYNONYM_BYTES - 1); + GRN_TEXT_PUTC(ctx, value, '\0'); + } + grn_memcpy(value_location, GRN_TEXT_VALUE(value), GRN_TEXT_LEN(value)); } } static void load_synonyms(grn_ctx *ctx) { + static char path_env[GRN_ENV_BUFFER_SIZE]; const char *path; FILE *file; int number_of_lines; grn_encoding encoding; grn_obj line, key, value; - path = getenv("GRN_QUERY_EXPANDER_TSV_SYNONYMS_FILE"); - if (!path) { + grn_getenv("GRN_QUERY_EXPANDER_TSV_SYNONYMS_FILE", + path_env, + GRN_ENV_BUFFER_SIZE); + if (path_env[0]) { + path = path_env; + } else { path = get_system_synonyms_file(); } - file = fopen(path, "r"); + file = grn_fopen(path, "r"); if (!file) { GRN_LOG(ctx, GRN_LOG_WARNING, "[plugin][query-expander][tsv] " @@ -219,6 +228,13 @@ load_synonyms(grn_ctx *ctx) const char *line_value = GRN_TEXT_VALUE(&line); size_t line_length = GRN_TEXT_LEN(&line); + if (line_length > 0 && line_value[line_length - 1] == '\n') { + if (line_length > 1 && line_value[line_length - 2] == '\r') { + line_length -= 2; + } else { + line_length -= 1; + } + } number_of_lines++; if (number_of_lines == 1) { encoding = guess_encoding(ctx, &line_value, &line_length); diff --git a/storage/mroonga/vendor/groonga/plugins/sharding.rb b/storage/mroonga/vendor/groonga/plugins/sharding.rb index 8902d7e6c27..e369c3798da 100644 --- a/storage/mroonga/vendor/groonga/plugins/sharding.rb +++ b/storage/mroonga/vendor/groonga/plugins/sharding.rb @@ -1,3 +1,4 @@ +require "sharding/range_expression_builder" require "sharding/logical_enumerator" require "sharding/logical_count" require "sharding/logical_range_filter" diff --git a/storage/mroonga/vendor/groonga/plugins/sharding/logical_count.rb b/storage/mroonga/vendor/groonga/plugins/sharding/logical_count.rb index ef993b40dc8..610dae834d3 100644 --- a/storage/mroonga/vendor/groonga/plugins/sharding/logical_count.rb +++ b/storage/mroonga/vendor/groonga/plugins/sharding/logical_count.rb @@ -32,96 +32,68 @@ module Groonga cover_type = target_range.cover_type(shard_range) return 0 if cover_type == :none + expression_builder = RangeExpressionBuilder.new(shard_key, + target_range, + filter) if cover_type == :all if filter.nil? return table.size else - return filtered_count_n_records(table, filter) + return filtered_count_n_records(table) do |expression| + expression_builder.build_all(expression) + end end end - use_range_index = false range_index = nil if filter.nil? index_info = shard_key.find_index(Operator::LESS) if index_info range_index = index_info.index - use_range_index = true end end case cover_type when :partial_min - if use_range_index + if range_index count_n_records_in_range(range_index, target_range.min, target_range.min_border, nil, nil) else - filtered_count_n_records(table, filter) do |expression| - expression.append_object(shard_key, Operator::PUSH, 1) - expression.append_operator(Operator::GET_VALUE, 1) - expression.append_constant(target_range.min, Operator::PUSH, 1) - if target_range.min_border == :include - expression.append_operator(Operator::GREATER_EQUAL, 2) - else - expression.append_operator(Operator::GREATER, 2) - end + filtered_count_n_records(table) do |expression| + expression_builder.build_partial_min(expression) end end when :partial_max - if use_range_index + if range_index count_n_records_in_range(range_index, nil, nil, target_range.max, target_range.max_border) else - filtered_count_n_records(table, filter) do |expression| - expression.append_object(shard_key, Operator::PUSH, 1) - expression.append_operator(Operator::GET_VALUE, 1) - expression.append_constant(target_range.max, Operator::PUSH, 1) - if target_range.max_border == :include - expression.append_operator(Operator::LESS_EQUAL, 2) - else - expression.append_operator(Operator::LESS, 2) - end + filtered_count_n_records(table) do |expression| + expression_builder.build_partial_max(expression) end end when :partial_min_and_max - if use_range_index + if range_index count_n_records_in_range(range_index, target_range.min, target_range.min_border, target_range.max, target_range.max_border) else - filtered_count_n_records(table, filter) do |expression| - expression.append_object(context["between"], Operator::PUSH, 1) - expression.append_object(shard_key, Operator::PUSH, 1) - expression.append_operator(Operator::GET_VALUE, 1) - expression.append_constant(target_range.min, Operator::PUSH, 1) - expression.append_constant(target_range.min_border, - Operator::PUSH, 1) - expression.append_constant(target_range.max, Operator::PUSH, 1) - expression.append_constant(target_range.max_border, - Operator::PUSH, 1) - expression.append_operator(Operator::CALL, 5) + filtered_count_n_records(table) do |expression| + expression_builder.build_partial_min_and_max(expression) end end end end - def filtered_count_n_records(table, filter) + def filtered_count_n_records(table) expression = nil filtered_table = nil begin expression = Expression.create(table) - if block_given? - yield(expression) - if filter - expression.parse(filter) - expression.append_operator(Operator::AND, 2) - end - else - expression.parse(filter) - end + yield(expression) filtered_table = table.select(expression) filtered_table.size ensure diff --git a/storage/mroonga/vendor/groonga/plugins/sharding/logical_enumerator.rb b/storage/mroonga/vendor/groonga/plugins/sharding/logical_enumerator.rb index 7ad5393ae77..1aeafef5438 100644 --- a/storage/mroonga/vendor/groonga/plugins/sharding/logical_enumerator.rb +++ b/storage/mroonga/vendor/groonga/plugins/sharding/logical_enumerator.rb @@ -2,6 +2,7 @@ module Groonga module Sharding class LogicalEnumerator attr_reader :target_range + attr_reader :logical_table attr_reader :shard_key_name def initialize(command_name, input) @command_name = command_name @@ -9,12 +10,21 @@ module Groonga initialize_parameters end - def each + def each(&block) + each_internal(:ascending, &block) + end + + def reverse_each(&block) + each_internal(:descending, &block) + end + + private + def each_internal(order) prefix = "#{@logical_table}_" context = Context.instance context.database.each_table(:prefix => prefix, :order_by => :key, - :order => :ascending) do |table| + :order => order) do |table| shard_range_raw = table.name[prefix.size..-1] next unless /\A(\d{4})(\d{2})(\d{2})\z/ =~ shard_range_raw @@ -142,10 +152,10 @@ module Groonga return true if @min_border == :exclude - @min.hour != 0 and - @min.min != 0 and - @min.sec != 0 and - @min.usec != 0 + not (@min.hour == 0 and + @min.min == 0 and + @min.sec == 0 and + @min.usec == 0) end def in_max?(shard_range) diff --git a/storage/mroonga/vendor/groonga/plugins/sharding/logical_range_filter.rb b/storage/mroonga/vendor/groonga/plugins/sharding/logical_range_filter.rb index 7e765f7d3c2..94ae8600d2b 100644 --- a/storage/mroonga/vendor/groonga/plugins/sharding/logical_range_filter.rb +++ b/storage/mroonga/vendor/groonga/plugins/sharding/logical_range_filter.rb @@ -17,168 +17,424 @@ module Groonga ]) def run_body(input) - enumerator = LogicalEnumerator.new("logical_range_filter", input) - filter = input[:filter] - offset = (input[:offset] || 0).to_i - limit = (input[:limit] || 10).to_i output_columns = input[:output_columns] || "_key, *" - result_sets = [] - n_records = 0 - enumerator.each do |table, shard_key, shard_range| - result_set = filter_shard(table, filter, - shard_key, shard_range, - enumerator.target_range) - next if result_set.nil? - if result_set.empty? - result_set.close if result_set.temporary? - next - end - result_sets << result_set - n_records += result_set.size - break if n_records >= offset + limit - end + context = ExecuteContext.new(input) + begin + executor = Executor.new(context) + executor.execute - if result_sets.empty? - n_elements = 0 - else + result_sets = context.result_sets n_elements = 1 # for columns result_sets.each do |result_set| n_elements += result_set.size end + + writer.array("RESULTSET", n_elements) do + first_result_set = result_sets.first + if first_result_set + writer.write_table_columns(first_result_set, output_columns) + end + limit = context.limit + if limit < 0 + n_records = result_sets.inject(0) do |n, result_set| + n + result_set.size + end + limit = n_records + limit + 1 + end + options = {} + result_sets.each do |result_set| + options[:limit] = limit + writer.write_table_records(result_set, output_columns, options) + limit -= result_set.size + break if limit <= 0 + end + end + ensure + context.close + end + end + + class ExecuteContext + attr_reader :enumerator + attr_reader :order + attr_reader :filter + attr_reader :offset + attr_reader :limit + attr_accessor :current_offset + attr_accessor :current_limit + attr_reader :result_sets + attr_reader :unsorted_result_sets + attr_reader :threshold + def initialize(input) + @input = input + @enumerator = LogicalEnumerator.new("logical_range_filter", @input) + @order = parse_order(@input, :order) + @filter = @input[:filter] + @offset = (@input[:offset] || 0).to_i + @limit = (@input[:limit] || 10).to_i + + @current_offset = @offset + @current_limit = @limit + + @result_sets = [] + @unsorted_result_sets = [] + + @threshold = compute_threshold end - sort_keys = [ - { - :key => enumerator.shard_key_name, - :order => :ascending, - }, - ] - current_offset = offset - current_limit = limit - writer.array("RESULTSET", n_elements) do - first_result_set = result_sets.first - if first_result_set - writer.write_table_columns(first_result_set, output_columns) + def close + @unsorted_result_sets.each do |result_set| + result_set.close if result_set.temporary? end - result_sets.each do |result_set| - if result_set.size <= current_offset - current_offset -= result_set.size - next - end - sorted_result_set = result_set.sort(sort_keys, - :offset => current_offset, - :limit => current_limit) - writer.write_table_records(sorted_result_set, output_columns) - current_limit -= sorted_result_set.size - sorted_result_set.close + @result_sets.each do |result_set| + result_set.close if result_set.temporary? end end - result_sets.each do |result_set| - result_set.close if result_set.temporary? + private + def parse_order(input, name) + order = input[name] + return :ascending if order.nil? + + case order + when "ascending" + :ascending + when "descending" + :descending + else + message = + "[logical_range_filter] #{name} must be " + + "\"ascending\" or \"descending\": <#{order}>" + raise InvalidArgument, message + end + end + + def compute_threshold + threshold_env = ENV["GRN_LOGICAL_RANGE_FILTER_THRESHOLD"] + default_threshold = 0.2 + (threshold_env || default_threshold).to_f end end - def filter_shard(table, filter, shard_key, shard_range, target_range) - cover_type = target_range.cover_type(shard_range) - return nil if cover_type == :none + class Executor + def initialize(context) + @context = context + end - if cover_type == :all - if filter.nil? - return table + def execute + first_table = nil + enumerator = @context.enumerator + if @context.order == :descending + each_method = :reverse_each else - return filter_table(table, filter) - end - end - - use_range_index = false - range_index = nil - # TODO - # if filter.nil? - # index_info = shard_key.find_index(Operator::LESS) - # if index_info - # range_index = index_info.index - # use_range_index = true - # end - # end - - case cover_type - when :partial_min - if use_range_index - # TODO - # count_n_records_in_range(range_index, - # target_range.min, target_range.min_border, - # nil, nil) + each_method = :each + end + enumerator.send(each_method) do |table, shard_key, shard_range| + first_table ||= table + next if table.empty? + + shard_executor = ShardExecutor.new(@context, + table, shard_key, shard_range) + shard_executor.execute + break if @context.current_limit == 0 + end + if first_table.nil? + message = + "[logical_range_filter] no shard exists: " + + "logical_table: <#{enumerator.logical_table}>: " + + "shard_key: <#{enumerator.shard_key_name}>" + raise InvalidArgument, message + end + if @context.result_sets.empty? + result_set = HashTable.create(:flags => ObjectFlags::WITH_SUBREC, + :key_type => first_table) + @context.result_sets << result_set + end + end + end + + class ShardExecutor + def initialize(context, table, shard_key, shard_range) + @context = context + @table = table + @shard_key = shard_key + @shard_range = shard_range + + @filter = @context.filter + @result_sets = @context.result_sets + @unsorted_result_sets = @context.unsorted_result_sets + + @target_range = @context.enumerator.target_range + + @cover_type = @target_range.cover_type(@shard_range) + + @expression_builder = RangeExpressionBuilder.new(@shard_key, + @target_range, + @filter) + end + + def execute + return if @cover_type == :none + + index_info = @shard_key.find_index(Operator::LESS) + if index_info + range_index = index_info.index + range_index = nil unless use_range_index?(range_index) else - filter_table(table, filter) do |expression| - expression.append_object(shard_key, Operator::PUSH, 1) - expression.append_operator(Operator::GET_VALUE, 1) - expression.append_constant(target_range.min, Operator::PUSH, 1) - if target_range.min_border == :include - expression.append_operator(Operator::GREATER_EQUAL, 2) - else - expression.append_operator(Operator::GREATER, 2) + range_index = nil + end + + case @cover_type + when :all + filter_shard_all(range_index) + when :partial_min + if range_index + filter_by_range(range_index, + @target_range.min, @target_range.min_border, + nil, nil) + else + filter_table do |expression| + @expression_builder.build_partial_min(expression) + end + end + when :partial_max + if range_index + filter_by_range(range_index, + nil, nil, + @target_range.max, @target_range.max_border) + else + filter_table do |expression| + @expression_builder.build_partial_max(expression) + end + end + when :partial_min_and_max + if range_index + filter_by_range(range_index, + @target_range.min, @target_range.min_border, + @target_range.max, @target_range.max_border) + else + filter_table do |expression| + @expression_builder.build_partial_min_and_max(expression) end end end - when :partial_max - if use_range_index - # TODO - # count_n_records_in_range(range_index, - # nil, nil, - # target_range.max, target_range.max_border) - else - filter_table(table, filter) do |expression| - expression.append_object(shard_key, Operator::PUSH, 1) - expression.append_operator(Operator::GET_VALUE, 1) - expression.append_constant(target_range.max, Operator::PUSH, 1) - if target_range.max_border == :include - expression.append_operator(Operator::LESS_EQUAL, 2) - else - expression.append_operator(Operator::LESS, 2) + end + + private + def use_range_index?(range_index) + current_limit = @context.current_limit + if current_limit < 0 + return false + end + + required_n_records = @context.current_offset + current_limit + max_n_records = @table.size + if max_n_records <= required_n_records + return false + end + + threshold = @context.threshold + if threshold <= 0.0 + return true + end + if threshold >= 1.0 + return false + end + + estimated_n_records = 0 + case @cover_type + when :all + if @filter + create_expression(@table) do |expression| + @expression_builder.build_all(expression) + estimated_n_records = expression.estimate_size(@table) end + else + estimated_n_records = max_n_records + end + when :partial_min + create_expression(@table) do |expression| + @expression_builder.build_partial_min(expression) + estimated_n_records = expression.estimate_size(@table) end + when :partial_max + create_expression(@table) do |expression| + @expression_builder.build_partial_max(expression) + estimated_n_records = expression.estimate_size(@table) + end + when :partial_min_and_max + create_expression(@table) do |expression| + @expression_builder.build_partial_min_and_max(expression) + estimated_n_records = expression.estimate_size(@table) + end + end + + if estimated_n_records <= required_n_records + return false end - when :partial_min_and_max - if use_range_index - # TODO - # count_n_records_in_range(range_index, - # target_range.min, target_range.min_border, - # target_range.max, target_range.max_border) + + hit_ratio = estimated_n_records / max_n_records.to_f + hit_ratio >= threshold + end + + def filter_shard_all(range_index) + if @filter.nil? + if @table.size <= @context.current_offset + @context.current_offset -= @table.size + return + end + if range_index + filter_by_range(range_index, + nil, nil, + nil, nil) + else + sort_result_set(@table) + end else - filter_table(table, filter) do |expression| - expression.append_object(context["between"], Operator::PUSH, 1) - expression.append_object(shard_key, Operator::PUSH, 1) - expression.append_operator(Operator::GET_VALUE, 1) - expression.append_constant(target_range.min, Operator::PUSH, 1) - expression.append_constant(target_range.min_border, - Operator::PUSH, 1) - expression.append_constant(target_range.max, Operator::PUSH, 1) - expression.append_constant(target_range.max_border, - Operator::PUSH, 1) - expression.append_operator(Operator::CALL, 5) + if range_index + filter_by_range(range_index, + nil, nil, + nil, nil) + else + filter_table do |expression| + @expression_builder.build_all(expression) + end end end end - end - def filter_table(table, filter) - expression = nil - begin + def create_expression(table) expression = Expression.create(table) - if block_given? + begin yield(expression) - if filter - expression.parse(filter) - expression.append_operator(Operator::AND, 2) + ensure + expression.close + end + end + + def filter_by_range(range_index, + min, min_border, max, max_border) + lexicon = range_index.domain + data_table = range_index.range + flags = build_range_search_flags(min_border, max_border) + + result_set = HashTable.create(:flags => ObjectFlags::WITH_SUBREC, + :key_type => data_table) + n_matched_records = 0 + begin + TableCursor.open(lexicon, + :min => min, + :max => max, + :flags => flags) do |table_cursor| + options = { + :offset => @context.current_offset, + } + current_limit = @context.current_limit + if current_limit < 0 + options[:limit] = data_table.size + else + options[:limit] = current_limit + end + if @filter + create_expression(data_table) do |expression| + expression.parse(@filter) + options[:expression] = expression + IndexCursor.open(table_cursor, range_index) do |index_cursor| + n_matched_records = index_cursor.select(result_set, options) + end + end + else + IndexCursor.open(table_cursor, range_index) do |index_cursor| + n_matched_records = index_cursor.select(result_set, options) + end + end end + rescue + result_set.close + raise + end + + if n_matched_records <= @context.current_offset + @context.current_offset -= n_matched_records + result_set.close + return + end + + if @context.current_offset > 0 + @context.current_offset = 0 + end + if @context.current_limit > 0 + @context.current_limit -= result_set.size + end + @result_sets << result_set + end + + def build_range_search_flags(min_border, max_border) + flags = TableCursorFlags::BY_KEY + case @context.order + when :ascending + flags |= TableCursorFlags::ASCENDING + when :descending + flags |= TableCursorFlags::DESCENDING + end + case min_border + when :include + flags |= TableCursorFlags::GE + when :exclude + flags |= TableCursorFlags::GT + end + case max_border + when :include + flags |= TableCursorFlags::LE + when :exclude + flags |= TableCursorFlags::LT + end + flags + end + + def filter_table + create_expression(@table) do |expression| + yield(expression) + result_set = @table.select(expression) + sort_result_set(result_set) + end + end + + def sort_result_set(result_set) + if result_set.empty? + result_set.close if result_set.temporary? + return + end + + if result_set.size <= @context.current_offset + @context.current_offset -= result_set.size + result_set.close if result_set.temporary? + return + end + + @unsorted_result_sets << result_set if result_set.temporary? + sort_keys = [ + { + :key => @context.enumerator.shard_key_name, + :order => @context.order, + }, + ] + if @context.current_limit > 0 + limit = @context.current_limit else - expression.parse(filter) + limit = result_set.size + end + sorted_result_set = result_set.sort(sort_keys, + :offset => @context.current_offset, + :limit => limit) + @result_sets << sorted_result_set + if @context.current_offset > 0 + @context.current_offset = 0 + end + if @context.current_limit > 0 + @context.current_limit -= sorted_result_set.size end - table.select(expression) - ensure - expression.close if expression end end end diff --git a/storage/mroonga/vendor/groonga/plugins/sharding/range_expression_builder.rb b/storage/mroonga/vendor/groonga/plugins/sharding/range_expression_builder.rb new file mode 100644 index 00000000000..2e2dd29c51c --- /dev/null +++ b/storage/mroonga/vendor/groonga/plugins/sharding/range_expression_builder.rb @@ -0,0 +1,65 @@ +module Groonga + module Sharding + class RangeExpressionBuilder + def initialize(key, target_range, filter) + @key = key + @target_range = target_range + @filter = filter + end + + def build_all(expression) + return if @filter.nil? + + expression.parse(@filter) + end + + def build_partial_min(expression) + expression.append_object(@key, Operator::PUSH, 1) + expression.append_operator(Operator::GET_VALUE, 1) + expression.append_constant(@target_range.min, Operator::PUSH, 1) + if @target_range.min_border == :include + expression.append_operator(Operator::GREATER_EQUAL, 2) + else + expression.append_operator(Operator::GREATER, 2) + end + if @filter + expression.parse(@filter) + expression.append_operator(Operator::AND, 2) + end + end + + def build_partial_max(expression) + expression.append_object(@key, Operator::PUSH, 1) + expression.append_operator(Operator::GET_VALUE, 1) + expression.append_constant(@target_range.max, Operator::PUSH, 1) + if @target_range.max_border == :include + expression.append_operator(Operator::LESS_EQUAL, 2) + else + expression.append_operator(Operator::LESS, 2) + end + if @filter + expression.parse(@filter) + expression.append_operator(Operator::AND, 2) + end + end + + def build_partial_min_and_max(expression) + between = Groonga::Context.instance["between"] + expression.append_object(between, Operator::PUSH, 1) + expression.append_object(@key, Operator::PUSH, 1) + expression.append_operator(Operator::GET_VALUE, 1) + expression.append_constant(@target_range.min, Operator::PUSH, 1) + expression.append_constant(@target_range.min_border, + Operator::PUSH, 1) + expression.append_constant(@target_range.max, Operator::PUSH, 1) + expression.append_constant(@target_range.max_border, + Operator::PUSH, 1) + expression.append_operator(Operator::CALL, 5) + if @filter + expression.parse(@filter) + expression.append_operator(Operator::AND, 2) + end + end + end + end +end diff --git a/storage/mroonga/vendor/groonga/plugins/sharding/sources.am b/storage/mroonga/vendor/groonga/plugins/sharding/sources.am index 11b5382a6df..5defd4880f1 100644 --- a/storage/mroonga/vendor/groonga/plugins/sharding/sources.am +++ b/storage/mroonga/vendor/groonga/plugins/sharding/sources.am @@ -1,4 +1,5 @@ sharding_scripts = \ logical_count.rb \ logical_enumerator.rb \ - logical_range_filter.rb + logical_range_filter.rb \ + range_expression_builder.rb diff --git a/storage/mroonga/vendor/groonga/plugins/token_filters/stop_word.c b/storage/mroonga/vendor/groonga/plugins/token_filters/stop_word.c index 9b8cc68200a..054cd65aa37 100644 --- a/storage/mroonga/vendor/groonga/plugins/token_filters/stop_word.c +++ b/storage/mroonga/vendor/groonga/plugins/token_filters/stop_word.c @@ -38,6 +38,10 @@ stop_word_init(grn_ctx *ctx, grn_obj *table, grn_token_mode mode) { grn_stop_word_token_filter *token_filter; + if (mode != GRN_TOKEN_GET) { + return NULL; + } + token_filter = GRN_PLUGIN_MALLOC(ctx, sizeof(grn_stop_word_token_filter)); if (!token_filter) { GRN_PLUGIN_ERROR(ctx, GRN_NO_MEMORY_AVAILABLE, @@ -83,28 +87,29 @@ stop_word_filter(grn_ctx *ctx, void *user_data) { grn_stop_word_token_filter *token_filter = user_data; + grn_id id; + grn_obj *data; + + if (!token_filter) { + return; + } - if (token_filter->mode == GRN_TOKEN_GET) { - grn_id id; - grn_obj *data; - - data = grn_token_get_data(ctx, current_token); - id = grn_table_get(ctx, - token_filter->table, - GRN_TEXT_VALUE(data), - GRN_TEXT_LEN(data)); - if (id != GRN_ID_NIL) { - GRN_BULK_REWIND(&(token_filter->value)); - grn_obj_get_value(ctx, - token_filter->column, - id, - &(token_filter->value)); - if (GRN_BOOL_VALUE(&(token_filter->value))) { - grn_tokenizer_status status; - status = grn_token_get_status(ctx, current_token); - status |= GRN_TOKEN_SKIP; - grn_token_set_status(ctx, next_token, status); - } + data = grn_token_get_data(ctx, current_token); + id = grn_table_get(ctx, + token_filter->table, + GRN_TEXT_VALUE(data), + GRN_TEXT_LEN(data)); + if (id != GRN_ID_NIL) { + GRN_BULK_REWIND(&(token_filter->value)); + grn_obj_get_value(ctx, + token_filter->column, + id, + &(token_filter->value)); + if (GRN_BOOL_VALUE(&(token_filter->value))) { + grn_tokenizer_status status; + status = grn_token_get_status(ctx, current_token); + status |= GRN_TOKEN_SKIP; + grn_token_set_status(ctx, next_token, status); } } } diff --git a/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c b/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c index 49e4b5c877e..9207f94229e 100644 --- a/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c +++ b/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c @@ -30,6 +30,9 @@ static mecab_t *sole_mecab = NULL; static grn_plugin_mutex *sole_mecab_mutex = NULL; static grn_encoding sole_mecab_encoding = GRN_ENC_NONE; +static grn_bool grn_mecab_chunked_tokenize_enabled = GRN_FALSE; +static int grn_mecab_chunk_size_threshold = 8192; + typedef struct { mecab_t *mecab; grn_obj buf; @@ -83,6 +86,187 @@ get_mecab_encoding(mecab_t *mecab) return encoding; } +static inline grn_bool +is_delimiter_character(grn_ctx *ctx, const char *character, int character_bytes) +{ + switch (character_bytes) { + case 1 : + switch (character[0]) { + case ',' : + case '.' : + case '!' : + case '?' : + return GRN_TRUE; + default : + return GRN_FALSE; + } + case 3 : + switch ((unsigned char)(character[0])) { + case 0xE3 : + switch ((unsigned char)(character[1])) { + case 0x80 : + switch ((unsigned char)(character[2])) { + case 0x81 : /* U+3001 (0xE3 0x80 0x81 in UTF-8) IDEOGRAPHIC COMMA */ + case 0x82 : /* U+3002 (0xE3 0x80 0x82 in UTF-8) IDEOGRAPHIC FULL STOP */ + return GRN_TRUE; + default : + return GRN_FALSE; + } + default : + return GRN_FALSE; + } + return GRN_FALSE; + case 0xEF : + switch ((unsigned char)(character[1])) { + case 0xBC : + switch ((unsigned char)(character[2])) { + case 0x81 : + /* U+FF01 (0xEF 0xBC 0x81 in UTF-8) FULLWIDTH EXCLAMATION MARK */ + case 0x9F : + /* U+FF1F (0xEF 0xBC 0x9F in UTF-8) FULLWIDTH QUESTION MARK */ + return GRN_TRUE; + default : + return GRN_FALSE; + } + default : + return GRN_FALSE; + } + return GRN_FALSE; + default : + return GRN_FALSE; + } + default : + return GRN_FALSE; + } +} + +static grn_bool +chunked_tokenize_utf8_chunk(grn_ctx *ctx, + grn_mecab_tokenizer *tokenizer, + const char *chunk, + unsigned int chunk_bytes) +{ + const char *tokenized_chunk; + size_t tokenized_chunk_length; + + tokenized_chunk = mecab_sparse_tostr2(tokenizer->mecab, chunk, chunk_bytes); + if (!tokenized_chunk) { + GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR, + "[tokenizer][mecab][chunk] " + "mecab_sparse_tostr2() failed len=%d err=%s", + chunk_bytes, + mecab_strerror(tokenizer->mecab)); + return GRN_FALSE; + } + + if (GRN_TEXT_LEN(&(tokenizer->buf)) > 0) { + GRN_TEXT_PUTS(ctx, &(tokenizer->buf), " "); + } + + tokenized_chunk_length = strlen(tokenized_chunk); + if (tokenized_chunk_length >= 1 && + isspace(tokenized_chunk[tokenized_chunk_length - 1])) { + GRN_TEXT_PUT(ctx, &(tokenizer->buf), + tokenized_chunk, tokenized_chunk_length - 1); + } else { + GRN_TEXT_PUT(ctx, &(tokenizer->buf), + tokenized_chunk, tokenized_chunk_length); + } + + return GRN_TRUE; +} + +static grn_bool +chunked_tokenize_utf8(grn_ctx *ctx, + grn_mecab_tokenizer *tokenizer, + const char *string, + unsigned int string_bytes) +{ + const char *chunk_start; + const char *current; + const char *last_delimiter; + const char *string_end = string + string_bytes; + grn_encoding encoding = tokenizer->query->encoding; + + if (string_bytes < grn_mecab_chunk_size_threshold) { + return chunked_tokenize_utf8_chunk(ctx, + tokenizer, + string, + string_bytes); + } + + chunk_start = current = string; + last_delimiter = NULL; + while (current < string_end) { + int space_bytes; + int character_bytes; + const char *current_character; + + space_bytes = grn_isspace(current, encoding); + if (space_bytes > 0) { + if (chunk_start != current) { + grn_bool succeeded; + succeeded = chunked_tokenize_utf8_chunk(ctx, + tokenizer, + chunk_start, + current - chunk_start); + if (!succeeded) { + return succeeded; + } + } + current += space_bytes; + chunk_start = current; + last_delimiter = NULL; + continue; + } + + character_bytes = grn_charlen_(ctx, current, string_end, encoding); + if (character_bytes == 0) { + GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR, + "[tokenizer][mecab][chunk] " + "invalid byte sequence: position=%d", + (int)(current - string)); + return GRN_FALSE; + } + + current_character = current; + current += character_bytes; + if (is_delimiter_character(ctx, current_character, character_bytes)) { + last_delimiter = current; + } + + if ((current - chunk_start) >= grn_mecab_chunk_size_threshold) { + grn_bool succeeded; + if (last_delimiter) { + succeeded = chunked_tokenize_utf8_chunk(ctx, + tokenizer, + chunk_start, + last_delimiter - chunk_start); + chunk_start = last_delimiter; + } else { + succeeded = chunked_tokenize_utf8_chunk(ctx, + tokenizer, + chunk_start, + current - chunk_start); + chunk_start = current; + } + if (!succeeded) { + return succeeded; + } + last_delimiter = NULL; + } + } + + if (current == chunk_start) { + return GRN_TRUE; + } else { + return chunked_tokenize_utf8_chunk(ctx, + tokenizer, + chunk_start, + current - chunk_start); + } +} + /* This function is called for a full text search query or a document to be indexed. This means that both short/long strings are given. @@ -92,7 +276,6 @@ get_mecab_encoding(mecab_t *mecab) static grn_obj * mecab_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) { - const char *s; grn_mecab_tokenizer *tokenizer; unsigned int normalizer_flags = 0; grn_tokenizer_query *query; @@ -159,21 +342,33 @@ mecab_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) tokenizer->next = ""; tokenizer->end = tokenizer->next; } else { + grn_bool succeeded; grn_plugin_mutex_lock(ctx, sole_mecab_mutex); - s = mecab_sparse_tostr2(tokenizer->mecab, - normalized_string, - normalized_string_length); - if (!s) { - GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR, - "[tokenizer][mecab] " - "mecab_sparse_tostr() failed len=%d err=%s", - normalized_string_length, - mecab_strerror(tokenizer->mecab)); + if (grn_mecab_chunked_tokenize_enabled && + ctx->encoding == GRN_ENC_UTF8) { + succeeded = chunked_tokenize_utf8(ctx, + tokenizer, + normalized_string, + normalized_string_length); } else { - GRN_TEXT_PUTS(ctx, &(tokenizer->buf), s); + const char *s; + s = mecab_sparse_tostr2(tokenizer->mecab, + normalized_string, + normalized_string_length); + if (!s) { + succeeded = GRN_FALSE; + GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR, + "[tokenizer][mecab] " + "mecab_sparse_tostr() failed len=%d err=%s", + normalized_string_length, + mecab_strerror(tokenizer->mecab)); + } else { + succeeded = GRN_TRUE; + GRN_TEXT_PUTS(ctx, &(tokenizer->buf), s); + } } grn_plugin_mutex_unlock(ctx, sole_mecab_mutex); - if (!s) { + if (!succeeded) { grn_tokenizer_query_close(ctx, tokenizer->query); GRN_PLUGIN_FREE(ctx, tokenizer); return NULL; @@ -223,19 +418,31 @@ mecab_next(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data) grn_tokenizer_status status; for (r = p; r < e; r += cl) { + int space_len; + + space_len = grn_isspace(r, encoding); + if (space_len > 0 && r == p) { + cl = space_len; + p = r + cl; + continue; + } + if (!(cl = grn_charlen_(ctx, r, e, encoding))) { tokenizer->next = e; break; } - if (grn_isspace(r, encoding)) { - const char *q = r; - while ((cl = grn_isspace(q, encoding))) { q += cl; } + + if (space_len > 0) { + const char *q = r + space_len; + while (q < e && (space_len = grn_isspace(q, encoding))) { + q += space_len; + } tokenizer->next = q; break; } } - if (r == e) { + if (r == e || tokenizer->next == e) { status = GRN_TOKENIZER_LAST; } else { status = GRN_TOKENIZER_CONTINUE; @@ -272,10 +479,10 @@ check_mecab_dictionary_encoding(grn_ctx *ctx) mecab = mecab_new2("-Owakati"); if (mecab) { grn_encoding encoding; - int have_same_encoding_dictionary = 0; + grn_bool have_same_encoding_dictionary; encoding = GRN_CTX_GET_ENCODING(ctx); - have_same_encoding_dictionary = encoding == get_mecab_encoding(mecab); + have_same_encoding_dictionary = (encoding == get_mecab_encoding(mecab)); mecab_destroy(mecab); if (!have_same_encoding_dictionary) { @@ -301,6 +508,30 @@ check_mecab_dictionary_encoding(grn_ctx *ctx) grn_rc GRN_PLUGIN_INIT(grn_ctx *ctx) { + { + const char *env; + + env = getenv("GRN_MECAB_CHUNKED_TOKENIZE_ENABLED"); + grn_mecab_chunked_tokenize_enabled = (env && strcmp(env, "yes") == 0); + } + + { + const char *env; + + env = getenv("GRN_MECAB_CHUNK_SIZE_THRESHOLD"); + if (env) { + int threshold = -1; + const char *end; + const char *rest; + + end = env + strlen(env); + threshold = grn_atoi(env, end, &rest); + if (end > env && end == rest) { + grn_mecab_chunk_size_threshold = threshold; + } + } + } + sole_mecab = NULL; sole_mecab_mutex = grn_plugin_mutex_open(ctx); if (!sole_mecab_mutex) { diff --git a/storage/mroonga/vendor/groonga/src/grnslap.c b/storage/mroonga/vendor/groonga/src/grnslap.c index be2f181f3e7..2f5562fe6a9 100644 --- a/storage/mroonga/vendor/groonga/src/grnslap.c +++ b/storage/mroonga/vendor/groonga/src/grnslap.c @@ -54,7 +54,7 @@ lprint(grn_ctx *ctx, const char *fmt, ...) int len; va_list argp; grn_timeval_now(ctx, &tv); - grn_timeval2str(ctx, &tv, buf); + grn_timeval2str(ctx, &tv, buf, 1024); len = strlen(buf); buf[len++] = '|'; va_start(argp, fmt); diff --git a/storage/mroonga/vendor/groonga/src/groonga.c b/storage/mroonga/vendor/groonga/src/groonga.c index b03e70e0258..262c4de2e86 100644 --- a/storage/mroonga/vendor/groonga/src/groonga.c +++ b/storage/mroonga/vendor/groonga/src/groonga.c @@ -20,6 +20,7 @@ #include <stdio.h> #include <ctype.h> #include <fcntl.h> +#include <sys/stat.h> #ifdef WIN32 # define GROONGA_MAIN @@ -142,8 +143,8 @@ line_editor_init(int argc __attribute__((unused)), char *argv[]) setlocale(LC_ALL, ""); if (strlen(HOME_PATH) + strlen(HISTORY_PATH) < PATH_MAX) { - strcpy(line_editor_history_path, HOME_PATH); - strcat(line_editor_history_path, HISTORY_PATH); + grn_strcpy(line_editor_history_path, PATH_MAX, HOME_PATH); + grn_strcat(line_editor_history_path, PATH_MAX, HISTORY_PATH); } else { line_editor_history_path[0] = '\0'; } @@ -341,10 +342,14 @@ s_output(grn_ctx *ctx, int flags, void *arg) { FILE *stream = (FILE *)arg; - if (grn_ctx_get_output_type(ctx) == GRN_CONTENT_NONE) { + switch (grn_ctx_get_output_type(ctx)) { + case GRN_CONTENT_GROONGA_COMMAND_LIST : + case GRN_CONTENT_NONE : s_output_raw(ctx, flags, stream); - } else { + break; + default : s_output_typed(ctx, flags, stream); + break; } } @@ -501,12 +506,39 @@ send_ready_notify(void) close_ready_notify_pipe(); } +static void +create_pid_file(void) +{ +#ifndef WIN32 + FILE *pid_file = NULL; + pid_t pid; + + if (!pid_file_path) { + return; + } + + pid_file = fopen(pid_file_path, "w"); + pid = getpid(); + fprintf(pid_file, "%d\n", pid); + fclose(pid_file); +#endif +} + +static void +clean_pid_file(void) +{ +#ifndef WIN32 + if (pid_file_path) { + unlink(pid_file_path); + } +#endif +} + static int daemonize(void) { int exit_code = EXIT_SUCCESS; #ifndef WIN32 - pid_t pid; if (pipe(ready_notify_pipe) == -1) { reset_ready_notify_pipe(); @@ -533,19 +565,12 @@ daemonize(void) } switch (fork()) { case 0: - { - FILE *pid_file = NULL; - if (pid_file_path) { - pid_file = fopen(pid_file_path, "w"); - } + if (pid_file_path) { + create_pid_file(); + } else { + pid_t pid; pid = getpid(); - if (!pid_file) { - fprintf(stderr, "%d\n", pid); - } else { - fprintf(pid_file, "%d\n", pid); - fclose(pid_file); - pid_file = NULL; - } + fprintf(stderr, "%d\n", pid); } break; case -1: @@ -556,12 +581,13 @@ daemonize(void) _exit(EXIT_SUCCESS); } { - int null_fd = GRN_OPEN("/dev/null", O_RDWR, 0); + int null_fd; + grn_open(null_fd, "/dev/null", O_RDWR); if (null_fd != -1) { dup2(null_fd, STDIN_FILENO); dup2(null_fd, STDOUT_FILENO); dup2(null_fd, STDERR_FILENO); - if (null_fd > STDERR_FILENO) { GRN_CLOSE(null_fd); } + if (null_fd > STDERR_FILENO) { grn_close(null_fd); } } } #endif /* WIN32 */ @@ -569,16 +595,6 @@ daemonize(void) } static void -clean_pid_file(void) -{ -#ifndef WIN32 - if (pid_file_path) { - unlink(pid_file_path); - } -#endif -} - -static void run_server_loop(grn_ctx *ctx, grn_com_event *ev) { while (!grn_com_event_poll(ctx, ev, 1000) && grn_gctx.stat != GRN_CTX_QUIT) { @@ -666,6 +682,8 @@ start_service(grn_ctx *ctx, const char *db_path, if (exit_code != EXIT_SUCCESS) { return exit_code; } + } else { + create_pid_file(); } if (!grn_com_event_init(ctx, &ev, MAX_CON, sizeof(grn_com))) { @@ -686,9 +704,7 @@ start_service(grn_ctx *ctx, const char *db_path, send_ready_notify(); } - if (is_daemon_mode) { - clean_pid_file(); - } + clean_pid_file(); return exit_code; } @@ -936,10 +952,14 @@ h_output(grn_ctx *ctx, int flags, void *arg) { ht_context *hc = (ht_context *)arg; - if (grn_ctx_get_output_type(ctx) == GRN_CONTENT_NONE) { + switch (grn_ctx_get_output_type(ctx)) { + case GRN_CONTENT_GROONGA_COMMAND_LIST : + case GRN_CONTENT_NONE : h_output_raw(ctx, flags, hc); - } else { + break; + default : h_output_typed(ctx, flags, hc); + break; } } @@ -970,7 +990,7 @@ do_htreq_get(grn_ctx *ctx, grn_msg *msg) typedef struct { const char *path_start; int path_length; - int content_length; + long long int content_length; grn_bool have_100_continue; const char *body_start; } h_post_header; @@ -1089,7 +1109,7 @@ do_htreq_post_parse_header_values(grn_ctx *ctx, } if (STRING_EQUAL_CI(name, name_length, "Content-Length")) { const char *rest; - header->content_length = grn_atoi(value, value + value_length, &rest); + header->content_length = grn_atoll(value, value + value_length, &rest); if (rest != value + value_length) { /* Invalid Content-Length value. TODO: report error. */ header->content_length = -1; @@ -1188,13 +1208,12 @@ do_htreq_post(grn_ctx *ctx, grn_msg *msg) } { - grn_obj line_buffer; - int read_content_length = 0; + grn_obj chunk_buffer; + long long int read_content_length = 0; - GRN_TEXT_INIT(&line_buffer, 0); + GRN_TEXT_INIT(&chunk_buffer, 0); while (read_content_length < header.content_length) { #define POST_BUFFER_SIZE 8192 - grn_rc rc; char buffer[POST_BUFFER_SIZE]; const char *buffer_start, *buffer_current, *buffer_end; @@ -1218,42 +1237,56 @@ do_htreq_post(grn_ctx *ctx, grn_msg *msg) } read_content_length += buffer_end - buffer_start; - rc = GRN_SUCCESS; - buffer_current = buffer_start; - for (; rc == GRN_SUCCESS && buffer_current < buffer_end; buffer_current++) { - if (buffer_current[0] != '\n') { + buffer_current = buffer_end - 1; + for (; buffer_current > buffer_start; buffer_current--) { + grn_bool is_separator; + switch (buffer_current[0]) { + case '\n' : + case ',' : + is_separator = GRN_TRUE; + break; + default : + is_separator = GRN_FALSE; + break; + } + if (!is_separator) { continue; } + GRN_TEXT_PUT(ctx, - &line_buffer, + &chunk_buffer, buffer_start, - buffer_current - buffer_start); + buffer_current + 1 - buffer_start); { int flags = 0; if (!(read_content_length == header.content_length && buffer_current + 1 == buffer_end)) { flags |= GRN_CTX_QUIET; } - rc = grn_ctx_send(ctx, - GRN_TEXT_VALUE(&line_buffer), - GRN_TEXT_LEN(&line_buffer), - flags); + grn_ctx_send(ctx, + GRN_TEXT_VALUE(&chunk_buffer), + GRN_TEXT_LEN(&chunk_buffer), + flags); } buffer_start = buffer_current + 1; - GRN_BULK_REWIND(&line_buffer); + GRN_BULK_REWIND(&chunk_buffer); + break; + } + if (buffer_end > buffer_start) { + GRN_TEXT_PUT(ctx, &chunk_buffer, + buffer_start, buffer_end - buffer_start); } - GRN_TEXT_PUT(ctx, &line_buffer, buffer_start, buffer_end - buffer_start); #undef POST_BUFFER_SIZE } - if (GRN_TEXT_LEN(&line_buffer) > 0) { + if (GRN_TEXT_LEN(&chunk_buffer) > 0) { grn_ctx_send(ctx, - GRN_TEXT_VALUE(&line_buffer), - GRN_TEXT_LEN(&line_buffer), + GRN_TEXT_VALUE(&chunk_buffer), + GRN_TEXT_LEN(&chunk_buffer), 0); } - GRN_OBJ_FIN(ctx, &line_buffer); + GRN_OBJ_FIN(ctx, &chunk_buffer); } } @@ -2108,7 +2141,8 @@ enum { ACTION_ERROR }; -#define ACTION_MASK (0x0f) +#define ACTION_MASK (0x0f) +#define MODE_MASK (0xf0) #define FLAG_MODE_ALONE (1 << 4) #define FLAG_MODE_CLIENT (1 << 5) #define FLAG_MODE_DAEMON (1 << 6) @@ -2195,7 +2229,7 @@ config_file_register(const char *path, const grn_str_getopt_opt *opts, char *args[4]; name_buf[0] = name_buf[1] = '-'; - strcpy(name_buf + 2, name); + grn_strcpy(name_buf + 2, CONFIG_FILE_MAX_NAME_LENGTH + 1, name); if (value) { const size_t entry_size = sizeof(config_file_entry) + value_length + 1; @@ -2205,7 +2239,7 @@ config_file_register(const char *path, const grn_str_getopt_opt *opts, (unsigned int)entry_size); return CONFIG_FILE_MALLOC_ERROR; } - strcpy((char *)(entry + 1), value); + grn_strcpy((char *)(entry + 1), value_length + 1, value); entry->next = config_file_entry_head; if (!config_file_entry_head) { if (atexit(config_file_clear)) { @@ -2405,9 +2439,10 @@ init_default_settings(void) if (document_root_length >= PATH_MAX) { fprintf(stderr, "can't use default root: too long path\n"); } else { - strcpy(win32_default_document_root, grn_win32_base_dir()); - strcat(win32_default_document_root, "/"); - strcat(win32_default_document_root, GRN_DEFAULT_RELATIVE_DOCUMENT_ROOT); + grn_strcpy(win32_default_document_root, PATH_MAX, grn_win32_base_dir()); + grn_strcat(win32_default_document_root, PATH_MAX, "/"); + grn_strcat(win32_default_document_root, PATH_MAX, + GRN_DEFAULT_RELATIVE_DOCUMENT_ROOT); default_document_root = win32_default_document_root; } } @@ -2484,6 +2519,12 @@ show_version(void) #ifdef GRN_WITH_MESSAGE_PACK printf(",msgpack"); #endif +#ifdef GRN_WITH_MRUBY + printf(",mruby"); +#endif +#ifdef GRN_WITH_ONIGMO + printf(",onigmo"); +#endif #ifdef GRN_WITH_ZLIB printf(",zlib"); #endif @@ -2558,9 +2599,21 @@ show_usage(FILE *output) " specify log level (default: %d)\n" " --log-path <path>: specify log path\n" " (default: %s)\n" + " --log-rotate-threshold-size <threshold>:\n" + " specify threshold for log rotate\n" + " Log file is rotated when\n" + " log file size is larger than or\n" + " equals to the threshold\n" + " (default: 0; disabled)\n" " --query-log-path <path>:\n" " specify query log path\n" " (default: %s)\n" + " --query-log-rotate-threshold-size <threshold>:\n" + " specify threshold for query log rotate\n" + " Query log file is rotated when\n" + " query log file size is larger than or\n" + " equals to the threshold\n" + " (default: 0; disabled)\n" "\n" "Common options:\n" " --working-directory <path>:\n" @@ -2596,20 +2649,30 @@ show_usage(FILE *output) int main(int argc, char **argv) { - const char *port_arg = NULL, *encoding_arg = NULL, - *max_num_threads_arg = NULL, *log_level_arg = NULL, - *bind_address_arg = NULL, *hostname_arg = NULL, *protocol_arg = NULL, - *log_path_arg = NULL, *query_log_path_arg = NULL, - *cache_limit_arg = NULL, *document_root_arg = NULL, - *default_command_version_arg = NULL, - *default_match_escalation_threshold_arg = NULL, - *input_fd_arg = NULL, *output_fd_arg = NULL, - *working_directory_arg = NULL; + const char *port_arg = NULL; + const char *encoding_arg = NULL; + const char *max_num_threads_arg = NULL; + const char *log_level_arg = NULL; + const char *bind_address_arg = NULL; + const char *hostname_arg = NULL; + const char *protocol_arg = NULL; + const char *log_path_arg = NULL; + const char *log_rotate_threshold_size_arg = NULL; + const char *query_log_path_arg = NULL; + const char *query_log_rotate_threshold_size_arg = NULL; + const char *cache_limit_arg = NULL; + const char *document_root_arg = NULL; + const char *default_command_version_arg = NULL; + const char *default_match_escalation_threshold_arg = NULL; + const char *input_fd_arg = NULL; + const char *output_fd_arg = NULL; + const char *working_directory_arg = NULL; const char *config_path = NULL; int exit_code = EXIT_SUCCESS; int i; int flags = 0; uint32_t cache_limit = 0; + grn_bool need_line_editor = GRN_FALSE; static grn_str_getopt_opt opts[] = { {'p', "port", NULL, 0, GETOPT_OP_NONE}, {'e', "encoding", NULL, 0, GETOPT_OP_NONE}, @@ -2624,7 +2687,9 @@ main(int argc, char **argv) {'\0', "protocol", NULL, 0, GETOPT_OP_NONE}, {'\0', "version", NULL, ACTION_VERSION, GETOPT_OP_UPDATE}, {'\0', "log-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "log-rotate-threshold-size", NULL, 0, GETOPT_OP_NONE}, {'\0', "query-log-path", NULL, 0, GETOPT_OP_NONE}, + {'\0', "query-log-rotate-threshold-size", NULL, 0, GETOPT_OP_NONE}, {'\0', "pid-path", NULL, 0, GETOPT_OP_NONE}, {'\0', "config-path", NULL, 0, GETOPT_OP_NONE}, {'\0', "show-config", NULL, ACTION_SHOW_CONFIG, GETOPT_OP_UPDATE}, @@ -2646,18 +2711,20 @@ main(int argc, char **argv) opts[8].arg = &hostname_arg; opts[10].arg = &protocol_arg; opts[12].arg = &log_path_arg; - opts[13].arg = &query_log_path_arg; - opts[14].arg = &pid_file_path; - opts[15].arg = &config_path; - opts[17].arg = &cache_limit_arg; - opts[18].arg = &input_path; - opts[19].arg = &document_root_arg; - opts[20].arg = &default_command_version_arg; - opts[21].arg = &default_match_escalation_threshold_arg; - opts[22].arg = &bind_address_arg; - opts[23].arg = &input_fd_arg; - opts[24].arg = &output_fd_arg; - opts[25].arg = &working_directory_arg; + opts[13].arg = &log_rotate_threshold_size_arg; + opts[14].arg = &query_log_path_arg; + opts[15].arg = &query_log_rotate_threshold_size_arg; + opts[16].arg = &pid_file_path; + opts[17].arg = &config_path; + opts[19].arg = &cache_limit_arg; + opts[20].arg = &input_path; + opts[21].arg = &document_root_arg; + opts[22].arg = &default_command_version_arg; + opts[23].arg = &default_match_escalation_threshold_arg; + opts[24].arg = &bind_address_arg; + opts[25].arg = &input_fd_arg; + opts[26].arg = &output_fd_arg; + opts[27].arg = &working_directory_arg; reset_ready_notify_pipe(); @@ -2721,6 +2788,10 @@ main(int argc, char **argv) return EXIT_FAILURE; } + if ((flags & MODE_MASK) == 0) { + flags |= FLAG_MODE_ALONE; + } + if (port_arg) { const char * const end = port_arg + strlen(port_arg); const char *rest = NULL; @@ -2807,10 +2878,39 @@ main(int argc, char **argv) grn_default_logger_set_path(log_path_arg); } + if (log_rotate_threshold_size_arg) { + const char * const end = + log_rotate_threshold_size_arg + + strlen(log_rotate_threshold_size_arg); + const char *rest = NULL; + const uint64_t value = grn_atoull(log_rotate_threshold_size_arg, end, &rest); + if (end != rest) { + fprintf(stderr, "invalid log rotate threshold size: <%s>\n", + log_rotate_threshold_size_arg); + return EXIT_FAILURE; + } + grn_default_logger_set_rotate_threshold_size(value); + } + if (query_log_path_arg) { grn_default_query_logger_set_path(query_log_path_arg); } + if (query_log_rotate_threshold_size_arg) { + const char * const end = + query_log_rotate_threshold_size_arg + + strlen(query_log_rotate_threshold_size_arg); + const char *rest = NULL; + const uint64_t value = + grn_atoull(query_log_rotate_threshold_size_arg, end, &rest); + if (end != rest) { + fprintf(stderr, "invalid query log rotate threshold size: <%s>\n", + query_log_rotate_threshold_size_arg); + return EXIT_FAILURE; + } + grn_default_query_logger_set_rotate_threshold_size(value); + } + if (log_level_arg) { const char * const end = log_level_arg + strlen(log_level_arg); const char *rest = NULL; @@ -2870,6 +2970,11 @@ main(int argc, char **argv) } } + if ((flags & (FLAG_MODE_ALONE | FLAG_MODE_CLIENT)) && + !batchmode) { + need_line_editor = GRN_TRUE; + } + if (output_fd_arg) { const char * const end = output_fd_arg + strlen(output_fd_arg); const char *rest = NULL; @@ -2895,9 +3000,9 @@ main(int argc, char **argv) bind_address_arg, (unsigned int)bind_address_length, HOST_NAME_MAX); return EXIT_FAILURE; } - strcpy(bind_address, bind_address_arg); + grn_strcpy(bind_address, HOST_NAME_MAX + 1, bind_address_arg); } else { - strcpy(bind_address, default_bind_address); + grn_strcpy(bind_address, HOST_NAME_MAX + 1, default_bind_address); } if (hostname_arg) { @@ -2908,9 +3013,9 @@ main(int argc, char **argv) hostname_arg, (unsigned int)hostname_length, HOST_NAME_MAX); return EXIT_FAILURE; } - strcpy(hostname, hostname_arg); + grn_strcpy(hostname, HOST_NAME_MAX + 1, hostname_arg); } else { - strcpy(hostname, default_hostname); + grn_strcpy(hostname, HOST_NAME_MAX + 1, default_hostname); } if (document_root_arg) { @@ -2971,7 +3076,7 @@ main(int argc, char **argv) } #ifdef GRN_WITH_LIBEDIT - if (!batchmode) { + if (need_line_editor) { line_editor_init(argc, argv); } #endif @@ -3008,7 +3113,7 @@ main(int argc, char **argv) } #ifdef GRN_WITH_LIBEDIT - if (!batchmode) { + if (need_line_editor) { line_editor_fin(); } #endif diff --git a/storage/mroonga/vendor/groonga/src/groonga_benchmark.c b/storage/mroonga/vendor/groonga/src/groonga_benchmark.c index e7e426e748f..656984a7695 100644 --- a/storage/mroonga/vendor/groonga/src/groonga_benchmark.c +++ b/storage/mroonga/vendor/groonga/src/groonga_benchmark.c @@ -1115,7 +1115,7 @@ exit: job->max, job->qnum); if (grntest_jobdone < grntest_jobnum) { - strcat(tmpbuf, ","); + grn_strcat(tmpbuf, BUF_LEN, ","); } } GRN_TEXT_PUTS(ctx, log, tmpbuf); @@ -1261,19 +1261,19 @@ get_sysinfo(const char *path, char *result, int olen) if (grntest_outtype == OUT_TSV) { result[0] = '\0'; sprintf(tmpbuf, "script\t%s\n", grntest_scriptname); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, "user\t%s\n", grntest_username); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, "date\t%s\n", grntest_date); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); } else { - strcpy(result, "{"); + grn_strcpy(result, olen, "{"); sprintf(tmpbuf, "\"script\": \"%s.scr\",\n", grntest_scriptname); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, " \"user\": \"%s\",\n", grntest_username); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, " \"date\": \"%s\",\n", grntest_date); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); } memset(cpustring, 0, 64); @@ -1294,7 +1294,7 @@ get_sysinfo(const char *path, char *result, int olen) } else { sprintf(tmpbuf, " \"CPU\": \"%s\",\n", cpustring); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (sizeof(int *) == 8) { grntest_osinfo = OS_WINDOWS64; @@ -1311,7 +1311,7 @@ get_sysinfo(const char *path, char *result, int olen) sprintf(tmpbuf, " \"BIT\": 32,\n"); } } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); GetSystemInfo(&sinfo); if (grntest_outtype == OUT_TSV) { @@ -1319,7 +1319,7 @@ get_sysinfo(const char *path, char *result, int olen) } else { sprintf(tmpbuf, " \"CORE\": %lu,\n", sinfo.dwNumberOfProcessors); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); minfo.dwLength = sizeof(MEMORYSTATUSEX); GlobalMemoryStatusEx(&minfo); @@ -1328,7 +1328,7 @@ get_sysinfo(const char *path, char *result, int olen) } else { sprintf(tmpbuf, " \"RAM\": \"%I64dMByte\",\n", minfo.ullTotalPhys/(1024*1024)); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); GetDiskFreeSpaceEx(NULL, NULL, &dinfo, NULL); if (grntest_outtype == OUT_TSV) { @@ -1336,7 +1336,7 @@ get_sysinfo(const char *path, char *result, int olen) } else { sprintf(tmpbuf, " \"HDD\": \"%I64dKBytes\",\n", dinfo.QuadPart/1024 ); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); osinfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); GetVersionEx(&osinfo); if (grntest_outtype == OUT_TSV) { @@ -1346,21 +1346,21 @@ get_sysinfo(const char *path, char *result, int olen) sprintf(tmpbuf, " \"OS\": \"Windows %lu.%lu\",\n", osinfo.dwMajorVersion, osinfo.dwMinorVersion); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (grntest_outtype == OUT_TSV) { sprintf(tmpbuf, "%s\n", grntest_serverhost); } else { sprintf(tmpbuf, " \"HOST\": \"%s\",\n", grntest_serverhost); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (grntest_outtype == OUT_TSV) { sprintf(tmpbuf, "%d\n", grntest_serverport); } else { sprintf(tmpbuf, " \"PORT\": \"%d\",\n", grntest_serverport); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (grntest_outtype == OUT_TSV) { sprintf(tmpbuf, "%s\"\n", grn_get_version()); @@ -1368,9 +1368,9 @@ get_sysinfo(const char *path, char *result, int olen) sprintf(tmpbuf, " \"VERSION\": \"%s\"\n", grn_get_version()); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (grntest_outtype != OUT_TSV) { - strcat(result, "}"); + grn_strcat(result, olen, "}"); } #else /* linux only */ @@ -1380,26 +1380,27 @@ get_sysinfo(const char *path, char *result, int olen) int minfo = 0; int unevictable = 0; int mlocked = 0; - char cpustring[256]; +#define CPU_STRING_SIZE 256 + char cpu_string[CPU_STRING_SIZE]; struct utsname ubuf; struct statvfs vfsbuf; if (grntest_outtype == OUT_TSV) { result[0] = '\0'; sprintf(tmpbuf, "sctipt\t%s\n", grntest_scriptname); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, "user\t%s\n", grntest_username); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, "date\t%s\n", grntest_date); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); } else { - strcpy(result, "{"); + grn_strcpy(result, olen, "{"); sprintf(tmpbuf, "\"script\": \"%s.scr\",\n", grntest_scriptname); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, " \"user\": \"%s\",\n", grntest_username); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, " \"date\": \"%s\",\n", grntest_date); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); } fp = fopen("/proc/cpuinfo", "r"); @@ -1410,18 +1411,20 @@ get_sysinfo(const char *path, char *result, int olen) while (fgets(tmpbuf, 256, fp) != NULL) { tmpbuf[strlen(tmpbuf)-1] = '\0'; if (!strncmp(tmpbuf, "model name\t: ", 13)) { - strcpy(cpustring, &tmpbuf[13]); + grn_strcpy(cpu_string, CPU_STRING_SIZE, &tmpbuf[13]); } } fclose(fp); +#undef CPU_STRING_SIZE + cpunum = sysconf(_SC_NPROCESSORS_CONF); if (grntest_outtype == OUT_TSV) { - sprintf(tmpbuf, "%s\n", cpustring); + sprintf(tmpbuf, "%s\n", cpu_string); } else { - sprintf(tmpbuf, " \"CPU\": \"%s\",\n", cpustring); + sprintf(tmpbuf, " \"CPU\": \"%s\",\n", cpu_string); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (sizeof(int *) == 8) { grntest_osinfo = OS_LINUX64; @@ -1438,14 +1441,14 @@ get_sysinfo(const char *path, char *result, int olen) sprintf(tmpbuf, " \"BIT\": 32,\n"); } } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (grntest_outtype == OUT_TSV) { sprintf(tmpbuf, "CORE\t%d\n", cpunum); } else { sprintf(tmpbuf, " \"CORE\": %d,\n", cpunum); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); fp = fopen("/proc/meminfo", "r"); if (!fp) { @@ -1467,18 +1470,18 @@ get_sysinfo(const char *path, char *result, int olen) fclose(fp); if (grntest_outtype == OUT_TSV) { sprintf(tmpbuf, "%dMBytes\n", minfo/1024); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, "%dMBytes_Unevictable\n", unevictable/1024); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, "%dMBytes_Mlocked\n", mlocked/1024); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); } else { sprintf(tmpbuf, " \"RAM\": \"%dMBytes\",\n", minfo/1024); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, " \"Unevictable\": \"%dMBytes\",\n", unevictable/1024); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); sprintf(tmpbuf, " \"Mlocked\": \"%dMBytes\",\n", mlocked/1024); - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); } ret = statvfs(path, &vfsbuf); @@ -1494,7 +1497,7 @@ get_sysinfo(const char *path, char *result, int olen) " \"HDD\": \"%" GRN_FMT_INT64U "KBytes\",\n", vfsbuf.f_blocks * 4); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); uname(&ubuf); if (grntest_outtype == OUT_TSV) { @@ -1502,31 +1505,31 @@ get_sysinfo(const char *path, char *result, int olen) } else { sprintf(tmpbuf, " \"OS\": \"%s %s\",\n", ubuf.sysname, ubuf.release); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (grntest_outtype == OUT_TSV) { sprintf(tmpbuf, "%s\n", grntest_serverhost); } else { sprintf(tmpbuf, " \"HOST\": \"%s\",\n", grntest_serverhost); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (grntest_outtype == OUT_TSV) { sprintf(tmpbuf, "%d\n", grntest_serverport); } else { sprintf(tmpbuf, " \"PORT\": \"%d\",\n", grntest_serverport); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (grntest_outtype == OUT_TSV) { sprintf(tmpbuf, "%s\n", grn_get_version()); } else { sprintf(tmpbuf, " \"VERSION\": \"%s\"\n", grn_get_version()); } - strcat(result, tmpbuf); + grn_strcat(result, olen, tmpbuf); if (grntest_outtype != OUT_TSV) { - strcat(result, "},"); + grn_strcat(result, olen, "},"); } #endif /* WIN32 */ if (strlen(result) >= olen) { @@ -1551,13 +1554,13 @@ start_server(const char *dbpath, int r) exit(1); } - strcpy(tmpbuf, groonga_path); - strcat(tmpbuf, " -s --protocol "); - strcat(tmpbuf, groonga_protocol); - strcat(tmpbuf, " -p "); + grn_strcpy(tmpbuf, BUF_LEN, groonga_path); + grn_strcat(tmpbuf, BUF_LEN, " -s --protocol "); + grn_strcat(tmpbuf, BUF_LEN, groonga_protocol); + grn_strcat(tmpbuf, BUF_LEN, " -p "); sprintf(optbuf, "%d ", grntest_serverport); - strcat(tmpbuf, optbuf); - strcat(tmpbuf, dbpath); + grn_strcat(tmpbuf, BUF_LEN, optbuf); + grn_strcat(tmpbuf, BUF_LEN, dbpath); memset(&si, 0, sizeof(STARTUPINFO)); si.cb=sizeof(STARTUPINFO); ret = CreateProcess(NULL, tmpbuf, NULL, NULL, FALSE, @@ -1767,7 +1770,7 @@ parse_line(char *buf, int start, int end, int num) return 15; } } - strcpy(grntest_job[num].logfile, tmpbuf); + grn_strcpy(grntest_job[num].logfile, BUF_LEN, tmpbuf); return 0; } else { grntest_job[num].concurrency = grntest_atoi(tmpbuf, tmpbuf + j, NULL); @@ -2427,10 +2430,10 @@ ftp_sub(const char *user, const char *passwd, const char *host, #ifdef WIN32 _splitpath(filename, NULL, NULL, fname, ext); - strcpy(base, fname); + grn_strcpy(base, BUF_LEN, fname); strcat(base, ext); #else - strcpy(buf, filename); + grn_strcpy(buf, BUF_LEN, filename); base = basename(buf); #endif /* WIN32 */ @@ -2471,7 +2474,7 @@ ftp_sub(const char *user, const char *passwd, const char *host, } if (!strncmp(buf, "213", 3)) { retval[BUF_LEN-2] = '\0'; - strcpy(retval, get_ftp_date(buf)); + grn_strcpy(retval, BUF_LEN - 2, get_ftp_date(buf)); if (retval[BUF_LEN-2] != '\0' ) { fprintf(stderr, "buffer over run in ftp\n"); exit(1); @@ -2534,7 +2537,7 @@ static int get_username(char *name, int maxlen) { char *env=NULL; - strcpy(name, "nobody"); + grn_strcpy(name, maxlen, "nobody"); #ifdef WIN32 env = getenv("USERNAME"); #else @@ -2545,7 +2548,7 @@ get_username(char *name, int maxlen) exit(1); } if (env) { - strcpy(name, env); + grn_strcpy(name, maxlen, env); } return 0; } @@ -2577,7 +2580,7 @@ get_date(char *date, time_t *sec) } static int -get_scriptname(const char *path, char *name, const char *suffix) +get_scriptname(const char *path, char *name, size_t name_len, const char *suffix) { int slen = strlen(suffix); int len = strlen(path); @@ -2591,7 +2594,7 @@ get_scriptname(const char *path, char *name, const char *suffix) exit(1); } - strcpy(name, path); + grn_strcpy(name, name_len, path); if (strncmp(&name[len-slen], suffix, slen)) { name[0] = '\0'; return 0; @@ -2874,7 +2877,7 @@ check_script(grn_ctx *ctx, const char *script_file_path) while (grn_text_fgets(ctx, &line, script_file) == GRN_SUCCESS) { GRN_TEXT_VALUE(&line)[GRN_TEXT_LEN(&line) - 1] = '\0'; get_token(GRN_TEXT_VALUE(&line), token, BUF_LEN, &next); - strcpy(prev, token); + grn_strcpy(prev, BUF_LEN, token); while (next) { get_token(next, token, BUF_LEN, &next); @@ -2882,10 +2885,10 @@ check_script(grn_ctx *ctx, const char *script_file_path) grntest_serverport = grn_atoi(token, token + strlen(token), NULL); } if (!strncmp(prev, "SET_HOST", 8)) { - strcpy(grntest_serverhost, token); + grn_strcpy(grntest_serverhost, BUF_LEN, token); grntest_remote_mode = 1; } - strcpy(prev, token); + grn_strcpy(prev, BUF_LEN, token); } } grn_obj_unlink(ctx, &line); @@ -3045,10 +3048,10 @@ main(int argc, char **argv) usage(); } - strcpy(grntest_serverhost, DEFAULT_DEST); + grn_strcpy(grntest_serverhost, BUF_LEN, DEFAULT_DEST); if (hoststr) { grntest_remote_mode = 1; - strcpy(grntest_serverhost, hoststr); + grn_strcpy(grntest_serverhost, BUF_LEN, hoststr); } grntest_serverport = DEFAULT_PORT; if (portstr) { @@ -3086,7 +3089,7 @@ main(int argc, char **argv) } } - get_scriptname(scrname, grntest_scriptname, ".scr"); + get_scriptname(scrname, grntest_scriptname, BUF_LEN, ".scr"); get_username(grntest_username, 256); GRN_TIME_INIT(&grntest_starttime, 0); diff --git a/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c b/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c index 115de1338a4..6ba1df4a9e8 100644 --- a/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c +++ b/storage/mroonga/vendor/groonga/src/httpd/nginx-module/ngx_http_groonga_module.c @@ -529,8 +529,8 @@ ngx_http_groonga_context_receive_handler_typed(grn_ctx *context, ngx_pid = getppid(); } - ngx_rc = ngx_os_signal_process((ngx_cycle_t*)ngx_cycle, - "stop", + ngx_rc = ngx_os_signal_process((ngx_cycle_t *)ngx_cycle, + "quit", ngx_pid); if (ngx_rc == NGX_OK) { context->stat &= ~GRN_CTX_QUIT; @@ -568,10 +568,14 @@ ngx_http_groonga_context_receive_handler(grn_ctx *context, { ngx_http_groonga_handler_data_t *data = callback_data; - if (grn_ctx_get_output_type(context) == GRN_CONTENT_NONE) { + switch (grn_ctx_get_output_type(context)) { + case GRN_CONTENT_GROONGA_COMMAND_LIST : + case GRN_CONTENT_NONE : ngx_http_groonga_context_receive_handler_raw(context, flags, data); - } else { + break; + default : ngx_http_groonga_context_receive_handler_typed(context, flags, data); + break; } } diff --git a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c index 98ae0958908..f3127288c73 100644 --- a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c +++ b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_httpd.c @@ -1,5 +1,5 @@ /* -*- c-basic-offset: 2 -*- */ -/* Copyright(C) 2010-2013 Brazil +/* Copyright(C) 2010-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -17,6 +17,7 @@ /* groonga origin headers */ #include <grn_str.h> +#include <grn_msgpack.h> #include <stdio.h> #include <signal.h> @@ -38,7 +39,6 @@ #include "zmq_compatible.h" #include <event.h> #include <evhttp.h> -#include <msgpack.h> #include <groonga.h> #include <pthread.h> @@ -177,35 +177,35 @@ log_send(struct evkeyvalq *output_headers, struct evbuffer *res_buf, msgpack_pack_map(&pk, cnt); c = 'i'; - msgpack_pack_raw(&pk, 1); - msgpack_pack_raw_body(&pk, &c, 1); + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); l = strlen(client_id); - msgpack_pack_raw(&pk, l); - msgpack_pack_raw_body(&pk, client_id, l); + msgpack_pack_str(&pk, l); + msgpack_pack_str_body(&pk, client_id, l); c = 'q'; - msgpack_pack_raw(&pk, 1); - msgpack_pack_raw_body(&pk, &c, 1); + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); l = strlen(query); - msgpack_pack_raw(&pk, l); - msgpack_pack_raw_body(&pk, query, l); + msgpack_pack_str(&pk, l); + msgpack_pack_str_body(&pk, query, l); c = 's'; - msgpack_pack_raw(&pk, 1); - msgpack_pack_raw_body(&pk, &c, 1); + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); msgpack_pack_uint64(&pk, millisec); c = 'l'; - msgpack_pack_raw(&pk, 1); - msgpack_pack_raw_body(&pk, &c, 1); + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); l = strlen(learn_target_name); - msgpack_pack_raw(&pk, l); - msgpack_pack_raw_body(&pk, learn_target_name, l); + msgpack_pack_str(&pk, l); + msgpack_pack_str_body(&pk, learn_target_name, l); if (submit_flag) { c = 't'; - msgpack_pack_raw(&pk, 1); - msgpack_pack_raw_body(&pk, &c, 1); + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, &c, 1); msgpack_pack_true(&pk); } { @@ -249,9 +249,11 @@ log_send(struct evkeyvalq *output_headers, struct evbuffer *res_buf, &(thd->pass_through_parameters)); } if (content_length >= 0) { - char num_buf[16]; - snprintf(num_buf, 16, "%d", content_length); +#define NUM_BUF_SIZE 16 + char num_buf[NUM_BUF_SIZE]; + grn_snprintf(num_buf, NUM_BUF_SIZE, NUM_BUF_SIZE, "%d", content_length); evhttp_add_header(output_headers, "Content-Length", num_buf); +#undef NUM_BUF_SIZE } } } @@ -321,10 +323,18 @@ generic_handler(struct evhttp_request *req, void *arg) time(&n); t_st = localtime(&n); - snprintf(p, PATH_MAX, "%s%04d%02d%02d%02d%02d%02d-%02d", - thd->log_base_path, - t_st->tm_year + 1900, t_st->tm_mon + 1, t_st->tm_mday, - t_st->tm_hour, t_st->tm_min, t_st->tm_sec, thd->thread_id); + grn_snprintf(p, + PATH_MAX, + PATH_MAX, + "%s%04d%02d%02d%02d%02d%02d-%02d", + thd->log_base_path, + t_st->tm_year + 1900, + t_st->tm_mon + 1, + t_st->tm_mday, + t_st->tm_hour, + t_st->tm_min, + t_st->tm_sec, + thd->thread_id); if (!(thd->log_file = fopen(p, "a"))) { print_error("cannot open log_file %s.", p); @@ -420,8 +430,10 @@ msgpack2json(msgpack_object *o, grn_ctx *ctx, grn_obj *buf) case MSGPACK_OBJECT_POSITIVE_INTEGER: grn_text_ulltoa(ctx, buf, o->via.u64); break; - case MSGPACK_OBJECT_RAW: - grn_text_esc(ctx, buf, o->via.raw.ptr, o->via.raw.size); + case MSGPACK_OBJECT_STR: + grn_text_esc(ctx, buf, + MSGPACK_OBJECT_STR_PTR(o), + MSGPACK_OBJECT_STR_SIZE(o)); break; case MSGPACK_OBJECT_ARRAY: GRN_TEXT_PUTC(ctx, buf, '['); @@ -433,8 +445,8 @@ msgpack2json(msgpack_object *o, grn_ctx *ctx, grn_obj *buf) } GRN_TEXT_PUTC(ctx, buf, ']'); break; - case MSGPACK_OBJECT_DOUBLE: - grn_text_ftoa(ctx, buf, o->via.dec); + case MSGPACK_OBJECT_FLOAT: + grn_text_ftoa(ctx, buf, MSGPACK_OBJECT_FLOAT_VALUE(o)); break; default: print_error("cannot handle this msgpack type."); @@ -446,19 +458,25 @@ load_from_learner(msgpack_object *o, grn_ctx *ctx, grn_obj *cmd_buf) { if (o->type == MSGPACK_OBJECT_MAP && o->via.map.size) { msgpack_object_kv *kv; + msgpack_object *key; + msgpack_object *value; kv = &(o->via.map.ptr[0]); - if (kv->key.type == MSGPACK_OBJECT_RAW && kv->key.via.raw.size == 6 && - !memcmp(kv->key.via.raw.ptr, CONST_STR_LEN("target"))) { - if (kv->val.type == MSGPACK_OBJECT_RAW) { + key = &(kv->key); + value = &(kv->val); + if (key->type == MSGPACK_OBJECT_STR && MSGPACK_OBJECT_STR_SIZE(key) == 6 && + !memcmp(MSGPACK_OBJECT_STR_PTR(key), CONST_STR_LEN("target"))) { + if (value->type == MSGPACK_OBJECT_STR) { int i; GRN_BULK_REWIND(cmd_buf); GRN_TEXT_PUTS(ctx, cmd_buf, "load --table "); - GRN_TEXT_PUT(ctx, cmd_buf, kv->val.via.raw.ptr, kv->val.via.raw.size); + GRN_TEXT_PUT(ctx, cmd_buf, + MSGPACK_OBJECT_STR_PTR(value), + MSGPACK_OBJECT_STR_SIZE(value)); grn_ctx_send(ctx, GRN_TEXT_VALUE(cmd_buf), GRN_TEXT_LEN(cmd_buf), GRN_CTX_MORE); grn_ctx_send(ctx, CONST_STR_LEN("["), GRN_CTX_MORE); - if (kv->val.via.raw.size > 5) { - if (!memcmp(kv->val.via.raw.ptr, CONST_STR_LEN("item_")) || - !memcmp(kv->val.via.raw.ptr, CONST_STR_LEN("pair_"))) { + if (MSGPACK_OBJECT_STR_SIZE(value) > 5) { + if (!memcmp(MSGPACK_OBJECT_STR_PTR(value), CONST_STR_LEN("item_")) || + !memcmp(MSGPACK_OBJECT_STR_PTR(value), CONST_STR_LEN("pair_"))) { char delim = '{'; GRN_BULK_REWIND(cmd_buf); for (i = 1; i < o->via.map.size; i++) { diff --git a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c index 18b91d438e2..03d889f5b01 100644 --- a/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c +++ b/storage/mroonga/vendor/groonga/src/suggest/groonga_suggest_learner.c @@ -1,5 +1,5 @@ /* -*- c-basic-offset: 2 -*- */ -/* Copyright(C) 2010-2014 Brazil +/* Copyright(C) 2010-2015 Brazil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -17,12 +17,12 @@ /* for grn_str_getopt() */ #include <grn_str.h> +#include <grn_msgpack.h> #include "zmq_compatible.h" #include <stdio.h> #include <signal.h> #include <unistd.h> -#include <msgpack.h> #include <pthread.h> #include <groonga.h> #include <inttypes.h> @@ -142,14 +142,14 @@ load_to_multi_targets(grn_ctx *ctx, int _k_len; \ char _k_buf[GRN_TABLE_MAX_KEY_SIZE]; \ _k_len = grn_table_get_key(ctx, ref_table, (id), _k_buf, GRN_TABLE_MAX_KEY_SIZE); \ - msgpack_pack_raw(&pk, _k_len); \ - msgpack_pack_raw_body(&pk, _k_buf, _k_len); \ + msgpack_pack_str(&pk, _k_len); \ + msgpack_pack_str_body(&pk, _k_buf, _k_len); \ } while (0) #define PACK_MAP_ITEM(col_name) do { \ grn_obj _v; \ - msgpack_pack_raw(&pk, sizeof(#col_name) - 1); \ - msgpack_pack_raw_body(&pk, CONST_STR_LEN(#col_name)); \ + msgpack_pack_str(&pk, sizeof(#col_name) - 1); \ + msgpack_pack_str_body(&pk, #col_name, sizeof(#col_name) - 1); \ switch (col_##col_name->header.type) { \ case GRN_COLUMN_FIX_SIZE: \ GRN_VALUE_FIX_SIZE_INIT(&_v, 0, grn_obj_get_range(ctx, col_##col_name)); \ @@ -168,8 +168,8 @@ load_to_multi_targets(grn_ctx *ctx, case GRN_BULK: \ switch (_v.header.domain) { \ case GRN_DB_SHORT_TEXT: \ - msgpack_pack_raw(&pk, GRN_TEXT_LEN(&_v)); \ - msgpack_pack_raw_body(&pk, GRN_TEXT_VALUE(&_v), GRN_TEXT_LEN(&_v)); \ + msgpack_pack_str(&pk, GRN_TEXT_LEN(&_v)); \ + msgpack_pack_str_body(&pk, GRN_TEXT_VALUE(&_v), GRN_TEXT_LEN(&_v)); \ break; \ case GRN_DB_INT32: \ msgpack_pack_int32(&pk, GRN_INT32_VALUE(&_v)); \ @@ -268,18 +268,18 @@ send_handler(void *zmq_send_sock, grn_ctx *ctx) msgpack_pack_map(&pk, 8); /* ["_key","ShortText"],["last","Time"],["kana","kana"],["freq2","Int32"],["freq","Int32"],["co","pair_all"],["buzz","Int32"],["boost","Int32"] */ - msgpack_pack_raw(&pk, 6); - msgpack_pack_raw_body(&pk, CONST_STR_LEN("target")); - msgpack_pack_raw(&pk, name_len); - msgpack_pack_raw_body(&pk, name_buf, name_len); + msgpack_pack_str(&pk, 6); + msgpack_pack_str_body(&pk, "target", strlen("target")); + msgpack_pack_str(&pk, name_len); + msgpack_pack_str_body(&pk, name_buf, name_len); - msgpack_pack_raw(&pk, 4); - msgpack_pack_raw_body(&pk, + msgpack_pack_str(&pk, 4); + msgpack_pack_str_body(&pk, GRN_COLUMN_NAME_KEY, GRN_COLUMN_NAME_KEY_LEN); key_len = grn_table_cursor_get_key(ctx, tc, (void **)&key); - msgpack_pack_raw(&pk, key_len); - msgpack_pack_raw_body(&pk, key, key_len); + msgpack_pack_str(&pk, key_len); + msgpack_pack_str_body(&pk, key, key_len); PACK_MAP_ITEM(last); PACK_MAP_ITEM(kana); @@ -339,13 +339,13 @@ send_handler(void *zmq_send_sock, grn_ctx *ctx) msgpack_pack_map(&pk, 7); /* ["_key","UInt64"],["pre","item_all"],["post","item_all"],["freq2","Int32"],["freq1","Int32"],["freq0","Int32"] */ - msgpack_pack_raw(&pk, 6); - msgpack_pack_raw_body(&pk, CONST_STR_LEN("target")); - msgpack_pack_raw(&pk, name_len); - msgpack_pack_raw_body(&pk, name_buf, name_len); + msgpack_pack_str(&pk, 6); + msgpack_pack_str_body(&pk, "target", strlen("target")); + msgpack_pack_str(&pk, name_len); + msgpack_pack_str_body(&pk, name_buf, name_len); - msgpack_pack_raw(&pk, 4); - msgpack_pack_raw_body(&pk, + msgpack_pack_str(&pk, 4); + msgpack_pack_str_body(&pk, GRN_COLUMN_NAME_KEY, GRN_COLUMN_NAME_KEY_LEN); grn_table_cursor_get_key(ctx, tc, (void **)&key); @@ -419,25 +419,29 @@ handle_msg(msgpack_object *obj, grn_ctx *ctx, grn_obj *buf) int i; for (i = 0; i < obj->via.map.size; i++) { msgpack_object_kv *kv; + msgpack_object *key; + msgpack_object *value; kv = &(obj->via.map.ptr[i]); - if (kv->key.type == MSGPACK_OBJECT_RAW && kv->key.via.raw.size) { - switch (kv->key.via.raw.ptr[0]) { + key = &(kv->key); + value = &(kv->val); + if (key->type == MSGPACK_OBJECT_STR && MSGPACK_OBJECT_STR_SIZE(key) > 0) { + switch (MSGPACK_OBJECT_STR_PTR(key)[0]) { case 'i': - if (kv->val.type == MSGPACK_OBJECT_RAW) { - client_id_len = kv->val.via.raw.size; - client_id = kv->val.via.raw.ptr; + if (value->type == MSGPACK_OBJECT_STR) { + client_id_len = MSGPACK_OBJECT_STR_SIZE(value); + client_id = MSGPACK_OBJECT_STR_PTR(value); } break; case 'q': - if (kv->val.type == MSGPACK_OBJECT_RAW) { - query_len = kv->val.via.raw.size; - query = kv->val.via.raw.ptr; + if (value->type == MSGPACK_OBJECT_STR) { + query_len = MSGPACK_OBJECT_STR_SIZE(value); + query = MSGPACK_OBJECT_STR_PTR(value); } break; case 'l': - if (kv->val.type == MSGPACK_OBJECT_RAW) { - learn_target_names_len = kv->val.via.raw.size; - learn_target_names = kv->val.via.raw.ptr; + if (value->type == MSGPACK_OBJECT_STR) { + learn_target_names_len = MSGPACK_OBJECT_STR_SIZE(value); + learn_target_names = MSGPACK_OBJECT_STR_PTR(value); } break; case 's': diff --git a/storage/mroonga/vendor/groonga/tools/travis-before-script.sh b/storage/mroonga/vendor/groonga/tools/travis-before-script.sh index 71d5a816b87..ac56e5bd5f0 100755 --- a/storage/mroonga/vendor/groonga/tools/travis-before-script.sh +++ b/storage/mroonga/vendor/groonga/tools/travis-before-script.sh @@ -17,9 +17,6 @@ case "${BUILD_TOOL}" in if [ "$ENABLE_MRUBY" = "yes" ]; then configure_args="${configure_args} --with-ruby --enable-mruby" fi - if [ "$ENABLE_JEMALLOC" = "yes" ]; then - configure_args="${configure_args} --with-jemalloc" - fi ./configure --prefix=${prefix} --with-ruby ${configure_args} ;; diff --git a/storage/mroonga/vendor/groonga/tools/travis-install.sh b/storage/mroonga/vendor/groonga/tools/travis-install.sh index e64e17b9872..943c6220ca6 100755 --- a/storage/mroonga/vendor/groonga/tools/travis-install.sh +++ b/storage/mroonga/vendor/groonga/tools/travis-install.sh @@ -13,9 +13,6 @@ case "${TRAVIS_OS_NAME}" in libmecab-dev \ mecab-naist-jdic \ cmake - if [ "${ENABLE_JEMALLOC}" = "yes" ]; then - sudo apt-get install -qq -y libjemalloc-dev - fi ;; osx) brew install \ diff --git a/storage/mroonga/vendor/groonga/tools/travis-script.sh b/storage/mroonga/vendor/groonga/tools/travis-script.sh index 35bf0bd1c9e..1c4a1a7e1a9 100755 --- a/storage/mroonga/vendor/groonga/tools/travis-script.sh +++ b/storage/mroonga/vendor/groonga/tools/travis-script.sh @@ -4,18 +4,22 @@ set -e prefix=/tmp/local +command_test_options="--n-workers=4" + +set -x + case "${BUILD_TOOL}" in autotools) test/unit/run-test.sh - test/command/run-test.sh + test/command/run-test.sh ${command_test_options} if [ "${ENABLE_MRUBY}" = "yes" ]; then test/query_optimizer/run-test.rb fi - test/command/run-test.sh --interface http + test/command/run-test.sh ${command_test_options} --interface http mkdir -p ${prefix}/var/log/groonga/httpd - test/command/run-test.sh --testee groonga-httpd + test/command/run-test.sh ${command_test_options} --testee groonga-httpd ;; cmake) - test/command/run-test.sh + test/command/run-test.sh ${command_test_options} ;; esac diff --git a/storage/mroonga/vendor/groonga/vendor/CMakeLists.txt b/storage/mroonga/vendor/groonga/vendor/CMakeLists.txt index 30c9befebb6..91a806e0429 100644 --- a/storage/mroonga/vendor/groonga/vendor/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/vendor/CMakeLists.txt @@ -13,4 +13,5 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +add_subdirectory(onigmo) add_subdirectory(mruby) diff --git a/storage/mroonga/vendor/groonga/vendor/mruby/CMakeLists.txt b/storage/mroonga/vendor/groonga/vendor/mruby/CMakeLists.txt index 994ad5e0e6c..c78fde5d258 100644 --- a/storage/mroonga/vendor/groonga/vendor/mruby/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/vendor/mruby/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright(C) 2013 Brazil +# Copyright(C) 2013-2015 Brazil # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,21 +13,60 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -add_definitions( - -DDISABLE_GEMS - ) - -include_directories( - "${CMAKE_CURRENT_SOURCE_DIR}/../mruby-${MRUBY_VERSION}/include" - "${CMAKE_CURRENT_SOURCE_DIR}/../mruby-${MRUBY_VERSION}/src" +if(GRN_WITH_MRUBY) + include_directories( + "${CMAKE_CURRENT_SOURCE_DIR}/../mruby-source/include" + "${CMAKE_CURRENT_SOURCE_DIR}/../mruby-source/src" + "${CMAKE_CURRENT_SOURCE_DIR}/../onigmo-source" ) -if(GRN_WITH_MRUBY) read_file_list("${CMAKE_CURRENT_SOURCE_DIR}/sources.am" MRUBY_SOURCES) string(REGEX REPLACE "\\.\\./" "${CMAKE_CURRENT_SOURCE_DIR}/../" MRUBY_SOURCES "${MRUBY_SOURCES}") - add_library(mruby OBJECT ${MRUBY_SOURCES}) - set_source_files_properties(${MRUBY_SOURCES} + + read_file_list("${CMAKE_CURRENT_SOURCE_DIR}/built_sources.am" + MRUBY_BUILT_SOURCES) + set(mruby_pre_build_timestamp + "${CMAKE_CURRENT_SOURCE_DIR}/mruby_build.timestamp") + if(EXISTS "${mruby_pre_build_timestamp}") + string(REGEX REPLACE "([^;]+)" "${CMAKE_CURRENT_SOURCE_DIR}/\\1" + MRUBY_BUILT_SOURCES "${MRUBY_BUILT_SOURCES}") + include_directories( + "${CMAKE_CURRENT_SOURCE_DIR}/mruby-io/include" + ) + else() + set(mruby_build_timestamp + "${CMAKE_CURRENT_BINARY_DIR}/mruby_build.timestamp") + if(NOT EXISTS "${mruby_build_timestamp}") + execute_process( + COMMAND + "${RUBY}" + "${CMAKE_CURRENT_SOURCE_DIR}/mruby_build.rb" + "${CMAKE_CURRENT_SOURCE_DIR}/build_config.rb" + "${CMAKE_CURRENT_SOURCE_DIR}/../mruby-source" + "${CMAKE_CURRENT_BINARY_DIR}/../mruby-build" + "${CMAKE_CURRENT_SOURCE_DIR}/../onigmo-source" + "${mruby_build_timestamp}" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + RESULT_VARIABLE MRUBY_BUILD_RESULT + ) + if(NOT MRUBY_BUILD_RESULT EQUAL "0") + message(FATAL_ERROR "Failed to build mruby files") + endif() + endif() + string(REGEX REPLACE "([^;]+)" "${CMAKE_CURRENT_BINARY_DIR}/\\1" + MRUBY_BUILT_SOURCES "${MRUBY_BUILT_SOURCES}") + include_directories( + "${CMAKE_CURRENT_BINARY_DIR}/mruby-io/include" + ) + endif() + + set(MRUBY_ALL_SOURCES + ${MRUBY_SOURCES} + ${MRUBY_BUILT_SOURCES} + ) + add_library(mruby STATIC ${MRUBY_ALL_SOURCES}) + set_source_files_properties(${MRUBY_ALL_SOURCES} PROPERTIES COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") set_target_properties( diff --git a/storage/mroonga/vendor/groonga/vendor/mruby/Makefile.am b/storage/mroonga/vendor/groonga/vendor/mruby/Makefile.am index dff8a235b49..d995ccfcce1 100644 --- a/storage/mroonga/vendor/groonga/vendor/mruby/Makefile.am +++ b/storage/mroonga/vendor/groonga/vendor/mruby/Makefile.am @@ -1,6 +1,7 @@ EXTRA_DIST = \ build_config.rb \ - mruby-build.timestamp + mruby_build.rb \ + mruby_build.timestamp DEFAULT_INCLUDES = \ -I$(srcdir)/../mruby-source/include \ @@ -8,6 +9,8 @@ DEFAULT_INCLUDES = \ -Imruby-io/include \ -I$(srcdir)/mruby-io/include +CFLAGS += $(NO_FLOAT_EQUAL_CFLAGS) + if WITH_MRUBY noinst_LTLIBRARIES = libmruby.la @@ -21,59 +24,32 @@ AM_CPPFLAGS += \ endif include sources.am - -BUILT_SOURCES = \ - parse.c \ - mrblib.c \ - mrbgems_init.c -BUILT_SOURCES += \ - mruby-onig-regexp/src/mruby_onig_regexp.c -BUILT_SOURCES += \ - mruby-io/include/mruby/ext/io.h \ - mruby-io/src/file.c \ - mruby-io/src/file_test.c \ - mruby-io/src/io.c \ - mruby-io/src/mruby_io_gem.c +include built_sources.am libmruby_la_SOURCES += $(BUILT_SOURCES) -parse.c: mruby-build.timestamp -mrblib.c: mruby-build.timestamp -mrbgems_init.c: mruby-build.timestamp -mruby-onig-regexp/src/mruby_onig_regexp.c: mruby-build.timestamp -mruby-io/include/mruby/ext/io.h: mruby-build.timestamp -mruby-io/src/file.c: mruby-build.timestamp -mruby-io/src/file_test.c: mruby-build.timestamp -mruby-io/src/io.c: mruby-build.timestamp -mruby-io/src/mruby_io_gem.c: mruby-build.timestamp +parse.c: mruby_build.timestamp +mrblib.c: mruby_build.timestamp +mrbgems_init.c: mruby_build.timestamp +mruby-onig-regexp/src/mruby_onig_regexp.c: mruby_build.timestamp +mruby-env/src/env.c: mruby_build.timestamp +mruby-io/include/mruby/ext/io.h: mruby_build.timestamp +mruby-io/src/file.c: mruby_build.timestamp +mruby-io/src/file_test.c: mruby_build.timestamp +mruby-io/src/io.c: mruby_build.timestamp +mruby-io/src/mruby_io_gem.c: mruby_build.timestamp MRUBY_CONFIG = $(abs_srcdir)/build_config.rb MRUBY_BUILD_DIR = $(abs_top_builddir)/vendor/mruby-build -MRUBY_ONIG_REGEXP_DIR = $(MRUBY_BUILD_DIR)/mrbgems/mruby-onig-regexp -MRUBY_IO_DIR = $(MRUBY_BUILD_DIR)/mrbgems/mruby-io -mruby-build.timestamp: build_config.rb - rm -rf $(MRUBY_BUILD_DIR) - cd $(srcdir)/../mruby-source && \ - $(RUBY) minirake \ - MRUBY_BUILD_DIR=$(MRUBY_BUILD_DIR) \ - MRUBY_CONFIG=$(MRUBY_CONFIG) - touch $@ - cp $(MRUBY_BUILD_DIR)/host/src/y.tab.c parse.c - cp $(MRUBY_BUILD_DIR)/host/mrblib/mrblib.c ./ - ( \ - cat $(MRUBY_BUILD_DIR)/host/mrbgems/gem_init.c; \ - cat $(MRUBY_BUILD_DIR)/host/mrbgems/*/gem_init.c; \ - ) > mrbgems_init.c - $(MKDIR_P) mruby-onig-regexp/ - cp -r $(MRUBY_ONIG_REGEXP_DIR)/src/ mruby-onig-regexp/ - $(MKDIR_P) mruby-io/ - cp -r $(MRUBY_IO_DIR)/include/ mruby-io/ - cp -r $(MRUBY_IO_DIR)/src/ mruby-io/ - -EXTRA_DIST += \ - mruby-build.timestamp +mruby_build.timestamp: build_config.rb + $(RUBY) "$(srcdir)/mruby_build.rb" \ + "$(srcdir)/build_config.rb" \ + "$(srcdir)/../mruby-source" \ + "$(builddir)/../mruby-build" \ + "$(srcdir)/../onigmo-source" \ + "mruby_build.timestamp" CLEANFILES = *.gcno *gcda -MAINTAINERCLEANFILES = mruby-build.timestamp +MAINTAINERCLEANFILES = mruby_build.timestamp endif update: diff --git a/storage/mroonga/vendor/groonga/vendor/mruby/build_config.rb b/storage/mroonga/vendor/groonga/vendor/mruby/build_config.rb index 65992c20340..5a963f9c7e7 100644 --- a/storage/mroonga/vendor/groonga/vendor/mruby/build_config.rb +++ b/storage/mroonga/vendor/groonga/vendor/mruby/build_config.rb @@ -5,6 +5,11 @@ MRuby::Build.new do |conf| toolchain :gcc end + oniguruma_include_path = ENV["MRUBY_ONIGURUMA_INCLUDE_PATH"] + if oniguruma_include_path + conf.cc.include_paths << oniguruma_include_path + end + enable_debug conf.gem :core => "mruby-sprintf" @@ -30,6 +35,7 @@ MRuby::Build.new do |conf| conf.gem :core => "mruby-kernel-ext" conf.gem :github => "mattn/mruby-onig-regexp" + conf.gem :github => "iij/mruby-env" conf.gem :github => "iij/mruby-io" conf.gem :github => "kou/mruby-pp" conf.gem :github => "kou/mruby-slop" diff --git a/storage/mroonga/vendor/groonga/vendor/mruby/built_sources.am b/storage/mroonga/vendor/groonga/vendor/mruby/built_sources.am new file mode 100644 index 00000000000..08c5e3a9453 --- /dev/null +++ b/storage/mroonga/vendor/groonga/vendor/mruby/built_sources.am @@ -0,0 +1,11 @@ +BUILT_SOURCES = \ + parse.c \ + mrblib.c \ + mrbgems_init.c \ + mruby-onig-regexp/src/mruby_onig_regexp.c \ + mruby-env/src/env.c \ + mruby-io/include/mruby/ext/io.h \ + mruby-io/src/file.c \ + mruby-io/src/file_test.c \ + mruby-io/src/io.c \ + mruby-io/src/mruby_io_gem.c diff --git a/storage/mroonga/vendor/groonga/vendor/mruby/mruby_build.rb b/storage/mroonga/vendor/groonga/vendor/mruby/mruby_build.rb new file mode 100755 index 00000000000..f0db931ed38 --- /dev/null +++ b/storage/mroonga/vendor/groonga/vendor/mruby/mruby_build.rb @@ -0,0 +1,51 @@ +#!/usr/bin/env ruby + +if ARGV.size != 5 + puts("Usage: #{$0} BUILD_COFNIG.RB MRUBY_SOURCE_DIR MRUBY_BUILD_DIR ONIGURUMA_INCLUDE_PATH TIMESTAMP_FILE") + exit(false) +end + +require "rbconfig" +require "fileutils" + +build_config_rb = File.expand_path(ARGV.shift) +mruby_source_dir = ARGV.shift +mruby_build_dir = File.expand_path(ARGV.shift) +oniguruma_include_path = File.expand_path(ARGV.shift) +timestamp_file = File.expand_path(ARGV.shift) + +FileUtils.rm_rf(mruby_build_dir) + +Dir.chdir(mruby_source_dir) do + unless system(RbConfig.ruby, + "minirake", + "MRUBY_CONFIG=#{build_config_rb}", + "MRUBY_BUILD_DIR=#{mruby_build_dir}", + "MRUBY_ONIGURUMA_INCLUDE_PATH=#{oniguruma_include_path}") + exit(false) + end +end + +FileUtils.touch(timestamp_file) + +FileUtils.cp("#{mruby_build_dir}/host/src/y.tab.c", "parse.c") +FileUtils.cp("#{mruby_build_dir}/host/mrblib/mrblib.c", "./") + +File.open("mrbgems_init.c", "w") do |mrbgems_init| + Dir.glob("#{mruby_build_dir}/host/mrbgems/**/gem_init.c") do |gem_init| + mrbgems_init.puts(File.read(gem_init)) + end +end + +mruby_onig_regexp_dir = "#{mruby_build_dir}/mrbgems/mruby-onig-regexp" +FileUtils.mkdir_p("mruby-onig-regexp/") +FileUtils.cp_r("#{mruby_onig_regexp_dir}/src/", "mruby-onig-regexp/") + +mruby_env_dir = "#{mruby_build_dir}/mrbgems/mruby-env" +FileUtils.mkdir_p("mruby-env/") +FileUtils.cp_r("#{mruby_env_dir}/src/", "mruby-env/") + +mruby_io_dir = "#{mruby_build_dir}/mrbgems/mruby-io" +FileUtils.mkdir_p("mruby-io/") +FileUtils.cp_r("#{mruby_io_dir}/include/", "mruby-io/") +FileUtils.cp_r("#{mruby_io_dir}/src/", "mruby-io/") diff --git a/storage/mroonga/vendor/groonga/vendor/mruby/sources.am b/storage/mroonga/vendor/groonga/vendor/mruby/sources.am index b470d58b803..68f15c64571 100644 --- a/storage/mroonga/vendor/groonga/vendor/mruby/sources.am +++ b/storage/mroonga/vendor/groonga/vendor/mruby/sources.am @@ -11,6 +11,7 @@ libmruby_la_SOURCES = \ ../mruby-source/src/error.c \ ../mruby-source/src/error.h \ ../mruby-source/src/etc.c \ + ../mruby-source/src/fmt_fp.c \ ../mruby-source/src/gc.c \ ../mruby-source/src/hash.c \ ../mruby-source/src/init.c \ diff --git a/storage/mroonga/vendor/groonga/vendor/onigmo/CMakeLists.txt b/storage/mroonga/vendor/groonga/vendor/onigmo/CMakeLists.txt new file mode 100644 index 00000000000..0f79347aa73 --- /dev/null +++ b/storage/mroonga/vendor/groonga/vendor/onigmo/CMakeLists.txt @@ -0,0 +1,110 @@ +# Copyright(C) 2015 Brazil +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +if(GRN_WITH_ONIGMO) + set(ONIGMO_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../onigmo-source") + set(ONIGMO_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/../onigmo-source") + + include(CheckTypeSize) + + macro(ac_check_sizeof type) + string(TOUPPER "${type}" output_variable_base_name) + set(output_variable_name "ONIG_SIZEOF_${output_variable_base_name}") + check_type_size(${type} ${output_variable_name}) + if(HAVE_${output_variable_name}) + add_definitions( + -DSIZEOF_${output_variable_base_name}=${${output_variable_name}} + ) + endif() + endmacro() + + ac_check_sizeof(short) + ac_check_sizeof(int) + ac_check_sizeof(long) + + if(MSVC) + add_definitions(-Dinline=__inline) + add_definitions(-D_CRT_SECURE_NO_WARNINGS) + endif() + + include_directories( + ${ONIGMO_BINARY_DIR} + ${ONIGMO_SOURCE_DIR} + ) + + set(ONIGMO_SOURCES + "${ONIGMO_SOURCE_DIR}/regint.h" + "${ONIGMO_SOURCE_DIR}/regparse.h" + "${ONIGMO_SOURCE_DIR}/regenc.h" + "${ONIGMO_SOURCE_DIR}/st.h" + "${ONIGMO_SOURCE_DIR}/regerror.c" + "${ONIGMO_SOURCE_DIR}/regparse.c" + "${ONIGMO_SOURCE_DIR}/regext.c" + "${ONIGMO_SOURCE_DIR}/regcomp.c" + "${ONIGMO_SOURCE_DIR}/regexec.c" + "${ONIGMO_SOURCE_DIR}/reggnu.c" + "${ONIGMO_SOURCE_DIR}/regenc.c" + "${ONIGMO_SOURCE_DIR}/regsyntax.c" + "${ONIGMO_SOURCE_DIR}/regtrav.c" + "${ONIGMO_SOURCE_DIR}/regversion.c" + "${ONIGMO_SOURCE_DIR}/st.c" + "${ONIGMO_SOURCE_DIR}/regposix.c" + "${ONIGMO_SOURCE_DIR}/regposerr.c" + "${ONIGMO_SOURCE_DIR}/enc/unicode.c" + "${ONIGMO_SOURCE_DIR}/enc/ascii.c" + "${ONIGMO_SOURCE_DIR}/enc/utf8.c" + "${ONIGMO_SOURCE_DIR}/enc/utf16_be.c" + "${ONIGMO_SOURCE_DIR}/enc/utf16_le.c" + "${ONIGMO_SOURCE_DIR}/enc/utf32_be.c" + "${ONIGMO_SOURCE_DIR}/enc/utf32_le.c" + "${ONIGMO_SOURCE_DIR}/enc/unicode/casefold.h" + "${ONIGMO_SOURCE_DIR}/enc/unicode/name2ctype.h" + "${ONIGMO_SOURCE_DIR}/enc/euc_jp.c" + "${ONIGMO_SOURCE_DIR}/enc/sjis.c" + "${ONIGMO_SOURCE_DIR}/enc/cp932.c" + "${ONIGMO_SOURCE_DIR}/enc/jis/props.h" +# "${ONIGMO_SOURCE_DIR}/enc/jis/props.kwd" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_1.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_2.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_3.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_4.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_5.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_6.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_7.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_8.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_9.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_10.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_11.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_13.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_14.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_15.c" + "${ONIGMO_SOURCE_DIR}/enc/iso8859_16.c" + "${ONIGMO_SOURCE_DIR}/enc/euc_tw.c" + "${ONIGMO_SOURCE_DIR}/enc/euc_kr.c" + "${ONIGMO_SOURCE_DIR}/enc/big5.c" + "${ONIGMO_SOURCE_DIR}/enc/gb18030.c" + "${ONIGMO_SOURCE_DIR}/enc/koi8_r.c" + "${ONIGMO_SOURCE_DIR}/enc/cp1251.c" + ) + + add_library(onigmo STATIC ${ONIGMO_SOURCES}) + set_source_files_properties(${ONIGMO_SOURCES} + PROPERTIES + COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS}") + set_target_properties( + onigmo + PROPERTIES + POSITION_INDEPENDENT_CODE ON) +endif() diff --git a/storage/mroonga/vendor/groonga/vendor/onigmo/Makefile.am b/storage/mroonga/vendor/groonga/vendor/onigmo/Makefile.am index 9219783bbd7..73632ebd3bd 100644 --- a/storage/mroonga/vendor/groonga/vendor/onigmo/Makefile.am +++ b/storage/mroonga/vendor/groonga/vendor/onigmo/Makefile.am @@ -1,5 +1,6 @@ EXTRA_DIST = \ - configure + configure \ + CMakeLists.txt CONFIGURE_DEPENDENCIES = \ configure diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/README.md b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/README.md index a2fcf4c9f84..fae74b6ce13 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/README.md +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/README.md @@ -7,20 +7,30 @@ groonga-normalizer-mysql ## Description Groonga-normalizer-mysql is a Groonga plugin. It provides MySQL -compatible normalizers and a custom normalizer to Groonga. +compatible normalizers and a custom normalizers to Groonga. -MySQL compatible normalizers are `NormalizerMySQLGeneralCI` and -`NormalizerMySQLUnicodeCI`. `NormalizerMySQLGeneralCI` corresponds to -`utf8mb4_general_ci`. `NormalizerMySQLUnicodeCI` corresponds to -`utf8mb4_unicode_ci`. +Here are MySQL compatible normalizers: -A custom normalizer is -`NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark`. It is -self-descriptive name but long. It is a variant normalizer of -`NormalizerMySQLUnicode`. It has different behaviors. The followings -are the different behaviors. +* `NormalizerMySQLGeneralCI` for `utf8mb4_general_ci` +* `NormalizerMySQLUnicodeCI` for `utf8mb4_unicode_ci` +* `NormalizerMySQLUnicode520CI` for `utf8mb4_unicode_520_ci` -* `NormalizerMySQLUnicode` normalizes all small Hiragana such as `ぁ`, +Here are custom normalizers: + +* `NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark` + * It's based on `NormalizerMySQLUnicodeCI` +* `NormalizerMySQLUnicode520CIExceptKanaCIKanaWithVoicedSoundMark` + * It's based on `NormalizerMySQLUnicode520CI` + +They are self-descriptive name but long. They are variant normalizers +of `NormalizerMySQLUnicodeCI` and `NormalizerMySQLUnicode520CI`. They +have different behaviors. The followings are the different +behaviors. They describes with +`NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark` but they +are true for +`NormalizerMySQLUnicode520CIExceptKanaCIKanaWithVoicedSoundMark`. + +* `NormalizerMySQLUnicodeCI` normalizes all small Hiragana such as `ぁ`, `っ` to Hiragana such as `あ`, `つ`. `NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark` doesn't normalize `ぁ` to `あ` nor `っ` to `つ`. `ぁ` and `あ` are @@ -51,11 +61,12 @@ are the different behaviors. normalizes all halfwidth Katakana with voided sound mark such as `ｶﾞ` to fullwidth Katakana with voiced sound mark such as `ガ`. -`NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark` is MySQL -incompatible normalizer but it is useful for Japanese text. For -example, `ふらつく` and `ブラック` has different -means. `NormalizerMySQLUnicodeCI` identifies `ふらつく` with `ブラック -` but `NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark +`NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark` and +`NormalizerMySQLUnicode520CIExceptKanaCIKanaWithVoicedSoundMark` and +are MySQL incompatible normalizers but they are useful for Japanese +text. For example, `ふらつく` and `ブラック` has different +means. `NormalizerMySQLUnicodeCI` identifies `ふらつく` with `ブラック` +but `NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark` doesn't identify them. ## Install @@ -65,15 +76,14 @@ doesn't identify them. [Add apt-line for the Groonga deb package repository](http://groonga.org/docs/install/debian.html) and install `groonga-normalizer-mysql` package: - % sudo aptitude -V -D -y install groonga-normalizer-mysql + % sudo apt-get -y install groonga-normalizer-mysql ### Ubuntu [Add apt-line for the Groonga deb package repository](http://groonga.org/docs/install/ubuntu.html) and install `groonga-normalizer-mysql` package: - % sudo aptitude -V -D -y install groonga-normalizer-mysql - + % sudo apt-get -y install groonga-normalizer-mysql ### CentOS @@ -125,7 +135,7 @@ Extract the source and move to the source folder: Run CMake. Here is a command line to install Groonga to `C:\groonga` folder: - groonga-X.Y.Z> cmake . -G "Visual Studio 10 Win64" -DCMAKE_INSTALL_PREFIX=C:\groonga + groonga-X.Y.Z> cmake . -G "Visual Studio 12 Win64" -DCMAKE_INSTALL_PREFIX=C:\groonga Build: @@ -150,7 +160,7 @@ IMPORTANT!!!: Set `PKG_CONFIG_PATH` environment variable: Run CMake. Here is a command line to install Groonga to `C:\groonga` folder: - groonga-normalizer-mysql-X.Y.Z> cmake . -G "Visual Studio 10 Win64" -DCMAKE_INSTALL_PREFIX=C:\groonga + groonga-normalizer-mysql-X.Y.Z> cmake . -G "Visual Studio 12 Win64" -DCMAKE_INSTALL_PREFIX=C:\groonga Build: diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/configure.ac b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/configure.ac index 1ef007bb6df..fd52203837e 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/configure.ac +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/configure.ac @@ -111,7 +111,7 @@ AM_CONDITIONAL([WITH_TEST], [test "$ac_cv_ruby_available" = "yes"]) # For package AC_ARG_WITH(rsync-path, [AS_HELP_STRING([--with-rsync-path=PATH], - [specify rsync path to upload groonga packages.])], + [specify rsync path to upload Groonga packages.])], [RSYNC_PATH="$withval"], [RSYNC_PATH="packages@packages.groonga.org:public"]) AC_SUBST(RSYNC_PATH) @@ -188,7 +188,7 @@ echo " CFLAGS: ${CFLAGS}" echo " CXXFLAGS: ${CXXFLAGS}" echo " Libraries: ${LIBS}" echo -echo "groonga" +echo "Groonga" echo " CFLAGS: ${GROONGA_CFLAGS}" echo " Libraries: ${GROONGA_LIBS}" echo " install directory: ${normalizers_pluginsdir}" diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/doc/text/news.md b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/doc/text/news.md index 9f367a56671..22a7fdb17cf 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/doc/text/news.md +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/doc/text/news.md @@ -1,5 +1,12 @@ # News +## 1.0.9: 2015-03-29 + +### Improves + + * Added `NormalizerMySQLUnicode520CI` + * Added `NormalizerMySQLUnicode520CIExceptKanaCIKanaWithVoicedSoundMark` + ## 1.0.8: 2015-02-10 ### Fixes diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/CMakeLists.txt b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/CMakeLists.txt index db8ffedb082..8d79558a120 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/CMakeLists.txt @@ -18,9 +18,13 @@ set(NORMALIZERS_DIR "${GROONGA_PLUGINS_DIR}/normalizers") read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/mysql_sources.am MYSQL_SOURCES) if(GROONGA_NORMALIZER_MYSQL_EMBED) - add_convenience_library(mysql_normalizer ${MYSQL_SOURCES}) + add_library(mysql_normalizer STATIC ${MYSQL_SOURCES}) set_property(TARGET mysql_normalizer APPEND PROPERTY COMPILE_DEFINITIONS "GROONGA_NORMALIZER_MYSQL_EMBED") + set_target_properties( + mysql_normalizer + PROPERTIES + POSITION_INDEPENDENT_CODE ON) else() add_library(mysql_normalizer MODULE ${MYSQL_SOURCES}) set_target_properties(mysql_normalizer PROPERTIES diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/Makefile.am b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/Makefile.am index c5c131ce2af..64cfe6552fa 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/Makefile.am +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/Makefile.am @@ -39,10 +39,12 @@ ensure-mysql-source-dir: exit 1; \ fi -UPDATE_TABLES_TARGETS = \ - update-general-ci-table \ - update-unicode-ci-table \ - update-unicode-ci-except-kana-ci-kana-with-voiced-sound-mark-table +UPDATE_TABLES_TARGETS = \ + update-general-ci-table \ + update-unicode-ci-table \ + update-unicode-ci-except-kana-ci-kana-with-voiced-sound-mark-table \ + update-unicode-520-ci-table \ + update-unicode-520-ci-except-kana-ci-kana-with-voiced-sound-mark-table update-tables: $(UPDATE_TABLES_TARGETS) @@ -67,3 +69,22 @@ update-unicode-ci-except-kana-ci-kana-with-voiced-sound-mark-table: ensure-mysql --suffix _except_kana_ci_kana_with_voiced_sound_mark \ $(MYSQL_SOURCE_DIR)/strings/ctype-uca.c > \ $(srcdir)/mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h + +update-unicode-520-ci-table: ensure-mysql-source-dir + $(RUBY) \ + $(top_srcdir)/tool/generate_uca_table.rb \ + --version=520 \ + $(MYSQL_SOURCE_DIR)/strings/ctype-uca.c > \ + $(srcdir)/mysql_unicode_520_ci_table.h + +update-unicode-520-ci-except-kana-ci-kana-with-voiced-sound-mark-table: ensure-mysql-source-dir + $(RUBY) \ + $(top_srcdir)/tool/generate_uca_table.rb \ + --version=520 \ + --split-small-kana \ + --split-kana-with-voiced-sound-mark \ + --split-kana-with-semi-voiced-sound-mark \ + --suffix _except_kana_ci_kana_with_voiced_sound_mark \ + $(MYSQL_SOURCE_DIR)/strings/ctype-uca.c > \ + $(srcdir)/mysql_unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h + diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql.c b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql.c index d9e8de4d4a0..c7eb102d0e1 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql.c +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql.c @@ -35,6 +35,8 @@ #include "mysql_general_ci_table.h" #include "mysql_unicode_ci_table.h" #include "mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h" +#include "mysql_unicode_520_ci_table.h" +#include "mysql_unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h" #ifdef __GNUC__ # define GNUC_UNUSED __attribute__((__unused__)) @@ -155,7 +157,7 @@ utf8_to_unichar(const char *utf8, int byte_size) static inline void decompose_character(const char *rest, int character_length, - int *page, uint32_t *low_code) + size_t *page, uint32_t *low_code) { switch (character_length) { case 1 : @@ -175,10 +177,27 @@ decompose_character(const char *rest, int character_length, ((rest[0] & 0x07) << 10) + ((rest[1] & 0x3f) << 4) + ((rest[2] & 0x3c) >> 2); - *low_code = ((rest[1] & 0x03) << 6) + (rest[2] & 0x3f); + *low_code = ((rest[2] & 0x03) << 6) + (rest[3] & 0x3f); + break; + case 5 : + *page = + ((rest[0] & 0x03) << 16) + + ((rest[1] & 0x3f) << 10) + + ((rest[2] & 0x3f) << 4) + + ((rest[3] & 0x3c) >> 2); + *low_code = ((rest[3] & 0x03) << 6) + (rest[4] & 0x3f); + break; + case 6 : + *page = + ((rest[0] & 0x01) << 22) + + ((rest[1] & 0x3f) << 16) + + ((rest[2] & 0x3f) << 10) + + ((rest[3] & 0x3f) << 4) + + ((rest[4] & 0x3c) >> 2); + *low_code = ((rest[4] & 0x03) << 6) + (rest[5] & 0x3f); break; default : - *page = -1; + *page = (size_t)-1; *low_code = 0x00; break; } @@ -187,15 +206,16 @@ decompose_character(const char *rest, int character_length, static inline void normalize_character(const char *utf8, int character_length, uint32_t **normalize_table, + size_t normalize_table_size, char *normalized, unsigned int *normalized_character_length, unsigned int *normalized_length_in_bytes, unsigned int *normalized_n_characters) { - int page; + size_t page; uint32_t low_code; decompose_character(utf8, character_length, &page, &low_code); - if ((0x00 <= page && page <= 0xff) && normalize_table[page]) { + if (page < normalize_table_size && normalize_table[page]) { uint32_t normalized_code; unsigned int n_bytes; normalized_code = normalize_table[page][low_code]; @@ -304,6 +324,7 @@ static void normalize(grn_ctx *ctx, grn_obj *string, const char *normalizer_type_label, uint32_t **normalize_table, + size_t normalize_table_size, normalizer_func custom_normalizer) { const char *original, *rest; @@ -371,7 +392,8 @@ normalize(grn_ctx *ctx, grn_obj *string, &normalized_n_characters); } if (!custom_normalized) { - normalize_character(rest, character_length, normalize_table, + normalize_character(rest, character_length, + normalize_table, normalize_table_size, normalized, &normalized_character_length, &normalized_length_in_bytes, @@ -448,7 +470,9 @@ mysql_general_ci_next(GNUC_UNUSED grn_ctx *ctx, grn_encoding_to_string(encoding)); return NULL; } - normalize(ctx, string, normalizer_type_label, general_ci_table, NULL); + normalize(ctx, string, normalizer_type_label, + general_ci_table, sizeof(general_ci_table) / sizeof(uint32_t *), + NULL); return NULL; } @@ -472,7 +496,9 @@ mysql_unicode_ci_next(GNUC_UNUSED grn_ctx *ctx, grn_encoding_to_string(encoding)); return NULL; } - normalize(ctx, string, normalizer_type_label, unicode_ci_table, NULL); + normalize(ctx, string, normalizer_type_label, + unicode_ci_table, sizeof(unicode_ci_table) / sizeof(uint32_t *), + NULL); return NULL; } @@ -638,6 +664,64 @@ mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_next( normalize(ctx, string, normalizer_type_label, unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table, + sizeof(unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table) / sizeof(uint32_t *), + normalize_halfwidth_katakana_with_voiced_sound_mark); + return NULL; +} + +static grn_obj * +mysql_unicode_520_ci_next(GNUC_UNUSED grn_ctx *ctx, + GNUC_UNUSED int nargs, + grn_obj **args, + GNUC_UNUSED grn_user_data *user_data) +{ + grn_obj *string = args[0]; + grn_encoding encoding; + const char *normalizer_type_label = "mysql-unicode-520-ci"; + + encoding = grn_string_get_encoding(ctx, string); + if (encoding != GRN_ENC_UTF8) { + GRN_PLUGIN_ERROR(ctx, + GRN_FUNCTION_NOT_IMPLEMENTED, + "[normalizer][%s] " + "UTF-8 encoding is only supported: %s", + normalizer_type_label, + grn_encoding_to_string(encoding)); + return NULL; + } + normalize(ctx, string, normalizer_type_label, + unicode_520_ci_table, + sizeof(unicode_520_ci_table) / sizeof(uint32_t *), + NULL); + return NULL; +} + +static grn_obj * +mysql_unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_next( + GNUC_UNUSED grn_ctx *ctx, + GNUC_UNUSED int nargs, + grn_obj **args, + GNUC_UNUSED grn_user_data *user_data) +{ + grn_obj *string = args[0]; + grn_encoding encoding; + const char *normalizer_type_label = + "mysql-unicode-520-ci-except-kana-ci-kana-with-voiced-sound-mark"; + + encoding = grn_string_get_encoding(ctx, string); + if (encoding != GRN_ENC_UTF8) { + GRN_PLUGIN_ERROR(ctx, + GRN_FUNCTION_NOT_IMPLEMENTED, + "[normalizer][%s] " + "UTF-8 encoding is only supported: %s", + normalizer_type_label, + grn_encoding_to_string(encoding)); + return NULL; + } + normalize(ctx, string, + normalizer_type_label, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_table, + sizeof(unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_table) / sizeof(uint32_t *), normalize_halfwidth_katakana_with_voiced_sound_mark); return NULL; } @@ -664,6 +748,17 @@ GRN_PLUGIN_REGISTER(grn_ctx *ctx) NULL, mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_next, NULL); + grn_normalizer_register(ctx, "NormalizerMySQLUnicode520CI", -1, + NULL, mysql_unicode_520_ci_next, NULL); + grn_normalizer_register(ctx, + "NormalizerMySQLUnicode520CI" + "Except" + "KanaCI" + "KanaWithVoicedSoundMark", + -1, + NULL, + mysql_unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_next, + NULL); return GRN_SUCCESS; } diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_general_ci_table.h b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_general_ci_table.h index 15ed558e2dc..a84e449910d 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_general_ci_table.h +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_general_ci_table.h @@ -17,7 +17,7 @@ MA 02110-1301, USA This file uses normalization table defined in - mysql-5.5.29/strings/ctype-utf8.c. + mysql-5.6.23/strings/ctype-utf8.c. The following is the header of the file: Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_sources.am b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_sources.am index 6f1d9b02018..07f30b30f5b 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_sources.am +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_sources.am @@ -1,5 +1,7 @@ -mysql_la_SOURCES = \ - mysql.c \ - mysql_general_ci_table.h \ - mysql_unicode_ci_table.h \ - mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h +mysql_la_SOURCES = \ + mysql.c \ + mysql_general_ci_table.h \ + mysql_unicode_ci_table.h \ + mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h \ + mysql_unicode_520_ci_table.h \ + mysql_unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h new file mode 100644 index 00000000000..6832d3a3b66 --- /dev/null +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h @@ -0,0 +1,5028 @@ +/* + Copyright(C) 2013-2015 Kouhei Sutou <kou@clear-code.com> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; version 2 + of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + MA 02110-1301, USA + + This file uses normalization table defined in + mysql-5.6.23/strings/ctype-uca.c. + The following is the header of the file: + + Copyright (c) 2004, 2014, Oracle and/or its affiliates. All rights reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; version 2 + of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + MA 02110-1301, USA + + UCA (Unicode Collation Algorithm) support. + Written by Alexander Barkov <bar@mysql.com> +*/ + +#ifndef MYSQL_UCA_520_EXCEPT_KANA_CI_KANA_WITH_VOICED_SOUND_MARK_H +#define MYSQL_UCA_520_EXCEPT_KANA_CI_KANA_WITH_VOICED_SOUND_MARK_H + +#include <stdint.h> + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_00[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00009, 0x0000a, 0x0000b, 0x0000c, 0x0000d, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00020, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, + 0x00028, 0x00029, 0x0002a, 0x0002b, 0x0002c, 0x0002d, 0x0002e, 0x0002f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0003a, 0x0003b, 0x0003c, 0x0003d, 0x0003e, 0x0003f, + 0x00040, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, + 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, + 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, + 0x00058, 0x00059, 0x0005a, 0x0005b, 0x0005c, 0x0005d, 0x0005e, 0x0005f, + 0x00060, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, + 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, + 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, + 0x00058, 0x00059, 0x0005a, 0x0007b, 0x0007c, 0x0007d, 0x0007e, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00085, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00020, 0x000a1, 0x000a2, 0x000a3, 0x000a4, 0x000a5, 0x000a6, 0x000a7, + 0x000a8, 0x000a9, 0x00041, 0x000ab, 0x000ac, 0x000ad, 0x000ae, 0x000af, + 0x000b0, 0x000b1, 0x00032, 0x00033, 0x000b4, 0x0039c, 0x000b6, 0x000b7, + 0x000b8, 0x00031, 0x0004f, 0x000bb, 0x000bc, 0x000bd, 0x000be, 0x000bf, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x000c6, 0x00043, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00049, 0x00049, 0x00049, 0x00049, + 0x00044, 0x0004e, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x000d7, + 0x0004f, 0x00055, 0x00055, 0x00055, 0x00055, 0x00059, 0x000de, 0x000df, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x000c6, 0x00043, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00049, 0x00049, 0x00049, 0x00049, + 0x00044, 0x0004e, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x000f7, + 0x0004f, 0x00055, 0x00055, 0x00055, 0x00055, 0x00059, 0x000fe, 0x000ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_01[] = { + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00043, 0x00043, + 0x00043, 0x00043, 0x00043, 0x00043, 0x00043, 0x00043, 0x00044, 0x00044, + 0x00044, 0x00044, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00047, 0x00047, 0x00047, 0x00047, + 0x00047, 0x00047, 0x00047, 0x00047, 0x00048, 0x00048, 0x00048, 0x00048, + 0x00049, 0x00049, 0x00049, 0x00049, 0x00049, 0x00049, 0x00049, 0x00049, + 0x00049, 0x00131, 0x00132, 0x00132, 0x0004a, 0x0004a, 0x0004b, 0x0004b, + 0x00138, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, + 0x0004c, 0x0004c, 0x0004c, 0x0004e, 0x0004e, 0x0004e, 0x0004e, 0x0004e, + 0x0004e, 0x00149, 0x0014a, 0x0014a, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x00152, 0x00152, 0x00052, 0x00052, 0x00052, 0x00052, + 0x00052, 0x00052, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, + 0x00053, 0x00053, 0x00054, 0x00054, 0x00054, 0x00054, 0x00166, 0x00166, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00057, 0x00057, 0x00059, 0x00059, + 0x00059, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x00053, + 0x00180, 0x00181, 0x00182, 0x00182, 0x00184, 0x00184, 0x00186, 0x00187, + 0x00187, 0x00189, 0x0018a, 0x0018b, 0x0018b, 0x0018d, 0x0018e, 0x0018f, + 0x00190, 0x00191, 0x00191, 0x00193, 0x00194, 0x00195, 0x00196, 0x00197, + 0x00198, 0x00198, 0x0019a, 0x0019b, 0x0019c, 0x0019d, 0x0019e, 0x0019f, + 0x0004f, 0x0004f, 0x001a2, 0x001a2, 0x001a4, 0x001a4, 0x001a6, 0x001a7, + 0x001a7, 0x001a9, 0x001aa, 0x001ab, 0x001ac, 0x001ac, 0x001ae, 0x00055, + 0x00055, 0x001b1, 0x001b2, 0x001b3, 0x001b3, 0x001b5, 0x001b5, 0x001b7, + 0x001b8, 0x001b8, 0x001ba, 0x001bb, 0x001bc, 0x001bc, 0x001be, 0x001bf, + 0x001c0, 0x001c1, 0x001c2, 0x001c3, 0x001c4, 0x001c4, 0x001c4, 0x001c7, + 0x001c7, 0x001c7, 0x001ca, 0x001ca, 0x001ca, 0x00041, 0x00041, 0x00049, + 0x00049, 0x0004f, 0x0004f, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x0018e, 0x00041, 0x00041, + 0x00041, 0x00041, 0x000c6, 0x000c6, 0x001e4, 0x001e4, 0x00047, 0x00047, + 0x0004b, 0x0004b, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x001b7, 0x001b7, + 0x0004a, 0x001c4, 0x001c4, 0x001c4, 0x00047, 0x00047, 0x00195, 0x001bf, + 0x0004e, 0x0004e, 0x00041, 0x00041, 0x000c6, 0x000c6, 0x0004f, 0x001ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_02[] = { + 0x00041, 0x00041, 0x00041, 0x00041, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00049, 0x00049, 0x00049, 0x00049, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x00052, 0x00052, 0x00052, 0x00052, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00053, 0x00053, 0x00054, 0x00054, 0x0021c, 0x0021c, 0x00048, 0x00048, + 0x0019e, 0x00221, 0x00222, 0x00222, 0x00224, 0x00224, 0x00041, 0x00041, + 0x00045, 0x00045, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x00059, 0x00059, 0x00234, 0x00235, 0x00236, 0x00237, + 0x00238, 0x00239, 0x0023a, 0x0023b, 0x0023b, 0x0019a, 0x0023e, 0x0023f, + 0x00240, 0x00241, 0x00241, 0x00180, 0x00244, 0x00245, 0x00246, 0x00246, + 0x00248, 0x00248, 0x0024a, 0x0024a, 0x0024c, 0x0024c, 0x0024e, 0x0024e, + 0x00250, 0x00251, 0x00252, 0x00181, 0x00186, 0x00255, 0x00189, 0x0018a, + 0x00258, 0x0018f, 0x0025a, 0x00190, 0x0025c, 0x0025d, 0x0025e, 0x0025f, + 0x00193, 0x00261, 0x00262, 0x00194, 0x00264, 0x00265, 0x00266, 0x00267, + 0x00197, 0x00196, 0x0026a, 0x0026b, 0x0026c, 0x0026d, 0x0026e, 0x0019c, + 0x00270, 0x00271, 0x0019d, 0x00273, 0x00274, 0x0019f, 0x00276, 0x00277, + 0x00278, 0x00279, 0x0027a, 0x0027b, 0x0027c, 0x0027d, 0x0027e, 0x0027f, + 0x001a6, 0x00281, 0x00282, 0x001a9, 0x00284, 0x00285, 0x00286, 0x00287, + 0x001ae, 0x00244, 0x001b1, 0x001b2, 0x00245, 0x0028d, 0x0028e, 0x0028f, + 0x00290, 0x00291, 0x001b7, 0x00293, 0x00294, 0x00295, 0x00296, 0x00297, + 0x00298, 0x00299, 0x0029a, 0x0029b, 0x0029c, 0x0029d, 0x0029e, 0x0029f, + 0x002a0, 0x002a1, 0x002a2, 0x001c4, 0x002a4, 0x002a5, 0x001be, 0x002a7, + 0x002a8, 0x002a9, 0x002aa, 0x002ab, 0x002ac, 0x002ad, 0x002ae, 0x002af, + 0x00048, 0x00266, 0x0004a, 0x00052, 0x00279, 0x0027b, 0x00281, 0x00057, + 0x00059, 0x002b9, 0x002ba, 0x002bb, 0x002bc, 0x002bd, 0x002be, 0x002bf, + 0x002c0, 0x002c1, 0x002c2, 0x002c3, 0x002c4, 0x002c5, 0x002c6, 0x002c7, + 0x002c8, 0x002c9, 0x002ca, 0x002cb, 0x002cc, 0x002cd, 0x002ce, 0x002cf, + 0x002d0, 0x002d1, 0x002d2, 0x002d3, 0x002d4, 0x002d5, 0x002d6, 0x002d7, + 0x002d8, 0x002d9, 0x002da, 0x002db, 0x002dc, 0x002dd, 0x002de, 0x002df, + 0x00194, 0x0004c, 0x00053, 0x00058, 0x00295, 0x002e5, 0x002e6, 0x002e7, + 0x002e8, 0x002e9, 0x002ea, 0x002eb, 0x002ec, 0x002ed, 0x002ee, 0x002ef, + 0x002f0, 0x002f1, 0x002f2, 0x002f3, 0x002f4, 0x002f5, 0x002f6, 0x002f7, + 0x002f8, 0x002f9, 0x002fa, 0x002fb, 0x002fc, 0x002fd, 0x002fe, 0x002ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_03[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00041, 0x00045, 0x00049, 0x0004f, 0x00055, + 0x00043, 0x00044, 0x00048, 0x0004d, 0x00052, 0x00054, 0x00056, 0x00058, + 0x00370, 0x00370, 0x00372, 0x00372, 0x002b9, 0x00375, 0x00376, 0x00376, + 0x00378, 0x00379, 0x00399, 0x0037b, 0x0037c, 0x0037d, 0x0003b, 0x0037f, + 0x00380, 0x00381, 0x00382, 0x00383, 0x000b4, 0x000a8, 0x00391, 0x000b7, + 0x00395, 0x00397, 0x00399, 0x0038b, 0x0039f, 0x0038d, 0x003a5, 0x003a9, + 0x00399, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, + 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, + 0x003a0, 0x003a1, 0x003a2, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, + 0x003a8, 0x003a9, 0x00399, 0x003a5, 0x00391, 0x00395, 0x00397, 0x00399, + 0x003a5, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, + 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, + 0x003a0, 0x003a1, 0x003a3, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, + 0x003a8, 0x003a9, 0x00399, 0x003a5, 0x0039f, 0x003a5, 0x003a9, 0x003cf, + 0x00392, 0x00398, 0x003a5, 0x003a5, 0x003a5, 0x003a6, 0x003a0, 0x003cf, + 0x003d8, 0x003d8, 0x003da, 0x003da, 0x003dc, 0x003dc, 0x003de, 0x003de, + 0x003e0, 0x003e0, 0x003e2, 0x003e2, 0x003e4, 0x003e4, 0x003e6, 0x003e6, + 0x003e8, 0x003e8, 0x003ea, 0x003ea, 0x003ec, 0x003ec, 0x003ee, 0x003ee, + 0x0039a, 0x003a1, 0x003a3, 0x003f3, 0x00398, 0x00395, 0x003f6, 0x003f7, + 0x003f7, 0x003a3, 0x003fa, 0x003fa, 0x003fc, 0x0037b, 0x003fe, 0x003ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_04[] = { + 0x00400, 0x00400, 0x00402, 0x00403, 0x00404, 0x00405, 0x00406, 0x00407, + 0x00408, 0x00409, 0x0040a, 0x0040b, 0x0040c, 0x0040d, 0x0040e, 0x0040f, + 0x00410, 0x00411, 0x00412, 0x00413, 0x00414, 0x00400, 0x00416, 0x00417, + 0x0040d, 0x00419, 0x0041a, 0x0041b, 0x0041c, 0x0041d, 0x0041e, 0x0041f, + 0x00420, 0x00421, 0x00422, 0x00423, 0x00424, 0x00425, 0x00426, 0x00427, + 0x00428, 0x00429, 0x0042a, 0x0042b, 0x0042c, 0x0042d, 0x0042e, 0x0042f, + 0x00410, 0x00411, 0x00412, 0x00413, 0x00414, 0x00400, 0x00416, 0x00417, + 0x0040d, 0x00419, 0x0041a, 0x0041b, 0x0041c, 0x0041d, 0x0041e, 0x0041f, + 0x00420, 0x00421, 0x00422, 0x00423, 0x00424, 0x00425, 0x00426, 0x00427, + 0x00428, 0x00429, 0x0042a, 0x0042b, 0x0042c, 0x0042d, 0x0042e, 0x0042f, + 0x00400, 0x00400, 0x00402, 0x00403, 0x00404, 0x00405, 0x00406, 0x00407, + 0x00408, 0x00409, 0x0040a, 0x0040b, 0x0040c, 0x0040d, 0x0040e, 0x0040f, + 0x00460, 0x00460, 0x00462, 0x00462, 0x00464, 0x00464, 0x00466, 0x00466, + 0x00468, 0x00468, 0x0046a, 0x0046a, 0x0046c, 0x0046c, 0x0046e, 0x0046e, + 0x00470, 0x00470, 0x00472, 0x00472, 0x00474, 0x00474, 0x00476, 0x00476, + 0x00478, 0x00478, 0x0047a, 0x0047a, 0x0047c, 0x0047c, 0x0047e, 0x0047e, + 0x00480, 0x00480, 0x00482, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x0048a, 0x0048a, 0x0048c, 0x0048c, 0x0048e, 0x0048e, + 0x00413, 0x00413, 0x00492, 0x00492, 0x00494, 0x00494, 0x00496, 0x00496, + 0x00498, 0x00498, 0x0049a, 0x0049a, 0x0049c, 0x0049c, 0x0049e, 0x0049e, + 0x004a0, 0x004a0, 0x004a2, 0x004a2, 0x004a4, 0x004a4, 0x004a6, 0x004a6, + 0x004a8, 0x004a8, 0x004aa, 0x004aa, 0x004ac, 0x004ac, 0x004ae, 0x004ae, + 0x004b0, 0x004b0, 0x004b2, 0x004b2, 0x004b4, 0x004b4, 0x004b6, 0x004b6, + 0x004b8, 0x004b8, 0x004ba, 0x004ba, 0x004bc, 0x004bc, 0x004be, 0x004be, + 0x004c0, 0x00416, 0x00416, 0x004c3, 0x004c3, 0x004c5, 0x004c5, 0x004c7, + 0x004c7, 0x004c9, 0x004c9, 0x004cb, 0x004cb, 0x004cd, 0x004cd, 0x004c0, + 0x004d0, 0x004d0, 0x004d2, 0x004d2, 0x004d4, 0x004d4, 0x004d6, 0x004d6, + 0x004d8, 0x004d8, 0x004da, 0x004da, 0x004dc, 0x004dc, 0x004de, 0x004de, + 0x004e0, 0x004e0, 0x0040d, 0x0040d, 0x004e4, 0x004e4, 0x004e6, 0x004e6, + 0x004e8, 0x004e8, 0x004ea, 0x004ea, 0x004ec, 0x004ec, 0x00423, 0x00423, + 0x004f0, 0x004f0, 0x004f2, 0x004f2, 0x004f4, 0x004f4, 0x004f6, 0x004f6, + 0x004f8, 0x004f8, 0x004fa, 0x004fa, 0x004fc, 0x004fd, 0x004fe, 0x004ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_05[] = { + 0x00500, 0x00500, 0x00502, 0x00502, 0x00504, 0x00504, 0x00506, 0x00506, + 0x00508, 0x00508, 0x0050a, 0x0050a, 0x0050c, 0x0050c, 0x0050e, 0x0050e, + 0x00510, 0x00510, 0x00512, 0x00512, 0x00514, 0x00514, 0x00516, 0x00516, + 0x00518, 0x00518, 0x0051a, 0x0051a, 0x0051c, 0x0051c, 0x0051e, 0x0051e, + 0x00520, 0x00520, 0x00522, 0x00522, 0x00524, 0x00524, 0x00526, 0x00527, + 0x00528, 0x00529, 0x0052a, 0x0052b, 0x0052c, 0x0052d, 0x0052e, 0x0052f, + 0x00530, 0x00531, 0x00532, 0x00533, 0x00534, 0x00535, 0x00536, 0x00537, + 0x00538, 0x00539, 0x0053a, 0x0053b, 0x0053c, 0x0053d, 0x0053e, 0x0053f, + 0x00540, 0x00541, 0x00542, 0x00543, 0x00544, 0x00545, 0x00546, 0x00547, + 0x00548, 0x00549, 0x0054a, 0x0054b, 0x0054c, 0x0054d, 0x0054e, 0x0054f, + 0x00550, 0x00551, 0x00552, 0x00553, 0x00554, 0x00555, 0x00556, 0x00557, + 0x00558, 0x00559, 0x0055a, 0x0055b, 0x0055c, 0x0055d, 0x0055e, 0x0055f, + 0x00560, 0x00531, 0x00532, 0x00533, 0x00534, 0x00535, 0x00536, 0x00537, + 0x00538, 0x00539, 0x0053a, 0x0053b, 0x0053c, 0x0053d, 0x0053e, 0x0053f, + 0x00540, 0x00541, 0x00542, 0x00543, 0x00544, 0x00545, 0x00546, 0x00547, + 0x00548, 0x00549, 0x0054a, 0x0054b, 0x0054c, 0x0054d, 0x0054e, 0x0054f, + 0x00550, 0x00551, 0x00552, 0x00553, 0x00554, 0x00555, 0x00556, 0x00587, + 0x00588, 0x00589, 0x0058a, 0x0058b, 0x0058c, 0x0058d, 0x0058e, 0x0058f, + 0x00590, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x005be, 0x00000, + 0x005c0, 0x00000, 0x00000, 0x005c3, 0x00000, 0x00000, 0x005c6, 0x00000, + 0x005c8, 0x005c9, 0x005ca, 0x005cb, 0x005cc, 0x005cd, 0x005ce, 0x005cf, + 0x005d0, 0x005d1, 0x005d2, 0x005d3, 0x005d4, 0x005d5, 0x005d6, 0x005d7, + 0x005d8, 0x005d9, 0x005da, 0x005da, 0x005dc, 0x005dd, 0x005dd, 0x005df, + 0x005df, 0x005e1, 0x005e2, 0x005e3, 0x005e3, 0x005e5, 0x005e5, 0x005e7, + 0x005e8, 0x005e9, 0x005ea, 0x005eb, 0x005ec, 0x005ed, 0x005ee, 0x005ef, + 0x005f0, 0x005f1, 0x005f2, 0x005f3, 0x005f4, 0x005f5, 0x005f6, 0x005f7, + 0x005f8, 0x005f9, 0x005fa, 0x005fb, 0x005fc, 0x005fd, 0x005fe, 0x005ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_06[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00604, 0x00605, 0x00606, 0x00607, + 0x00608, 0x00609, 0x0060a, 0x0060b, 0x0060c, 0x0060d, 0x0060e, 0x0060f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x0061b, 0x0061c, 0x0061d, 0x0061e, 0x0061f, + 0x00620, 0x00621, 0x00622, 0x00623, 0x00624, 0x00625, 0x00626, 0x00627, + 0x00628, 0x00629, 0x0062a, 0x0062b, 0x0062c, 0x0062d, 0x0062e, 0x0062f, + 0x00630, 0x00631, 0x00632, 0x00633, 0x00634, 0x00635, 0x00636, 0x00637, + 0x00638, 0x00639, 0x0063a, 0x0063b, 0x0063c, 0x0063d, 0x0063e, 0x0063f, + 0x00640, 0x00641, 0x00642, 0x00643, 0x00644, 0x00645, 0x00646, 0x00647, + 0x00648, 0x00649, 0x0064a, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x0065f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0066a, 0x0066b, 0x0066c, 0x0066d, 0x0066e, 0x0066f, + 0x00000, 0x00671, 0x00672, 0x00673, 0x00621, 0x00675, 0x00676, 0x00677, + 0x00678, 0x00679, 0x0067a, 0x0067b, 0x0067c, 0x0067d, 0x0067e, 0x0067f, + 0x00680, 0x00681, 0x00682, 0x00683, 0x00684, 0x00685, 0x00686, 0x00687, + 0x00688, 0x00689, 0x0068a, 0x0068b, 0x0068c, 0x0068d, 0x0068e, 0x0068f, + 0x00690, 0x00691, 0x00692, 0x00693, 0x00694, 0x00695, 0x00696, 0x00697, + 0x00698, 0x00699, 0x0069a, 0x0069b, 0x0069c, 0x0069d, 0x0069e, 0x0069f, + 0x006a0, 0x006a1, 0x006a2, 0x006a3, 0x006a4, 0x006a5, 0x006a6, 0x006a7, + 0x006a8, 0x006a9, 0x006aa, 0x006ab, 0x006ac, 0x006ad, 0x006ae, 0x006af, + 0x006b0, 0x006b1, 0x006b2, 0x006b3, 0x006b4, 0x006b5, 0x006b6, 0x006b7, + 0x006b8, 0x006b9, 0x006ba, 0x006bb, 0x006bc, 0x006bd, 0x006be, 0x006bf, + 0x006c0, 0x006c1, 0x006c1, 0x006c3, 0x006c4, 0x006c5, 0x006c6, 0x006c7, + 0x006c8, 0x006c9, 0x006ca, 0x006cb, 0x006cc, 0x006cd, 0x006ce, 0x006cf, + 0x006d0, 0x006d1, 0x006d2, 0x006d2, 0x006d4, 0x006c0, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00648, 0x0064a, 0x00000, + 0x00000, 0x006e9, 0x00000, 0x00000, 0x00000, 0x00000, 0x006ee, 0x006ef, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x006fa, 0x006fb, 0x006fc, 0x00621, 0x00645, 0x006ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_07[] = { + 0x00700, 0x00701, 0x00702, 0x00703, 0x00704, 0x00705, 0x00706, 0x00707, + 0x00708, 0x00709, 0x0070a, 0x0070b, 0x0070c, 0x0070d, 0x0070e, 0x00000, + 0x00710, 0x00000, 0x00712, 0x00713, 0x00713, 0x00715, 0x00716, 0x00717, + 0x00718, 0x00719, 0x0071a, 0x0071b, 0x0071b, 0x0071d, 0x0071e, 0x0071f, + 0x00720, 0x00721, 0x00722, 0x00723, 0x00723, 0x00725, 0x00726, 0x00726, + 0x00728, 0x00729, 0x0072a, 0x0072b, 0x0072c, 0x00712, 0x00713, 0x00715, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x0074b, 0x0074c, 0x0074d, 0x0074e, 0x0074f, + 0x00750, 0x00751, 0x00752, 0x00753, 0x00754, 0x00755, 0x00756, 0x00757, + 0x00758, 0x00759, 0x0075a, 0x0075b, 0x0075c, 0x0075d, 0x0075e, 0x0075f, + 0x00760, 0x00761, 0x00762, 0x00763, 0x00764, 0x00765, 0x00766, 0x00767, + 0x00768, 0x00769, 0x0076a, 0x0076b, 0x0076c, 0x0076d, 0x0076e, 0x0076f, + 0x00770, 0x00771, 0x00772, 0x00773, 0x00774, 0x00775, 0x00776, 0x00777, + 0x00778, 0x00779, 0x0077a, 0x0077b, 0x0077c, 0x0077d, 0x0077e, 0x0077f, + 0x00780, 0x00781, 0x00782, 0x00783, 0x00784, 0x00785, 0x00786, 0x00787, + 0x00788, 0x00789, 0x0078a, 0x0078b, 0x0078c, 0x0078d, 0x0078e, 0x0078f, + 0x00790, 0x00791, 0x00792, 0x00793, 0x00794, 0x00795, 0x00796, 0x00797, + 0x00798, 0x00799, 0x0079a, 0x0079b, 0x0079c, 0x0079d, 0x0079e, 0x0079f, + 0x007a0, 0x007a1, 0x007a2, 0x007a3, 0x007a4, 0x007a5, 0x007a6, 0x007a7, + 0x007a8, 0x007a9, 0x007aa, 0x007ab, 0x007ac, 0x007ad, 0x007ae, 0x007af, + 0x007b0, 0x007b1, 0x007b2, 0x007b3, 0x007b4, 0x007b5, 0x007b6, 0x007b7, + 0x007b8, 0x007b9, 0x007ba, 0x007bb, 0x007bc, 0x007bd, 0x007be, 0x007bf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x007ca, 0x007cb, 0x007cc, 0x007cd, 0x007ce, 0x007cf, + 0x007d0, 0x007d1, 0x007d2, 0x007d3, 0x007d4, 0x007d5, 0x007d6, 0x007d7, + 0x007d8, 0x007d9, 0x007da, 0x007db, 0x007dc, 0x007dd, 0x007de, 0x007df, + 0x007e0, 0x007e1, 0x007e2, 0x007e3, 0x007e4, 0x007e5, 0x007e6, 0x007e7, + 0x007d6, 0x007d7, 0x007d9, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x007f4, 0x007f5, 0x007f6, 0x007f7, + 0x007f8, 0x007f9, 0x007fa, 0x007fb, 0x007fc, 0x007fd, 0x007fe, 0x007ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_08[] = { + 0x00800, 0x00801, 0x00802, 0x00803, 0x00804, 0x00805, 0x00806, 0x00807, + 0x00808, 0x00809, 0x0080a, 0x0080b, 0x0080c, 0x0080d, 0x0080e, 0x0080f, + 0x00810, 0x00811, 0x00812, 0x00813, 0x00814, 0x00815, 0x00816, 0x00817, + 0x00000, 0x00000, 0x0081a, 0x0081b, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x0082e, 0x0082f, + 0x00830, 0x00831, 0x00832, 0x00833, 0x00834, 0x00835, 0x00836, 0x00837, + 0x00838, 0x00839, 0x0083a, 0x0083b, 0x0083c, 0x0083d, 0x0083e, 0x0083f, + 0x00840, 0x00841, 0x00842, 0x00843, 0x00844, 0x00845, 0x00846, 0x00847, + 0x00848, 0x00849, 0x0084a, 0x0084b, 0x0084c, 0x0084d, 0x0084e, 0x0084f, + 0x00850, 0x00851, 0x00852, 0x00853, 0x00854, 0x00855, 0x00856, 0x00857, + 0x00858, 0x00859, 0x0085a, 0x0085b, 0x0085c, 0x0085d, 0x0085e, 0x0085f, + 0x00860, 0x00861, 0x00862, 0x00863, 0x00864, 0x00865, 0x00866, 0x00867, + 0x00868, 0x00869, 0x0086a, 0x0086b, 0x0086c, 0x0086d, 0x0086e, 0x0086f, + 0x00870, 0x00871, 0x00872, 0x00873, 0x00874, 0x00875, 0x00876, 0x00877, + 0x00878, 0x00879, 0x0087a, 0x0087b, 0x0087c, 0x0087d, 0x0087e, 0x0087f, + 0x00880, 0x00881, 0x00882, 0x00883, 0x00884, 0x00885, 0x00886, 0x00887, + 0x00888, 0x00889, 0x0088a, 0x0088b, 0x0088c, 0x0088d, 0x0088e, 0x0088f, + 0x00890, 0x00891, 0x00892, 0x00893, 0x00894, 0x00895, 0x00896, 0x00897, + 0x00898, 0x00899, 0x0089a, 0x0089b, 0x0089c, 0x0089d, 0x0089e, 0x0089f, + 0x008a0, 0x008a1, 0x008a2, 0x008a3, 0x008a4, 0x008a5, 0x008a6, 0x008a7, + 0x008a8, 0x008a9, 0x008aa, 0x008ab, 0x008ac, 0x008ad, 0x008ae, 0x008af, + 0x008b0, 0x008b1, 0x008b2, 0x008b3, 0x008b4, 0x008b5, 0x008b6, 0x008b7, + 0x008b8, 0x008b9, 0x008ba, 0x008bb, 0x008bc, 0x008bd, 0x008be, 0x008bf, + 0x008c0, 0x008c1, 0x008c2, 0x008c3, 0x008c4, 0x008c5, 0x008c6, 0x008c7, + 0x008c8, 0x008c9, 0x008ca, 0x008cb, 0x008cc, 0x008cd, 0x008ce, 0x008cf, + 0x008d0, 0x008d1, 0x008d2, 0x008d3, 0x008d4, 0x008d5, 0x008d6, 0x008d7, + 0x008d8, 0x008d9, 0x008da, 0x008db, 0x008dc, 0x008dd, 0x008de, 0x008df, + 0x008e0, 0x008e1, 0x008e2, 0x008e3, 0x008e4, 0x008e5, 0x008e6, 0x008e7, + 0x008e8, 0x008e9, 0x008ea, 0x008eb, 0x008ec, 0x008ed, 0x008ee, 0x008ef, + 0x008f0, 0x008f1, 0x008f2, 0x008f3, 0x008f4, 0x008f5, 0x008f6, 0x008f7, + 0x008f8, 0x008f9, 0x008fa, 0x008fb, 0x008fc, 0x008fd, 0x008fe, 0x008ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_09[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00904, 0x00905, 0x00906, 0x00907, + 0x00908, 0x00909, 0x0090a, 0x0090b, 0x0090c, 0x0090d, 0x0090e, 0x0090f, + 0x00910, 0x00911, 0x00912, 0x00913, 0x00914, 0x00915, 0x00916, 0x00917, + 0x00918, 0x00919, 0x0091a, 0x0091b, 0x0091c, 0x0091d, 0x0091e, 0x0091f, + 0x00920, 0x00921, 0x00922, 0x00923, 0x00924, 0x00925, 0x00926, 0x00927, + 0x00928, 0x00928, 0x0092a, 0x0092b, 0x0092c, 0x0092d, 0x0092e, 0x0092f, + 0x00930, 0x00930, 0x00932, 0x00933, 0x00933, 0x00935, 0x00936, 0x00937, + 0x00938, 0x00939, 0x0093a, 0x0093b, 0x00000, 0x0093d, 0x0093e, 0x0093f, + 0x00940, 0x00941, 0x00942, 0x00943, 0x00944, 0x00945, 0x00946, 0x00947, + 0x00948, 0x00949, 0x0094a, 0x0094b, 0x0094c, 0x0094d, 0x0094e, 0x0094f, + 0x00950, 0x00000, 0x00000, 0x00000, 0x00000, 0x00955, 0x00956, 0x00957, + 0x00915, 0x00916, 0x00917, 0x0091c, 0x00921, 0x00922, 0x0092b, 0x0092f, + 0x00960, 0x00961, 0x00962, 0x00963, 0x00964, 0x00965, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00970, 0x00971, 0x00972, 0x00973, 0x00974, 0x00975, 0x00976, 0x00977, + 0x00978, 0x00979, 0x0097a, 0x0097b, 0x0097c, 0x0097d, 0x0097e, 0x0097f, + 0x00980, 0x00000, 0x00000, 0x00000, 0x00984, 0x00985, 0x00986, 0x00987, + 0x00988, 0x00989, 0x0098a, 0x0098b, 0x0098c, 0x0098d, 0x0098e, 0x0098f, + 0x00990, 0x00991, 0x00992, 0x00993, 0x00994, 0x00995, 0x00996, 0x00997, + 0x00998, 0x00999, 0x0099a, 0x0099b, 0x0099c, 0x0099d, 0x0099e, 0x0099f, + 0x009a0, 0x009a1, 0x009a2, 0x009a3, 0x009a4, 0x009a5, 0x009a6, 0x009a7, + 0x009a8, 0x009a9, 0x009aa, 0x009ab, 0x009ac, 0x009ad, 0x009ae, 0x009af, + 0x009b0, 0x009b1, 0x009b2, 0x009b3, 0x009b4, 0x009b5, 0x009b6, 0x009b7, + 0x009b8, 0x009b9, 0x009ba, 0x009bb, 0x00000, 0x009bd, 0x009be, 0x009bf, + 0x009c0, 0x009c1, 0x009c2, 0x009c3, 0x009c4, 0x009c5, 0x009c6, 0x009c7, + 0x009c8, 0x009c9, 0x009ca, 0x009cb, 0x009cc, 0x009cd, 0x009ce, 0x009cf, + 0x009d0, 0x009d1, 0x009d2, 0x009d3, 0x009d4, 0x009d5, 0x009d6, 0x009d7, + 0x009d8, 0x009d9, 0x009da, 0x009db, 0x009a1, 0x009a2, 0x009de, 0x009af, + 0x009e0, 0x009e1, 0x009e2, 0x009e3, 0x009e4, 0x009e5, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x009f0, 0x009f1, 0x009f2, 0x009f3, 0x009f4, 0x009f5, 0x009f6, 0x009f7, + 0x009f8, 0x009f9, 0x009fa, 0x009fb, 0x009fc, 0x009fd, 0x009fe, 0x009ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0a[] = { + 0x00a00, 0x00000, 0x00000, 0x00000, 0x00a04, 0x00a05, 0x00a06, 0x00a07, + 0x00a08, 0x00a09, 0x00a0a, 0x00a0b, 0x00a0c, 0x00a0d, 0x00a0e, 0x00a0f, + 0x00a10, 0x00a11, 0x00a12, 0x00a13, 0x00a14, 0x00a15, 0x00a16, 0x00a17, + 0x00a18, 0x00a19, 0x00a1a, 0x00a1b, 0x00a1c, 0x00a1d, 0x00a1e, 0x00a1f, + 0x00a20, 0x00a21, 0x00a22, 0x00a23, 0x00a24, 0x00a25, 0x00a26, 0x00a27, + 0x00a28, 0x00a29, 0x00a2a, 0x00a2b, 0x00a2c, 0x00a2d, 0x00a2e, 0x00a2f, + 0x00a30, 0x00a31, 0x00a32, 0x00a32, 0x00a34, 0x00a35, 0x00a36, 0x00a37, + 0x00a36, 0x00a39, 0x00a3a, 0x00a3b, 0x00000, 0x00a3d, 0x00a3e, 0x00a3f, + 0x00a40, 0x00a41, 0x00a42, 0x00a43, 0x00a44, 0x00a45, 0x00a46, 0x00a47, + 0x00a48, 0x00a49, 0x00a4a, 0x00a4b, 0x00a4c, 0x00a4d, 0x00a4e, 0x00a4f, + 0x00a50, 0x00a51, 0x00a52, 0x00a53, 0x00a54, 0x00a55, 0x00a56, 0x00a57, + 0x00a58, 0x00a16, 0x00a17, 0x00a1c, 0x00a5c, 0x00a5d, 0x00a2b, 0x00a5f, + 0x00a60, 0x00a61, 0x00a62, 0x00a63, 0x00a64, 0x00a65, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00000, 0x00000, 0x00a72, 0x00a73, 0x00a74, 0x00a75, 0x00a76, 0x00a77, + 0x00a78, 0x00a79, 0x00a7a, 0x00a7b, 0x00a7c, 0x00a7d, 0x00a7e, 0x00a7f, + 0x00a80, 0x00000, 0x00000, 0x00000, 0x00a84, 0x00a85, 0x00a86, 0x00a87, + 0x00a88, 0x00a89, 0x00a8a, 0x00a8b, 0x00a8c, 0x00a8d, 0x00a8e, 0x00a8f, + 0x00a90, 0x00a91, 0x00a92, 0x00a93, 0x00a94, 0x00a95, 0x00a96, 0x00a97, + 0x00a98, 0x00a99, 0x00a9a, 0x00a9b, 0x00a9c, 0x00a9d, 0x00a9e, 0x00a9f, + 0x00aa0, 0x00aa1, 0x00aa2, 0x00aa3, 0x00aa4, 0x00aa5, 0x00aa6, 0x00aa7, + 0x00aa8, 0x00aa9, 0x00aaa, 0x00aab, 0x00aac, 0x00aad, 0x00aae, 0x00aaf, + 0x00ab0, 0x00ab1, 0x00ab2, 0x00ab3, 0x00ab4, 0x00ab5, 0x00ab6, 0x00ab7, + 0x00ab8, 0x00ab9, 0x00aba, 0x00abb, 0x00000, 0x00abd, 0x00abe, 0x00abf, + 0x00ac0, 0x00ac1, 0x00ac2, 0x00ac3, 0x00ac4, 0x00ac5, 0x00ac6, 0x00ac7, + 0x00ac8, 0x00ac9, 0x00aca, 0x00acb, 0x00acc, 0x00acd, 0x00ace, 0x00acf, + 0x00ad0, 0x00ad1, 0x00ad2, 0x00ad3, 0x00ad4, 0x00ad5, 0x00ad6, 0x00ad7, + 0x00ad8, 0x00ad9, 0x00ada, 0x00adb, 0x00adc, 0x00add, 0x00ade, 0x00adf, + 0x00ae0, 0x00ae1, 0x00ae2, 0x00ae3, 0x00ae4, 0x00ae5, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00af0, 0x00af1, 0x00af2, 0x00af3, 0x00af4, 0x00af5, 0x00af6, 0x00af7, + 0x00af8, 0x00af9, 0x00afa, 0x00afb, 0x00afc, 0x00afd, 0x00afe, 0x00aff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0b[] = { + 0x00b00, 0x00000, 0x00000, 0x00000, 0x00b04, 0x00b05, 0x00b06, 0x00b07, + 0x00b08, 0x00b09, 0x00b0a, 0x00b0b, 0x00b0c, 0x00b0d, 0x00b0e, 0x00b0f, + 0x00b10, 0x00b11, 0x00b12, 0x00b13, 0x00b14, 0x00b15, 0x00b16, 0x00b17, + 0x00b18, 0x00b19, 0x00b1a, 0x00b1b, 0x00b1c, 0x00b1d, 0x00b1e, 0x00b1f, + 0x00b20, 0x00b21, 0x00b22, 0x00b23, 0x00b24, 0x00b25, 0x00b26, 0x00b27, + 0x00b28, 0x00b29, 0x00b2a, 0x00b2b, 0x00b2c, 0x00b2d, 0x00b2e, 0x00b2f, + 0x00b30, 0x00b31, 0x00b32, 0x00b33, 0x00b34, 0x00b35, 0x00b36, 0x00b37, + 0x00b38, 0x00b39, 0x00b3a, 0x00b3b, 0x00000, 0x00b3d, 0x00b3e, 0x00b3f, + 0x00b40, 0x00b41, 0x00b42, 0x00b43, 0x00b44, 0x00b45, 0x00b46, 0x00b47, + 0x00b48, 0x00b49, 0x00b4a, 0x00b4b, 0x00b4c, 0x00b4d, 0x00b4e, 0x00b4f, + 0x00b50, 0x00b51, 0x00b52, 0x00b53, 0x00b54, 0x00b55, 0x00b56, 0x00b57, + 0x00b58, 0x00b59, 0x00b5a, 0x00b5b, 0x00b21, 0x00b22, 0x00b5e, 0x00b5f, + 0x00b60, 0x00b61, 0x00b62, 0x00b63, 0x00b64, 0x00b65, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00b70, 0x00b71, 0x00b72, 0x00b73, 0x00b74, 0x00b75, 0x00b76, 0x00b77, + 0x00b78, 0x00b79, 0x00b7a, 0x00b7b, 0x00b7c, 0x00b7d, 0x00b7e, 0x00b7f, + 0x00b80, 0x00b81, 0x00000, 0x00b83, 0x00b84, 0x00b85, 0x00b86, 0x00b87, + 0x00b88, 0x00b89, 0x00b8a, 0x00b8b, 0x00b8c, 0x00b8d, 0x00b8e, 0x00b8f, + 0x00b90, 0x00b91, 0x00b92, 0x00b93, 0x00b94, 0x00b95, 0x00b96, 0x00b97, + 0x00b98, 0x00b99, 0x00b9a, 0x00b9b, 0x00b9c, 0x00b9d, 0x00b9e, 0x00b9f, + 0x00ba0, 0x00ba1, 0x00ba2, 0x00ba3, 0x00ba4, 0x00ba5, 0x00ba6, 0x00ba7, + 0x00ba8, 0x00ba9, 0x00baa, 0x00bab, 0x00bac, 0x00bad, 0x00bae, 0x00baf, + 0x00bb0, 0x00bb1, 0x00bb2, 0x00bb3, 0x00bb4, 0x00bb5, 0x00bb6, 0x00bb7, + 0x00bb8, 0x00bb9, 0x00bba, 0x00bbb, 0x00bbc, 0x00bbd, 0x00bbe, 0x00bbf, + 0x00bc0, 0x00bc1, 0x00bc2, 0x00bc3, 0x00bc4, 0x00bc5, 0x00bc6, 0x00bc7, + 0x00bc8, 0x00bc9, 0x00bca, 0x00bcb, 0x00bcc, 0x00bcd, 0x00bce, 0x00bcf, + 0x00bd0, 0x00bd1, 0x00bd2, 0x00bd3, 0x00bd4, 0x00bd5, 0x00bd6, 0x00bd7, + 0x00bd8, 0x00bd9, 0x00bda, 0x00bdb, 0x00bdc, 0x00bdd, 0x00bde, 0x00bdf, + 0x00be0, 0x00be1, 0x00be2, 0x00be3, 0x00be4, 0x00be5, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00bf0, 0x00bf1, 0x00bf2, 0x00bf3, 0x00bf4, 0x00bf5, 0x00bf6, 0x00bf7, + 0x00bf8, 0x00bf9, 0x00bfa, 0x00bfb, 0x00bfc, 0x00bfd, 0x00bfe, 0x00bff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0c[] = { + 0x00c00, 0x00000, 0x00000, 0x00000, 0x00c04, 0x00c05, 0x00c06, 0x00c07, + 0x00c08, 0x00c09, 0x00c0a, 0x00c0b, 0x00c0c, 0x00c0d, 0x00c0e, 0x00c0f, + 0x00c10, 0x00c11, 0x00c12, 0x00c13, 0x00c14, 0x00c15, 0x00c16, 0x00c17, + 0x00c18, 0x00c19, 0x00c1a, 0x00c1b, 0x00c1c, 0x00c1d, 0x00c1e, 0x00c1f, + 0x00c20, 0x00c21, 0x00c22, 0x00c23, 0x00c24, 0x00c25, 0x00c26, 0x00c27, + 0x00c28, 0x00c29, 0x00c2a, 0x00c2b, 0x00c2c, 0x00c2d, 0x00c2e, 0x00c2f, + 0x00c30, 0x00c31, 0x00c32, 0x00c33, 0x00c34, 0x00c35, 0x00c36, 0x00c37, + 0x00c38, 0x00c39, 0x00c3a, 0x00c3b, 0x00c3c, 0x00c3d, 0x00c3e, 0x00c3f, + 0x00c40, 0x00c41, 0x00c42, 0x00c43, 0x00c44, 0x00c45, 0x00c46, 0x00c47, + 0x00c48, 0x00c49, 0x00c4a, 0x00c4b, 0x00c4c, 0x00c4d, 0x00c4e, 0x00c4f, + 0x00c50, 0x00c51, 0x00c52, 0x00c53, 0x00c54, 0x00c55, 0x00c56, 0x00c57, + 0x00c58, 0x00c59, 0x00c5a, 0x00c5b, 0x00c5c, 0x00c5d, 0x00c5e, 0x00c5f, + 0x00c60, 0x00c61, 0x00c62, 0x00c63, 0x00c64, 0x00c65, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00c70, 0x00c71, 0x00c72, 0x00c73, 0x00c74, 0x00c75, 0x00c76, 0x00c77, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00031, 0x00032, 0x00033, 0x00c7f, + 0x00c80, 0x00c81, 0x00000, 0x00000, 0x00c84, 0x00c85, 0x00c86, 0x00c87, + 0x00c88, 0x00c89, 0x00c8a, 0x00c8b, 0x00c8c, 0x00c8d, 0x00c8e, 0x00c8f, + 0x00c90, 0x00c91, 0x00c92, 0x00c93, 0x00c94, 0x00c95, 0x00c96, 0x00c97, + 0x00c98, 0x00c99, 0x00c9a, 0x00c9b, 0x00c9c, 0x00c9d, 0x00c9e, 0x00c9f, + 0x00ca0, 0x00ca1, 0x00ca2, 0x00ca3, 0x00ca4, 0x00ca5, 0x00ca6, 0x00ca7, + 0x00ca8, 0x00ca9, 0x00caa, 0x00cab, 0x00cac, 0x00cad, 0x00cae, 0x00caf, + 0x00cb0, 0x00cb1, 0x00cb2, 0x00cb3, 0x00cb4, 0x00cb5, 0x00cb6, 0x00cb7, + 0x00cb8, 0x00cb9, 0x00cba, 0x00cbb, 0x00000, 0x00cbd, 0x00cbe, 0x00cbf, + 0x00cc0, 0x00cc1, 0x00cc2, 0x00cc3, 0x00cc4, 0x00cc5, 0x00cc6, 0x00cc7, + 0x00cc8, 0x00cc9, 0x00cca, 0x00ccb, 0x00ccc, 0x00ccd, 0x00cce, 0x00ccf, + 0x00cd0, 0x00cd1, 0x00cd2, 0x00cd3, 0x00cd4, 0x00cd5, 0x00cd6, 0x00cd7, + 0x00cd8, 0x00cd9, 0x00cda, 0x00cdb, 0x00cdc, 0x00cdd, 0x00cde, 0x00cdf, + 0x00ce0, 0x00ce1, 0x00ce2, 0x00ce3, 0x00ce4, 0x00ce5, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00cf0, 0x00cf1, 0x00cf2, 0x00cf3, 0x00cf4, 0x00cf5, 0x00cf6, 0x00cf7, + 0x00cf8, 0x00cf9, 0x00cfa, 0x00cfb, 0x00cfc, 0x00cfd, 0x00cfe, 0x00cff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0d[] = { + 0x00d00, 0x00d01, 0x00000, 0x00000, 0x00d04, 0x00d05, 0x00d06, 0x00d07, + 0x00d08, 0x00d09, 0x00d0a, 0x00d0b, 0x00d0c, 0x00d0d, 0x00d0e, 0x00d0f, + 0x00d10, 0x00d11, 0x00d12, 0x00d13, 0x00d14, 0x00d15, 0x00d16, 0x00d17, + 0x00d18, 0x00d19, 0x00d1a, 0x00d1b, 0x00d1c, 0x00d1d, 0x00d1e, 0x00d1f, + 0x00d20, 0x00d21, 0x00d22, 0x00d23, 0x00d24, 0x00d25, 0x00d26, 0x00d27, + 0x00d28, 0x00d29, 0x00d2a, 0x00d2b, 0x00d2c, 0x00d2d, 0x00d2e, 0x00d2f, + 0x00d30, 0x00d31, 0x00d32, 0x00d33, 0x00d34, 0x00d35, 0x00d36, 0x00d37, + 0x00d38, 0x00d39, 0x00d3a, 0x00d3b, 0x00d3c, 0x00d3d, 0x00d3e, 0x00d3f, + 0x00d40, 0x00d41, 0x00d42, 0x00d43, 0x00d44, 0x00d45, 0x00d46, 0x00d47, + 0x00d48, 0x00d49, 0x00d4a, 0x00d4b, 0x00d4c, 0x00d4d, 0x00d4e, 0x00d4f, + 0x00d50, 0x00d51, 0x00d52, 0x00d53, 0x00d54, 0x00d55, 0x00d56, 0x00d57, + 0x00d58, 0x00d59, 0x00d5a, 0x00d5b, 0x00d5c, 0x00d5d, 0x00d5e, 0x00d5f, + 0x00d60, 0x00d61, 0x00d62, 0x00d63, 0x00d64, 0x00d65, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00d70, 0x00d71, 0x00d72, 0x00d73, 0x00d74, 0x00d75, 0x00d76, 0x00d77, + 0x00d78, 0x00d79, 0x00d7a, 0x00d7b, 0x00d7c, 0x00d7d, 0x00d7e, 0x00d7f, + 0x00d80, 0x00d81, 0x00000, 0x00000, 0x00d84, 0x00d85, 0x00d86, 0x00d87, + 0x00d88, 0x00d89, 0x00d8a, 0x00d8b, 0x00d8c, 0x00d8d, 0x00d8e, 0x00d8f, + 0x00d90, 0x00d91, 0x00d92, 0x00d93, 0x00d94, 0x00d95, 0x00d96, 0x00d97, + 0x00d98, 0x00d99, 0x00d9a, 0x00d9b, 0x00d9c, 0x00d9d, 0x00d9e, 0x00d9f, + 0x00da0, 0x00da1, 0x00da2, 0x00da3, 0x00da4, 0x00da5, 0x00da6, 0x00da7, + 0x00da8, 0x00da9, 0x00daa, 0x00dab, 0x00dac, 0x00dad, 0x00dae, 0x00daf, + 0x00db0, 0x00db1, 0x00db2, 0x00db3, 0x00db4, 0x00db5, 0x00db6, 0x00db7, + 0x00db8, 0x00db9, 0x00dba, 0x00dbb, 0x00dbc, 0x00dbd, 0x00dbe, 0x00dbf, + 0x00dc0, 0x00dc1, 0x00dc2, 0x00dc3, 0x00dc4, 0x00dc5, 0x00dc6, 0x00dc7, + 0x00dc8, 0x00dc9, 0x00dca, 0x00dcb, 0x00dcc, 0x00dcd, 0x00dce, 0x00dcf, + 0x00dd0, 0x00dd1, 0x00dd2, 0x00dd3, 0x00dd4, 0x00dd5, 0x00dd6, 0x00dd7, + 0x00dd8, 0x00dd9, 0x00dda, 0x00ddb, 0x00ddc, 0x00ddd, 0x00dde, 0x00ddf, + 0x00de0, 0x00de1, 0x00de2, 0x00de3, 0x00de4, 0x00de5, 0x00de6, 0x00de7, + 0x00de8, 0x00de9, 0x00dea, 0x00deb, 0x00dec, 0x00ded, 0x00dee, 0x00def, + 0x00df0, 0x00df1, 0x00df2, 0x00df3, 0x00df4, 0x00df5, 0x00df6, 0x00df7, + 0x00df8, 0x00df9, 0x00dfa, 0x00dfb, 0x00dfc, 0x00dfd, 0x00dfe, 0x00dff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0e[] = { + 0x00e00, 0x00e01, 0x00e02, 0x00e03, 0x00e04, 0x00e05, 0x00e06, 0x00e07, + 0x00e08, 0x00e09, 0x00e0a, 0x00e0b, 0x00e0c, 0x00e0d, 0x00e0e, 0x00e0f, + 0x00e10, 0x00e11, 0x00e12, 0x00e13, 0x00e14, 0x00e15, 0x00e16, 0x00e17, + 0x00e18, 0x00e19, 0x00e1a, 0x00e1b, 0x00e1c, 0x00e1d, 0x00e1e, 0x00e1f, + 0x00e20, 0x00e21, 0x00e22, 0x00e23, 0x00e24, 0x00e25, 0x00e26, 0x00e27, + 0x00e28, 0x00e29, 0x00e2a, 0x00e2b, 0x00e2c, 0x00e2d, 0x00e2e, 0x00e2f, + 0x00e30, 0x00e31, 0x00e32, 0x00e33, 0x00e34, 0x00e35, 0x00e36, 0x00e37, + 0x00e38, 0x00e39, 0x00e3a, 0x00e3b, 0x00e3c, 0x00e3d, 0x00e3e, 0x00e3f, + 0x00e40, 0x00e41, 0x00e42, 0x00e43, 0x00e44, 0x00e45, 0x00e46, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00e4f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00e5a, 0x00e5b, 0x00e5c, 0x00e5d, 0x00e5e, 0x00e5f, + 0x00e60, 0x00e61, 0x00e62, 0x00e63, 0x00e64, 0x00e65, 0x00e66, 0x00e67, + 0x00e68, 0x00e69, 0x00e6a, 0x00e6b, 0x00e6c, 0x00e6d, 0x00e6e, 0x00e6f, + 0x00e70, 0x00e71, 0x00e72, 0x00e73, 0x00e74, 0x00e75, 0x00e76, 0x00e77, + 0x00e78, 0x00e79, 0x00e7a, 0x00e7b, 0x00e7c, 0x00e7d, 0x00e7e, 0x00e7f, + 0x00e80, 0x00e81, 0x00e82, 0x00e83, 0x00e84, 0x00e85, 0x00e86, 0x00e87, + 0x00e88, 0x00e89, 0x00e8a, 0x00e8b, 0x00e8c, 0x00e8d, 0x00e8e, 0x00e8f, + 0x00e90, 0x00e91, 0x00e92, 0x00e93, 0x00e94, 0x00e95, 0x00e96, 0x00e97, + 0x00e98, 0x00e99, 0x00e9a, 0x00e9b, 0x00e9c, 0x00e9d, 0x00e9e, 0x00e9f, + 0x00ea0, 0x00ea1, 0x00ea2, 0x00ea3, 0x00ea4, 0x00ea5, 0x00ea6, 0x00ea7, + 0x00ea8, 0x00ea9, 0x00eaa, 0x00eab, 0x00eac, 0x00ead, 0x00eae, 0x00eaf, + 0x00eb0, 0x00eb1, 0x00eb2, 0x00eb3, 0x00eb4, 0x00eb5, 0x00eb6, 0x00eb7, + 0x00eb8, 0x00eb9, 0x00eba, 0x00ebb, 0x00ebc, 0x00ebd, 0x00ebe, 0x00ebf, + 0x00ec0, 0x00ec1, 0x00ec2, 0x00ec3, 0x00ec4, 0x00ec5, 0x00ec6, 0x00ec7, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00ece, 0x00ecf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00eda, 0x00edb, 0x00edc, 0x00edd, 0x00ede, 0x00edf, + 0x00ee0, 0x00ee1, 0x00ee2, 0x00ee3, 0x00ee4, 0x00ee5, 0x00ee6, 0x00ee7, + 0x00ee8, 0x00ee9, 0x00eea, 0x00eeb, 0x00eec, 0x00eed, 0x00eee, 0x00eef, + 0x00ef0, 0x00ef1, 0x00ef2, 0x00ef3, 0x00ef4, 0x00ef5, 0x00ef6, 0x00ef7, + 0x00ef8, 0x00ef9, 0x00efa, 0x00efb, 0x00efc, 0x00efd, 0x00efe, 0x00eff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0f[] = { + 0x00f00, 0x00f01, 0x00f02, 0x00f03, 0x00f04, 0x00f05, 0x00f06, 0x00f07, + 0x00f08, 0x00f09, 0x00f0a, 0x00f0b, 0x00f0b, 0x00f0d, 0x00f0e, 0x00f0f, + 0x00f10, 0x00f11, 0x00f12, 0x00f13, 0x00f14, 0x00f15, 0x00f16, 0x00f17, + 0x00000, 0x00000, 0x00f1a, 0x00f1b, 0x00f1c, 0x00f1d, 0x00f1e, 0x00f1f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, + 0x00037, 0x00038, 0x00039, 0x00030, 0x00f34, 0x00000, 0x00f36, 0x00000, + 0x00f38, 0x00000, 0x00f3a, 0x00f3b, 0x00f3c, 0x00f3d, 0x00f3e, 0x00f3f, + 0x00f40, 0x00f41, 0x00f42, 0x00f43, 0x00f44, 0x00f45, 0x00f46, 0x00f47, + 0x00f48, 0x00f49, 0x00f4a, 0x00f4b, 0x00f4c, 0x00f4d, 0x00f4e, 0x00f4f, + 0x00f50, 0x00f51, 0x00f52, 0x00f53, 0x00f54, 0x00f55, 0x00f56, 0x00f57, + 0x00f58, 0x00f59, 0x00f5a, 0x00f5b, 0x00f5c, 0x00f5d, 0x00f5e, 0x00f5f, + 0x00f60, 0x00f61, 0x00f62, 0x00f63, 0x00f64, 0x00f65, 0x00f66, 0x00f67, + 0x00f68, 0x00f69, 0x00f62, 0x00f6b, 0x00f6c, 0x00f6d, 0x00f6e, 0x00f6f, + 0x00f70, 0x00f71, 0x00f72, 0x00f73, 0x00f74, 0x00f75, 0x00f76, 0x00f77, + 0x00f78, 0x00f79, 0x00f7a, 0x00f7b, 0x00f7c, 0x00f7d, 0x00000, 0x00000, + 0x00f80, 0x00f81, 0x00000, 0x00000, 0x00f84, 0x00f85, 0x00000, 0x00000, + 0x00f88, 0x00f89, 0x00f8a, 0x00f8b, 0x00f8c, 0x00f8d, 0x00f8e, 0x00f8f, + 0x00f90, 0x00f91, 0x00f92, 0x00f93, 0x00f94, 0x00f95, 0x00f96, 0x00f97, + 0x00f98, 0x00f99, 0x00f9a, 0x00f9b, 0x00f9c, 0x00f9d, 0x00f9e, 0x00f9f, + 0x00fa0, 0x00fa1, 0x00fa2, 0x00fa3, 0x00fa4, 0x00fa5, 0x00fa6, 0x00fa7, + 0x00fa8, 0x00fa9, 0x00faa, 0x00fab, 0x00fac, 0x00fad, 0x00fae, 0x00faf, + 0x00fb0, 0x00fb1, 0x00fb2, 0x00fb3, 0x00fb4, 0x00fb5, 0x00fb6, 0x00fb7, + 0x00fb8, 0x00fb9, 0x00fad, 0x00fb1, 0x00fb2, 0x00fbd, 0x00fbe, 0x00fbf, + 0x00fc0, 0x00fc1, 0x00fc2, 0x00fc3, 0x00fc4, 0x00fc5, 0x00000, 0x00fc7, + 0x00fc8, 0x00fc9, 0x00fca, 0x00fcb, 0x00fcc, 0x00fcd, 0x00fce, 0x00fcf, + 0x00fd0, 0x00fd1, 0x00fd2, 0x00fd3, 0x00fd4, 0x00fd5, 0x00fd6, 0x00fd7, + 0x00fd8, 0x00fd9, 0x00fda, 0x00fdb, 0x00fdc, 0x00fdd, 0x00fde, 0x00fdf, + 0x00fe0, 0x00fe1, 0x00fe2, 0x00fe3, 0x00fe4, 0x00fe5, 0x00fe6, 0x00fe7, + 0x00fe8, 0x00fe9, 0x00fea, 0x00feb, 0x00fec, 0x00fed, 0x00fee, 0x00fef, + 0x00ff0, 0x00ff1, 0x00ff2, 0x00ff3, 0x00ff4, 0x00ff5, 0x00ff6, 0x00ff7, + 0x00ff8, 0x00ff9, 0x00ffa, 0x00ffb, 0x00ffc, 0x00ffd, 0x00ffe, 0x00fff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10[] = { + 0x01000, 0x01001, 0x01002, 0x01003, 0x01004, 0x01005, 0x01006, 0x01007, + 0x01008, 0x01009, 0x0100a, 0x0100b, 0x0100c, 0x0100d, 0x0100e, 0x0100f, + 0x01010, 0x01011, 0x01012, 0x01013, 0x01014, 0x01015, 0x01016, 0x01017, + 0x01018, 0x01019, 0x0101a, 0x0101b, 0x0101c, 0x0101d, 0x0101e, 0x0101f, + 0x01020, 0x01021, 0x01022, 0x01023, 0x01024, 0x01025, 0x01026, 0x01027, + 0x01028, 0x01029, 0x0102a, 0x0102b, 0x0102b, 0x0102d, 0x0102e, 0x0102f, + 0x01030, 0x01031, 0x01032, 0x01033, 0x01034, 0x01035, 0x00000, 0x00000, + 0x00000, 0x01039, 0x0103a, 0x0103b, 0x0103c, 0x0103d, 0x0103e, 0x0103f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0104a, 0x0104b, 0x0104c, 0x0104d, 0x0104e, 0x0104f, + 0x01050, 0x01051, 0x01052, 0x01053, 0x01054, 0x01055, 0x01056, 0x01057, + 0x01058, 0x01059, 0x0105a, 0x0105b, 0x0105c, 0x0105d, 0x0105e, 0x0105f, + 0x01060, 0x01061, 0x01062, 0x01063, 0x01064, 0x01065, 0x01066, 0x01067, + 0x01068, 0x01069, 0x0106a, 0x0106b, 0x0106c, 0x0106d, 0x0106e, 0x0106f, + 0x01070, 0x01071, 0x01072, 0x01073, 0x01074, 0x01075, 0x01076, 0x01077, + 0x01078, 0x01079, 0x0107a, 0x0107b, 0x0107c, 0x0107d, 0x0107e, 0x0107f, + 0x01080, 0x01081, 0x01082, 0x01083, 0x01084, 0x01085, 0x01086, 0x01087, + 0x01088, 0x01089, 0x0108a, 0x0108b, 0x0108c, 0x00000, 0x0108e, 0x0108f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0109a, 0x0109b, 0x0109c, 0x0109d, 0x0109e, 0x0109f, + 0x010a0, 0x010a1, 0x010a2, 0x010a3, 0x010a4, 0x010a5, 0x010a6, 0x010a7, + 0x010a8, 0x010a9, 0x010aa, 0x010ab, 0x010ac, 0x010ad, 0x010ae, 0x010af, + 0x010b0, 0x010b1, 0x010b2, 0x010b3, 0x010b4, 0x010b5, 0x010b6, 0x010b7, + 0x010b8, 0x010b9, 0x010ba, 0x010bb, 0x010bc, 0x010bd, 0x010be, 0x010bf, + 0x010c0, 0x010c1, 0x010c2, 0x010c3, 0x010c4, 0x010c5, 0x010c6, 0x010c7, + 0x010c8, 0x010c9, 0x010ca, 0x010cb, 0x010cc, 0x010cd, 0x010ce, 0x010cf, + 0x010d0, 0x010d1, 0x010d2, 0x010d3, 0x010d4, 0x010d5, 0x010d6, 0x010d7, + 0x010d8, 0x010d9, 0x010da, 0x010db, 0x010dc, 0x010dd, 0x010de, 0x010df, + 0x010e0, 0x010e1, 0x010e2, 0x010e3, 0x010e4, 0x010e5, 0x010e6, 0x010e7, + 0x010e8, 0x010e9, 0x010ea, 0x010eb, 0x010ec, 0x010ed, 0x010ee, 0x010ef, + 0x010f0, 0x010f1, 0x010f2, 0x010f3, 0x010f4, 0x010f5, 0x010f6, 0x010f7, + 0x010f8, 0x010f9, 0x010fa, 0x010fb, 0x010dc, 0x010fd, 0x010fe, 0x010ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_13[] = { + 0x01300, 0x01301, 0x01302, 0x01303, 0x01304, 0x01305, 0x01306, 0x01307, + 0x01308, 0x01309, 0x0130a, 0x0130b, 0x0130c, 0x0130d, 0x0130e, 0x0130f, + 0x01310, 0x01311, 0x01312, 0x01313, 0x01314, 0x01315, 0x01316, 0x01317, + 0x01318, 0x01319, 0x0131a, 0x0131b, 0x0131c, 0x0131d, 0x0131e, 0x0131f, + 0x01320, 0x01321, 0x01322, 0x01323, 0x01324, 0x01325, 0x01326, 0x01327, + 0x01328, 0x01329, 0x0132a, 0x0132b, 0x0132c, 0x0132d, 0x0132e, 0x0132f, + 0x01330, 0x01331, 0x01332, 0x01333, 0x01334, 0x01335, 0x01336, 0x01337, + 0x01338, 0x01339, 0x0133a, 0x0133b, 0x0133c, 0x0133d, 0x0133e, 0x0133f, + 0x01340, 0x01341, 0x01342, 0x01343, 0x01344, 0x01345, 0x01346, 0x01347, + 0x01348, 0x01349, 0x0134a, 0x0134b, 0x0134c, 0x0134d, 0x0134e, 0x0134f, + 0x01350, 0x01351, 0x01352, 0x01353, 0x01354, 0x01355, 0x01356, 0x01357, + 0x01358, 0x01359, 0x0135a, 0x0135b, 0x0135c, 0x0135d, 0x0135e, 0x00000, + 0x01360, 0x01361, 0x01362, 0x01363, 0x01364, 0x01365, 0x01366, 0x01367, + 0x01368, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01372, 0x01373, 0x01374, 0x01375, 0x01376, 0x01377, + 0x01378, 0x01379, 0x0137a, 0x0137b, 0x0137c, 0x0137d, 0x0137e, 0x0137f, + 0x01380, 0x01381, 0x01382, 0x01383, 0x01384, 0x01385, 0x01386, 0x01387, + 0x01388, 0x01389, 0x0138a, 0x0138b, 0x0138c, 0x0138d, 0x0138e, 0x0138f, + 0x01390, 0x01391, 0x01392, 0x01393, 0x01394, 0x01395, 0x01396, 0x01397, + 0x01398, 0x01399, 0x0139a, 0x0139b, 0x0139c, 0x0139d, 0x0139e, 0x0139f, + 0x013a0, 0x013a1, 0x013a2, 0x013a3, 0x013a4, 0x013a5, 0x013a6, 0x013a7, + 0x013a8, 0x013a9, 0x013aa, 0x013ab, 0x013ac, 0x013ad, 0x013ae, 0x013af, + 0x013b0, 0x013b1, 0x013b2, 0x013b3, 0x013b4, 0x013b5, 0x013b6, 0x013b7, + 0x013b8, 0x013b9, 0x013ba, 0x013bb, 0x013bc, 0x013bd, 0x013be, 0x013bf, + 0x013c0, 0x013c1, 0x013c2, 0x013c3, 0x013c4, 0x013c5, 0x013c6, 0x013c7, + 0x013c8, 0x013c9, 0x013ca, 0x013cb, 0x013cc, 0x013cd, 0x013ce, 0x013cf, + 0x013d0, 0x013d1, 0x013d2, 0x013d3, 0x013d4, 0x013d5, 0x013d6, 0x013d7, + 0x013d8, 0x013d9, 0x013da, 0x013db, 0x013dc, 0x013dd, 0x013de, 0x013df, + 0x013e0, 0x013e1, 0x013e2, 0x013e3, 0x013e4, 0x013e5, 0x013e6, 0x013e7, + 0x013e8, 0x013e9, 0x013ea, 0x013eb, 0x013ec, 0x013ed, 0x013ee, 0x013ef, + 0x013f0, 0x013f1, 0x013f2, 0x013f3, 0x013f4, 0x013f5, 0x013f6, 0x013f7, + 0x013f8, 0x013f9, 0x013fa, 0x013fb, 0x013fc, 0x013fd, 0x013fe, 0x013ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_16[] = { + 0x01600, 0x01601, 0x01602, 0x01603, 0x01604, 0x01605, 0x01606, 0x01607, + 0x01608, 0x01609, 0x0160a, 0x0160b, 0x0160c, 0x0160d, 0x0160e, 0x0160f, + 0x01610, 0x01611, 0x01612, 0x01613, 0x01614, 0x01615, 0x01616, 0x01617, + 0x01618, 0x01619, 0x0161a, 0x0161b, 0x0161c, 0x0161d, 0x0161e, 0x0161f, + 0x01620, 0x01621, 0x01622, 0x01623, 0x01624, 0x01625, 0x01626, 0x01627, + 0x01628, 0x01629, 0x0162a, 0x0162b, 0x0162c, 0x0162d, 0x0162e, 0x0162f, + 0x01630, 0x01631, 0x01632, 0x01633, 0x01634, 0x01635, 0x01636, 0x01637, + 0x01638, 0x01639, 0x0163a, 0x0163b, 0x0163c, 0x0163d, 0x0163e, 0x0163f, + 0x01640, 0x01641, 0x01642, 0x01643, 0x01644, 0x01645, 0x01646, 0x01647, + 0x01648, 0x01649, 0x0164a, 0x0164b, 0x0164c, 0x0164d, 0x0164e, 0x0164f, + 0x01650, 0x01651, 0x01652, 0x01653, 0x01654, 0x01655, 0x01656, 0x01657, + 0x01658, 0x01659, 0x0165a, 0x0165b, 0x0165c, 0x0165d, 0x0165e, 0x0165f, + 0x01660, 0x01661, 0x01662, 0x01663, 0x01664, 0x01665, 0x01666, 0x01667, + 0x01668, 0x01669, 0x0166a, 0x0166b, 0x0166c, 0x0166d, 0x0166e, 0x0166f, + 0x01670, 0x01671, 0x01672, 0x01673, 0x01674, 0x01675, 0x01676, 0x01677, + 0x01678, 0x01679, 0x0167a, 0x0167b, 0x0167c, 0x0167d, 0x0167e, 0x0167f, + 0x01680, 0x01681, 0x01682, 0x01683, 0x01684, 0x01685, 0x01686, 0x01687, + 0x01688, 0x01689, 0x0168a, 0x0168b, 0x0168c, 0x0168d, 0x0168e, 0x0168f, + 0x01690, 0x01691, 0x01692, 0x01693, 0x01694, 0x01695, 0x01696, 0x01697, + 0x01698, 0x01699, 0x0169a, 0x0169b, 0x0169c, 0x0169d, 0x0169e, 0x0169f, + 0x016a0, 0x016a0, 0x016a2, 0x016a3, 0x016a2, 0x016a2, 0x016a6, 0x016a6, + 0x016a8, 0x016a8, 0x016aa, 0x016ab, 0x016a8, 0x016a8, 0x016a8, 0x016af, + 0x016b0, 0x016b1, 0x016b2, 0x016b2, 0x016b2, 0x016b2, 0x016b2, 0x016b7, + 0x016b8, 0x016b9, 0x016ba, 0x016ba, 0x016ba, 0x016ba, 0x016be, 0x016be, + 0x016be, 0x016c1, 0x016c1, 0x016c3, 0x016c3, 0x016c5, 0x016c5, 0x016c7, + 0x016c8, 0x016c9, 0x016ca, 0x016ca, 0x016ca, 0x016ca, 0x016ca, 0x016cf, + 0x016cf, 0x016cf, 0x016d2, 0x016d2, 0x016d2, 0x016c8, 0x016d6, 0x016d7, + 0x016d7, 0x016d7, 0x016da, 0x016da, 0x016dc, 0x016dc, 0x016de, 0x016df, + 0x016e0, 0x016e1, 0x016e2, 0x016e3, 0x016e4, 0x016e5, 0x016e6, 0x016e6, + 0x016e6, 0x016b9, 0x016ca, 0x016eb, 0x016ec, 0x016ed, 0x016ee, 0x016ef, + 0x016f0, 0x016f1, 0x016f2, 0x016f3, 0x016f4, 0x016f5, 0x016f6, 0x016f7, + 0x016f8, 0x016f9, 0x016fa, 0x016fb, 0x016fc, 0x016fd, 0x016fe, 0x016ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_17[] = { + 0x01700, 0x01701, 0x01702, 0x01703, 0x01704, 0x01705, 0x01706, 0x01707, + 0x01708, 0x01709, 0x0170a, 0x0170b, 0x0170c, 0x0170d, 0x0170e, 0x0170f, + 0x01710, 0x01711, 0x01712, 0x01713, 0x01714, 0x01715, 0x01716, 0x01717, + 0x01718, 0x01719, 0x0171a, 0x0171b, 0x0171c, 0x0171d, 0x0171e, 0x0171f, + 0x01720, 0x01721, 0x01722, 0x01723, 0x01724, 0x01725, 0x01726, 0x01727, + 0x01728, 0x01729, 0x0172a, 0x0172b, 0x0172c, 0x0172d, 0x0172e, 0x0172f, + 0x01730, 0x01731, 0x01732, 0x01733, 0x01734, 0x01735, 0x01736, 0x01737, + 0x01738, 0x01739, 0x0173a, 0x0173b, 0x0173c, 0x0173d, 0x0173e, 0x0173f, + 0x01740, 0x01741, 0x01742, 0x01743, 0x01744, 0x01745, 0x01746, 0x01747, + 0x01748, 0x01749, 0x0174a, 0x0174b, 0x0174c, 0x0174d, 0x0174e, 0x0174f, + 0x01750, 0x01751, 0x01752, 0x01753, 0x01754, 0x01755, 0x01756, 0x01757, + 0x01758, 0x01759, 0x0175a, 0x0175b, 0x0175c, 0x0175d, 0x0175e, 0x0175f, + 0x01760, 0x01761, 0x01762, 0x01763, 0x01764, 0x01765, 0x01766, 0x01767, + 0x01768, 0x01769, 0x0176a, 0x0176b, 0x0176c, 0x0176d, 0x0176e, 0x0176f, + 0x01770, 0x01771, 0x01772, 0x01773, 0x01774, 0x01775, 0x01776, 0x01777, + 0x01778, 0x01779, 0x0177a, 0x0177b, 0x0177c, 0x0177d, 0x0177e, 0x0177f, + 0x01780, 0x01781, 0x01782, 0x01783, 0x01784, 0x01785, 0x01786, 0x01787, + 0x01788, 0x01789, 0x0178a, 0x0178b, 0x0178c, 0x0178d, 0x0178e, 0x0178f, + 0x01790, 0x01791, 0x01792, 0x01793, 0x01794, 0x01795, 0x01796, 0x01797, + 0x01798, 0x01799, 0x0179a, 0x0179b, 0x0179c, 0x0179d, 0x0179e, 0x0179f, + 0x017a0, 0x017a1, 0x017a2, 0x017a3, 0x017a4, 0x017a5, 0x017a6, 0x017a7, + 0x017a8, 0x017a9, 0x017aa, 0x017ab, 0x017ac, 0x017ad, 0x017ae, 0x017af, + 0x017b0, 0x017b1, 0x017b2, 0x017b3, 0x017b4, 0x017b5, 0x017b6, 0x017b7, + 0x017b8, 0x017b9, 0x017ba, 0x017bb, 0x017bc, 0x017bd, 0x017be, 0x017bf, + 0x017c0, 0x017c1, 0x017c2, 0x017c3, 0x017c4, 0x017c5, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x017d2, 0x00000, 0x017d4, 0x017d5, 0x017d6, 0x017d7, + 0x017d8, 0x017d9, 0x017da, 0x017db, 0x017dc, 0x00000, 0x017de, 0x017df, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x017ea, 0x017eb, 0x017ec, 0x017ed, 0x017ee, 0x017ef, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x017fa, 0x017fb, 0x017fc, 0x017fd, 0x017fe, 0x017ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_18[] = { + 0x01800, 0x01801, 0x01802, 0x01803, 0x01804, 0x01805, 0x01806, 0x01807, + 0x01808, 0x01809, 0x0180a, 0x00000, 0x00000, 0x00000, 0x0180e, 0x0180f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0181a, 0x0181b, 0x0181c, 0x0181d, 0x0181e, 0x0181f, + 0x01820, 0x01821, 0x01822, 0x01823, 0x01824, 0x01825, 0x01826, 0x01827, + 0x01828, 0x01829, 0x0182a, 0x0182b, 0x0182c, 0x0182d, 0x0182e, 0x0182f, + 0x01830, 0x01831, 0x01832, 0x01833, 0x01834, 0x01835, 0x01836, 0x01837, + 0x01838, 0x01839, 0x0183a, 0x0183b, 0x0183c, 0x0183d, 0x0183e, 0x0183f, + 0x01840, 0x01841, 0x01842, 0x01843, 0x01844, 0x01845, 0x01846, 0x01847, + 0x01848, 0x01849, 0x0184a, 0x0184b, 0x0184c, 0x0184d, 0x0184e, 0x0184f, + 0x01850, 0x01851, 0x01852, 0x01853, 0x01854, 0x01855, 0x01856, 0x01857, + 0x01858, 0x01859, 0x0185a, 0x0185b, 0x0185c, 0x0185d, 0x0185e, 0x0185f, + 0x01860, 0x01861, 0x01862, 0x01863, 0x01864, 0x01865, 0x01866, 0x01867, + 0x01868, 0x01869, 0x0186a, 0x0186b, 0x0186c, 0x0186d, 0x0186e, 0x0186f, + 0x01870, 0x01871, 0x01872, 0x01873, 0x01874, 0x01875, 0x01876, 0x01877, + 0x01878, 0x01879, 0x0187a, 0x0187b, 0x0187c, 0x0187d, 0x0187e, 0x0187f, + 0x01880, 0x01881, 0x01882, 0x01883, 0x01884, 0x01885, 0x01886, 0x01887, + 0x01888, 0x01889, 0x0188a, 0x0188b, 0x0188c, 0x0188d, 0x0188e, 0x0188f, + 0x01890, 0x01891, 0x01892, 0x01893, 0x01894, 0x01895, 0x01896, 0x01897, + 0x01898, 0x01899, 0x0189a, 0x0189b, 0x0189c, 0x0189d, 0x0189e, 0x0189f, + 0x018a0, 0x018a1, 0x018a2, 0x018a3, 0x018a4, 0x018a5, 0x018a6, 0x018a7, + 0x018a8, 0x018a9, 0x018aa, 0x018ab, 0x018ac, 0x018ad, 0x018ae, 0x018af, + 0x018b0, 0x018b1, 0x018b2, 0x018b3, 0x018b4, 0x018b5, 0x018b6, 0x018b7, + 0x018b8, 0x018b9, 0x018ba, 0x018bb, 0x018bc, 0x018bd, 0x018be, 0x018bf, + 0x018c0, 0x018c1, 0x018c2, 0x018c3, 0x018c4, 0x018c5, 0x018c6, 0x018c7, + 0x018c8, 0x018c9, 0x018ca, 0x018cb, 0x018cc, 0x018cd, 0x018ce, 0x018cf, + 0x018d0, 0x018d1, 0x018d2, 0x018d3, 0x018d4, 0x018d5, 0x018d6, 0x018d7, + 0x018d8, 0x018d9, 0x018da, 0x018db, 0x018dc, 0x018dd, 0x018de, 0x018df, + 0x018e0, 0x018e1, 0x018e2, 0x018e3, 0x018e4, 0x018e5, 0x018e6, 0x018e7, + 0x018e8, 0x018e9, 0x018ea, 0x018eb, 0x018ec, 0x018ed, 0x018ee, 0x018ef, + 0x018f0, 0x018f1, 0x018f2, 0x018f3, 0x018f4, 0x018f5, 0x018f6, 0x018f7, + 0x018f8, 0x018f9, 0x018fa, 0x018fb, 0x018fc, 0x018fd, 0x018fe, 0x018ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_19[] = { + 0x01900, 0x01901, 0x01902, 0x01903, 0x01904, 0x01905, 0x01906, 0x01907, + 0x01908, 0x01909, 0x0190a, 0x0190b, 0x0190c, 0x0190d, 0x0190e, 0x0190f, + 0x01910, 0x01911, 0x01912, 0x01913, 0x01914, 0x01915, 0x01916, 0x01917, + 0x01918, 0x01919, 0x0191a, 0x0191b, 0x0191c, 0x0191d, 0x0191e, 0x0191f, + 0x01920, 0x01921, 0x01922, 0x01923, 0x01924, 0x01925, 0x01926, 0x01927, + 0x01928, 0x01929, 0x0192a, 0x0192b, 0x0192c, 0x0192d, 0x0192e, 0x0192f, + 0x01930, 0x01931, 0x01932, 0x01933, 0x01934, 0x01935, 0x01936, 0x01937, + 0x01938, 0x00000, 0x00000, 0x00000, 0x0193c, 0x0193d, 0x0193e, 0x0193f, + 0x01940, 0x01941, 0x01942, 0x01943, 0x01944, 0x01945, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x01950, 0x01951, 0x01952, 0x01953, 0x01954, 0x01955, 0x01956, 0x01957, + 0x01958, 0x01959, 0x0195a, 0x0195b, 0x0195c, 0x0195d, 0x0195e, 0x0195f, + 0x01960, 0x01961, 0x01962, 0x01963, 0x01964, 0x01965, 0x01966, 0x01967, + 0x01968, 0x01969, 0x0196a, 0x0196b, 0x0196c, 0x0196d, 0x0196e, 0x0196f, + 0x01970, 0x01971, 0x01972, 0x01973, 0x01974, 0x01975, 0x01976, 0x01977, + 0x01978, 0x01979, 0x0197a, 0x0197b, 0x0197c, 0x0197d, 0x0197e, 0x0197f, + 0x01980, 0x01981, 0x01982, 0x01983, 0x01984, 0x01985, 0x01986, 0x01987, + 0x01988, 0x01989, 0x0198a, 0x0198b, 0x0198c, 0x0198d, 0x0198e, 0x0198f, + 0x01990, 0x01991, 0x01992, 0x01993, 0x01994, 0x01995, 0x01996, 0x01997, + 0x01998, 0x01999, 0x0199a, 0x0199b, 0x0199c, 0x0199d, 0x0199e, 0x0199f, + 0x019a0, 0x019a1, 0x019a2, 0x019a3, 0x019a4, 0x019a5, 0x019a6, 0x019a7, + 0x019a8, 0x019a9, 0x019aa, 0x019ab, 0x019ac, 0x019ad, 0x019ae, 0x019af, + 0x019b0, 0x019b1, 0x019b2, 0x019b3, 0x019b4, 0x019b5, 0x019b6, 0x019b7, + 0x019b8, 0x019b9, 0x019ba, 0x019bb, 0x019bc, 0x019bd, 0x019be, 0x019bf, + 0x019c0, 0x019c1, 0x019c2, 0x019c3, 0x019c4, 0x019c5, 0x019c6, 0x019c7, + 0x019c8, 0x019c9, 0x019ca, 0x019cb, 0x019cc, 0x019cd, 0x019ce, 0x019cf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00031, 0x019db, 0x019dc, 0x019dd, 0x019de, 0x019df, + 0x019e0, 0x019e1, 0x019e2, 0x019e3, 0x019e4, 0x019e5, 0x019e6, 0x019e7, + 0x019e8, 0x019e9, 0x019ea, 0x019eb, 0x019ec, 0x019ed, 0x019ee, 0x019ef, + 0x019f0, 0x019f1, 0x019f2, 0x019f3, 0x019f4, 0x019f5, 0x019f6, 0x019f7, + 0x019f8, 0x019f9, 0x019fa, 0x019fb, 0x019fc, 0x019fd, 0x019fe, 0x019ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1a[] = { + 0x01a00, 0x01a01, 0x01a02, 0x01a03, 0x01a04, 0x01a05, 0x01a06, 0x01a07, + 0x01a08, 0x01a09, 0x01a0a, 0x01a0b, 0x01a0c, 0x01a0d, 0x01a0e, 0x01a0f, + 0x01a10, 0x01a11, 0x01a12, 0x01a13, 0x01a14, 0x01a15, 0x01a16, 0x01a17, + 0x01a18, 0x01a19, 0x01a1a, 0x01a1b, 0x01a1c, 0x01a1d, 0x01a1e, 0x01a1f, + 0x01a20, 0x01a21, 0x01a22, 0x01a23, 0x01a24, 0x01a25, 0x01a26, 0x01a27, + 0x01a28, 0x01a29, 0x01a2a, 0x01a2b, 0x01a2c, 0x01a2d, 0x01a2e, 0x01a2f, + 0x01a30, 0x01a31, 0x01a32, 0x01a33, 0x01a34, 0x01a35, 0x01a36, 0x01a37, + 0x01a38, 0x01a39, 0x01a3a, 0x01a3b, 0x01a3c, 0x01a3d, 0x01a3e, 0x01a3f, + 0x01a40, 0x01a41, 0x01a42, 0x01a43, 0x01a44, 0x01a45, 0x01a46, 0x01a47, + 0x01a48, 0x01a49, 0x01a4a, 0x01a4b, 0x01a4c, 0x01a4d, 0x01a4e, 0x01a4f, + 0x01a50, 0x01a51, 0x01a52, 0x01a53, 0x01a54, 0x01a55, 0x01a56, 0x01a57, + 0x01a26, 0x01a26, 0x01a3b, 0x01a3b, 0x01a5c, 0x01a5d, 0x01a5e, 0x01a5f, + 0x01a60, 0x01a61, 0x01a62, 0x01a63, 0x01a63, 0x01a65, 0x01a66, 0x01a67, + 0x01a68, 0x01a69, 0x01a6a, 0x01a6b, 0x01a6c, 0x01a6d, 0x01a6e, 0x01a6f, + 0x01a70, 0x01a71, 0x01a72, 0x01a73, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x01a7d, 0x01a7e, 0x00000, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01a8a, 0x01a8b, 0x01a8c, 0x01a8d, 0x01a8e, 0x01a8f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01a9a, 0x01a9b, 0x01a9c, 0x01a9d, 0x01a9e, 0x01a9f, + 0x01aa0, 0x01aa1, 0x01aa2, 0x01aa3, 0x01aa4, 0x01aa5, 0x01aa6, 0x01aa7, + 0x01aa8, 0x01aa9, 0x01aaa, 0x01aab, 0x01aac, 0x01aad, 0x01aae, 0x01aaf, + 0x01ab0, 0x01ab1, 0x01ab2, 0x01ab3, 0x01ab4, 0x01ab5, 0x01ab6, 0x01ab7, + 0x01ab8, 0x01ab9, 0x01aba, 0x01abb, 0x01abc, 0x01abd, 0x01abe, 0x01abf, + 0x01ac0, 0x01ac1, 0x01ac2, 0x01ac3, 0x01ac4, 0x01ac5, 0x01ac6, 0x01ac7, + 0x01ac8, 0x01ac9, 0x01aca, 0x01acb, 0x01acc, 0x01acd, 0x01ace, 0x01acf, + 0x01ad0, 0x01ad1, 0x01ad2, 0x01ad3, 0x01ad4, 0x01ad5, 0x01ad6, 0x01ad7, + 0x01ad8, 0x01ad9, 0x01ada, 0x01adb, 0x01adc, 0x01add, 0x01ade, 0x01adf, + 0x01ae0, 0x01ae1, 0x01ae2, 0x01ae3, 0x01ae4, 0x01ae5, 0x01ae6, 0x01ae7, + 0x01ae8, 0x01ae9, 0x01aea, 0x01aeb, 0x01aec, 0x01aed, 0x01aee, 0x01aef, + 0x01af0, 0x01af1, 0x01af2, 0x01af3, 0x01af4, 0x01af5, 0x01af6, 0x01af7, + 0x01af8, 0x01af9, 0x01afa, 0x01afb, 0x01afc, 0x01afd, 0x01afe, 0x01aff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1b[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x01b05, 0x01b06, 0x01b07, + 0x01b08, 0x01b09, 0x01b0a, 0x01b0b, 0x01b0c, 0x01b0d, 0x01b0e, 0x01b0f, + 0x01b10, 0x01b11, 0x01b12, 0x01b13, 0x01b14, 0x01b15, 0x01b16, 0x01b17, + 0x01b18, 0x01b19, 0x01b1a, 0x01b1b, 0x01b1c, 0x01b1d, 0x01b1e, 0x01b1f, + 0x01b20, 0x01b21, 0x01b22, 0x01b23, 0x01b24, 0x01b25, 0x01b26, 0x01b27, + 0x01b28, 0x01b29, 0x01b2a, 0x01b2b, 0x01b2c, 0x01b2d, 0x01b2e, 0x01b2f, + 0x01b30, 0x01b31, 0x01b32, 0x01b33, 0x00000, 0x01b35, 0x01b36, 0x01b37, + 0x01b38, 0x01b39, 0x01b3a, 0x01b3b, 0x01b3c, 0x01b3d, 0x01b3e, 0x01b3f, + 0x01b40, 0x01b41, 0x01b42, 0x01b43, 0x01b44, 0x01b45, 0x01b46, 0x01b47, + 0x01b48, 0x01b49, 0x01b4a, 0x01b4b, 0x01b4c, 0x01b4d, 0x01b4e, 0x01b4f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01b5a, 0x01b5b, 0x01b5c, 0x01b5d, 0x01b5e, 0x01b5f, + 0x01b60, 0x01b61, 0x01b62, 0x01b63, 0x01b64, 0x01b65, 0x01b66, 0x01b67, + 0x01b68, 0x01b69, 0x01b6a, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x01b74, 0x01b75, 0x01b76, 0x01b77, + 0x01b78, 0x01b79, 0x01b7a, 0x01b7b, 0x01b7c, 0x01b7d, 0x01b7e, 0x01b7f, + 0x00000, 0x00000, 0x00000, 0x01b83, 0x01b84, 0x01b85, 0x01b86, 0x01b87, + 0x01b88, 0x01b89, 0x01b8a, 0x01b8b, 0x01b8c, 0x01b8d, 0x01b8e, 0x01b8f, + 0x01b90, 0x01b91, 0x01b92, 0x01b93, 0x01b94, 0x01b95, 0x01b96, 0x01b97, + 0x01b98, 0x01b99, 0x01b9a, 0x01b9b, 0x01b9c, 0x01b9d, 0x01b9e, 0x01b9f, + 0x01ba0, 0x01ba1, 0x01ba2, 0x01ba3, 0x01ba4, 0x01ba5, 0x01ba6, 0x01ba7, + 0x01ba8, 0x01ba9, 0x01baa, 0x01bab, 0x01bac, 0x01bad, 0x01bae, 0x01baf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01bba, 0x01bbb, 0x01bbc, 0x01bbd, 0x01bbe, 0x01bbf, + 0x01bc0, 0x01bc1, 0x01bc2, 0x01bc3, 0x01bc4, 0x01bc5, 0x01bc6, 0x01bc7, + 0x01bc8, 0x01bc9, 0x01bca, 0x01bcb, 0x01bcc, 0x01bcd, 0x01bce, 0x01bcf, + 0x01bd0, 0x01bd1, 0x01bd2, 0x01bd3, 0x01bd4, 0x01bd5, 0x01bd6, 0x01bd7, + 0x01bd8, 0x01bd9, 0x01bda, 0x01bdb, 0x01bdc, 0x01bdd, 0x01bde, 0x01bdf, + 0x01be0, 0x01be1, 0x01be2, 0x01be3, 0x01be4, 0x01be5, 0x01be6, 0x01be7, + 0x01be8, 0x01be9, 0x01bea, 0x01beb, 0x01bec, 0x01bed, 0x01bee, 0x01bef, + 0x01bf0, 0x01bf1, 0x01bf2, 0x01bf3, 0x01bf4, 0x01bf5, 0x01bf6, 0x01bf7, + 0x01bf8, 0x01bf9, 0x01bfa, 0x01bfb, 0x01bfc, 0x01bfd, 0x01bfe, 0x01bff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1c[] = { + 0x01c00, 0x01c01, 0x01c02, 0x01c03, 0x01c04, 0x01c05, 0x01c06, 0x01c07, + 0x01c08, 0x01c09, 0x01c0a, 0x01c0b, 0x01c0c, 0x01c0d, 0x01c0e, 0x01c0f, + 0x01c10, 0x01c11, 0x01c12, 0x01c13, 0x01c14, 0x01c15, 0x01c16, 0x01c17, + 0x01c18, 0x01c19, 0x01c1a, 0x01c1b, 0x01c1c, 0x01c1d, 0x01c1e, 0x01c1f, + 0x01c20, 0x01c21, 0x01c22, 0x01c23, 0x01c24, 0x01c25, 0x01c26, 0x01c27, + 0x01c28, 0x01c29, 0x01c2a, 0x01c2b, 0x01c2c, 0x01c2d, 0x01c2e, 0x01c2f, + 0x01c30, 0x01c31, 0x01c32, 0x01c33, 0x01c34, 0x01c35, 0x01c36, 0x00000, + 0x01c38, 0x01c39, 0x01c3a, 0x01c3b, 0x01c3c, 0x01c3d, 0x01c3e, 0x01c3f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01c4a, 0x01c4b, 0x01c4c, 0x01c4d, 0x01c4e, 0x01c4f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01c5a, 0x01c5b, 0x01c5c, 0x01c5d, 0x01c5e, 0x01c5f, + 0x01c60, 0x01c61, 0x01c62, 0x01c63, 0x01c64, 0x01c65, 0x01c66, 0x01c67, + 0x01c68, 0x01c69, 0x01c6a, 0x01c6b, 0x01c6c, 0x01c6d, 0x01c6e, 0x01c6f, + 0x01c70, 0x01c71, 0x01c72, 0x01c73, 0x01c74, 0x01c75, 0x01c76, 0x01c77, + 0x01c78, 0x01c79, 0x01c7a, 0x01c7b, 0x01c7c, 0x01c7d, 0x01c7e, 0x01c7f, + 0x01c80, 0x01c81, 0x01c82, 0x01c83, 0x01c84, 0x01c85, 0x01c86, 0x01c87, + 0x01c88, 0x01c89, 0x01c8a, 0x01c8b, 0x01c8c, 0x01c8d, 0x01c8e, 0x01c8f, + 0x01c90, 0x01c91, 0x01c92, 0x01c93, 0x01c94, 0x01c95, 0x01c96, 0x01c97, + 0x01c98, 0x01c99, 0x01c9a, 0x01c9b, 0x01c9c, 0x01c9d, 0x01c9e, 0x01c9f, + 0x01ca0, 0x01ca1, 0x01ca2, 0x01ca3, 0x01ca4, 0x01ca5, 0x01ca6, 0x01ca7, + 0x01ca8, 0x01ca9, 0x01caa, 0x01cab, 0x01cac, 0x01cad, 0x01cae, 0x01caf, + 0x01cb0, 0x01cb1, 0x01cb2, 0x01cb3, 0x01cb4, 0x01cb5, 0x01cb6, 0x01cb7, + 0x01cb8, 0x01cb9, 0x01cba, 0x01cbb, 0x01cbc, 0x01cbd, 0x01cbe, 0x01cbf, + 0x01cc0, 0x01cc1, 0x01cc2, 0x01cc3, 0x01cc4, 0x01cc5, 0x01cc6, 0x01cc7, + 0x01cc8, 0x01cc9, 0x01cca, 0x01ccb, 0x01ccc, 0x01ccd, 0x01cce, 0x01ccf, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x01ce9, 0x01ce9, 0x01ce9, 0x01ce9, 0x00000, 0x01ce9, 0x01ce9, + 0x01ce9, 0x01ce9, 0x00000, 0x01cf3, 0x01cf4, 0x01cf5, 0x01cf6, 0x01cf7, + 0x01cf8, 0x01cf9, 0x01cfa, 0x01cfb, 0x01cfc, 0x01cfd, 0x01cfe, 0x01cff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d[] = { + 0x01d00, 0x01d01, 0x01d02, 0x01d03, 0x01d04, 0x01d05, 0x01d06, 0x01d07, + 0x01d08, 0x01d09, 0x01d0a, 0x01d0b, 0x01d0c, 0x01d0d, 0x01d0e, 0x01d0f, + 0x01d10, 0x01d11, 0x01d12, 0x01d13, 0x01d14, 0x01d15, 0x01d16, 0x01d17, + 0x01d18, 0x01d19, 0x01d1a, 0x01d1b, 0x01d1c, 0x01d1d, 0x01d1e, 0x01d1f, + 0x01d20, 0x01d21, 0x01d22, 0x01d23, 0x01d24, 0x01d25, 0x01d26, 0x01d27, + 0x01d28, 0x01d29, 0x01d2a, 0x01d2b, 0x00041, 0x000c6, 0x00042, 0x01d2f, + 0x00044, 0x00045, 0x0018e, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, + 0x0004c, 0x0004d, 0x0004e, 0x01d3b, 0x0004f, 0x00222, 0x00050, 0x00052, + 0x00054, 0x00055, 0x00057, 0x00041, 0x00250, 0x00251, 0x01d02, 0x00042, + 0x00044, 0x00045, 0x0018f, 0x00190, 0x01d08, 0x00047, 0x01d09, 0x0004b, + 0x0004d, 0x0014a, 0x0004f, 0x00186, 0x01d16, 0x01d17, 0x00050, 0x00054, + 0x00055, 0x01d1d, 0x0019c, 0x00056, 0x01d25, 0x00392, 0x00393, 0x00394, + 0x003a6, 0x003a7, 0x00049, 0x00052, 0x00055, 0x00056, 0x00392, 0x00393, + 0x003a1, 0x003a6, 0x003a7, 0x01d6b, 0x01d6c, 0x01d6d, 0x01d6e, 0x01d6f, + 0x01d70, 0x01d71, 0x01d72, 0x01d73, 0x01d74, 0x01d75, 0x01d76, 0x01d77, + 0x0041d, 0x00047, 0x01d7a, 0x01d7b, 0x01d7c, 0x01d7d, 0x01d7e, 0x01d7f, + 0x01d80, 0x01d81, 0x01d82, 0x01d83, 0x01d84, 0x01d85, 0x01d86, 0x01d87, + 0x01d88, 0x01d89, 0x01d8a, 0x01d8b, 0x01d8c, 0x01d8d, 0x01d8e, 0x01d8f, + 0x01d90, 0x01d91, 0x01d92, 0x01d93, 0x01d94, 0x01d95, 0x01d96, 0x01d97, + 0x01d98, 0x01d99, 0x01d9a, 0x00252, 0x00043, 0x00255, 0x00044, 0x0025c, + 0x00046, 0x0025f, 0x00261, 0x00265, 0x00197, 0x00196, 0x0026a, 0x01d7b, + 0x0029d, 0x0026d, 0x01d85, 0x0029f, 0x00271, 0x00270, 0x0019d, 0x00273, + 0x00274, 0x0019f, 0x00278, 0x00282, 0x001a9, 0x001ab, 0x00244, 0x001b1, + 0x01d1c, 0x001b2, 0x00245, 0x0005a, 0x00290, 0x00291, 0x001b7, 0x00398, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00052, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x01dd2, 0x00041, 0x000c6, 0x01dd5, 0x01dd6, 0x00043, + 0x00044, 0x00044, 0x00047, 0x00262, 0x0004b, 0x0004c, 0x0029f, 0x01d0d, + 0x0004e, 0x00274, 0x001a6, 0x01de3, 0x00053, 0x00053, 0x0005a, 0x01de7, + 0x01de8, 0x01de9, 0x01dea, 0x01deb, 0x01dec, 0x01ded, 0x01dee, 0x01def, + 0x01df0, 0x01df1, 0x01df2, 0x01df3, 0x01df4, 0x01df5, 0x01df6, 0x01df7, + 0x01df8, 0x01df9, 0x01dfa, 0x01dfb, 0x01dfc, 0x00000, 0x00000, 0x01dff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1e[] = { + 0x00041, 0x00041, 0x00042, 0x00042, 0x00042, 0x00042, 0x00042, 0x00042, + 0x00043, 0x00043, 0x00044, 0x00044, 0x00044, 0x00044, 0x00044, 0x00044, + 0x00044, 0x00044, 0x00044, 0x00044, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00046, 0x00046, + 0x00047, 0x00047, 0x00048, 0x00048, 0x00048, 0x00048, 0x00048, 0x00048, + 0x00048, 0x00048, 0x00048, 0x00048, 0x00049, 0x00049, 0x00049, 0x00049, + 0x0004b, 0x0004b, 0x0004b, 0x0004b, 0x0004b, 0x0004b, 0x0004c, 0x0004c, + 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004d, 0x0004d, + 0x0004d, 0x0004d, 0x0004d, 0x0004d, 0x0004e, 0x0004e, 0x0004e, 0x0004e, + 0x0004e, 0x0004e, 0x0004e, 0x0004e, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x00050, 0x00050, 0x00050, 0x00050, + 0x00052, 0x00052, 0x00052, 0x00052, 0x00052, 0x00052, 0x00052, 0x00052, + 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, + 0x00053, 0x00053, 0x00054, 0x00054, 0x00054, 0x00054, 0x00054, 0x00054, + 0x00054, 0x00054, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00056, 0x00056, 0x00056, 0x00056, + 0x00057, 0x00057, 0x00057, 0x00057, 0x00057, 0x00057, 0x00057, 0x00057, + 0x00057, 0x00057, 0x00058, 0x00058, 0x00058, 0x00058, 0x00059, 0x00059, + 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x00048, 0x00054, + 0x00057, 0x00059, 0x01e9a, 0x00053, 0x01e9c, 0x01e9d, 0x000df, 0x01e9f, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00049, 0x00049, 0x00049, 0x00049, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00059, 0x00059, 0x00059, 0x00059, 0x00059, 0x00059, + 0x00059, 0x00059, 0x01efa, 0x01efa, 0x01efc, 0x01efc, 0x01efe, 0x01eff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1f[] = { + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, + 0x00395, 0x00395, 0x00395, 0x00395, 0x00395, 0x00395, 0x01f16, 0x01f17, + 0x00395, 0x00395, 0x00395, 0x00395, 0x00395, 0x00395, 0x01f1e, 0x01f1f, + 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, + 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, + 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, + 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, + 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x01f46, 0x01f47, + 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x01f4e, 0x01f4f, + 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a5, + 0x01f58, 0x003a5, 0x01f5a, 0x003a5, 0x01f5c, 0x003a5, 0x01f5e, 0x003a5, + 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, + 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, + 0x00391, 0x00391, 0x00395, 0x00395, 0x00397, 0x00397, 0x00399, 0x00399, + 0x0039f, 0x0039f, 0x003a5, 0x003a5, 0x003a9, 0x003a9, 0x01f7e, 0x01f7f, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, + 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, + 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, + 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, + 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x01fb5, 0x00391, 0x00391, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x01fbd, 0x00399, 0x01fbd, + 0x01fc0, 0x000a8, 0x00397, 0x00397, 0x00397, 0x01fc5, 0x00397, 0x00397, + 0x00395, 0x00395, 0x00397, 0x00397, 0x00397, 0x01fbd, 0x01fbd, 0x01fbd, + 0x00399, 0x00399, 0x00399, 0x00399, 0x01fd4, 0x01fd5, 0x00399, 0x00399, + 0x00399, 0x00399, 0x00399, 0x00399, 0x01fdc, 0x01fdd, 0x01fdd, 0x01fdd, + 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a1, 0x003a1, 0x003a5, 0x003a5, + 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a1, 0x000a8, 0x000a8, 0x00060, + 0x01ff0, 0x01ff1, 0x003a9, 0x003a9, 0x003a9, 0x01ff5, 0x003a9, 0x003a9, + 0x0039f, 0x0039f, 0x003a9, 0x003a9, 0x003a9, 0x000b4, 0x01fdd, 0x01fff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_20[] = { + 0x00020, 0x00020, 0x00020, 0x00020, 0x00020, 0x00020, 0x00020, 0x00020, + 0x00020, 0x00020, 0x00020, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x02010, 0x02010, 0x02012, 0x02013, 0x02014, 0x02015, 0x02016, 0x02017, + 0x02018, 0x02019, 0x0201a, 0x0201b, 0x0201c, 0x0201d, 0x0201e, 0x0201f, + 0x02020, 0x02021, 0x02022, 0x02023, 0x0002e, 0x02025, 0x02026, 0x02027, + 0x02028, 0x02029, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00020, + 0x02030, 0x02031, 0x02032, 0x02033, 0x02034, 0x02035, 0x02036, 0x02037, + 0x02038, 0x02039, 0x0203a, 0x0203b, 0x0203c, 0x0203d, 0x0203e, 0x0203f, + 0x02040, 0x02041, 0x02042, 0x02043, 0x02044, 0x02045, 0x02046, 0x02047, + 0x02048, 0x02049, 0x0204a, 0x0204b, 0x0204c, 0x0204d, 0x0204e, 0x0204f, + 0x02050, 0x02051, 0x02052, 0x02053, 0x02054, 0x02055, 0x02056, 0x02057, + 0x02058, 0x02059, 0x0205a, 0x0205b, 0x0205c, 0x0205d, 0x0205e, 0x00020, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x02065, 0x02066, 0x02067, + 0x02068, 0x02069, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00030, 0x00049, 0x02072, 0x02073, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0002b, 0x0207b, 0x0003d, 0x00028, 0x00029, 0x0004e, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0002b, 0x0207b, 0x0003d, 0x00028, 0x00029, 0x0208f, + 0x00041, 0x00045, 0x0004f, 0x00058, 0x0018f, 0x02095, 0x02096, 0x02097, + 0x02098, 0x02099, 0x0209a, 0x0209b, 0x0209c, 0x0209d, 0x0209e, 0x0209f, + 0x020a0, 0x020a1, 0x020a2, 0x020a3, 0x020a4, 0x020a5, 0x020a6, 0x020a7, + 0x020a8, 0x020a9, 0x020aa, 0x020ab, 0x020ac, 0x020ad, 0x020ae, 0x020af, + 0x020b0, 0x020b1, 0x020b2, 0x020b3, 0x020b4, 0x020b5, 0x020b6, 0x020b7, + 0x020b8, 0x020b9, 0x020ba, 0x020bb, 0x020bc, 0x020bd, 0x020be, 0x020bf, + 0x020c0, 0x020c1, 0x020c2, 0x020c3, 0x020c4, 0x020c5, 0x020c6, 0x020c7, + 0x020c8, 0x020c9, 0x020ca, 0x020cb, 0x020cc, 0x020cd, 0x020ce, 0x020cf, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x020f1, 0x020f2, 0x020f3, 0x020f4, 0x020f5, 0x020f6, 0x020f7, + 0x020f8, 0x020f9, 0x020fa, 0x020fb, 0x020fc, 0x020fd, 0x020fe, 0x020ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_21[] = { + 0x02100, 0x02101, 0x00043, 0x02103, 0x02104, 0x02105, 0x02106, 0x00190, + 0x02108, 0x02109, 0x00047, 0x00048, 0x00048, 0x00048, 0x00048, 0x00048, + 0x00049, 0x00049, 0x0004c, 0x0004c, 0x02114, 0x0004e, 0x02116, 0x02117, + 0x02118, 0x00050, 0x00051, 0x00052, 0x00052, 0x00052, 0x0211e, 0x0211f, + 0x02120, 0x02121, 0x02122, 0x02123, 0x0005a, 0x02125, 0x003a9, 0x02127, + 0x0005a, 0x02129, 0x0004b, 0x00041, 0x00042, 0x00043, 0x0212e, 0x00045, + 0x00045, 0x00046, 0x02132, 0x0004d, 0x0004f, 0x005d0, 0x005d1, 0x005d2, + 0x005d3, 0x00049, 0x0213a, 0x0213b, 0x003a0, 0x00393, 0x00393, 0x003a0, + 0x02140, 0x02141, 0x02142, 0x02143, 0x02144, 0x00044, 0x00044, 0x00045, + 0x00049, 0x0004a, 0x0214a, 0x0214b, 0x0214c, 0x0214d, 0x02132, 0x0214f, + 0x02150, 0x02151, 0x02152, 0x02153, 0x02154, 0x02155, 0x02156, 0x02157, + 0x02158, 0x02159, 0x0215a, 0x0215b, 0x0215c, 0x0215d, 0x0215e, 0x0215f, + 0x00049, 0x02161, 0x02162, 0x02163, 0x00056, 0x02165, 0x02166, 0x02167, + 0x02168, 0x00058, 0x0216a, 0x0216b, 0x0004c, 0x00043, 0x00044, 0x0004d, + 0x00049, 0x02161, 0x02162, 0x02163, 0x00056, 0x02165, 0x02166, 0x02167, + 0x02168, 0x00058, 0x0216a, 0x0216b, 0x0004c, 0x00043, 0x00044, 0x0004d, + 0x02180, 0x02181, 0x02182, 0x02183, 0x02183, 0x00036, 0x02186, 0x02187, + 0x02188, 0x02189, 0x0218a, 0x0218b, 0x0218c, 0x0218d, 0x0218e, 0x0218f, + 0x02190, 0x02191, 0x02192, 0x02193, 0x02194, 0x02195, 0x02196, 0x02197, + 0x02198, 0x02199, 0x02190, 0x02192, 0x0219c, 0x0219d, 0x0219e, 0x0219f, + 0x021a0, 0x021a1, 0x021a2, 0x021a3, 0x021a4, 0x021a5, 0x021a6, 0x021a7, + 0x021a8, 0x021a9, 0x021aa, 0x021ab, 0x021ac, 0x021ad, 0x02194, 0x021af, + 0x021b0, 0x021b1, 0x021b2, 0x021b3, 0x021b4, 0x021b5, 0x021b6, 0x021b7, + 0x021b8, 0x021b9, 0x021ba, 0x021bb, 0x021bc, 0x021bd, 0x021be, 0x021bf, + 0x021c0, 0x021c1, 0x021c2, 0x021c3, 0x021c4, 0x021c5, 0x021c6, 0x021c7, + 0x021c8, 0x021c9, 0x021ca, 0x021cb, 0x021cc, 0x021cd, 0x021ce, 0x021cf, + 0x021cd, 0x021d1, 0x021cf, 0x021d3, 0x021ce, 0x021d5, 0x021d6, 0x021d7, + 0x021d8, 0x021d9, 0x021da, 0x021db, 0x021dc, 0x021dd, 0x021de, 0x021df, + 0x021e0, 0x021e1, 0x021e2, 0x021e3, 0x021e4, 0x021e5, 0x021e6, 0x021e7, + 0x021e8, 0x021e9, 0x021ea, 0x021eb, 0x021ec, 0x021ed, 0x021ee, 0x021ef, + 0x021f0, 0x021f1, 0x021f2, 0x021f3, 0x021f4, 0x021f5, 0x021f6, 0x021f7, + 0x021f8, 0x021f9, 0x021fa, 0x021fb, 0x021fc, 0x021fd, 0x021fe, 0x021ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_22[] = { + 0x02200, 0x02201, 0x02202, 0x02203, 0x02203, 0x02205, 0x02206, 0x02207, + 0x02208, 0x02208, 0x0220a, 0x0220b, 0x0220b, 0x0220d, 0x0220e, 0x0220f, + 0x02210, 0x02140, 0x0207b, 0x02213, 0x02214, 0x02215, 0x02216, 0x02217, + 0x02218, 0x02219, 0x0221a, 0x0221b, 0x0221c, 0x0221d, 0x0221e, 0x0221f, + 0x02220, 0x02221, 0x02222, 0x02223, 0x02223, 0x02225, 0x02225, 0x02227, + 0x02228, 0x02229, 0x0222a, 0x0222b, 0x0222c, 0x0222d, 0x0222e, 0x0222f, + 0x02230, 0x02231, 0x02232, 0x02233, 0x02234, 0x02235, 0x02236, 0x02237, + 0x02238, 0x02239, 0x0223a, 0x0223b, 0x0223c, 0x0223d, 0x0223e, 0x0223f, + 0x02240, 0x0223c, 0x02242, 0x02243, 0x02243, 0x02245, 0x02246, 0x02245, + 0x02248, 0x02248, 0x0224a, 0x0224b, 0x0224c, 0x0224d, 0x0224e, 0x0224f, + 0x02250, 0x02251, 0x02252, 0x02253, 0x02254, 0x02255, 0x02256, 0x02257, + 0x02258, 0x02259, 0x0225a, 0x0225b, 0x0225c, 0x0225d, 0x0225e, 0x0225f, + 0x0003d, 0x02261, 0x02261, 0x02263, 0x02264, 0x02265, 0x02266, 0x02267, + 0x02268, 0x02269, 0x0226a, 0x0226b, 0x0226c, 0x0224d, 0x0003c, 0x0003e, + 0x02264, 0x02265, 0x02272, 0x02273, 0x02272, 0x02273, 0x02276, 0x02277, + 0x02276, 0x02277, 0x0227a, 0x0227b, 0x0227c, 0x0227d, 0x0227e, 0x0227f, + 0x0227a, 0x0227b, 0x02282, 0x02283, 0x02282, 0x02283, 0x02286, 0x02287, + 0x02286, 0x02287, 0x0228a, 0x0228b, 0x0228c, 0x0228d, 0x0228e, 0x0228f, + 0x02290, 0x02291, 0x02292, 0x02293, 0x02294, 0x02295, 0x02296, 0x02297, + 0x02298, 0x02299, 0x0229a, 0x0229b, 0x0229c, 0x0229d, 0x0229e, 0x0229f, + 0x022a0, 0x022a1, 0x022a2, 0x022a3, 0x022a4, 0x022a5, 0x022a6, 0x022a7, + 0x022a8, 0x022a9, 0x022aa, 0x022ab, 0x022a2, 0x022a8, 0x022a9, 0x022ab, + 0x022b0, 0x022b1, 0x022b2, 0x022b3, 0x022b4, 0x022b5, 0x022b6, 0x022b7, + 0x022b8, 0x022b9, 0x022ba, 0x022bb, 0x022bc, 0x022bd, 0x022be, 0x022bf, + 0x022c0, 0x022c1, 0x022c2, 0x022c3, 0x022c4, 0x022c5, 0x022c6, 0x022c7, + 0x022c8, 0x022c9, 0x022ca, 0x022cb, 0x022cc, 0x022cd, 0x022ce, 0x022cf, + 0x022d0, 0x022d1, 0x022d2, 0x022d3, 0x022d4, 0x022d5, 0x022d6, 0x022d7, + 0x022d8, 0x022d9, 0x022da, 0x022db, 0x022dc, 0x022dd, 0x022de, 0x022df, + 0x0227c, 0x0227d, 0x02291, 0x02292, 0x022e4, 0x022e5, 0x022e6, 0x022e7, + 0x022e8, 0x022e9, 0x022b2, 0x022b3, 0x022b4, 0x022b5, 0x022ee, 0x022ef, + 0x022f0, 0x022f1, 0x022f2, 0x022f3, 0x022f4, 0x022f5, 0x022f6, 0x022f7, + 0x022f8, 0x022f9, 0x022fa, 0x022fb, 0x022fc, 0x022fd, 0x022fe, 0x022ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_24[] = { + 0x02400, 0x02401, 0x02402, 0x02403, 0x02404, 0x02405, 0x02406, 0x02407, + 0x02408, 0x02409, 0x0240a, 0x0240b, 0x0240c, 0x0240d, 0x0240e, 0x0240f, + 0x02410, 0x02411, 0x02412, 0x02413, 0x02414, 0x02415, 0x02416, 0x02417, + 0x02418, 0x02419, 0x0241a, 0x0241b, 0x0241c, 0x0241d, 0x0241e, 0x0241f, + 0x02420, 0x02421, 0x02422, 0x02423, 0x02424, 0x02425, 0x02426, 0x02427, + 0x02428, 0x02429, 0x0242a, 0x0242b, 0x0242c, 0x0242d, 0x0242e, 0x0242f, + 0x02430, 0x02431, 0x02432, 0x02433, 0x02434, 0x02435, 0x02436, 0x02437, + 0x02438, 0x02439, 0x0243a, 0x0243b, 0x0243c, 0x0243d, 0x0243e, 0x0243f, + 0x02440, 0x02441, 0x02442, 0x02443, 0x02444, 0x02445, 0x02446, 0x02447, + 0x02448, 0x02449, 0x0244a, 0x0244b, 0x0244c, 0x0244d, 0x0244e, 0x0244f, + 0x02450, 0x02451, 0x02452, 0x02453, 0x02454, 0x02455, 0x02456, 0x02457, + 0x02458, 0x02459, 0x0245a, 0x0245b, 0x0245c, 0x0245d, 0x0245e, 0x0245f, + 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, + 0x00039, 0x02469, 0x0246a, 0x0246b, 0x0246c, 0x0246d, 0x0246e, 0x0246f, + 0x02470, 0x02471, 0x02472, 0x02473, 0x02474, 0x02475, 0x02476, 0x02477, + 0x02478, 0x02479, 0x0247a, 0x0247b, 0x0247c, 0x0247d, 0x0247e, 0x0247f, + 0x02480, 0x02481, 0x02482, 0x02483, 0x02484, 0x02485, 0x02486, 0x02487, + 0x02488, 0x02489, 0x0248a, 0x0248b, 0x0248c, 0x0248d, 0x0248e, 0x0248f, + 0x02490, 0x02491, 0x02492, 0x02493, 0x02494, 0x02495, 0x02496, 0x02497, + 0x02498, 0x02499, 0x0249a, 0x0249b, 0x0249c, 0x0249d, 0x0249e, 0x0249f, + 0x024a0, 0x024a1, 0x024a2, 0x024a3, 0x024a4, 0x024a5, 0x024a6, 0x024a7, + 0x024a8, 0x024a9, 0x024aa, 0x024ab, 0x024ac, 0x024ad, 0x024ae, 0x024af, + 0x024b0, 0x024b1, 0x024b2, 0x024b3, 0x024b4, 0x024b5, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00030, 0x0246a, 0x0246b, 0x0246c, 0x0246d, 0x0246e, + 0x0246f, 0x02470, 0x02471, 0x02472, 0x02473, 0x00031, 0x00032, 0x00033, + 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x02469, 0x024ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_27[] = { + 0x02700, 0x02701, 0x02702, 0x02703, 0x02704, 0x02705, 0x02706, 0x02707, + 0x02708, 0x02709, 0x0270a, 0x0270b, 0x0270c, 0x0270d, 0x0270e, 0x0270f, + 0x02710, 0x02711, 0x02712, 0x02713, 0x02714, 0x02715, 0x02716, 0x02717, + 0x02718, 0x02719, 0x0271a, 0x0271b, 0x0271c, 0x0271d, 0x0271e, 0x0271f, + 0x02720, 0x02721, 0x02722, 0x02723, 0x02724, 0x02725, 0x02726, 0x02727, + 0x02728, 0x02729, 0x0272a, 0x0272b, 0x0272c, 0x0272d, 0x0272e, 0x0272f, + 0x02730, 0x02731, 0x02732, 0x02733, 0x02734, 0x02735, 0x02736, 0x02737, + 0x02738, 0x02739, 0x0273a, 0x0273b, 0x0273c, 0x0273d, 0x0273e, 0x0273f, + 0x02740, 0x02741, 0x02742, 0x02743, 0x02744, 0x02745, 0x02746, 0x02747, + 0x02748, 0x02749, 0x0274a, 0x0274b, 0x0274c, 0x0274d, 0x0274e, 0x0274f, + 0x02750, 0x02751, 0x02752, 0x02753, 0x02754, 0x02755, 0x02756, 0x02757, + 0x02758, 0x02759, 0x0275a, 0x0275b, 0x0275c, 0x0275d, 0x0275e, 0x0275f, + 0x02760, 0x02761, 0x02762, 0x02763, 0x02764, 0x02765, 0x02766, 0x02767, + 0x02768, 0x02769, 0x0276a, 0x0276b, 0x0276c, 0x0276d, 0x0276e, 0x0276f, + 0x02770, 0x02771, 0x02772, 0x02773, 0x02774, 0x02775, 0x00031, 0x00032, + 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x02469, + 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, + 0x00039, 0x02469, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, + 0x00037, 0x00038, 0x00039, 0x02469, 0x02794, 0x02795, 0x02796, 0x02797, + 0x02798, 0x02799, 0x0279a, 0x0279b, 0x0279c, 0x0279d, 0x0279e, 0x0279f, + 0x027a0, 0x027a1, 0x027a2, 0x027a3, 0x027a4, 0x027a5, 0x027a6, 0x027a7, + 0x027a8, 0x027a9, 0x027aa, 0x027ab, 0x027ac, 0x027ad, 0x027ae, 0x027af, + 0x027b0, 0x027b1, 0x027b2, 0x027b3, 0x027b4, 0x027b5, 0x027b6, 0x027b7, + 0x027b8, 0x027b9, 0x027ba, 0x027bb, 0x027bc, 0x027bd, 0x027be, 0x027bf, + 0x027c0, 0x027c1, 0x027c2, 0x027c3, 0x027c4, 0x027c5, 0x027c6, 0x027c7, + 0x027c8, 0x027c9, 0x027ca, 0x027cb, 0x027cc, 0x027cd, 0x027ce, 0x027cf, + 0x027d0, 0x027d1, 0x027d2, 0x027d3, 0x027d4, 0x027d5, 0x027d6, 0x027d7, + 0x027d8, 0x027d9, 0x027da, 0x027db, 0x027dc, 0x027dd, 0x027de, 0x027df, + 0x027e0, 0x027e1, 0x027e2, 0x027e3, 0x027e4, 0x027e5, 0x027e6, 0x027e7, + 0x027e8, 0x027e9, 0x027ea, 0x027eb, 0x027ec, 0x027ed, 0x027ee, 0x027ef, + 0x027f0, 0x027f1, 0x027f2, 0x027f3, 0x027f4, 0x027f5, 0x027f6, 0x027f7, + 0x027f8, 0x027f9, 0x027fa, 0x027fb, 0x027fc, 0x027fd, 0x027fe, 0x027ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2a[] = { + 0x02a00, 0x02a01, 0x02a02, 0x02a03, 0x02a04, 0x02a05, 0x02a06, 0x02a07, + 0x02a08, 0x02a09, 0x02a0a, 0x02a0b, 0x02a0c, 0x02a0d, 0x02a0e, 0x02a0f, + 0x02a10, 0x02a11, 0x02a12, 0x02a13, 0x02a14, 0x02a15, 0x02a16, 0x02a17, + 0x02a18, 0x02a19, 0x02a1a, 0x02a1b, 0x02a1c, 0x02a1d, 0x02a1e, 0x02a1f, + 0x02a20, 0x02a21, 0x02a22, 0x02a23, 0x02a24, 0x02a25, 0x02a26, 0x02a27, + 0x02a28, 0x02a29, 0x02a2a, 0x02a2b, 0x02a2c, 0x02a2d, 0x02a2e, 0x02a2f, + 0x02a30, 0x02a31, 0x02a32, 0x02a33, 0x02a34, 0x02a35, 0x02a36, 0x02a37, + 0x02a38, 0x02a39, 0x02a3a, 0x02a3b, 0x02a3c, 0x02a3d, 0x02a3e, 0x02a3f, + 0x02a40, 0x02a41, 0x02a42, 0x02a43, 0x02a44, 0x02a45, 0x02a46, 0x02a47, + 0x02a48, 0x02a49, 0x02a4a, 0x02a4b, 0x02a4c, 0x02a4d, 0x02a4e, 0x02a4f, + 0x02a50, 0x02a51, 0x02a52, 0x02a53, 0x02a54, 0x02a55, 0x02a56, 0x02a57, + 0x02a58, 0x02a59, 0x02a5a, 0x02a5b, 0x02a5c, 0x02a5d, 0x02a5e, 0x02a5f, + 0x02a60, 0x02a61, 0x02a62, 0x02a63, 0x02a64, 0x02a65, 0x02a66, 0x02a67, + 0x02a68, 0x02a69, 0x02a6a, 0x02a6b, 0x02a6c, 0x02a6d, 0x02a6e, 0x02a6f, + 0x02a70, 0x02a71, 0x02a72, 0x02a73, 0x02a74, 0x02a75, 0x02a76, 0x02a77, + 0x02a78, 0x02a79, 0x02a7a, 0x02a7b, 0x02a7c, 0x02a7d, 0x02a7e, 0x02a7f, + 0x02a80, 0x02a81, 0x02a82, 0x02a83, 0x02a84, 0x02a85, 0x02a86, 0x02a87, + 0x02a88, 0x02a89, 0x02a8a, 0x02a8b, 0x02a8c, 0x02a8d, 0x02a8e, 0x02a8f, + 0x02a90, 0x02a91, 0x02a92, 0x02a93, 0x02a94, 0x02a95, 0x02a96, 0x02a97, + 0x02a98, 0x02a99, 0x02a9a, 0x02a9b, 0x02a9c, 0x02a9d, 0x02a9e, 0x02a9f, + 0x02aa0, 0x02aa1, 0x02aa2, 0x02aa3, 0x02aa4, 0x02aa5, 0x02aa6, 0x02aa7, + 0x02aa8, 0x02aa9, 0x02aaa, 0x02aab, 0x02aac, 0x02aad, 0x02aae, 0x02aaf, + 0x02ab0, 0x02ab1, 0x02ab2, 0x02ab3, 0x02ab4, 0x02ab5, 0x02ab6, 0x02ab7, + 0x02ab8, 0x02ab9, 0x02aba, 0x02abb, 0x02abc, 0x02abd, 0x02abe, 0x02abf, + 0x02ac0, 0x02ac1, 0x02ac2, 0x02ac3, 0x02ac4, 0x02ac5, 0x02ac6, 0x02ac7, + 0x02ac8, 0x02ac9, 0x02aca, 0x02acb, 0x02acc, 0x02acd, 0x02ace, 0x02acf, + 0x02ad0, 0x02ad1, 0x02ad2, 0x02ad3, 0x02ad4, 0x02ad5, 0x02ad6, 0x02ad7, + 0x02ad8, 0x02ad9, 0x02ada, 0x02adb, 0x02adc, 0x02adc, 0x02ade, 0x02adf, + 0x02ae0, 0x02ae1, 0x02ae2, 0x02ae3, 0x02ae4, 0x02ae5, 0x02ae6, 0x02ae7, + 0x02ae8, 0x02ae9, 0x02aea, 0x02aeb, 0x02aec, 0x02aed, 0x02aee, 0x02aef, + 0x02af0, 0x02af1, 0x02af2, 0x02af3, 0x02af4, 0x02af5, 0x02af6, 0x02af7, + 0x02af8, 0x02af9, 0x02afa, 0x02afb, 0x02afc, 0x02afd, 0x02afe, 0x02aff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2c[] = { + 0x02c00, 0x02c01, 0x02c02, 0x02c03, 0x02c04, 0x02c05, 0x02c06, 0x02c07, + 0x02c08, 0x02c09, 0x02c0a, 0x02c0b, 0x02c0c, 0x02c0d, 0x02c0e, 0x02c0f, + 0x02c10, 0x02c11, 0x02c12, 0x02c13, 0x02c14, 0x02c15, 0x02c16, 0x02c17, + 0x02c18, 0x02c19, 0x02c1a, 0x02c1b, 0x02c1c, 0x02c1d, 0x02c1e, 0x02c1f, + 0x02c20, 0x02c21, 0x02c22, 0x02c23, 0x02c24, 0x02c25, 0x02c26, 0x02c27, + 0x02c28, 0x02c29, 0x02c2a, 0x02c2b, 0x02c2c, 0x02c2d, 0x02c2e, 0x02c2f, + 0x02c00, 0x02c01, 0x02c02, 0x02c03, 0x02c04, 0x02c05, 0x02c06, 0x02c07, + 0x02c08, 0x02c09, 0x02c0a, 0x02c0b, 0x02c0c, 0x02c0d, 0x02c0e, 0x02c0f, + 0x02c10, 0x02c11, 0x02c12, 0x02c13, 0x02c14, 0x02c15, 0x02c16, 0x02c17, + 0x02c18, 0x02c19, 0x02c1a, 0x02c1b, 0x02c1c, 0x02c1d, 0x02c1e, 0x02c1f, + 0x02c20, 0x02c21, 0x02c22, 0x02c23, 0x02c24, 0x02c25, 0x02c26, 0x02c27, + 0x02c28, 0x02c29, 0x02c2a, 0x02c2b, 0x02c2c, 0x02c2d, 0x02c2e, 0x02c5f, + 0x02c60, 0x02c60, 0x0026b, 0x01d7d, 0x0027d, 0x0023a, 0x0023e, 0x02c67, + 0x02c67, 0x02c69, 0x02c69, 0x02c6b, 0x02c6b, 0x00251, 0x00271, 0x00250, + 0x00252, 0x02c71, 0x02c72, 0x02c72, 0x02c74, 0x02c75, 0x02c75, 0x02c77, + 0x02c78, 0x02c79, 0x02c7a, 0x02c7b, 0x0004a, 0x00056, 0x0023f, 0x00240, + 0x02c80, 0x02c80, 0x02c82, 0x02c82, 0x02c84, 0x02c84, 0x02c86, 0x02c86, + 0x02c88, 0x02c88, 0x02c8a, 0x02c8a, 0x02c8c, 0x02c8c, 0x02c8e, 0x02c8e, + 0x02c90, 0x02c90, 0x02c92, 0x02c92, 0x02c94, 0x02c94, 0x02c96, 0x02c96, + 0x02c98, 0x02c98, 0x02c9a, 0x02c9a, 0x02c9c, 0x02c9c, 0x02c9e, 0x02c9e, + 0x02ca0, 0x02ca0, 0x02ca2, 0x02ca2, 0x02ca4, 0x02ca4, 0x02ca6, 0x02ca6, + 0x02ca8, 0x02ca8, 0x02caa, 0x02caa, 0x02cac, 0x02cac, 0x02cae, 0x02cae, + 0x02cb0, 0x02cb0, 0x02cb2, 0x02cb2, 0x02cb4, 0x02cb4, 0x02cb6, 0x02cb6, + 0x02cb8, 0x02cb8, 0x02cba, 0x02cba, 0x02cbc, 0x02cbc, 0x02cbe, 0x02cbe, + 0x02cc0, 0x02cc0, 0x02cc2, 0x02cc2, 0x02cc4, 0x02cc4, 0x02cc6, 0x02cc6, + 0x02cc8, 0x02cc8, 0x02cca, 0x02cca, 0x02ccc, 0x02ccc, 0x02cce, 0x02cce, + 0x02cd0, 0x02cd0, 0x02cd2, 0x02cd2, 0x02cd4, 0x02cd4, 0x02cd6, 0x02cd6, + 0x02cd8, 0x02cd8, 0x02cda, 0x02cda, 0x02cdc, 0x02cdc, 0x02cde, 0x02cde, + 0x02ce0, 0x02ce0, 0x02ce2, 0x02ce2, 0x02ce4, 0x02ce5, 0x02ce6, 0x02ce7, + 0x02ce8, 0x02ce9, 0x02cea, 0x02ceb, 0x02ceb, 0x02ced, 0x02ced, 0x00000, + 0x00000, 0x00000, 0x02cf2, 0x02cf3, 0x02cf4, 0x02cf5, 0x02cf6, 0x02cf7, + 0x02cf8, 0x02cf9, 0x02cfa, 0x02cfb, 0x02cfc, 0x02cfd, 0x02cfe, 0x02cff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2d[] = { + 0x010a0, 0x010a1, 0x010a2, 0x010a3, 0x010a4, 0x010a5, 0x010a6, 0x010a7, + 0x010a8, 0x010a9, 0x010aa, 0x010ab, 0x010ac, 0x010ad, 0x010ae, 0x010af, + 0x010b0, 0x010b1, 0x010b2, 0x010b3, 0x010b4, 0x010b5, 0x010b6, 0x010b7, + 0x010b8, 0x010b9, 0x010ba, 0x010bb, 0x010bc, 0x010bd, 0x010be, 0x010bf, + 0x010c0, 0x010c1, 0x010c2, 0x010c3, 0x010c4, 0x010c5, 0x02d26, 0x02d27, + 0x02d28, 0x02d29, 0x02d2a, 0x02d2b, 0x02d2c, 0x02d2d, 0x02d2e, 0x02d2f, + 0x02d30, 0x02d31, 0x02d32, 0x02d33, 0x02d34, 0x02d35, 0x02d36, 0x02d37, + 0x02d38, 0x02d39, 0x02d3a, 0x02d3b, 0x02d3c, 0x02d3d, 0x02d3e, 0x02d3f, + 0x02d40, 0x02d41, 0x02d42, 0x02d43, 0x02d44, 0x02d45, 0x02d46, 0x02d47, + 0x02d48, 0x02d49, 0x02d4a, 0x02d4b, 0x02d4c, 0x02d4d, 0x02d4e, 0x02d4f, + 0x02d50, 0x02d51, 0x02d52, 0x02d53, 0x02d54, 0x02d55, 0x02d56, 0x02d57, + 0x02d58, 0x02d59, 0x02d5a, 0x02d5b, 0x02d5c, 0x02d5d, 0x02d5e, 0x02d5f, + 0x02d60, 0x02d61, 0x02d62, 0x02d63, 0x02d64, 0x02d65, 0x02d66, 0x02d67, + 0x02d68, 0x02d69, 0x02d6a, 0x02d6b, 0x02d6c, 0x02d6d, 0x02d6e, 0x02d6f, + 0x02d70, 0x02d71, 0x02d72, 0x02d73, 0x02d74, 0x02d75, 0x02d76, 0x02d77, + 0x02d78, 0x02d79, 0x02d7a, 0x02d7b, 0x02d7c, 0x02d7d, 0x02d7e, 0x02d7f, + 0x02d80, 0x02d81, 0x02d82, 0x02d83, 0x02d84, 0x02d85, 0x02d86, 0x02d87, + 0x02d88, 0x02d89, 0x02d8a, 0x02d8b, 0x02d8c, 0x02d8d, 0x02d8e, 0x02d8f, + 0x02d90, 0x02d91, 0x02d92, 0x02d93, 0x02d94, 0x02d95, 0x02d96, 0x02d97, + 0x02d98, 0x02d99, 0x02d9a, 0x02d9b, 0x02d9c, 0x02d9d, 0x02d9e, 0x02d9f, + 0x02da0, 0x02da1, 0x02da2, 0x02da3, 0x02da4, 0x02da5, 0x02da6, 0x02da7, + 0x02da8, 0x02da9, 0x02daa, 0x02dab, 0x02dac, 0x02dad, 0x02dae, 0x02daf, + 0x02db0, 0x02db1, 0x02db2, 0x02db3, 0x02db4, 0x02db5, 0x02db6, 0x02db7, + 0x02db8, 0x02db9, 0x02dba, 0x02dbb, 0x02dbc, 0x02dbd, 0x02dbe, 0x02dbf, + 0x02dc0, 0x02dc1, 0x02dc2, 0x02dc3, 0x02dc4, 0x02dc5, 0x02dc6, 0x02dc7, + 0x02dc8, 0x02dc9, 0x02dca, 0x02dcb, 0x02dcc, 0x02dcd, 0x02dce, 0x02dcf, + 0x02dd0, 0x02dd1, 0x02dd2, 0x02dd3, 0x02dd4, 0x02dd5, 0x02dd6, 0x02dd7, + 0x02dd8, 0x02dd9, 0x02dda, 0x02ddb, 0x02ddc, 0x02ddd, 0x02dde, 0x02ddf, + 0x00411, 0x00412, 0x00413, 0x00414, 0x00416, 0x00417, 0x0041a, 0x0041b, + 0x0041c, 0x0041d, 0x0041e, 0x0041f, 0x00420, 0x00421, 0x00422, 0x00425, + 0x00426, 0x00427, 0x00428, 0x00429, 0x00472, 0x02df5, 0x00410, 0x00400, + 0x02df8, 0x02df9, 0x00462, 0x0042e, 0x02dfc, 0x00466, 0x0046a, 0x02dff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2e[] = { + 0x02e00, 0x02e01, 0x02e02, 0x02e03, 0x02e04, 0x02e05, 0x02e06, 0x02e07, + 0x02e08, 0x02e09, 0x02e0a, 0x02e0b, 0x02e0c, 0x02e0d, 0x02e0e, 0x02e0f, + 0x02e10, 0x02e11, 0x02e12, 0x02e13, 0x02e14, 0x02e15, 0x02e16, 0x02e17, + 0x02e18, 0x02e19, 0x02e1a, 0x02e1b, 0x02e1c, 0x02e1d, 0x02e1e, 0x02e1f, + 0x02e20, 0x02e21, 0x02e22, 0x02e23, 0x02e24, 0x02e25, 0x02e26, 0x02e27, + 0x02e28, 0x02e29, 0x02e2a, 0x02e2b, 0x02e2c, 0x02e2d, 0x02e2e, 0x02e2f, + 0x02e30, 0x02e31, 0x02e32, 0x02e33, 0x02e34, 0x02e35, 0x02e36, 0x02e37, + 0x02e38, 0x02e39, 0x02e3a, 0x02e3b, 0x02e3c, 0x02e3d, 0x02e3e, 0x02e3f, + 0x02e40, 0x02e41, 0x02e42, 0x02e43, 0x02e44, 0x02e45, 0x02e46, 0x02e47, + 0x02e48, 0x02e49, 0x02e4a, 0x02e4b, 0x02e4c, 0x02e4d, 0x02e4e, 0x02e4f, + 0x02e50, 0x02e51, 0x02e52, 0x02e53, 0x02e54, 0x02e55, 0x02e56, 0x02e57, + 0x02e58, 0x02e59, 0x02e5a, 0x02e5b, 0x02e5c, 0x02e5d, 0x02e5e, 0x02e5f, + 0x02e60, 0x02e61, 0x02e62, 0x02e63, 0x02e64, 0x02e65, 0x02e66, 0x02e67, + 0x02e68, 0x02e69, 0x02e6a, 0x02e6b, 0x02e6c, 0x02e6d, 0x02e6e, 0x02e6f, + 0x02e70, 0x02e71, 0x02e72, 0x02e73, 0x02e74, 0x02e75, 0x02e76, 0x02e77, + 0x02e78, 0x02e79, 0x02e7a, 0x02e7b, 0x02e7c, 0x02e7d, 0x02e7e, 0x02e7f, + 0x02e80, 0x02e81, 0x02e82, 0x02e83, 0x0319a, 0x02e85, 0x02e86, 0x02f0f, + 0x02f11, 0x02e89, 0x02f18, 0x02e8b, 0x02f29, 0x02f29, 0x02e8e, 0x02e8f, + 0x02e8e, 0x02e8f, 0x02e92, 0x02e93, 0x02e94, 0x02e95, 0x02e96, 0x02f3c, + 0x02e98, 0x02e99, 0x02e9a, 0x02e9b, 0x02f47, 0x02f49, 0x02e9e, 0x02e9f, + 0x02ea0, 0x02ea1, 0x02ea2, 0x02ea3, 0x02ea4, 0x02ea4, 0x02ea6, 0x02f5c, + 0x02ea8, 0x02ea9, 0x02eaa, 0x02f6c, 0x02f70, 0x02ead, 0x02f75, 0x02eaf, + 0x02eb0, 0x02eb1, 0x02eb2, 0x02eb1, 0x02eb1, 0x02eb2, 0x02eb7, 0x02eb7, + 0x02eb8, 0x02eb9, 0x02eba, 0x02f80, 0x02f81, 0x02f85, 0x02ebe, 0x02ebe, + 0x02ebe, 0x02ec1, 0x02ec2, 0x02ec3, 0x02ec4, 0x02ec5, 0x02f93, 0x02f93, + 0x02ec8, 0x02ec9, 0x02f9c, 0x02ecb, 0x02ecc, 0x02ecc, 0x02ecc, 0x02fa2, + 0x02ed0, 0x02fa7, 0x02ed2, 0x02ed3, 0x02ed4, 0x02fa9, 0x02ed6, 0x02fac, + 0x02ed8, 0x02ed9, 0x02eda, 0x02edb, 0x02edc, 0x02fb7, 0x02ede, 0x02ede, + 0x02ee0, 0x02fb8, 0x02ee2, 0x02fbb, 0x02fc1, 0x02ee5, 0x02ee6, 0x02ee7, + 0x02ee8, 0x02ee9, 0x02eea, 0x02eeb, 0x02eec, 0x02eed, 0x02eee, 0x02eef, + 0x02ef0, 0x02ef2, 0x02ef2, 0x02ef3, 0x02ef4, 0x02ef5, 0x02ef6, 0x02ef7, + 0x02ef8, 0x02ef9, 0x02efa, 0x02efb, 0x02efc, 0x02efd, 0x02efe, 0x02eff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2f[] = { + 0x03192, 0x02f01, 0x02e80, 0x02f03, 0x0319a, 0x02f05, 0x03193, 0x02f07, + 0x0319f, 0x02f09, 0x02f0a, 0x02f0b, 0x02e86, 0x02f0d, 0x02f0e, 0x02f0f, + 0x02f10, 0x02f11, 0x02f12, 0x02f13, 0x02f14, 0x02f15, 0x02f16, 0x02f17, + 0x02f18, 0x02e8b, 0x02e81, 0x02f1b, 0x02f1c, 0x02f1d, 0x02f1e, 0x02f1f, + 0x02f20, 0x02f21, 0x02f22, 0x02f23, 0x02f24, 0x02f25, 0x02f26, 0x02f27, + 0x02f28, 0x02f29, 0x02e8e, 0x02f2b, 0x02f2c, 0x02f2d, 0x02f2e, 0x02f2f, + 0x02f30, 0x02f31, 0x02f32, 0x02e93, 0x02f34, 0x02f35, 0x02f36, 0x02f37, + 0x02f38, 0x02e95, 0x02f3a, 0x02f3b, 0x02f3c, 0x02f3d, 0x02f3e, 0x02f3f, + 0x02f40, 0x02f41, 0x02f42, 0x02f43, 0x02f44, 0x02f45, 0x02f46, 0x02f47, + 0x02f48, 0x02f49, 0x02f4a, 0x02f4b, 0x02f4c, 0x02f4d, 0x02f4e, 0x02f4f, + 0x02f50, 0x02f51, 0x02f52, 0x02f53, 0x02f54, 0x02f55, 0x02f56, 0x02f57, + 0x02f58, 0x02f59, 0x02f5a, 0x02f5b, 0x02f5c, 0x02f5d, 0x02f5e, 0x02f5f, + 0x02f60, 0x02f61, 0x02f62, 0x02f63, 0x02f64, 0x02f65, 0x02eaa, 0x02f67, + 0x02f68, 0x02f69, 0x02f6a, 0x02f6b, 0x02f6c, 0x02f6d, 0x02f6e, 0x02f6f, + 0x02f70, 0x02f71, 0x02f72, 0x02f73, 0x02f74, 0x02f75, 0x02f76, 0x02f77, + 0x02f78, 0x02f79, 0x02eb7, 0x02f7b, 0x02f7c, 0x02f7d, 0x02f7e, 0x02f7f, + 0x02f80, 0x02f81, 0x02f82, 0x02f83, 0x02f84, 0x02f85, 0x02f86, 0x02f87, + 0x02f88, 0x02f89, 0x02f8a, 0x02f8b, 0x02f8c, 0x02f8d, 0x02f8e, 0x02f8f, + 0x02f90, 0x02f91, 0x02f92, 0x02f93, 0x02f94, 0x02f95, 0x02f96, 0x02f97, + 0x02f98, 0x02f99, 0x02f9a, 0x02f9b, 0x02f9c, 0x02f9d, 0x02f9e, 0x02f9f, + 0x02fa0, 0x02fa1, 0x02fa2, 0x02fa3, 0x02fa4, 0x02fa5, 0x02fa6, 0x02fa7, + 0x02fa8, 0x02fa9, 0x02faa, 0x02fab, 0x02fac, 0x02fad, 0x02fae, 0x02faf, + 0x02fb0, 0x02fb1, 0x02fb2, 0x02fb3, 0x02fb4, 0x02fb5, 0x02fb6, 0x02fb7, + 0x02fb8, 0x02fb9, 0x02fba, 0x02fbb, 0x02fbc, 0x02fbd, 0x02fbe, 0x02fbf, + 0x02fc0, 0x02fc1, 0x02fc2, 0x02fc3, 0x02ee7, 0x02fc5, 0x02fc6, 0x02fc7, + 0x02fc8, 0x02fc9, 0x02fca, 0x02fcb, 0x02fcc, 0x02fcd, 0x02fce, 0x02fcf, + 0x02fd0, 0x02eeb, 0x02eed, 0x02eef, 0x02ef2, 0x02fd5, 0x02fd6, 0x02fd7, + 0x02fd8, 0x02fd9, 0x02fda, 0x02fdb, 0x02fdc, 0x02fdd, 0x02fde, 0x02fdf, + 0x02fe0, 0x02fe1, 0x02fe2, 0x02fe3, 0x02fe4, 0x02fe5, 0x02fe6, 0x02fe7, + 0x02fe8, 0x02fe9, 0x02fea, 0x02feb, 0x02fec, 0x02fed, 0x02fee, 0x02fef, + 0x02ff0, 0x02ff1, 0x02ff2, 0x02ff3, 0x02ff4, 0x02ff5, 0x02ff6, 0x02ff7, + 0x02ff8, 0x02ff9, 0x02ffa, 0x02ffb, 0x02ffc, 0x02ffd, 0x02ffe, 0x02fff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_30[] = { + 0x00020, 0x03001, 0x03002, 0x03003, 0x03004, 0x03005, 0x03006, 0x00030, + 0x02329, 0x0232a, 0x0300a, 0x0300b, 0x0300c, 0x0300d, 0x0300e, 0x0300f, + 0x03010, 0x03011, 0x03012, 0x03013, 0x03014, 0x03015, 0x03016, 0x03017, + 0x03018, 0x03019, 0x0301a, 0x0301b, 0x0301c, 0x0301d, 0x0301e, 0x0301f, + 0x03020, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x03030, 0x03031, 0x03031, 0x03033, 0x03033, 0x03035, 0x03012, 0x03037, + 0x02f17, 0x03039, 0x0303a, 0x0303b, 0x0303c, 0x0303d, 0x0303e, 0x0303f, + 0x03040, 0x03041, 0x03042, 0x03043, 0x03044, 0x03045, 0x03046, 0x03047, + 0x03048, 0x03049, 0x0304a, 0x0304b, 0x0304c, 0x0304d, 0x0304e, 0x0304f, + 0x03050, 0x03051, 0x03052, 0x03053, 0x03054, 0x03055, 0x03056, 0x03057, + 0x03058, 0x03059, 0x0305a, 0x0305b, 0x0305c, 0x0305d, 0x0305e, 0x0305f, + 0x03060, 0x03061, 0x03062, 0x03063, 0x03064, 0x03065, 0x03066, 0x03067, + 0x03068, 0x03069, 0x0306a, 0x0306b, 0x0306c, 0x0306d, 0x0306e, 0x0306f, + 0x03070, 0x03071, 0x03072, 0x03073, 0x03074, 0x03075, 0x03076, 0x03077, + 0x03078, 0x03079, 0x0307a, 0x0307b, 0x0307c, 0x0307d, 0x0307e, 0x0307f, + 0x03080, 0x03081, 0x03082, 0x03083, 0x03084, 0x03085, 0x03086, 0x03087, + 0x03088, 0x03089, 0x0308a, 0x0308b, 0x0308c, 0x0308d, 0x0308e, 0x0308f, + 0x03090, 0x03091, 0x03092, 0x03093, 0x03046, 0x0304b, 0x03051, 0x03097, + 0x03098, 0x00000, 0x00000, 0x0309b, 0x0309c, 0x0309d, 0x0309d, 0x0309f, + 0x030a0, 0x03041, 0x03042, 0x03043, 0x03044, 0x03045, 0x03046, 0x03047, + 0x03048, 0x03049, 0x0304a, 0x0304b, 0x0304c, 0x0304d, 0x0304e, 0x0304f, + 0x03050, 0x03051, 0x03052, 0x03053, 0x03054, 0x03055, 0x03056, 0x03057, + 0x03058, 0x03059, 0x0305a, 0x0305b, 0x0305c, 0x0305d, 0x0305e, 0x0305f, + 0x03060, 0x03061, 0x03062, 0x03063, 0x03064, 0x03065, 0x03066, 0x03067, + 0x03068, 0x03069, 0x0306a, 0x0306b, 0x0306c, 0x0306d, 0x0306e, 0x0306f, + 0x03070, 0x03071, 0x03072, 0x03073, 0x03074, 0x03075, 0x03076, 0x03077, + 0x03078, 0x03079, 0x0307a, 0x0307b, 0x0307c, 0x0307d, 0x0307e, 0x0307f, + 0x03080, 0x03081, 0x03082, 0x03083, 0x03084, 0x03085, 0x03086, 0x03087, + 0x03088, 0x03089, 0x0308a, 0x0308b, 0x0308c, 0x0308d, 0x0308e, 0x0308f, + 0x03090, 0x03091, 0x03092, 0x03093, 0x03046, 0x0304b, 0x03051, 0x0308f, + 0x03090, 0x03091, 0x03092, 0x030fb, 0x030fc, 0x030fd, 0x030fd, 0x030ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_31[] = { + 0x03100, 0x03101, 0x03102, 0x03103, 0x03104, 0x03105, 0x03106, 0x03107, + 0x03108, 0x03109, 0x0310a, 0x0310b, 0x0310c, 0x0310d, 0x0310e, 0x0310f, + 0x03110, 0x03111, 0x03112, 0x03113, 0x03114, 0x03115, 0x03116, 0x03117, + 0x03118, 0x03119, 0x0311a, 0x0311b, 0x0311c, 0x0311d, 0x0311e, 0x0311f, + 0x03120, 0x03121, 0x03122, 0x03123, 0x03124, 0x03125, 0x03126, 0x03127, + 0x03128, 0x03129, 0x0312a, 0x0312b, 0x0312c, 0x0312d, 0x0312e, 0x0312f, + 0x03130, 0x01100, 0x01101, 0x011aa, 0x01102, 0x011ac, 0x011ad, 0x01103, + 0x01104, 0x01105, 0x011b0, 0x011b1, 0x011b2, 0x011b3, 0x011b4, 0x011b5, + 0x0111a, 0x01106, 0x01107, 0x01108, 0x01121, 0x01109, 0x0110a, 0x0110b, + 0x0110c, 0x0110d, 0x0110e, 0x0110f, 0x01110, 0x01111, 0x01112, 0x01161, + 0x01162, 0x01163, 0x01164, 0x01165, 0x01166, 0x01167, 0x01168, 0x01169, + 0x0116a, 0x0116b, 0x0116c, 0x0116d, 0x0116e, 0x0116f, 0x01170, 0x01171, + 0x01172, 0x01173, 0x01174, 0x01175, 0x01160, 0x01114, 0x01115, 0x011c7, + 0x011c8, 0x011cc, 0x011ce, 0x011d3, 0x011d7, 0x011d9, 0x0111c, 0x011dd, + 0x011df, 0x0111d, 0x0111e, 0x01120, 0x01122, 0x01123, 0x01127, 0x01129, + 0x0112b, 0x0112c, 0x0112d, 0x0112e, 0x0112f, 0x01132, 0x01136, 0x01140, + 0x01147, 0x0114c, 0x011f1, 0x011f2, 0x01157, 0x01158, 0x01159, 0x01184, + 0x01185, 0x01188, 0x01191, 0x01192, 0x01194, 0x0119e, 0x011a1, 0x0318f, + 0x03190, 0x03191, 0x03192, 0x03193, 0x03194, 0x03195, 0x03196, 0x03197, + 0x03198, 0x03199, 0x0319a, 0x0319b, 0x0319c, 0x0319d, 0x0319e, 0x0319f, + 0x03105, 0x03117, 0x03110, 0x0310d, 0x031a4, 0x031a4, 0x031a6, 0x0311b, + 0x03128, 0x0311a, 0x03127, 0x03128, 0x031ac, 0x031ad, 0x0311e, 0x03120, + 0x031b0, 0x031b1, 0x031b2, 0x03127, 0x03106, 0x0310a, 0x0310e, 0x0310f, + 0x031b8, 0x031b9, 0x031ba, 0x031bb, 0x031bc, 0x031bd, 0x031be, 0x031bf, + 0x031c0, 0x031c1, 0x031c2, 0x031c3, 0x031c4, 0x031c5, 0x031c6, 0x031c7, + 0x031c8, 0x031c9, 0x031ca, 0x031cb, 0x031cc, 0x031cd, 0x031ce, 0x031cf, + 0x031d0, 0x031d1, 0x031d2, 0x031d3, 0x031d4, 0x031d5, 0x031d6, 0x031d7, + 0x031d8, 0x031d9, 0x031da, 0x031db, 0x031dc, 0x031dd, 0x031de, 0x031df, + 0x031e0, 0x031e1, 0x031e2, 0x031e3, 0x031e4, 0x031e5, 0x031e6, 0x031e7, + 0x031e8, 0x031e9, 0x031ea, 0x031eb, 0x031ec, 0x031ed, 0x031ee, 0x031ef, + 0x0304f, 0x03057, 0x03059, 0x03068, 0x0306c, 0x0306f, 0x03072, 0x03075, + 0x03078, 0x0307b, 0x03080, 0x03089, 0x0308a, 0x0308b, 0x0308c, 0x031ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_32[] = { + 0x03200, 0x03201, 0x03202, 0x03203, 0x03204, 0x03205, 0x03206, 0x03207, + 0x03208, 0x03209, 0x0320a, 0x0320b, 0x0320c, 0x0320d, 0x0320e, 0x0320f, + 0x03210, 0x03211, 0x03212, 0x03213, 0x03214, 0x03215, 0x03216, 0x03217, + 0x03218, 0x03219, 0x0321a, 0x0321b, 0x0321c, 0x0321d, 0x0321e, 0x0321f, + 0x03220, 0x03221, 0x03222, 0x03223, 0x03224, 0x03225, 0x03226, 0x03227, + 0x03228, 0x03229, 0x0322a, 0x0322b, 0x0322c, 0x0322d, 0x0322e, 0x0322f, + 0x03230, 0x03231, 0x03232, 0x03233, 0x03234, 0x03235, 0x03236, 0x03237, + 0x03238, 0x03239, 0x0323a, 0x0323b, 0x0323c, 0x0323d, 0x0323e, 0x0323f, + 0x03240, 0x03241, 0x03242, 0x03243, 0x03244, 0x03245, 0x02f42, 0x03247, + 0x02469, 0x02473, 0x0324a, 0x0324b, 0x0324c, 0x0324d, 0x0324e, 0x0324f, + 0x03250, 0x03251, 0x03252, 0x03253, 0x03254, 0x03255, 0x03256, 0x03257, + 0x03258, 0x03259, 0x0324a, 0x0325b, 0x0325c, 0x0325d, 0x0325e, 0x0325f, + 0x01100, 0x01102, 0x01103, 0x01105, 0x01106, 0x01107, 0x01109, 0x0110b, + 0x0110c, 0x0110e, 0x0110f, 0x01110, 0x01111, 0x01112, 0x0326e, 0x0326f, + 0x03270, 0x03271, 0x03272, 0x03273, 0x03274, 0x03275, 0x03276, 0x03277, + 0x03278, 0x03279, 0x0327a, 0x0327b, 0x0327c, 0x0327d, 0x0327e, 0x0327f, + 0x03192, 0x03193, 0x03194, 0x03195, 0x03284, 0x03285, 0x03286, 0x02f0b, + 0x03288, 0x02f17, 0x02f49, 0x02f55, 0x02f54, 0x02f4a, 0x02fa6, 0x02f1f, + 0x02f47, 0x03291, 0x03292, 0x03293, 0x03294, 0x03295, 0x03296, 0x03297, + 0x03298, 0x03299, 0x0329a, 0x02f25, 0x0329c, 0x0329d, 0x0329e, 0x0329f, + 0x032a0, 0x032a1, 0x032a2, 0x032a3, 0x03196, 0x03197, 0x03198, 0x032a7, + 0x032a8, 0x032a9, 0x032aa, 0x032ab, 0x032ac, 0x032ad, 0x032ae, 0x032af, + 0x032b0, 0x032b1, 0x032b2, 0x032b3, 0x032b4, 0x0324b, 0x032b6, 0x032b7, + 0x032b8, 0x032b9, 0x032ba, 0x032bb, 0x032bc, 0x032bd, 0x032be, 0x0324c, + 0x032c0, 0x032c1, 0x032c2, 0x032c3, 0x032c4, 0x032c5, 0x032c6, 0x032c7, + 0x032c8, 0x032c9, 0x032ca, 0x032cb, 0x032cc, 0x032cd, 0x032ce, 0x032cf, + 0x03042, 0x03044, 0x03046, 0x03048, 0x0304a, 0x0304b, 0x0304d, 0x0304f, + 0x03051, 0x03053, 0x03055, 0x03057, 0x03059, 0x0305b, 0x0305d, 0x0305f, + 0x03061, 0x03064, 0x03066, 0x03068, 0x0306a, 0x0306b, 0x0306c, 0x0306d, + 0x0306e, 0x0306f, 0x03072, 0x03075, 0x03078, 0x0307b, 0x0307e, 0x0307f, + 0x03080, 0x03081, 0x03082, 0x03084, 0x03086, 0x03088, 0x03089, 0x0308a, + 0x0308b, 0x0308c, 0x0308d, 0x0308f, 0x03090, 0x03091, 0x03092, 0x032ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_33[] = { + 0x03300, 0x03301, 0x03302, 0x03303, 0x03304, 0x03305, 0x03306, 0x03307, + 0x03308, 0x03309, 0x0330a, 0x0330b, 0x0330c, 0x0330d, 0x0330e, 0x0330f, + 0x03310, 0x03311, 0x03312, 0x03313, 0x03314, 0x03315, 0x03316, 0x03317, + 0x03318, 0x03319, 0x0331a, 0x0331b, 0x0331c, 0x0331d, 0x0331e, 0x0331f, + 0x03320, 0x03321, 0x03322, 0x03323, 0x03324, 0x03325, 0x03326, 0x03327, + 0x03328, 0x03329, 0x0332a, 0x0332b, 0x0332c, 0x0332d, 0x0332e, 0x0332f, + 0x03330, 0x03331, 0x03332, 0x03333, 0x03334, 0x03335, 0x03336, 0x03337, + 0x03338, 0x03339, 0x0333a, 0x0333b, 0x0333c, 0x0333d, 0x0333e, 0x0333f, + 0x03340, 0x03341, 0x03342, 0x03343, 0x03344, 0x03345, 0x03346, 0x03347, + 0x03348, 0x03349, 0x0334a, 0x0334b, 0x0334c, 0x0334d, 0x0334e, 0x0334f, + 0x03350, 0x03351, 0x03352, 0x03353, 0x03354, 0x03355, 0x03356, 0x03357, + 0x03358, 0x03359, 0x0335a, 0x0335b, 0x0335c, 0x0335d, 0x0335e, 0x0335f, + 0x03360, 0x03361, 0x03362, 0x03363, 0x03364, 0x03365, 0x03366, 0x03367, + 0x03368, 0x03369, 0x0336a, 0x0336b, 0x0336c, 0x0336d, 0x0336e, 0x0336f, + 0x03370, 0x03371, 0x03372, 0x03373, 0x03374, 0x03375, 0x03376, 0x03377, + 0x03378, 0x03379, 0x0337a, 0x0337b, 0x0337c, 0x0337d, 0x0337e, 0x0337f, + 0x03380, 0x03381, 0x03382, 0x03383, 0x03384, 0x03385, 0x03386, 0x03387, + 0x03388, 0x03389, 0x0338a, 0x0338b, 0x0338c, 0x0338d, 0x0338e, 0x0338f, + 0x03390, 0x03391, 0x03392, 0x03393, 0x03394, 0x03395, 0x03396, 0x03397, + 0x03398, 0x03399, 0x0339a, 0x0339b, 0x0339c, 0x0339d, 0x0339e, 0x0339f, + 0x033a0, 0x033a1, 0x033a2, 0x033a3, 0x033a4, 0x033a5, 0x033a6, 0x033a7, + 0x033a8, 0x03380, 0x033aa, 0x033ab, 0x033ac, 0x033ad, 0x033ae, 0x033af, + 0x033b0, 0x033b1, 0x033b2, 0x033b3, 0x033b4, 0x033b5, 0x033b6, 0x033b7, + 0x033b8, 0x033b7, 0x033ba, 0x033bb, 0x033bc, 0x033bd, 0x033be, 0x033bd, + 0x033c0, 0x033c1, 0x033c2, 0x033c3, 0x033c4, 0x033c5, 0x033c6, 0x033c7, + 0x00238, 0x033c9, 0x033ca, 0x033cb, 0x033cc, 0x033cd, 0x0339e, 0x033cf, + 0x033d0, 0x033d1, 0x033d2, 0x033d3, 0x03386, 0x033d5, 0x033d6, 0x033d7, + 0x033d8, 0x033d9, 0x033da, 0x033db, 0x033dc, 0x033dd, 0x033de, 0x033df, + 0x033e0, 0x033e1, 0x033e2, 0x033e3, 0x033e4, 0x033e5, 0x033e6, 0x033e7, + 0x033e8, 0x033e9, 0x033ea, 0x033eb, 0x033ec, 0x033ed, 0x033ee, 0x033ef, + 0x033f0, 0x033f1, 0x033f2, 0x033f3, 0x033f4, 0x033f5, 0x033f6, 0x033f7, + 0x033f8, 0x033f9, 0x033fa, 0x033fb, 0x033fc, 0x033fd, 0x033fe, 0x033ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_a6[] = { + 0x0a600, 0x0a601, 0x0a602, 0x0a603, 0x0a604, 0x0a605, 0x0a606, 0x0a607, + 0x0a608, 0x0a609, 0x0a60a, 0x0a60b, 0x0a60c, 0x0a60d, 0x0a60e, 0x0a60f, + 0x0a558, 0x0a56a, 0x0a587, 0x0a613, 0x0a614, 0x0a615, 0x0a616, 0x0a617, + 0x0a618, 0x0a619, 0x0a61a, 0x0a61b, 0x0a61c, 0x0a61d, 0x0a61e, 0x0a61f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0a56e, 0x0a5d1, 0x0a62c, 0x0a62d, 0x0a62e, 0x0a62f, + 0x0a630, 0x0a631, 0x0a632, 0x0a633, 0x0a634, 0x0a635, 0x0a636, 0x0a637, + 0x0a638, 0x0a639, 0x0a63a, 0x0a63b, 0x0a63c, 0x0a63d, 0x0a63e, 0x0a63f, + 0x0a640, 0x0a640, 0x0a642, 0x0a642, 0x0a644, 0x0a644, 0x0a646, 0x0a646, + 0x02df8, 0x02df8, 0x02df9, 0x02df9, 0x0a64c, 0x0a64c, 0x0a64e, 0x0a64e, + 0x0a650, 0x0a650, 0x0a652, 0x0a652, 0x0a654, 0x0a654, 0x02dfc, 0x02dfc, + 0x0a658, 0x0a658, 0x0a65a, 0x0a65a, 0x0a65c, 0x0a65c, 0x0a65e, 0x0a65e, + 0x0a660, 0x0a661, 0x0a662, 0x0a662, 0x0a664, 0x0a664, 0x0a666, 0x0a666, + 0x0041e, 0x0041e, 0x0041e, 0x0041e, 0x0041e, 0x0041e, 0x0041e, 0x00000, + 0x00000, 0x00000, 0x00000, 0x0a673, 0x0a674, 0x0a675, 0x0a676, 0x0a677, + 0x0a678, 0x0a679, 0x0a67a, 0x0a67b, 0x00000, 0x00000, 0x0a67e, 0x0a67f, + 0x0a680, 0x0a680, 0x0a682, 0x0a682, 0x0a684, 0x0a684, 0x0a686, 0x0a686, + 0x0a688, 0x0a688, 0x0a68a, 0x0a68a, 0x0a68c, 0x0a68c, 0x0a68e, 0x0a68e, + 0x0a690, 0x0a690, 0x0a692, 0x0a692, 0x0a694, 0x0a694, 0x0a696, 0x0a696, + 0x0a698, 0x0a699, 0x0a69a, 0x0a69b, 0x0a69c, 0x0a69d, 0x0a69e, 0x0a69f, + 0x0a6a0, 0x0a6a1, 0x0a6a2, 0x0a6a3, 0x0a6a4, 0x0a6a5, 0x0a6a6, 0x0a6a7, + 0x0a6a8, 0x0a6a9, 0x0a6aa, 0x0a6ab, 0x0a6ac, 0x0a6ad, 0x0a6ae, 0x0a6af, + 0x0a6b0, 0x0a6b1, 0x0a6b2, 0x0a6b3, 0x0a6b4, 0x0a6b5, 0x0a6b6, 0x0a6b7, + 0x0a6b8, 0x0a6b9, 0x0a6ba, 0x0a6bb, 0x0a6bc, 0x0a6bd, 0x0a6be, 0x0a6bf, + 0x0a6c0, 0x0a6c1, 0x0a6c2, 0x0a6c3, 0x0a6c4, 0x0a6c5, 0x0a6c6, 0x0a6c7, + 0x0a6c8, 0x0a6c9, 0x0a6ca, 0x0a6cb, 0x0a6cc, 0x0a6cd, 0x0a6ce, 0x0a6cf, + 0x0a6d0, 0x0a6d1, 0x0a6d2, 0x0a6d3, 0x0a6d4, 0x0a6d5, 0x0a6d6, 0x0a6d7, + 0x0a6d8, 0x0a6d9, 0x0a6da, 0x0a6db, 0x0a6dc, 0x0a6dd, 0x0a6de, 0x0a6df, + 0x0a6e0, 0x0a6e1, 0x0a6e2, 0x0a6e3, 0x0a6e4, 0x0a6e5, 0x0a6e6, 0x0a6e7, + 0x0a6e8, 0x0a6e9, 0x0a6ea, 0x0a6eb, 0x0a6ec, 0x0a6ed, 0x0a6ee, 0x0a6ef, + 0x00000, 0x00000, 0x0a6f2, 0x0a6f3, 0x0a6f4, 0x0a6f5, 0x0a6f6, 0x0a6f7, + 0x0a6f8, 0x0a6f9, 0x0a6fa, 0x0a6fb, 0x0a6fc, 0x0a6fd, 0x0a6fe, 0x0a6ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_a7[] = { + 0x0a700, 0x0a701, 0x0a702, 0x0a703, 0x0a704, 0x0a705, 0x0a706, 0x0a707, + 0x0a708, 0x0a709, 0x0a70a, 0x0a70b, 0x0a70c, 0x0a70d, 0x0a70e, 0x0a70f, + 0x0a710, 0x0a711, 0x0a712, 0x0a713, 0x0a714, 0x0a715, 0x0a716, 0x0a717, + 0x0a718, 0x0a719, 0x0a71a, 0x0a71b, 0x0a71c, 0x0a71d, 0x0a71e, 0x0a71f, + 0x0a720, 0x0a721, 0x0a722, 0x0a722, 0x0a724, 0x0a724, 0x0a726, 0x0a726, + 0x0a728, 0x0a728, 0x0a72a, 0x0a72a, 0x0a72c, 0x0a72c, 0x0a72e, 0x0a72e, + 0x0a730, 0x0a731, 0x0a732, 0x0a732, 0x01dd5, 0x01dd5, 0x03373, 0x03373, + 0x01dd6, 0x01dd6, 0x01dd6, 0x01dd6, 0x0a73c, 0x0a73c, 0x0a73e, 0x0a73e, + 0x0a740, 0x0a740, 0x0a742, 0x0a742, 0x0a744, 0x0a744, 0x0a746, 0x0a746, + 0x0a748, 0x0a748, 0x0a74a, 0x0a74a, 0x0a74c, 0x0a74c, 0x0a74e, 0x0a74e, + 0x0a750, 0x0a750, 0x0a752, 0x0a752, 0x0a754, 0x0a754, 0x0a756, 0x0a756, + 0x0a758, 0x0a758, 0x01de3, 0x01de3, 0x0a75c, 0x0a75c, 0x0a75e, 0x0a75e, + 0x0a760, 0x0a760, 0x0a762, 0x0a762, 0x0a764, 0x0a764, 0x0a766, 0x0a766, + 0x0a768, 0x0a768, 0x0a76a, 0x0a76a, 0x0a76c, 0x0a76c, 0x01dd2, 0x01dd2, + 0x01dd2, 0x0a771, 0x0a772, 0x0a773, 0x0a774, 0x0a775, 0x0a776, 0x0a777, + 0x0a778, 0x00044, 0x00044, 0x00046, 0x00046, 0x00047, 0x0a77e, 0x0a77e, + 0x0a780, 0x0a780, 0x00052, 0x00052, 0x00053, 0x00053, 0x00054, 0x00054, + 0x0a788, 0x0a789, 0x0a78a, 0x0a78b, 0x0a78b, 0x0a78d, 0x0a78e, 0x0a78f, + 0x0a790, 0x0a791, 0x0a792, 0x0a793, 0x0a794, 0x0a795, 0x0a796, 0x0a797, + 0x0a798, 0x0a799, 0x0a79a, 0x0a79b, 0x0a79c, 0x0a79d, 0x0a79e, 0x0a79f, + 0x0a7a0, 0x0a7a1, 0x0a7a2, 0x0a7a3, 0x0a7a4, 0x0a7a5, 0x0a7a6, 0x0a7a7, + 0x0a7a8, 0x0a7a9, 0x0a7aa, 0x0a7ab, 0x0a7ac, 0x0a7ad, 0x0a7ae, 0x0a7af, + 0x0a7b0, 0x0a7b1, 0x0a7b2, 0x0a7b3, 0x0a7b4, 0x0a7b5, 0x0a7b6, 0x0a7b7, + 0x0a7b8, 0x0a7b9, 0x0a7ba, 0x0a7bb, 0x0a7bc, 0x0a7bd, 0x0a7be, 0x0a7bf, + 0x0a7c0, 0x0a7c1, 0x0a7c2, 0x0a7c3, 0x0a7c4, 0x0a7c5, 0x0a7c6, 0x0a7c7, + 0x0a7c8, 0x0a7c9, 0x0a7ca, 0x0a7cb, 0x0a7cc, 0x0a7cd, 0x0a7ce, 0x0a7cf, + 0x0a7d0, 0x0a7d1, 0x0a7d2, 0x0a7d3, 0x0a7d4, 0x0a7d5, 0x0a7d6, 0x0a7d7, + 0x0a7d8, 0x0a7d9, 0x0a7da, 0x0a7db, 0x0a7dc, 0x0a7dd, 0x0a7de, 0x0a7df, + 0x0a7e0, 0x0a7e1, 0x0a7e2, 0x0a7e3, 0x0a7e4, 0x0a7e5, 0x0a7e6, 0x0a7e7, + 0x0a7e8, 0x0a7e9, 0x0a7ea, 0x0a7eb, 0x0a7ec, 0x0a7ed, 0x0a7ee, 0x0a7ef, + 0x0a7f0, 0x0a7f1, 0x0a7f2, 0x0a7f3, 0x0a7f4, 0x0a7f5, 0x0a7f6, 0x0a7f7, + 0x0a7f8, 0x0a7f9, 0x0a7fa, 0x0a7fb, 0x0a7fc, 0x0a7fd, 0x0a7fe, 0x0a7ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_a8[] = { + 0x0a800, 0x0a801, 0x0a802, 0x0a803, 0x0a804, 0x0a805, 0x0a806, 0x0a807, + 0x0a808, 0x0a809, 0x0a80a, 0x00000, 0x0a80c, 0x0a80d, 0x0a80e, 0x0a80f, + 0x0a810, 0x0a811, 0x0a812, 0x0a813, 0x0a814, 0x0a815, 0x0a816, 0x0a817, + 0x0a818, 0x0a819, 0x0a81a, 0x0a81b, 0x0a81c, 0x0a81d, 0x0a81e, 0x0a81f, + 0x0a820, 0x0a821, 0x0a822, 0x0a823, 0x0a824, 0x0a825, 0x0a826, 0x0a827, + 0x0a828, 0x0a829, 0x0a82a, 0x0a82b, 0x0a82c, 0x0a82d, 0x0a82e, 0x0a82f, + 0x0a830, 0x0a831, 0x0a832, 0x0a833, 0x0a834, 0x0a835, 0x0a836, 0x0a837, + 0x0a838, 0x0a839, 0x0a83a, 0x0a83b, 0x0a83c, 0x0a83d, 0x0a83e, 0x0a83f, + 0x0a840, 0x0a841, 0x0a842, 0x0a843, 0x0a844, 0x0a845, 0x0a846, 0x0a847, + 0x0a848, 0x0a849, 0x0a84a, 0x0a84b, 0x0a84c, 0x0a84d, 0x0a84e, 0x0a84f, + 0x0a850, 0x0a851, 0x0a852, 0x0a853, 0x0a854, 0x0a855, 0x0a856, 0x0a857, + 0x0a858, 0x0a859, 0x0a85a, 0x0a85b, 0x0a85c, 0x0a85d, 0x0a85e, 0x0a85f, + 0x0a860, 0x0a861, 0x0a862, 0x0a863, 0x0a864, 0x0a865, 0x0a866, 0x0a867, + 0x0a868, 0x0a869, 0x0a86a, 0x0a86b, 0x0a86c, 0x0a86d, 0x0a86e, 0x0a86f, + 0x0a870, 0x0a871, 0x0a872, 0x0a873, 0x0a874, 0x0a875, 0x0a876, 0x0a877, + 0x0a878, 0x0a879, 0x0a87a, 0x0a87b, 0x0a87c, 0x0a87d, 0x0a87e, 0x0a87f, + 0x00000, 0x00000, 0x0a882, 0x0a883, 0x0a884, 0x0a885, 0x0a886, 0x0a887, + 0x0a888, 0x0a889, 0x0a88a, 0x0a88b, 0x0a88c, 0x0a88d, 0x0a88e, 0x0a88f, + 0x0a890, 0x0a891, 0x0a892, 0x0a893, 0x0a894, 0x0a895, 0x0a896, 0x0a897, + 0x0a898, 0x0a899, 0x0a89a, 0x0a89b, 0x0a89c, 0x0a89d, 0x0a89e, 0x0a89f, + 0x0a8a0, 0x0a8a1, 0x0a8a2, 0x0a8a3, 0x0a8a4, 0x0a8a5, 0x0a8a6, 0x0a8a7, + 0x0a8a8, 0x0a8a9, 0x0a8aa, 0x0a8ab, 0x0a8ac, 0x0a8ad, 0x0a8ae, 0x0a8af, + 0x0a8b0, 0x0a8b1, 0x0a8b2, 0x0a8b3, 0x0a8b4, 0x0a8b5, 0x0a8b6, 0x0a8b7, + 0x0a8b8, 0x0a8b9, 0x0a8ba, 0x0a8bb, 0x0a8bc, 0x0a8bd, 0x0a8be, 0x0a8bf, + 0x0a8c0, 0x0a8c1, 0x0a8c2, 0x0a8c3, 0x0a8c4, 0x0a8c5, 0x0a8c6, 0x0a8c7, + 0x0a8c8, 0x0a8c9, 0x0a8ca, 0x0a8cb, 0x0a8cc, 0x0a8cd, 0x0a8ce, 0x0a8cf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0a8da, 0x0a8db, 0x0a8dc, 0x0a8dd, 0x0a8de, 0x0a8df, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x0a8f2, 0x0a8f2, 0x0a8f2, 0x0a8f2, 0x0a8f2, 0x0a8f2, + 0x0a8f8, 0x0a8f9, 0x0a8fa, 0x0a8fb, 0x0a8fc, 0x0a8fd, 0x0a8fe, 0x0a8ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_a9[] = { + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0a90a, 0x0a90b, 0x0a90c, 0x0a90d, 0x0a90e, 0x0a90f, + 0x0a910, 0x0a911, 0x0a912, 0x0a913, 0x0a914, 0x0a915, 0x0a916, 0x0a917, + 0x0a918, 0x0a919, 0x0a91a, 0x0a91b, 0x0a91c, 0x0a91d, 0x0a91e, 0x0a91f, + 0x0a920, 0x0a921, 0x0a922, 0x0a923, 0x0a924, 0x0a925, 0x0a926, 0x0a927, + 0x0a928, 0x0a929, 0x0a92a, 0x00000, 0x00000, 0x00000, 0x0a92e, 0x0a92f, + 0x0a930, 0x0a931, 0x0a932, 0x0a933, 0x0a934, 0x0a935, 0x0a936, 0x0a937, + 0x0a938, 0x0a939, 0x0a93a, 0x0a93b, 0x0a93c, 0x0a93d, 0x0a93e, 0x0a93f, + 0x0a940, 0x0a941, 0x0a942, 0x0a943, 0x0a944, 0x0a945, 0x0a946, 0x0a947, + 0x0a948, 0x0a949, 0x0a94a, 0x0a94b, 0x0a94c, 0x0a94d, 0x0a94e, 0x0a94f, + 0x0a950, 0x0a951, 0x0a952, 0x0a953, 0x0a954, 0x0a955, 0x0a956, 0x0a957, + 0x0a958, 0x0a959, 0x0a95a, 0x0a95b, 0x0a95c, 0x0a95d, 0x0a95e, 0x0a95f, + 0x0a960, 0x0a961, 0x0a962, 0x0a963, 0x0a964, 0x0a965, 0x0a966, 0x0a967, + 0x0a968, 0x0a969, 0x0a96a, 0x0a96b, 0x0a96c, 0x0a96d, 0x0a96e, 0x0a96f, + 0x0a970, 0x0a971, 0x0a972, 0x0a973, 0x0a974, 0x0a975, 0x0a976, 0x0a977, + 0x0a978, 0x0a979, 0x0a97a, 0x0a97b, 0x0a97c, 0x0a97d, 0x0a97e, 0x0a97f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x0a984, 0x0a985, 0x0a986, 0x0a987, + 0x0a988, 0x0a989, 0x0a98a, 0x0a98b, 0x0a98c, 0x0a98d, 0x0a98e, 0x0a98f, + 0x0a990, 0x0a991, 0x0a992, 0x0a993, 0x0a994, 0x0a995, 0x0a996, 0x0a997, + 0x0a998, 0x0a999, 0x0a99a, 0x0a99b, 0x0a99c, 0x0a99d, 0x0a99e, 0x0a99f, + 0x0a9a0, 0x0a9a1, 0x0a9a2, 0x0a9a3, 0x0a9a4, 0x0a9a5, 0x0a9a6, 0x0a9a7, + 0x0a9a8, 0x0a9a9, 0x0a9aa, 0x0a9ab, 0x0a9ab, 0x0a9ad, 0x0a9ae, 0x0a9af, + 0x0a9b0, 0x0a9b1, 0x0a9b2, 0x00000, 0x0a9b4, 0x0a9b5, 0x0a9b6, 0x0a9b7, + 0x0a9b8, 0x0a9b9, 0x0a9ba, 0x0a9bb, 0x0a9bc, 0x0a9bd, 0x0a9be, 0x0a9bf, + 0x0a9c0, 0x0a9c1, 0x0a9c2, 0x0a9c3, 0x0a9c4, 0x0a9c5, 0x0a9c6, 0x0a9c7, + 0x0a9c8, 0x0a9c9, 0x0a9ca, 0x0a9cb, 0x0a9cc, 0x0a9cd, 0x0a9ce, 0x0a9cf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0a9da, 0x0a9db, 0x0a9dc, 0x0a9dd, 0x0a9de, 0x0a9df, + 0x0a9e0, 0x0a9e1, 0x0a9e2, 0x0a9e3, 0x0a9e4, 0x0a9e5, 0x0a9e6, 0x0a9e7, + 0x0a9e8, 0x0a9e9, 0x0a9ea, 0x0a9eb, 0x0a9ec, 0x0a9ed, 0x0a9ee, 0x0a9ef, + 0x0a9f0, 0x0a9f1, 0x0a9f2, 0x0a9f3, 0x0a9f4, 0x0a9f5, 0x0a9f6, 0x0a9f7, + 0x0a9f8, 0x0a9f9, 0x0a9fa, 0x0a9fb, 0x0a9fc, 0x0a9fd, 0x0a9fe, 0x0a9ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_aa[] = { + 0x0aa00, 0x0aa01, 0x0aa02, 0x0aa03, 0x0aa04, 0x0aa05, 0x0aa06, 0x0aa07, + 0x0aa08, 0x0aa09, 0x0aa0a, 0x0aa0b, 0x0aa0c, 0x0aa0d, 0x0aa0e, 0x0aa0f, + 0x0aa10, 0x0aa11, 0x0aa12, 0x0aa13, 0x0aa14, 0x0aa15, 0x0aa16, 0x0aa17, + 0x0aa18, 0x0aa19, 0x0aa1a, 0x0aa1b, 0x0aa1c, 0x0aa1d, 0x0aa1e, 0x0aa1f, + 0x0aa20, 0x0aa21, 0x0aa22, 0x0aa23, 0x0aa24, 0x0aa25, 0x0aa26, 0x0aa27, + 0x0aa28, 0x0aa29, 0x0aa2a, 0x0aa2b, 0x0aa2c, 0x0aa2d, 0x0aa2e, 0x0aa2f, + 0x0aa30, 0x0aa31, 0x0aa32, 0x0aa33, 0x0aa34, 0x0aa35, 0x0aa36, 0x0aa37, + 0x0aa38, 0x0aa39, 0x0aa3a, 0x0aa3b, 0x0aa3c, 0x0aa3d, 0x0aa3e, 0x0aa3f, + 0x0aa40, 0x0aa41, 0x0aa42, 0x0aa43, 0x0aa44, 0x0aa45, 0x0aa46, 0x0aa47, + 0x0aa48, 0x0aa49, 0x0aa4a, 0x0aa4b, 0x0aa4c, 0x0aa4d, 0x0aa4e, 0x0aa4f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0aa5a, 0x0aa5b, 0x0aa5c, 0x0aa5d, 0x0aa5e, 0x0aa5f, + 0x0aa60, 0x0aa61, 0x0aa62, 0x0aa63, 0x0aa64, 0x0aa65, 0x0aa66, 0x0aa67, + 0x0aa68, 0x0aa69, 0x0aa6a, 0x0aa6b, 0x0aa6c, 0x0aa6d, 0x0aa6e, 0x0aa6f, + 0x0aa70, 0x0aa71, 0x0aa72, 0x0aa73, 0x0aa74, 0x0aa75, 0x0aa76, 0x0aa77, + 0x0aa78, 0x0aa79, 0x0aa7a, 0x0aa7b, 0x0aa7c, 0x0aa7d, 0x0aa7e, 0x0aa7f, + 0x0aa80, 0x0aa81, 0x0aa82, 0x0aa83, 0x0aa84, 0x0aa85, 0x0aa86, 0x0aa87, + 0x0aa88, 0x0aa89, 0x0aa8a, 0x0aa8b, 0x0aa8c, 0x0aa8d, 0x0aa8e, 0x0aa8f, + 0x0aa90, 0x0aa91, 0x0aa92, 0x0aa93, 0x0aa94, 0x0aa95, 0x0aa96, 0x0aa97, + 0x0aa98, 0x0aa99, 0x0aa9a, 0x0aa9b, 0x0aa9c, 0x0aa9d, 0x0aa9e, 0x0aa9f, + 0x0aaa0, 0x0aaa1, 0x0aaa2, 0x0aaa3, 0x0aaa4, 0x0aaa5, 0x0aaa6, 0x0aaa7, + 0x0aaa8, 0x0aaa9, 0x0aaaa, 0x0aaab, 0x0aaac, 0x0aaad, 0x0aaae, 0x0aaaf, + 0x0aab0, 0x0aab1, 0x0aab2, 0x0aab3, 0x0aab4, 0x0aab5, 0x0aab6, 0x0aab7, + 0x0aab8, 0x0aab9, 0x0aaba, 0x0aabb, 0x0aabc, 0x0aabd, 0x0aabe, 0x00000, + 0x0aac0, 0x00000, 0x0aac2, 0x0aac3, 0x0aac4, 0x0aac5, 0x0aac6, 0x0aac7, + 0x0aac8, 0x0aac9, 0x0aaca, 0x0aacb, 0x0aacc, 0x0aacd, 0x0aace, 0x0aacf, + 0x0aad0, 0x0aad1, 0x0aad2, 0x0aad3, 0x0aad4, 0x0aad5, 0x0aad6, 0x0aad7, + 0x0aad8, 0x0aad9, 0x0aada, 0x0aadb, 0x0aadc, 0x0aadd, 0x0aade, 0x0aadf, + 0x0aae0, 0x0aae1, 0x0aae2, 0x0aae3, 0x0aae4, 0x0aae5, 0x0aae6, 0x0aae7, + 0x0aae8, 0x0aae9, 0x0aaea, 0x0aaeb, 0x0aaec, 0x0aaed, 0x0aaee, 0x0aaef, + 0x0aaf0, 0x0aaf1, 0x0aaf2, 0x0aaf3, 0x0aaf4, 0x0aaf5, 0x0aaf6, 0x0aaf7, + 0x0aaf8, 0x0aaf9, 0x0aafa, 0x0aafb, 0x0aafc, 0x0aafd, 0x0aafe, 0x0aaff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_ab[] = { + 0x0ab00, 0x0ab01, 0x0ab02, 0x0ab03, 0x0ab04, 0x0ab05, 0x0ab06, 0x0ab07, + 0x0ab08, 0x0ab09, 0x0ab0a, 0x0ab0b, 0x0ab0c, 0x0ab0d, 0x0ab0e, 0x0ab0f, + 0x0ab10, 0x0ab11, 0x0ab12, 0x0ab13, 0x0ab14, 0x0ab15, 0x0ab16, 0x0ab17, + 0x0ab18, 0x0ab19, 0x0ab1a, 0x0ab1b, 0x0ab1c, 0x0ab1d, 0x0ab1e, 0x0ab1f, + 0x0ab20, 0x0ab21, 0x0ab22, 0x0ab23, 0x0ab24, 0x0ab25, 0x0ab26, 0x0ab27, + 0x0ab28, 0x0ab29, 0x0ab2a, 0x0ab2b, 0x0ab2c, 0x0ab2d, 0x0ab2e, 0x0ab2f, + 0x0ab30, 0x0ab31, 0x0ab32, 0x0ab33, 0x0ab34, 0x0ab35, 0x0ab36, 0x0ab37, + 0x0ab38, 0x0ab39, 0x0ab3a, 0x0ab3b, 0x0ab3c, 0x0ab3d, 0x0ab3e, 0x0ab3f, + 0x0ab40, 0x0ab41, 0x0ab42, 0x0ab43, 0x0ab44, 0x0ab45, 0x0ab46, 0x0ab47, + 0x0ab48, 0x0ab49, 0x0ab4a, 0x0ab4b, 0x0ab4c, 0x0ab4d, 0x0ab4e, 0x0ab4f, + 0x0ab50, 0x0ab51, 0x0ab52, 0x0ab53, 0x0ab54, 0x0ab55, 0x0ab56, 0x0ab57, + 0x0ab58, 0x0ab59, 0x0ab5a, 0x0ab5b, 0x0ab5c, 0x0ab5d, 0x0ab5e, 0x0ab5f, + 0x0ab60, 0x0ab61, 0x0ab62, 0x0ab63, 0x0ab64, 0x0ab65, 0x0ab66, 0x0ab67, + 0x0ab68, 0x0ab69, 0x0ab6a, 0x0ab6b, 0x0ab6c, 0x0ab6d, 0x0ab6e, 0x0ab6f, + 0x0ab70, 0x0ab71, 0x0ab72, 0x0ab73, 0x0ab74, 0x0ab75, 0x0ab76, 0x0ab77, + 0x0ab78, 0x0ab79, 0x0ab7a, 0x0ab7b, 0x0ab7c, 0x0ab7d, 0x0ab7e, 0x0ab7f, + 0x0ab80, 0x0ab81, 0x0ab82, 0x0ab83, 0x0ab84, 0x0ab85, 0x0ab86, 0x0ab87, + 0x0ab88, 0x0ab89, 0x0ab8a, 0x0ab8b, 0x0ab8c, 0x0ab8d, 0x0ab8e, 0x0ab8f, + 0x0ab90, 0x0ab91, 0x0ab92, 0x0ab93, 0x0ab94, 0x0ab95, 0x0ab96, 0x0ab97, + 0x0ab98, 0x0ab99, 0x0ab9a, 0x0ab9b, 0x0ab9c, 0x0ab9d, 0x0ab9e, 0x0ab9f, + 0x0aba0, 0x0aba1, 0x0aba2, 0x0aba3, 0x0aba4, 0x0aba5, 0x0aba6, 0x0aba7, + 0x0aba8, 0x0aba9, 0x0abaa, 0x0abab, 0x0abac, 0x0abad, 0x0abae, 0x0abaf, + 0x0abb0, 0x0abb1, 0x0abb2, 0x0abb3, 0x0abb4, 0x0abb5, 0x0abb6, 0x0abb7, + 0x0abb8, 0x0abb9, 0x0abba, 0x0abbb, 0x0abbc, 0x0abbd, 0x0abbe, 0x0abbf, + 0x0abc0, 0x0abc1, 0x0abc2, 0x0abc3, 0x0abc4, 0x0abc5, 0x0abc6, 0x0abc7, + 0x0abc8, 0x0abc9, 0x0abca, 0x0abcb, 0x0abcc, 0x0abcd, 0x0abce, 0x0abcf, + 0x0abd0, 0x0abd1, 0x0abd2, 0x0abd3, 0x0abd4, 0x0abd5, 0x0abd6, 0x0abd7, + 0x0abd8, 0x0abd9, 0x0abda, 0x0abdb, 0x0abdc, 0x0abdd, 0x0abde, 0x0abdf, + 0x0abe0, 0x0abe1, 0x0abe2, 0x0abe3, 0x0abe4, 0x0abe5, 0x0abe6, 0x0abe7, + 0x0abe8, 0x0abe9, 0x0abea, 0x0abeb, 0x00000, 0x0abed, 0x0abee, 0x0abef, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0abfa, 0x0abfb, 0x0abfc, 0x0abfd, 0x0abfe, 0x0abff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_f9[] = { + 0x0f900, 0x0f901, 0x02f9e, 0x0f903, 0x0f904, 0x0f905, 0x0f906, 0x02ef2, + 0x02ef2, 0x0f909, 0x02fa6, 0x0f90b, 0x0f90c, 0x0f90d, 0x0f90e, 0x0f90f, + 0x0f910, 0x0f911, 0x0f912, 0x0f913, 0x0f914, 0x0f915, 0x0f916, 0x0f917, + 0x0f918, 0x0f919, 0x0f91a, 0x0f91b, 0x0f91c, 0x0f91d, 0x0f91e, 0x0f91f, + 0x0f920, 0x0f921, 0x0f922, 0x0f923, 0x0f924, 0x0f925, 0x0f926, 0x0f927, + 0x0f928, 0x0f929, 0x0f92a, 0x0f92b, 0x0f92c, 0x0f92d, 0x0f92e, 0x0f92f, + 0x0f930, 0x0f931, 0x0f932, 0x0f933, 0x02f7c, 0x0f935, 0x0f936, 0x0f937, + 0x0f938, 0x0f939, 0x0f93a, 0x0f93b, 0x0f93c, 0x0f93d, 0x0f93e, 0x0f93f, + 0x02fc5, 0x0f941, 0x0f942, 0x0f943, 0x0f944, 0x0f945, 0x0f946, 0x0f947, + 0x0f948, 0x0f949, 0x0f94a, 0x0f94b, 0x0f94c, 0x0f94d, 0x0f94e, 0x0f94f, + 0x0f950, 0x0f951, 0x0f952, 0x0f953, 0x0f954, 0x0f955, 0x0f956, 0x0f957, + 0x0f958, 0x0f959, 0x0f95a, 0x0f95b, 0x0f914, 0x0f95d, 0x0f95e, 0x0f95f, + 0x0f960, 0x0f961, 0x0f962, 0x0f963, 0x0f964, 0x0f965, 0x0f966, 0x0f967, + 0x0f968, 0x0f969, 0x0f96a, 0x0f96b, 0x0f96c, 0x0f96d, 0x0f96e, 0x0f96f, + 0x0f970, 0x02fa0, 0x0f972, 0x0f973, 0x0f974, 0x0f975, 0x0f976, 0x0f977, + 0x0f978, 0x0f979, 0x0f97a, 0x0f97b, 0x0f97c, 0x0f97d, 0x0f97e, 0x0f97f, + 0x0f980, 0x02f25, 0x0f982, 0x0f983, 0x0f984, 0x0f985, 0x0f986, 0x0f987, + 0x0f988, 0x0f989, 0x02f12, 0x0f98b, 0x0f98c, 0x0f98d, 0x0f98e, 0x0f98f, + 0x0f990, 0x0f991, 0x0f992, 0x0f993, 0x0f994, 0x0f995, 0x0f996, 0x0f997, + 0x0f998, 0x0f999, 0x0f99a, 0x0f99b, 0x0f99c, 0x0f99d, 0x0f99e, 0x0f99f, + 0x0f9a0, 0x0f96f, 0x0f9a2, 0x0f9a3, 0x0f9a4, 0x0f9a5, 0x0f9a6, 0x0f9a7, + 0x0f9a8, 0x0f9a9, 0x0f95f, 0x0f9ab, 0x0f9ac, 0x0f9ad, 0x0f9ae, 0x0f9af, + 0x0f9b0, 0x0f9b1, 0x0f9b2, 0x0f9b3, 0x0f9b4, 0x0f9b5, 0x0f9b6, 0x0f9b7, + 0x0f9b8, 0x0f9b9, 0x0f9ba, 0x0f9bb, 0x0f9bc, 0x0f9bd, 0x0f9be, 0x0f914, + 0x0f9c0, 0x0f9c1, 0x0f9c2, 0x0f9c3, 0x02eef, 0x0f9c5, 0x0f9c6, 0x0f9c7, + 0x0f9c8, 0x0f9c9, 0x0f9ca, 0x0f9cb, 0x0f9cc, 0x0f9cd, 0x0f9ce, 0x0f9cf, + 0x0f9d0, 0x03285, 0x0f9d2, 0x0f9d3, 0x0f9d4, 0x0f9d5, 0x0f9d6, 0x0f9d7, + 0x0f9d8, 0x0f9d9, 0x0f9da, 0x0f961, 0x0f9dc, 0x0f9dd, 0x0f9de, 0x0f9df, + 0x0f9e0, 0x0f9e1, 0x0f9e2, 0x0f9e3, 0x0f9e4, 0x0f9e5, 0x0f9e6, 0x0f9e7, + 0x0f9e8, 0x02fa5, 0x0f9ea, 0x0f9eb, 0x0f9ec, 0x0f9ed, 0x0f9ee, 0x0f9ef, + 0x0f9f0, 0x0f9f1, 0x0f9f2, 0x0f9f3, 0x0f9f4, 0x0f9f5, 0x0f9f6, 0x02f74, + 0x0f9f8, 0x0f9f9, 0x0f9fa, 0x0f9fb, 0x0f9fc, 0x0f9fd, 0x0f9fe, 0x0f9ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fa[] = { + 0x0fa00, 0x0fa01, 0x0fa02, 0x0fa03, 0x0fa04, 0x0fa05, 0x0fa06, 0x0fa07, + 0x02f8f, 0x0fa09, 0x02f92, 0x0fa0b, 0x0fa0c, 0x0fa0d, 0x0fa0e, 0x0fa0f, + 0x0fa10, 0x0fa11, 0x0fa12, 0x0fa13, 0x0fa14, 0x0fa15, 0x0fa16, 0x0fa17, + 0x0fa18, 0x0fa19, 0x0fa1a, 0x0fa1b, 0x0fa1c, 0x0fa1d, 0x02f7b, 0x0fa1f, + 0x0fa20, 0x0fa21, 0x0fa22, 0x0fa23, 0x0fa24, 0x0fa25, 0x0fa26, 0x0fa27, + 0x0fa28, 0x0fa29, 0x0fa2a, 0x0fa2b, 0x0fa2c, 0x0fa2d, 0x0fa2e, 0x0fa2f, + 0x0fa30, 0x0fa31, 0x0fa32, 0x0fa33, 0x0fa34, 0x0fa35, 0x0fa36, 0x0fa37, + 0x0fa38, 0x0fa39, 0x0fa3a, 0x0fa3b, 0x02f2c, 0x0fa3d, 0x0fa3e, 0x0fa3f, + 0x0fa40, 0x0fa41, 0x0fa42, 0x0fa43, 0x0fa44, 0x0fa45, 0x0fa46, 0x0fa47, + 0x0fa48, 0x02ea4, 0x0fa4a, 0x0fa4b, 0x03293, 0x0fa4d, 0x0fa4e, 0x0fa4f, + 0x0fa50, 0x03297, 0x0fa52, 0x0fa53, 0x0fa54, 0x0fa55, 0x0fa56, 0x0f996, + 0x0fa58, 0x0fa59, 0x0fa5a, 0x0fa5b, 0x0fa5c, 0x02ebe, 0x02ebe, 0x0fa5f, + 0x0fa60, 0x0fa61, 0x0fa62, 0x0fa63, 0x0fa64, 0x0fa65, 0x02ecc, 0x0fa25, + 0x0fa68, 0x0fa69, 0x0fa6a, 0x0fa6b, 0x0fa6c, 0x0fa6d, 0x0fa6e, 0x0fa6f, + 0x0fa70, 0x0fa71, 0x0fa72, 0x0fa73, 0x0fa74, 0x0fa75, 0x0fa76, 0x0fa77, + 0x0fa36, 0x0fa79, 0x0fa7a, 0x0fa7b, 0x0fa10, 0x0fa7d, 0x0fa7e, 0x0fa7f, + 0x0fa80, 0x0fa81, 0x0fa82, 0x0fa83, 0x0fa84, 0x0fa85, 0x0fa86, 0x0fa87, + 0x0fa88, 0x0fa3f, 0x0fa8a, 0x0fa40, 0x0fa8c, 0x0fa8d, 0x0fa8e, 0x0fa8f, + 0x0fa90, 0x0fa12, 0x0f929, 0x0fa93, 0x0fa94, 0x02f4d, 0x0f970, 0x0f9ca, + 0x0fa98, 0x0fa99, 0x0fa47, 0x0fa9b, 0x0fa48, 0x0fa9d, 0x0fa9e, 0x0fa9f, + 0x0fa16, 0x0faa1, 0x0faa2, 0x0faa3, 0x0faa4, 0x0faa5, 0x0fa17, 0x0faa7, + 0x0faa8, 0x0faa9, 0x0faaa, 0x0faab, 0x0faac, 0x0fa56, 0x0faae, 0x0faaf, + 0x0f996, 0x0fab1, 0x0fa5b, 0x0fab3, 0x0fab4, 0x0fab5, 0x0fab6, 0x0fab7, + 0x0fa61, 0x0fab9, 0x0fa22, 0x0fabb, 0x0fa62, 0x0f95d, 0x0fabe, 0x0fa63, + 0x0fac0, 0x0fa65, 0x0fac2, 0x0fac3, 0x0fac4, 0x0fac5, 0x0fac6, 0x0fa68, + 0x0fa1c, 0x0fac9, 0x0fa69, 0x0facb, 0x0fa6a, 0x0facd, 0x02ef2, 0x0facf, + 0x0fad0, 0x0fad1, 0x0fad2, 0x0fad3, 0x0fad4, 0x0fad5, 0x0fad6, 0x0fad7, + 0x0fad8, 0x0fad9, 0x0fada, 0x0fadb, 0x0fadc, 0x0fadd, 0x0fade, 0x0fadf, + 0x0fae0, 0x0fae1, 0x0fae2, 0x0fae3, 0x0fae4, 0x0fae5, 0x0fae6, 0x0fae7, + 0x0fae8, 0x0fae9, 0x0faea, 0x0faeb, 0x0faec, 0x0faed, 0x0faee, 0x0faef, + 0x0faf0, 0x0faf1, 0x0faf2, 0x0faf3, 0x0faf4, 0x0faf5, 0x0faf6, 0x0faf7, + 0x0faf8, 0x0faf9, 0x0fafa, 0x0fafb, 0x0fafc, 0x0fafd, 0x0fafe, 0x0faff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fb[] = { + 0x0fb00, 0x0fb01, 0x0fb02, 0x0fb03, 0x0fb04, 0x0fb05, 0x0fb05, 0x0fb07, + 0x0fb08, 0x0fb09, 0x0fb0a, 0x0fb0b, 0x0fb0c, 0x0fb0d, 0x0fb0e, 0x0fb0f, + 0x0fb10, 0x0fb11, 0x0fb12, 0x0fb13, 0x0fb14, 0x0fb15, 0x0fb16, 0x0fb17, + 0x0fb18, 0x0fb19, 0x0fb1a, 0x0fb1b, 0x0fb1c, 0x005d9, 0x00000, 0x005f2, + 0x005e2, 0x005d0, 0x005d3, 0x005d4, 0x005da, 0x005dc, 0x005dd, 0x005e8, + 0x005ea, 0x0002b, 0x005e9, 0x005e9, 0x005e9, 0x005e9, 0x005d0, 0x005d0, + 0x005d0, 0x005d1, 0x005d2, 0x005d3, 0x005d4, 0x005d5, 0x005d6, 0x0fb37, + 0x005d8, 0x005d9, 0x005da, 0x005da, 0x005dc, 0x0fb3d, 0x005dd, 0x0fb3f, + 0x005df, 0x005e1, 0x0fb42, 0x005e3, 0x005e3, 0x0fb45, 0x005e5, 0x005e7, + 0x005e8, 0x005e9, 0x005ea, 0x005d5, 0x005d1, 0x005da, 0x005e3, 0x0fb4f, + 0x00671, 0x00671, 0x0067b, 0x0067b, 0x0067b, 0x0067b, 0x0067e, 0x0067e, + 0x0067e, 0x0067e, 0x00680, 0x00680, 0x00680, 0x00680, 0x0067a, 0x0067a, + 0x0067a, 0x0067a, 0x0067f, 0x0067f, 0x0067f, 0x0067f, 0x00679, 0x00679, + 0x00679, 0x00679, 0x006a4, 0x006a4, 0x006a4, 0x006a4, 0x006a6, 0x006a6, + 0x006a6, 0x006a6, 0x00684, 0x00684, 0x00684, 0x00684, 0x00683, 0x00683, + 0x00683, 0x00683, 0x00686, 0x00686, 0x00686, 0x00686, 0x00687, 0x00687, + 0x00687, 0x00687, 0x0068d, 0x0068d, 0x0068c, 0x0068c, 0x0068e, 0x0068e, + 0x00688, 0x00688, 0x00698, 0x00698, 0x00691, 0x00691, 0x006a9, 0x006a9, + 0x006a9, 0x006a9, 0x006af, 0x006af, 0x006af, 0x006af, 0x006b3, 0x006b3, + 0x006b3, 0x006b3, 0x006b1, 0x006b1, 0x006b1, 0x006b1, 0x006ba, 0x006ba, + 0x006bb, 0x006bb, 0x006bb, 0x006bb, 0x006c0, 0x006c0, 0x006c1, 0x006c1, + 0x006c1, 0x006c1, 0x006be, 0x006be, 0x006be, 0x006be, 0x006d2, 0x006d2, + 0x006d2, 0x006d2, 0x0fbb2, 0x0fbb3, 0x0fbb4, 0x0fbb5, 0x0fbb6, 0x0fbb7, + 0x0fbb8, 0x0fbb9, 0x0fbba, 0x0fbbb, 0x0fbbc, 0x0fbbd, 0x0fbbe, 0x0fbbf, + 0x0fbc0, 0x0fbc1, 0x0fbc2, 0x0fbc3, 0x0fbc4, 0x0fbc5, 0x0fbc6, 0x0fbc7, + 0x0fbc8, 0x0fbc9, 0x0fbca, 0x0fbcb, 0x0fbcc, 0x0fbcd, 0x0fbce, 0x0fbcf, + 0x0fbd0, 0x0fbd1, 0x0fbd2, 0x006ad, 0x006ad, 0x006ad, 0x006ad, 0x006c7, + 0x006c7, 0x006c6, 0x006c6, 0x006c8, 0x006c8, 0x00677, 0x006cb, 0x006cb, + 0x006c5, 0x006c5, 0x006c9, 0x006c9, 0x006d0, 0x006d0, 0x006d0, 0x006d0, + 0x00649, 0x00649, 0x0fbea, 0x0fbea, 0x0fbec, 0x0fbec, 0x0fbee, 0x0fbee, + 0x0fbf0, 0x0fbf0, 0x0fbf2, 0x0fbf2, 0x0fbf4, 0x0fbf4, 0x0fbf6, 0x0fbf6, + 0x0fbf6, 0x0fbf9, 0x0fbf9, 0x0fbf9, 0x006cc, 0x006cc, 0x0fbfe, 0x0fbff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fc[] = { + 0x0fc00, 0x0fc01, 0x0fc02, 0x0fbf9, 0x0fc04, 0x0fc05, 0x0fc06, 0x0fc07, + 0x0fc08, 0x0fc09, 0x0fc0a, 0x0fc0b, 0x0fc0c, 0x0fc0d, 0x0fc0e, 0x0fc0f, + 0x0fc10, 0x0fc11, 0x0fc12, 0x0fc13, 0x0fc14, 0x0fc15, 0x0fc16, 0x0fc17, + 0x0fc18, 0x0fc19, 0x0fc1a, 0x0fc1b, 0x0fc1c, 0x0fc1d, 0x0fc1e, 0x0fc1f, + 0x0fc20, 0x0fc21, 0x0fc22, 0x0fc23, 0x0fc24, 0x0fc25, 0x0fc26, 0x0fc27, + 0x0fc28, 0x0fc29, 0x0fc2a, 0x0fc2b, 0x0fc2c, 0x0fc2d, 0x0fc2e, 0x0fc2f, + 0x0fc30, 0x0fc31, 0x0fc32, 0x0fc33, 0x0fc34, 0x0fc35, 0x0fc36, 0x0fc37, + 0x0fc38, 0x0fc39, 0x0fc3a, 0x0fc3b, 0x0fc3c, 0x0fc3d, 0x0fc3e, 0x0fc3f, + 0x0fc40, 0x0fc41, 0x0fc42, 0x0fc43, 0x0fc44, 0x0fc45, 0x0fc46, 0x0fc47, + 0x0fc48, 0x0fc49, 0x0fc4a, 0x0fc4b, 0x0fc4c, 0x0fc4d, 0x0fc4e, 0x0fc4f, + 0x0fc50, 0x0fc51, 0x0fc52, 0x0fc53, 0x0fc54, 0x0fc55, 0x0fc56, 0x0fc57, + 0x0fc58, 0x0fc59, 0x0fc5a, 0x00630, 0x00631, 0x00649, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x0fc64, 0x0fc65, 0x0fc02, 0x0fc67, + 0x0fbf9, 0x0fc04, 0x0fc6a, 0x0fc6b, 0x0fc08, 0x0fc6d, 0x0fc09, 0x0fc0a, + 0x0fc70, 0x0fc71, 0x0fc0e, 0x0fc73, 0x0fc0f, 0x0fc10, 0x0fc76, 0x0fc77, + 0x0fc12, 0x0fc79, 0x0fc13, 0x0fc14, 0x0fc31, 0x0fc32, 0x0fc35, 0x0fc36, + 0x0fc37, 0x0fc3b, 0x0fc3c, 0x0fc3d, 0x0fc3e, 0x0fc42, 0x0fc43, 0x0fc44, + 0x0fc88, 0x0fc48, 0x0fc8a, 0x0fc8b, 0x0fc4e, 0x0fc8d, 0x0fc4f, 0x0fc50, + 0x00649, 0x0fc91, 0x0fc92, 0x0fc58, 0x0fc94, 0x0fc59, 0x0fc5a, 0x0fc00, + 0x0fc01, 0x0fc99, 0x0fc02, 0x0fc9b, 0x0fc05, 0x0fc06, 0x0fc07, 0x0fc08, + 0x0fca0, 0x0fc0b, 0x0fc0c, 0x0fc0d, 0x0fc0e, 0x0fca5, 0x0fc12, 0x0fc15, + 0x0fc16, 0x0fc17, 0x0fc18, 0x0fc19, 0x0fc1b, 0x0fc1c, 0x0fc1d, 0x0fc1e, + 0x0fc1f, 0x0fc20, 0x0fcb2, 0x0fc21, 0x0fc22, 0x0fc23, 0x0fc24, 0x0fc25, + 0x0fc26, 0x0fc28, 0x0fc29, 0x0fc2a, 0x0fc2b, 0x0fc2c, 0x0fc2d, 0x0fc2e, + 0x0fc2f, 0x0fc30, 0x0fc33, 0x0fc34, 0x0fc38, 0x0fc39, 0x0fc3a, 0x0fc3b, + 0x0fc3c, 0x0fc3f, 0x0fc40, 0x0fc41, 0x0fc42, 0x0fccd, 0x0fc45, 0x0fc46, + 0x0fc47, 0x0fc48, 0x0fc4b, 0x0fc4c, 0x0fc4d, 0x0fc4e, 0x0fcd6, 0x0fc51, + 0x0fc52, 0x00647, 0x0fc55, 0x0fc56, 0x0fc57, 0x0fc58, 0x0fcde, 0x0fc02, + 0x0fc9b, 0x0fc08, 0x0fca0, 0x0fc0e, 0x0fca5, 0x0fc12, 0x0fce6, 0x0fc1f, + 0x0fce8, 0x0fce9, 0x0fcea, 0x0fc3b, 0x0fc3c, 0x0fc42, 0x0fc4e, 0x0fcd6, + 0x0fc58, 0x0fcde, 0x00000, 0x00000, 0x00000, 0x0fcf5, 0x0fcf6, 0x0fcf7, + 0x0fcf8, 0x0fcf9, 0x0fcfa, 0x0fcfb, 0x0fcfc, 0x0fcfd, 0x0fcfe, 0x0fcff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fd[] = { + 0x0fd00, 0x0fd01, 0x0fd02, 0x0fd03, 0x0fd04, 0x0fd05, 0x0fd06, 0x0fd07, + 0x0fd08, 0x0fd09, 0x0fd0a, 0x0fd0b, 0x0fce9, 0x0fd0d, 0x0fd0e, 0x0fd0f, + 0x0fd10, 0x0fcf5, 0x0fcf6, 0x0fcf7, 0x0fcf8, 0x0fcf9, 0x0fcfa, 0x0fcfb, + 0x0fcfc, 0x0fcfd, 0x0fcfe, 0x0fd1b, 0x0fd00, 0x0fd01, 0x0fd02, 0x0fd03, + 0x0fd04, 0x0fd05, 0x0fd06, 0x0fd07, 0x0fd08, 0x0fd09, 0x0fd0a, 0x0fd0b, + 0x0fce9, 0x0fd0d, 0x0fd0e, 0x0fd0f, 0x0fd10, 0x0fd09, 0x0fd0a, 0x0fd0b, + 0x0fce9, 0x0fce8, 0x0fcea, 0x0fc27, 0x0fc1c, 0x0fc1d, 0x0fc1e, 0x0fd09, + 0x0fd0a, 0x0fd0b, 0x0fc27, 0x0fc28, 0x00627, 0x00627, 0x0fd3e, 0x0fd3f, + 0x0fd40, 0x0fd41, 0x0fd42, 0x0fd43, 0x0fd44, 0x0fd45, 0x0fd46, 0x0fd47, + 0x0fd48, 0x0fd49, 0x0fd4a, 0x0fd4b, 0x0fd4c, 0x0fd4d, 0x0fd4e, 0x0fd4f, + 0x0fd50, 0x0fd51, 0x0fd51, 0x0fd53, 0x0fd54, 0x0fd55, 0x0fd56, 0x0fd57, + 0x0fd58, 0x0fd58, 0x0fd5a, 0x0fd5b, 0x0fd5c, 0x0fd5d, 0x0fd5e, 0x0fd5f, + 0x0fd5f, 0x0fd61, 0x0fd62, 0x0fd62, 0x0fd64, 0x0fd64, 0x0fd66, 0x0fd67, + 0x0fd67, 0x0fd69, 0x0fd6a, 0x0fd6a, 0x0fd6c, 0x0fd6c, 0x0fd6e, 0x0fd6f, + 0x0fd6f, 0x0fd71, 0x0fd71, 0x0fd73, 0x0fd74, 0x0fd75, 0x0fd76, 0x0fd76, + 0x0fd78, 0x0fd79, 0x0fd7a, 0x0fd7b, 0x0fd7c, 0x0fd7c, 0x0fd7e, 0x0fd7f, + 0x0fd80, 0x0fd81, 0x0fd82, 0x0fd83, 0x0fd83, 0x0fd85, 0x0fd85, 0x0fd87, + 0x0fd87, 0x0fd89, 0x0fd8a, 0x0fd8b, 0x0fd8c, 0x0fd8d, 0x0fd8e, 0x0fd8f, + 0x0fd90, 0x0fd91, 0x0fd92, 0x0fd93, 0x0fd94, 0x0fd95, 0x0fd96, 0x0fd97, + 0x0fd97, 0x0fd99, 0x0fd9a, 0x0fd9b, 0x0fd9c, 0x0fd9c, 0x0fd9e, 0x0fd9f, + 0x0fda0, 0x0fda1, 0x0fda2, 0x0fda3, 0x0fda4, 0x0fda5, 0x0fda6, 0x0fda7, + 0x0fda8, 0x0fda9, 0x0fdaa, 0x0fdab, 0x0fdac, 0x0fdad, 0x0fdae, 0x0fdaf, + 0x0fdb0, 0x0fdb1, 0x0fdb2, 0x0fdb3, 0x0fd7e, 0x0fd80, 0x0fdb6, 0x0fdb7, + 0x0fdb8, 0x0fdb9, 0x0fdba, 0x0fdbb, 0x0fdba, 0x0fdb8, 0x0fdbe, 0x0fdbf, + 0x0fdc0, 0x0fdc1, 0x0fdc2, 0x0fdbb, 0x0fd75, 0x0fd66, 0x0fdc6, 0x0fdc7, + 0x0fdc8, 0x0fdc9, 0x0fdca, 0x0fdcb, 0x0fdcc, 0x0fdcd, 0x0fdce, 0x0fdcf, + 0x0fdd0, 0x0fdd1, 0x0fdd2, 0x0fdd3, 0x0fdd4, 0x0fdd5, 0x0fdd6, 0x0fdd7, + 0x0fdd8, 0x0fdd9, 0x0fdda, 0x0fddb, 0x0fddc, 0x0fddd, 0x0fdde, 0x0fddf, + 0x0fde0, 0x0fde1, 0x0fde2, 0x0fde3, 0x0fde4, 0x0fde5, 0x0fde6, 0x0fde7, + 0x0fde8, 0x0fde9, 0x0fdea, 0x0fdeb, 0x0fdec, 0x0fded, 0x0fdee, 0x0fdef, + 0x0fdf0, 0x0fdf1, 0x0fdf2, 0x0fdf3, 0x0fdf4, 0x0fdf5, 0x0fdf6, 0x0fdf7, + 0x0fdf8, 0x0fdf9, 0x0fdfa, 0x0fdfb, 0x0fdfc, 0x0fdfd, 0x0fdfe, 0x0fdff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fe[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x0002c, 0x03001, 0x03002, 0x0003a, 0x0003b, 0x00021, 0x0003f, 0x03016, + 0x03017, 0x02026, 0x0fe1a, 0x0fe1b, 0x0fe1c, 0x0fe1d, 0x0fe1e, 0x0fe1f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x0fe27, + 0x0fe28, 0x0fe29, 0x0fe2a, 0x0fe2b, 0x0fe2c, 0x0fe2d, 0x0fe2e, 0x0fe2f, + 0x02025, 0x02014, 0x02013, 0x0005f, 0x0005f, 0x00028, 0x00029, 0x0007b, + 0x0007d, 0x03014, 0x03015, 0x03010, 0x03011, 0x0300a, 0x0300b, 0x02329, + 0x0232a, 0x0300c, 0x0300d, 0x0300e, 0x0300f, 0x0fe45, 0x0fe46, 0x0005b, + 0x0005d, 0x0203e, 0x0203e, 0x0203e, 0x0203e, 0x0005f, 0x0005f, 0x0005f, + 0x0002c, 0x03001, 0x0002e, 0x0fe53, 0x0003b, 0x0003a, 0x0003f, 0x00021, + 0x02014, 0x00028, 0x00029, 0x0007b, 0x0007d, 0x03014, 0x03015, 0x00023, + 0x00026, 0x0002a, 0x0002b, 0x0002d, 0x0003c, 0x0003e, 0x0003d, 0x0fe67, + 0x0005c, 0x00024, 0x00025, 0x00040, 0x0fe6c, 0x0fe6d, 0x0fe6e, 0x0fe6f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x0fe75, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00621, 0x00622, 0x00622, 0x00623, 0x00623, 0x00624, 0x00624, 0x00625, + 0x00625, 0x00626, 0x00626, 0x00626, 0x00626, 0x00627, 0x00627, 0x00628, + 0x00628, 0x00628, 0x00628, 0x00629, 0x00629, 0x0062a, 0x0062a, 0x0062a, + 0x0062a, 0x0062b, 0x0062b, 0x0062b, 0x0062b, 0x0062c, 0x0062c, 0x0062c, + 0x0062c, 0x0062d, 0x0062d, 0x0062d, 0x0062d, 0x0062e, 0x0062e, 0x0062e, + 0x0062e, 0x0062f, 0x0062f, 0x00630, 0x00630, 0x00631, 0x00631, 0x00632, + 0x00632, 0x00633, 0x00633, 0x00633, 0x00633, 0x00634, 0x00634, 0x00634, + 0x00634, 0x00635, 0x00635, 0x00635, 0x00635, 0x00636, 0x00636, 0x00636, + 0x00636, 0x00637, 0x00637, 0x00637, 0x00637, 0x00638, 0x00638, 0x00638, + 0x00638, 0x00639, 0x00639, 0x00639, 0x00639, 0x0063a, 0x0063a, 0x0063a, + 0x0063a, 0x00641, 0x00641, 0x00641, 0x00641, 0x00642, 0x00642, 0x00642, + 0x00642, 0x00643, 0x00643, 0x00643, 0x00643, 0x00644, 0x00644, 0x00644, + 0x00644, 0x00645, 0x00645, 0x00645, 0x00645, 0x00646, 0x00646, 0x00646, + 0x00646, 0x00647, 0x00647, 0x00647, 0x00647, 0x00648, 0x00648, 0x00649, + 0x00649, 0x0064a, 0x0064a, 0x0064a, 0x0064a, 0x0fef5, 0x0fef5, 0x0fef7, + 0x0fef7, 0x0fef9, 0x0fef9, 0x0fefb, 0x0fefb, 0x0fefd, 0x0fefe, 0x0feff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_ff[] = { + 0x0ff00, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, + 0x00028, 0x00029, 0x0002a, 0x0002b, 0x0002c, 0x0002d, 0x0002e, 0x0002f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0003a, 0x0003b, 0x0003c, 0x0003d, 0x0003e, 0x0003f, + 0x00040, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, + 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, + 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, + 0x00058, 0x00059, 0x0005a, 0x0005b, 0x0005c, 0x0005d, 0x0005e, 0x0005f, + 0x00060, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, + 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, + 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, + 0x00058, 0x00059, 0x0005a, 0x0007b, 0x0007c, 0x0007d, 0x0007e, 0x02985, + 0x02986, 0x03002, 0x0300c, 0x0300d, 0x03001, 0x030fb, 0x03092, 0x03041, + 0x03043, 0x03045, 0x03047, 0x03049, 0x03083, 0x03085, 0x03087, 0x03063, + 0x030fc, 0x03042, 0x03044, 0x03046, 0x03048, 0x0304a, 0x0304b, 0x0304d, + 0x0304f, 0x03051, 0x03053, 0x03055, 0x03057, 0x03059, 0x0305b, 0x0305d, + 0x0305f, 0x03061, 0x03064, 0x03066, 0x03068, 0x0306a, 0x0306b, 0x0306c, + 0x0306d, 0x0306e, 0x0306f, 0x03072, 0x03075, 0x03078, 0x0307b, 0x0307e, + 0x0307f, 0x03080, 0x03081, 0x03082, 0x03084, 0x03086, 0x03088, 0x03089, + 0x0308a, 0x0308b, 0x0308c, 0x0308d, 0x0308f, 0x03093, 0x00000, 0x00000, + 0x01160, 0x01100, 0x01101, 0x011aa, 0x01102, 0x011ac, 0x011ad, 0x01103, + 0x01104, 0x01105, 0x011b0, 0x011b1, 0x011b2, 0x011b3, 0x011b4, 0x011b5, + 0x0111a, 0x01106, 0x01107, 0x01108, 0x01121, 0x01109, 0x0110a, 0x0110b, + 0x0110c, 0x0110d, 0x0110e, 0x0110f, 0x01110, 0x01111, 0x01112, 0x0ffbf, + 0x0ffc0, 0x0ffc1, 0x01161, 0x01162, 0x01163, 0x01164, 0x01165, 0x01166, + 0x0ffc8, 0x0ffc9, 0x01167, 0x01168, 0x01169, 0x0116a, 0x0116b, 0x0116c, + 0x0ffd0, 0x0ffd1, 0x0116d, 0x0116e, 0x0116f, 0x01170, 0x01171, 0x01172, + 0x0ffd8, 0x0ffd9, 0x01173, 0x01174, 0x01175, 0x0ffdd, 0x0ffde, 0x0ffdf, + 0x000a2, 0x000a3, 0x000ac, 0x000af, 0x000a6, 0x000a5, 0x020a9, 0x0ffe7, + 0x02502, 0x02190, 0x02191, 0x02192, 0x02193, 0x025a0, 0x025cb, 0x0ffef, + 0x0fff0, 0x0fff1, 0x0fff2, 0x0fff3, 0x0fff4, 0x0fff5, 0x0fff6, 0x0fff7, + 0x0fff8, 0x00000, 0x00000, 0x00000, 0x0fffc, 0x0fffd, 0x0fffe, 0x0ffff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_101[] = { + 0x10100, 0x10101, 0x10102, 0x10103, 0x10104, 0x10105, 0x10106, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x10110, 0x10111, 0x10112, 0x10113, 0x10114, 0x10115, 0x10116, 0x10117, + 0x10118, 0x10119, 0x1011a, 0x1011b, 0x1011c, 0x1011d, 0x1011e, 0x1011f, + 0x10120, 0x10121, 0x10122, 0x10123, 0x10124, 0x10125, 0x10126, 0x10127, + 0x10128, 0x10129, 0x1012a, 0x1012b, 0x1012c, 0x1012d, 0x1012e, 0x1012f, + 0x10130, 0x10131, 0x10132, 0x10133, 0x10134, 0x10135, 0x10136, 0x10137, + 0x10138, 0x10139, 0x1013a, 0x1013b, 0x1013c, 0x1013d, 0x1013e, 0x1013f, + 0x10140, 0x10141, 0x00031, 0x00035, 0x10144, 0x10145, 0x10146, 0x10147, + 0x00035, 0x10149, 0x1014a, 0x1014b, 0x1014c, 0x1014d, 0x1014e, 0x00035, + 0x10150, 0x10151, 0x10152, 0x10153, 0x10154, 0x10155, 0x10156, 0x10157, + 0x00031, 0x00031, 0x00031, 0x00032, 0x00032, 0x00032, 0x00032, 0x00035, + 0x10160, 0x10161, 0x10162, 0x10163, 0x10164, 0x10165, 0x10166, 0x10167, + 0x10168, 0x10169, 0x1016a, 0x1016b, 0x1016c, 0x1016d, 0x1016e, 0x1016f, + 0x10170, 0x10171, 0x10172, 0x00035, 0x10174, 0x10175, 0x10176, 0x10177, + 0x10178, 0x10179, 0x1017a, 0x1017b, 0x1017c, 0x1017d, 0x1017e, 0x1017f, + 0x10180, 0x10181, 0x10182, 0x10183, 0x10184, 0x10185, 0x10186, 0x10187, + 0x10188, 0x10189, 0x00030, 0x1018b, 0x1018c, 0x1018d, 0x1018e, 0x1018f, + 0x10190, 0x10191, 0x10192, 0x10193, 0x10194, 0x10195, 0x10196, 0x10197, + 0x10198, 0x10199, 0x1019a, 0x1019b, 0x1019c, 0x1019d, 0x1019e, 0x1019f, + 0x101a0, 0x101a1, 0x101a2, 0x101a3, 0x101a4, 0x101a5, 0x101a6, 0x101a7, + 0x101a8, 0x101a9, 0x101aa, 0x101ab, 0x101ac, 0x101ad, 0x101ae, 0x101af, + 0x101b0, 0x101b1, 0x101b2, 0x101b3, 0x101b4, 0x101b5, 0x101b6, 0x101b7, + 0x101b8, 0x101b9, 0x101ba, 0x101bb, 0x101bc, 0x101bd, 0x101be, 0x101bf, + 0x101c0, 0x101c1, 0x101c2, 0x101c3, 0x101c4, 0x101c5, 0x101c6, 0x101c7, + 0x101c8, 0x101c9, 0x101ca, 0x101cb, 0x101cc, 0x101cd, 0x101ce, 0x101cf, + 0x101d0, 0x101d1, 0x101d2, 0x101d3, 0x101d4, 0x101d5, 0x101d6, 0x101d7, + 0x101d8, 0x101d9, 0x101da, 0x101db, 0x101dc, 0x101dd, 0x101de, 0x101df, + 0x101e0, 0x101e1, 0x101e2, 0x101e3, 0x101e4, 0x101e5, 0x101e6, 0x101e7, + 0x101e8, 0x101e9, 0x101ea, 0x101eb, 0x101ec, 0x101ed, 0x101ee, 0x101ef, + 0x101f0, 0x101f1, 0x101f2, 0x101f3, 0x101f4, 0x101f5, 0x101f6, 0x101f7, + 0x101f8, 0x101f9, 0x101fa, 0x101fb, 0x101fc, 0x00000, 0x101fe, 0x101ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_103[] = { + 0x10300, 0x10301, 0x10302, 0x10303, 0x10304, 0x10305, 0x10306, 0x10307, + 0x10308, 0x10309, 0x1030a, 0x1030b, 0x1030c, 0x1030d, 0x1030e, 0x1030f, + 0x10310, 0x10311, 0x10312, 0x10313, 0x10314, 0x10315, 0x10316, 0x10317, + 0x10318, 0x10319, 0x1031a, 0x1031b, 0x1031c, 0x1031d, 0x1031e, 0x1031f, + 0x00031, 0x00035, 0x10322, 0x10323, 0x10324, 0x10325, 0x10326, 0x10327, + 0x10328, 0x10329, 0x1032a, 0x1032b, 0x1032c, 0x1032d, 0x1032e, 0x1032f, + 0x10330, 0x10331, 0x10332, 0x10333, 0x10334, 0x10335, 0x10336, 0x10337, + 0x10338, 0x10339, 0x1033a, 0x1033b, 0x1033c, 0x1033d, 0x1033e, 0x1033f, + 0x10340, 0x10341, 0x10342, 0x10343, 0x10344, 0x10345, 0x10346, 0x10347, + 0x10348, 0x10349, 0x1034a, 0x1034b, 0x1034c, 0x1034d, 0x1034e, 0x1034f, + 0x10350, 0x10351, 0x10352, 0x10353, 0x10354, 0x10355, 0x10356, 0x10357, + 0x10358, 0x10359, 0x1035a, 0x1035b, 0x1035c, 0x1035d, 0x1035e, 0x1035f, + 0x10360, 0x10361, 0x10362, 0x10363, 0x10364, 0x10365, 0x10366, 0x10367, + 0x10368, 0x10369, 0x1036a, 0x1036b, 0x1036c, 0x1036d, 0x1036e, 0x1036f, + 0x10370, 0x10371, 0x10372, 0x10373, 0x10374, 0x10375, 0x10376, 0x10377, + 0x10378, 0x10379, 0x1037a, 0x1037b, 0x1037c, 0x1037d, 0x1037e, 0x1037f, + 0x10380, 0x10381, 0x10382, 0x10383, 0x10384, 0x10385, 0x10386, 0x10387, + 0x10388, 0x10389, 0x1038a, 0x1038b, 0x1038c, 0x1038d, 0x1038e, 0x1038f, + 0x10390, 0x10391, 0x10392, 0x10393, 0x10394, 0x10395, 0x10396, 0x10397, + 0x10398, 0x10399, 0x1039a, 0x1039b, 0x1039c, 0x1039d, 0x1039e, 0x1039f, + 0x103a0, 0x103a1, 0x103a2, 0x103a3, 0x103a4, 0x103a5, 0x103a6, 0x103a7, + 0x103a8, 0x103a9, 0x103aa, 0x103ab, 0x103ac, 0x103ad, 0x103ae, 0x103af, + 0x103b0, 0x103b1, 0x103b2, 0x103b3, 0x103b4, 0x103b5, 0x103b6, 0x103b7, + 0x103b8, 0x103b9, 0x103ba, 0x103bb, 0x103bc, 0x103bd, 0x103be, 0x103bf, + 0x103c0, 0x103c1, 0x103c2, 0x103c3, 0x103c4, 0x103c5, 0x103c6, 0x103c7, + 0x103c8, 0x103c9, 0x103ca, 0x103cb, 0x103cc, 0x103cd, 0x103ce, 0x103cf, + 0x103d0, 0x00031, 0x00032, 0x103d3, 0x103d4, 0x103d5, 0x103d6, 0x103d7, + 0x103d8, 0x103d9, 0x103da, 0x103db, 0x103dc, 0x103dd, 0x103de, 0x103df, + 0x103e0, 0x103e1, 0x103e2, 0x103e3, 0x103e4, 0x103e5, 0x103e6, 0x103e7, + 0x103e8, 0x103e9, 0x103ea, 0x103eb, 0x103ec, 0x103ed, 0x103ee, 0x103ef, + 0x103f0, 0x103f1, 0x103f2, 0x103f3, 0x103f4, 0x103f5, 0x103f6, 0x103f7, + 0x103f8, 0x103f9, 0x103fa, 0x103fb, 0x103fc, 0x103fd, 0x103fe, 0x103ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_104[] = { + 0x10400, 0x10401, 0x10402, 0x10403, 0x10404, 0x10405, 0x10406, 0x10407, + 0x10408, 0x10409, 0x1040a, 0x1040b, 0x1040c, 0x1040d, 0x1040e, 0x1040f, + 0x10410, 0x10411, 0x10412, 0x10413, 0x10414, 0x10415, 0x10416, 0x10417, + 0x10418, 0x10419, 0x1041a, 0x1041b, 0x1041c, 0x1041d, 0x1041e, 0x1041f, + 0x10420, 0x10421, 0x10422, 0x10423, 0x10424, 0x10425, 0x10426, 0x10427, + 0x10400, 0x10401, 0x10402, 0x10403, 0x10404, 0x10405, 0x10406, 0x10407, + 0x10408, 0x10409, 0x1040a, 0x1040b, 0x1040c, 0x1040d, 0x1040e, 0x1040f, + 0x10410, 0x10411, 0x10412, 0x10413, 0x10414, 0x10415, 0x10416, 0x10417, + 0x10418, 0x10419, 0x1041a, 0x1041b, 0x1041c, 0x1041d, 0x1041e, 0x1041f, + 0x10420, 0x10421, 0x10422, 0x10423, 0x10424, 0x10425, 0x10426, 0x10427, + 0x10450, 0x10451, 0x10452, 0x10453, 0x10454, 0x10455, 0x10456, 0x10457, + 0x10458, 0x10459, 0x1045a, 0x1045b, 0x1045c, 0x1045d, 0x1045e, 0x1045f, + 0x10460, 0x10461, 0x10462, 0x10463, 0x10464, 0x10465, 0x10466, 0x10467, + 0x10468, 0x10469, 0x1046a, 0x1046b, 0x1046c, 0x1046d, 0x1046e, 0x1046f, + 0x10470, 0x10471, 0x10472, 0x10473, 0x10474, 0x10475, 0x10476, 0x10477, + 0x10478, 0x10479, 0x1047a, 0x1047b, 0x1047c, 0x1047d, 0x1047e, 0x1047f, + 0x10480, 0x10481, 0x10482, 0x10483, 0x10484, 0x10485, 0x10486, 0x10487, + 0x10488, 0x10489, 0x1048a, 0x1048b, 0x1048c, 0x1048d, 0x1048e, 0x1048f, + 0x10490, 0x10491, 0x10492, 0x10493, 0x10494, 0x10495, 0x10496, 0x10497, + 0x10498, 0x10499, 0x1049a, 0x1049b, 0x1049c, 0x1049d, 0x1049e, 0x1049f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x104aa, 0x104ab, 0x104ac, 0x104ad, 0x104ae, 0x104af, + 0x104b0, 0x104b1, 0x104b2, 0x104b3, 0x104b4, 0x104b5, 0x104b6, 0x104b7, + 0x104b8, 0x104b9, 0x104ba, 0x104bb, 0x104bc, 0x104bd, 0x104be, 0x104bf, + 0x104c0, 0x104c1, 0x104c2, 0x104c3, 0x104c4, 0x104c5, 0x104c6, 0x104c7, + 0x104c8, 0x104c9, 0x104ca, 0x104cb, 0x104cc, 0x104cd, 0x104ce, 0x104cf, + 0x104d0, 0x104d1, 0x104d2, 0x104d3, 0x104d4, 0x104d5, 0x104d6, 0x104d7, + 0x104d8, 0x104d9, 0x104da, 0x104db, 0x104dc, 0x104dd, 0x104de, 0x104df, + 0x104e0, 0x104e1, 0x104e2, 0x104e3, 0x104e4, 0x104e5, 0x104e6, 0x104e7, + 0x104e8, 0x104e9, 0x104ea, 0x104eb, 0x104ec, 0x104ed, 0x104ee, 0x104ef, + 0x104f0, 0x104f1, 0x104f2, 0x104f3, 0x104f4, 0x104f5, 0x104f6, 0x104f7, + 0x104f8, 0x104f9, 0x104fa, 0x104fb, 0x104fc, 0x104fd, 0x104fe, 0x104ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_108[] = { + 0x10800, 0x10801, 0x10802, 0x10803, 0x10804, 0x10805, 0x10806, 0x10807, + 0x10808, 0x10809, 0x1080a, 0x1080b, 0x1080c, 0x1080d, 0x1080e, 0x1080f, + 0x10810, 0x10811, 0x10812, 0x10813, 0x10814, 0x10815, 0x10816, 0x10817, + 0x10818, 0x10819, 0x1081a, 0x1081b, 0x1081c, 0x1081d, 0x1081e, 0x1081f, + 0x10820, 0x10821, 0x10822, 0x10823, 0x10824, 0x10825, 0x10826, 0x10827, + 0x10828, 0x10829, 0x1082a, 0x1082b, 0x1082c, 0x1082d, 0x1082e, 0x1082f, + 0x10830, 0x10831, 0x10832, 0x10833, 0x10834, 0x10835, 0x10836, 0x10837, + 0x10838, 0x10839, 0x1083a, 0x1083b, 0x1083c, 0x1083d, 0x1083e, 0x1083f, + 0x10840, 0x10841, 0x10842, 0x10843, 0x10844, 0x10845, 0x10846, 0x10847, + 0x10848, 0x10849, 0x1084a, 0x1084b, 0x1084c, 0x1084d, 0x1084e, 0x1084f, + 0x10850, 0x10851, 0x10852, 0x10853, 0x10854, 0x10855, 0x10856, 0x10857, + 0x00031, 0x00032, 0x00033, 0x1085b, 0x1085c, 0x1085d, 0x1085e, 0x1085f, + 0x10860, 0x10861, 0x10862, 0x10863, 0x10864, 0x10865, 0x10866, 0x10867, + 0x10868, 0x10869, 0x1086a, 0x1086b, 0x1086c, 0x1086d, 0x1086e, 0x1086f, + 0x10870, 0x10871, 0x10872, 0x10873, 0x10874, 0x10875, 0x10876, 0x10877, + 0x10878, 0x10879, 0x1087a, 0x1087b, 0x1087c, 0x1087d, 0x1087e, 0x1087f, + 0x10880, 0x10881, 0x10882, 0x10883, 0x10884, 0x10885, 0x10886, 0x10887, + 0x10888, 0x10889, 0x1088a, 0x1088b, 0x1088c, 0x1088d, 0x1088e, 0x1088f, + 0x10890, 0x10891, 0x10892, 0x10893, 0x10894, 0x10895, 0x10896, 0x10897, + 0x10898, 0x10899, 0x1089a, 0x1089b, 0x1089c, 0x1089d, 0x1089e, 0x1089f, + 0x108a0, 0x108a1, 0x108a2, 0x108a3, 0x108a4, 0x108a5, 0x108a6, 0x108a7, + 0x108a8, 0x108a9, 0x108aa, 0x108ab, 0x108ac, 0x108ad, 0x108ae, 0x108af, + 0x108b0, 0x108b1, 0x108b2, 0x108b3, 0x108b4, 0x108b5, 0x108b6, 0x108b7, + 0x108b8, 0x108b9, 0x108ba, 0x108bb, 0x108bc, 0x108bd, 0x108be, 0x108bf, + 0x108c0, 0x108c1, 0x108c2, 0x108c3, 0x108c4, 0x108c5, 0x108c6, 0x108c7, + 0x108c8, 0x108c9, 0x108ca, 0x108cb, 0x108cc, 0x108cd, 0x108ce, 0x108cf, + 0x108d0, 0x108d1, 0x108d2, 0x108d3, 0x108d4, 0x108d5, 0x108d6, 0x108d7, + 0x108d8, 0x108d9, 0x108da, 0x108db, 0x108dc, 0x108dd, 0x108de, 0x108df, + 0x108e0, 0x108e1, 0x108e2, 0x108e3, 0x108e4, 0x108e5, 0x108e6, 0x108e7, + 0x108e8, 0x108e9, 0x108ea, 0x108eb, 0x108ec, 0x108ed, 0x108ee, 0x108ef, + 0x108f0, 0x108f1, 0x108f2, 0x108f3, 0x108f4, 0x108f5, 0x108f6, 0x108f7, + 0x108f8, 0x108f9, 0x108fa, 0x108fb, 0x108fc, 0x108fd, 0x108fe, 0x108ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_109[] = { + 0x10900, 0x10901, 0x10902, 0x10903, 0x10904, 0x10905, 0x10906, 0x10907, + 0x10908, 0x10909, 0x1090a, 0x1090b, 0x1090c, 0x1090d, 0x1090e, 0x1090f, + 0x10910, 0x10911, 0x10912, 0x10913, 0x10914, 0x10915, 0x00031, 0x10917, + 0x10918, 0x10919, 0x00032, 0x00033, 0x1091c, 0x1091d, 0x1091e, 0x1091f, + 0x10920, 0x10921, 0x10922, 0x10923, 0x10924, 0x10925, 0x10926, 0x10927, + 0x10928, 0x10929, 0x1092a, 0x1092b, 0x1092c, 0x1092d, 0x1092e, 0x1092f, + 0x10930, 0x10931, 0x10932, 0x10933, 0x10934, 0x10935, 0x10936, 0x10937, + 0x10938, 0x10939, 0x1093a, 0x1093b, 0x1093c, 0x1093d, 0x1093e, 0x1093f, + 0x10940, 0x10941, 0x10942, 0x10943, 0x10944, 0x10945, 0x10946, 0x10947, + 0x10948, 0x10949, 0x1094a, 0x1094b, 0x1094c, 0x1094d, 0x1094e, 0x1094f, + 0x10950, 0x10951, 0x10952, 0x10953, 0x10954, 0x10955, 0x10956, 0x10957, + 0x10958, 0x10959, 0x1095a, 0x1095b, 0x1095c, 0x1095d, 0x1095e, 0x1095f, + 0x10960, 0x10961, 0x10962, 0x10963, 0x10964, 0x10965, 0x10966, 0x10967, + 0x10968, 0x10969, 0x1096a, 0x1096b, 0x1096c, 0x1096d, 0x1096e, 0x1096f, + 0x10970, 0x10971, 0x10972, 0x10973, 0x10974, 0x10975, 0x10976, 0x10977, + 0x10978, 0x10979, 0x1097a, 0x1097b, 0x1097c, 0x1097d, 0x1097e, 0x1097f, + 0x10980, 0x10981, 0x10982, 0x10983, 0x10984, 0x10985, 0x10986, 0x10987, + 0x10988, 0x10989, 0x1098a, 0x1098b, 0x1098c, 0x1098d, 0x1098e, 0x1098f, + 0x10990, 0x10991, 0x10992, 0x10993, 0x10994, 0x10995, 0x10996, 0x10997, + 0x10998, 0x10999, 0x1099a, 0x1099b, 0x1099c, 0x1099d, 0x1099e, 0x1099f, + 0x109a0, 0x109a1, 0x109a2, 0x109a3, 0x109a4, 0x109a5, 0x109a6, 0x109a7, + 0x109a8, 0x109a9, 0x109aa, 0x109ab, 0x109ac, 0x109ad, 0x109ae, 0x109af, + 0x109b0, 0x109b1, 0x109b2, 0x109b3, 0x109b4, 0x109b5, 0x109b6, 0x109b7, + 0x109b8, 0x109b9, 0x109ba, 0x109bb, 0x109bc, 0x109bd, 0x109be, 0x109bf, + 0x109c0, 0x109c1, 0x109c2, 0x109c3, 0x109c4, 0x109c5, 0x109c6, 0x109c7, + 0x109c8, 0x109c9, 0x109ca, 0x109cb, 0x109cc, 0x109cd, 0x109ce, 0x109cf, + 0x109d0, 0x109d1, 0x109d2, 0x109d3, 0x109d4, 0x109d5, 0x109d6, 0x109d7, + 0x109d8, 0x109d9, 0x109da, 0x109db, 0x109dc, 0x109dd, 0x109de, 0x109df, + 0x109e0, 0x109e1, 0x109e2, 0x109e3, 0x109e4, 0x109e5, 0x109e6, 0x109e7, + 0x109e8, 0x109e9, 0x109ea, 0x109eb, 0x109ec, 0x109ed, 0x109ee, 0x109ef, + 0x109f0, 0x109f1, 0x109f2, 0x109f3, 0x109f4, 0x109f5, 0x109f6, 0x109f7, + 0x109f8, 0x109f9, 0x109fa, 0x109fb, 0x109fc, 0x109fd, 0x109fe, 0x109ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10a[] = { + 0x10a00, 0x10a01, 0x10a02, 0x10a03, 0x10a04, 0x10a05, 0x10a06, 0x10a07, + 0x10a08, 0x10a09, 0x10a0a, 0x10a0b, 0x10a0c, 0x00000, 0x00000, 0x00000, + 0x10a10, 0x10a11, 0x10a12, 0x10a13, 0x10a14, 0x10a15, 0x10a16, 0x10a17, + 0x10a18, 0x10a19, 0x10a1a, 0x10a1b, 0x10a1c, 0x10a1d, 0x10a1e, 0x10a1f, + 0x10a20, 0x10a21, 0x10a22, 0x10a23, 0x10a24, 0x10a25, 0x10a26, 0x10a27, + 0x10a28, 0x10a29, 0x10a2a, 0x10a2b, 0x10a2c, 0x10a2d, 0x10a2e, 0x10a2f, + 0x10a30, 0x10a31, 0x10a32, 0x10a33, 0x10a34, 0x10a35, 0x10a36, 0x10a37, + 0x00000, 0x00000, 0x00000, 0x10a3b, 0x10a3c, 0x10a3d, 0x10a3e, 0x10a3f, + 0x00031, 0x00032, 0x00033, 0x00034, 0x10a44, 0x10a45, 0x10a46, 0x10a47, + 0x10a48, 0x10a49, 0x10a4a, 0x10a4b, 0x10a4c, 0x10a4d, 0x10a4e, 0x10a4f, + 0x10a50, 0x10a51, 0x10a52, 0x10a53, 0x10a54, 0x10a55, 0x10a56, 0x10a57, + 0x10a58, 0x10a59, 0x10a5a, 0x10a5b, 0x10a5c, 0x10a5d, 0x10a5e, 0x10a5f, + 0x10a60, 0x10a61, 0x10a62, 0x10a63, 0x10a64, 0x10a65, 0x10a66, 0x10a67, + 0x10a68, 0x10a69, 0x10a6a, 0x10a6b, 0x10a6c, 0x10a6d, 0x10a6e, 0x10a6f, + 0x10a70, 0x10a71, 0x10a72, 0x10a73, 0x10a74, 0x10a75, 0x10a76, 0x10a77, + 0x10a78, 0x10a79, 0x10a7a, 0x10a7b, 0x10a7c, 0x00031, 0x10a7e, 0x10a7f, + 0x10a80, 0x10a81, 0x10a82, 0x10a83, 0x10a84, 0x10a85, 0x10a86, 0x10a87, + 0x10a88, 0x10a89, 0x10a8a, 0x10a8b, 0x10a8c, 0x10a8d, 0x10a8e, 0x10a8f, + 0x10a90, 0x10a91, 0x10a92, 0x10a93, 0x10a94, 0x10a95, 0x10a96, 0x10a97, + 0x10a98, 0x10a99, 0x10a9a, 0x10a9b, 0x10a9c, 0x10a9d, 0x10a9e, 0x10a9f, + 0x10aa0, 0x10aa1, 0x10aa2, 0x10aa3, 0x10aa4, 0x10aa5, 0x10aa6, 0x10aa7, + 0x10aa8, 0x10aa9, 0x10aaa, 0x10aab, 0x10aac, 0x10aad, 0x10aae, 0x10aaf, + 0x10ab0, 0x10ab1, 0x10ab2, 0x10ab3, 0x10ab4, 0x10ab5, 0x10ab6, 0x10ab7, + 0x10ab8, 0x10ab9, 0x10aba, 0x10abb, 0x10abc, 0x10abd, 0x10abe, 0x10abf, + 0x10ac0, 0x10ac1, 0x10ac2, 0x10ac3, 0x10ac4, 0x10ac5, 0x10ac6, 0x10ac7, + 0x10ac8, 0x10ac9, 0x10aca, 0x10acb, 0x10acc, 0x10acd, 0x10ace, 0x10acf, + 0x10ad0, 0x10ad1, 0x10ad2, 0x10ad3, 0x10ad4, 0x10ad5, 0x10ad6, 0x10ad7, + 0x10ad8, 0x10ad9, 0x10ada, 0x10adb, 0x10adc, 0x10add, 0x10ade, 0x10adf, + 0x10ae0, 0x10ae1, 0x10ae2, 0x10ae3, 0x10ae4, 0x10ae5, 0x10ae6, 0x10ae7, + 0x10ae8, 0x10ae9, 0x10aea, 0x10aeb, 0x10aec, 0x10aed, 0x10aee, 0x10aef, + 0x10af0, 0x10af1, 0x10af2, 0x10af3, 0x10af4, 0x10af5, 0x10af6, 0x10af7, + 0x10af8, 0x10af9, 0x10afa, 0x10afb, 0x10afc, 0x10afd, 0x10afe, 0x10aff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10b[] = { + 0x10b00, 0x10b01, 0x10b02, 0x10b03, 0x10b04, 0x10b05, 0x10b06, 0x10b07, + 0x10b08, 0x10b09, 0x10b0a, 0x10b0b, 0x10b0c, 0x10b0d, 0x10b0e, 0x10b0f, + 0x10b10, 0x10b11, 0x10b12, 0x10b13, 0x10b14, 0x10b15, 0x10b16, 0x10b17, + 0x10b18, 0x10b19, 0x10b1a, 0x10b1b, 0x10b1c, 0x10b1d, 0x10b1e, 0x10b1f, + 0x10b20, 0x10b21, 0x10b22, 0x10b23, 0x10b24, 0x10b25, 0x10b26, 0x10b27, + 0x10b28, 0x10b29, 0x10b2a, 0x10b2b, 0x10b2c, 0x10b2d, 0x10b2d, 0x10b2f, + 0x10b30, 0x10b31, 0x10b32, 0x10b33, 0x10b34, 0x10b35, 0x10b36, 0x10b37, + 0x10b38, 0x10b39, 0x10b3a, 0x10b3b, 0x10b3c, 0x10b3d, 0x10b3e, 0x10b3f, + 0x10b40, 0x10b41, 0x10b42, 0x10b43, 0x10b44, 0x10b45, 0x10b46, 0x10b47, + 0x10b48, 0x10b49, 0x10b4a, 0x10b4b, 0x10b4c, 0x10b4d, 0x10b4e, 0x10b4f, + 0x10b50, 0x10b51, 0x10b52, 0x10b53, 0x10b54, 0x10b55, 0x10b56, 0x10b57, + 0x00031, 0x00032, 0x00033, 0x00034, 0x10b5c, 0x10b5d, 0x10b5e, 0x10b5f, + 0x10b60, 0x10b61, 0x10b62, 0x10b63, 0x10b64, 0x10b65, 0x10b66, 0x10b67, + 0x10b68, 0x10b69, 0x10b6a, 0x10b6b, 0x10b6c, 0x10b6d, 0x10b6e, 0x10b6f, + 0x10b70, 0x10b71, 0x10b72, 0x10b73, 0x10b74, 0x10b75, 0x10b76, 0x10b77, + 0x00031, 0x00032, 0x00033, 0x00034, 0x10b7c, 0x10b7d, 0x10b7e, 0x10b7f, + 0x10b80, 0x10b81, 0x10b82, 0x10b83, 0x10b84, 0x10b85, 0x10b86, 0x10b87, + 0x10b88, 0x10b89, 0x10b8a, 0x10b8b, 0x10b8c, 0x10b8d, 0x10b8e, 0x10b8f, + 0x10b90, 0x10b91, 0x10b92, 0x10b93, 0x10b94, 0x10b95, 0x10b96, 0x10b97, + 0x10b98, 0x10b99, 0x10b9a, 0x10b9b, 0x10b9c, 0x10b9d, 0x10b9e, 0x10b9f, + 0x10ba0, 0x10ba1, 0x10ba2, 0x10ba3, 0x10ba4, 0x10ba5, 0x10ba6, 0x10ba7, + 0x10ba8, 0x10ba9, 0x10baa, 0x10bab, 0x10bac, 0x10bad, 0x10bae, 0x10baf, + 0x10bb0, 0x10bb1, 0x10bb2, 0x10bb3, 0x10bb4, 0x10bb5, 0x10bb6, 0x10bb7, + 0x10bb8, 0x10bb9, 0x10bba, 0x10bbb, 0x10bbc, 0x10bbd, 0x10bbe, 0x10bbf, + 0x10bc0, 0x10bc1, 0x10bc2, 0x10bc3, 0x10bc4, 0x10bc5, 0x10bc6, 0x10bc7, + 0x10bc8, 0x10bc9, 0x10bca, 0x10bcb, 0x10bcc, 0x10bcd, 0x10bce, 0x10bcf, + 0x10bd0, 0x10bd1, 0x10bd2, 0x10bd3, 0x10bd4, 0x10bd5, 0x10bd6, 0x10bd7, + 0x10bd8, 0x10bd9, 0x10bda, 0x10bdb, 0x10bdc, 0x10bdd, 0x10bde, 0x10bdf, + 0x10be0, 0x10be1, 0x10be2, 0x10be3, 0x10be4, 0x10be5, 0x10be6, 0x10be7, + 0x10be8, 0x10be9, 0x10bea, 0x10beb, 0x10bec, 0x10bed, 0x10bee, 0x10bef, + 0x10bf0, 0x10bf1, 0x10bf2, 0x10bf3, 0x10bf4, 0x10bf5, 0x10bf6, 0x10bf7, + 0x10bf8, 0x10bf9, 0x10bfa, 0x10bfb, 0x10bfc, 0x10bfd, 0x10bfe, 0x10bff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10c[] = { + 0x10c00, 0x10c00, 0x10c02, 0x10c03, 0x10c03, 0x10c05, 0x10c06, 0x10c07, + 0x10c07, 0x10c09, 0x10c09, 0x10c0b, 0x10c0b, 0x10c0d, 0x10c0d, 0x10c0f, + 0x10c0f, 0x10c11, 0x10c11, 0x10c13, 0x10c14, 0x10c14, 0x10c16, 0x10c16, + 0x10c18, 0x10c18, 0x10c1a, 0x10c1a, 0x10c1c, 0x10c1c, 0x10c1e, 0x10c1e, + 0x10c20, 0x10c21, 0x10c22, 0x10c23, 0x10c24, 0x10c24, 0x10c26, 0x10c26, + 0x10c28, 0x10c28, 0x10c2a, 0x10c2a, 0x10c2c, 0x10c2d, 0x10c2d, 0x10c2f, + 0x10c30, 0x10c31, 0x10c32, 0x10c32, 0x10c34, 0x10c34, 0x10c36, 0x10c36, + 0x10c38, 0x10c38, 0x10c3a, 0x10c3a, 0x10c3c, 0x10c3d, 0x10c3e, 0x10c3f, + 0x10c3f, 0x10c41, 0x10c41, 0x10c43, 0x10c43, 0x10c45, 0x10c45, 0x10c47, + 0x10c48, 0x10c49, 0x10c4a, 0x10c4b, 0x10c4c, 0x10c4d, 0x10c4e, 0x10c4f, + 0x10c50, 0x10c51, 0x10c52, 0x10c53, 0x10c54, 0x10c55, 0x10c56, 0x10c57, + 0x10c58, 0x10c59, 0x10c5a, 0x10c5b, 0x10c5c, 0x10c5d, 0x10c5e, 0x10c5f, + 0x10c60, 0x10c61, 0x10c62, 0x10c63, 0x10c64, 0x10c65, 0x10c66, 0x10c67, + 0x10c68, 0x10c69, 0x10c6a, 0x10c6b, 0x10c6c, 0x10c6d, 0x10c6e, 0x10c6f, + 0x10c70, 0x10c71, 0x10c72, 0x10c73, 0x10c74, 0x10c75, 0x10c76, 0x10c77, + 0x10c78, 0x10c79, 0x10c7a, 0x10c7b, 0x10c7c, 0x10c7d, 0x10c7e, 0x10c7f, + 0x10c80, 0x10c81, 0x10c82, 0x10c83, 0x10c84, 0x10c85, 0x10c86, 0x10c87, + 0x10c88, 0x10c89, 0x10c8a, 0x10c8b, 0x10c8c, 0x10c8d, 0x10c8e, 0x10c8f, + 0x10c90, 0x10c91, 0x10c92, 0x10c93, 0x10c94, 0x10c95, 0x10c96, 0x10c97, + 0x10c98, 0x10c99, 0x10c9a, 0x10c9b, 0x10c9c, 0x10c9d, 0x10c9e, 0x10c9f, + 0x10ca0, 0x10ca1, 0x10ca2, 0x10ca3, 0x10ca4, 0x10ca5, 0x10ca6, 0x10ca7, + 0x10ca8, 0x10ca9, 0x10caa, 0x10cab, 0x10cac, 0x10cad, 0x10cae, 0x10caf, + 0x10cb0, 0x10cb1, 0x10cb2, 0x10cb3, 0x10cb4, 0x10cb5, 0x10cb6, 0x10cb7, + 0x10cb8, 0x10cb9, 0x10cba, 0x10cbb, 0x10cbc, 0x10cbd, 0x10cbe, 0x10cbf, + 0x10cc0, 0x10cc1, 0x10cc2, 0x10cc3, 0x10cc4, 0x10cc5, 0x10cc6, 0x10cc7, + 0x10cc8, 0x10cc9, 0x10cca, 0x10ccb, 0x10ccc, 0x10ccd, 0x10cce, 0x10ccf, + 0x10cd0, 0x10cd1, 0x10cd2, 0x10cd3, 0x10cd4, 0x10cd5, 0x10cd6, 0x10cd7, + 0x10cd8, 0x10cd9, 0x10cda, 0x10cdb, 0x10cdc, 0x10cdd, 0x10cde, 0x10cdf, + 0x10ce0, 0x10ce1, 0x10ce2, 0x10ce3, 0x10ce4, 0x10ce5, 0x10ce6, 0x10ce7, + 0x10ce8, 0x10ce9, 0x10cea, 0x10ceb, 0x10cec, 0x10ced, 0x10cee, 0x10cef, + 0x10cf0, 0x10cf1, 0x10cf2, 0x10cf3, 0x10cf4, 0x10cf5, 0x10cf6, 0x10cf7, + 0x10cf8, 0x10cf9, 0x10cfa, 0x10cfb, 0x10cfc, 0x10cfd, 0x10cfe, 0x10cff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10e[] = { + 0x10e00, 0x10e01, 0x10e02, 0x10e03, 0x10e04, 0x10e05, 0x10e06, 0x10e07, + 0x10e08, 0x10e09, 0x10e0a, 0x10e0b, 0x10e0c, 0x10e0d, 0x10e0e, 0x10e0f, + 0x10e10, 0x10e11, 0x10e12, 0x10e13, 0x10e14, 0x10e15, 0x10e16, 0x10e17, + 0x10e18, 0x10e19, 0x10e1a, 0x10e1b, 0x10e1c, 0x10e1d, 0x10e1e, 0x10e1f, + 0x10e20, 0x10e21, 0x10e22, 0x10e23, 0x10e24, 0x10e25, 0x10e26, 0x10e27, + 0x10e28, 0x10e29, 0x10e2a, 0x10e2b, 0x10e2c, 0x10e2d, 0x10e2e, 0x10e2f, + 0x10e30, 0x10e31, 0x10e32, 0x10e33, 0x10e34, 0x10e35, 0x10e36, 0x10e37, + 0x10e38, 0x10e39, 0x10e3a, 0x10e3b, 0x10e3c, 0x10e3d, 0x10e3e, 0x10e3f, + 0x10e40, 0x10e41, 0x10e42, 0x10e43, 0x10e44, 0x10e45, 0x10e46, 0x10e47, + 0x10e48, 0x10e49, 0x10e4a, 0x10e4b, 0x10e4c, 0x10e4d, 0x10e4e, 0x10e4f, + 0x10e50, 0x10e51, 0x10e52, 0x10e53, 0x10e54, 0x10e55, 0x10e56, 0x10e57, + 0x10e58, 0x10e59, 0x10e5a, 0x10e5b, 0x10e5c, 0x10e5d, 0x10e5e, 0x10e5f, + 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, + 0x00039, 0x10e69, 0x10e6a, 0x10e6b, 0x10e6c, 0x10e6d, 0x10e6e, 0x10e6f, + 0x10e70, 0x10e71, 0x10e72, 0x10e73, 0x10e74, 0x10e75, 0x10e76, 0x10e77, + 0x10e78, 0x10e79, 0x10e7a, 0x10e7b, 0x10e7c, 0x10e7d, 0x10e7e, 0x10e7f, + 0x10e80, 0x10e81, 0x10e82, 0x10e83, 0x10e84, 0x10e85, 0x10e86, 0x10e87, + 0x10e88, 0x10e89, 0x10e8a, 0x10e8b, 0x10e8c, 0x10e8d, 0x10e8e, 0x10e8f, + 0x10e90, 0x10e91, 0x10e92, 0x10e93, 0x10e94, 0x10e95, 0x10e96, 0x10e97, + 0x10e98, 0x10e99, 0x10e9a, 0x10e9b, 0x10e9c, 0x10e9d, 0x10e9e, 0x10e9f, + 0x10ea0, 0x10ea1, 0x10ea2, 0x10ea3, 0x10ea4, 0x10ea5, 0x10ea6, 0x10ea7, + 0x10ea8, 0x10ea9, 0x10eaa, 0x10eab, 0x10eac, 0x10ead, 0x10eae, 0x10eaf, + 0x10eb0, 0x10eb1, 0x10eb2, 0x10eb3, 0x10eb4, 0x10eb5, 0x10eb6, 0x10eb7, + 0x10eb8, 0x10eb9, 0x10eba, 0x10ebb, 0x10ebc, 0x10ebd, 0x10ebe, 0x10ebf, + 0x10ec0, 0x10ec1, 0x10ec2, 0x10ec3, 0x10ec4, 0x10ec5, 0x10ec6, 0x10ec7, + 0x10ec8, 0x10ec9, 0x10eca, 0x10ecb, 0x10ecc, 0x10ecd, 0x10ece, 0x10ecf, + 0x10ed0, 0x10ed1, 0x10ed2, 0x10ed3, 0x10ed4, 0x10ed5, 0x10ed6, 0x10ed7, + 0x10ed8, 0x10ed9, 0x10eda, 0x10edb, 0x10edc, 0x10edd, 0x10ede, 0x10edf, + 0x10ee0, 0x10ee1, 0x10ee2, 0x10ee3, 0x10ee4, 0x10ee5, 0x10ee6, 0x10ee7, + 0x10ee8, 0x10ee9, 0x10eea, 0x10eeb, 0x10eec, 0x10eed, 0x10eee, 0x10eef, + 0x10ef0, 0x10ef1, 0x10ef2, 0x10ef3, 0x10ef4, 0x10ef5, 0x10ef6, 0x10ef7, + 0x10ef8, 0x10ef9, 0x10efa, 0x10efb, 0x10efc, 0x10efd, 0x10efe, 0x10eff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_110[] = { + 0x11000, 0x11001, 0x11002, 0x11003, 0x11004, 0x11005, 0x11006, 0x11007, + 0x11008, 0x11009, 0x1100a, 0x1100b, 0x1100c, 0x1100d, 0x1100e, 0x1100f, + 0x11010, 0x11011, 0x11012, 0x11013, 0x11014, 0x11015, 0x11016, 0x11017, + 0x11018, 0x11019, 0x1101a, 0x1101b, 0x1101c, 0x1101d, 0x1101e, 0x1101f, + 0x11020, 0x11021, 0x11022, 0x11023, 0x11024, 0x11025, 0x11026, 0x11027, + 0x11028, 0x11029, 0x1102a, 0x1102b, 0x1102c, 0x1102d, 0x1102e, 0x1102f, + 0x11030, 0x11031, 0x11032, 0x11033, 0x11034, 0x11035, 0x11036, 0x11037, + 0x11038, 0x11039, 0x1103a, 0x1103b, 0x1103c, 0x1103d, 0x1103e, 0x1103f, + 0x11040, 0x11041, 0x11042, 0x11043, 0x11044, 0x11045, 0x11046, 0x11047, + 0x11048, 0x11049, 0x1104a, 0x1104b, 0x1104c, 0x1104d, 0x1104e, 0x1104f, + 0x11050, 0x11051, 0x11052, 0x11053, 0x11054, 0x11055, 0x11056, 0x11057, + 0x11058, 0x11059, 0x1105a, 0x1105b, 0x1105c, 0x1105d, 0x1105e, 0x1105f, + 0x11060, 0x11061, 0x11062, 0x11063, 0x11064, 0x11065, 0x11066, 0x11067, + 0x11068, 0x11069, 0x1106a, 0x1106b, 0x1106c, 0x1106d, 0x1106e, 0x1106f, + 0x11070, 0x11071, 0x11072, 0x11073, 0x11074, 0x11075, 0x11076, 0x11077, + 0x11078, 0x11079, 0x1107a, 0x1107b, 0x1107c, 0x1107d, 0x1107e, 0x1107f, + 0x00000, 0x00000, 0x00000, 0x11083, 0x11084, 0x11085, 0x11086, 0x11087, + 0x11088, 0x11089, 0x1108a, 0x1108b, 0x1108c, 0x1108d, 0x1108e, 0x1108f, + 0x11090, 0x11091, 0x11092, 0x11093, 0x11094, 0x11095, 0x11096, 0x11097, + 0x11098, 0x11099, 0x11099, 0x1109b, 0x1109b, 0x1109d, 0x1109e, 0x1109f, + 0x110a0, 0x110a1, 0x110a2, 0x110a3, 0x110a4, 0x110a5, 0x110a6, 0x110a7, + 0x110a8, 0x110a9, 0x110aa, 0x110a5, 0x110ac, 0x110ad, 0x110ae, 0x110af, + 0x110b0, 0x110b1, 0x110b2, 0x110b3, 0x110b4, 0x110b5, 0x110b6, 0x110b7, + 0x110b8, 0x110b9, 0x00000, 0x110bb, 0x110bc, 0x00000, 0x110be, 0x110bf, + 0x110c0, 0x110c1, 0x110c2, 0x110c3, 0x110c4, 0x110c5, 0x110c6, 0x110c7, + 0x110c8, 0x110c9, 0x110ca, 0x110cb, 0x110cc, 0x110cd, 0x110ce, 0x110cf, + 0x110d0, 0x110d1, 0x110d2, 0x110d3, 0x110d4, 0x110d5, 0x110d6, 0x110d7, + 0x110d8, 0x110d9, 0x110da, 0x110db, 0x110dc, 0x110dd, 0x110de, 0x110df, + 0x110e0, 0x110e1, 0x110e2, 0x110e3, 0x110e4, 0x110e5, 0x110e6, 0x110e7, + 0x110e8, 0x110e9, 0x110ea, 0x110eb, 0x110ec, 0x110ed, 0x110ee, 0x110ef, + 0x110f0, 0x110f1, 0x110f2, 0x110f3, 0x110f4, 0x110f5, 0x110f6, 0x110f7, + 0x110f8, 0x110f9, 0x110fa, 0x110fb, 0x110fc, 0x110fd, 0x110fe, 0x110ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_124[] = { + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x00034, + 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x00031, 0x00032, 0x00033, + 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x00031, 0x00032, + 0x00033, 0x00034, 0x00035, 0x00032, 0x00033, 0x00033, 0x00034, 0x00035, + 0x00036, 0x00037, 0x00038, 0x00039, 0x00031, 0x00032, 0x00033, 0x00033, + 0x00034, 0x00035, 0x12432, 0x12433, 0x00031, 0x00032, 0x00033, 0x00033, + 0x00034, 0x00035, 0x00033, 0x00033, 0x00034, 0x00034, 0x00034, 0x00034, + 0x00036, 0x00037, 0x00037, 0x00037, 0x00038, 0x00038, 0x00039, 0x00039, + 0x00039, 0x00039, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00034, 0x00035, 0x00035, 0x12456, 0x12457, + 0x00031, 0x00032, 0x1245a, 0x1245b, 0x1245c, 0x1245d, 0x1245e, 0x1245f, + 0x12460, 0x12461, 0x12462, 0x12463, 0x12464, 0x12465, 0x12466, 0x12467, + 0x12468, 0x12469, 0x1246a, 0x1246b, 0x1246c, 0x1246d, 0x1246e, 0x1246f, + 0x12470, 0x12471, 0x12472, 0x12473, 0x12474, 0x12475, 0x12476, 0x12477, + 0x12478, 0x12479, 0x1247a, 0x1247b, 0x1247c, 0x1247d, 0x1247e, 0x1247f, + 0x12480, 0x12481, 0x12482, 0x12483, 0x12484, 0x12485, 0x12486, 0x12487, + 0x12488, 0x12489, 0x1248a, 0x1248b, 0x1248c, 0x1248d, 0x1248e, 0x1248f, + 0x12490, 0x12491, 0x12492, 0x12493, 0x12494, 0x12495, 0x12496, 0x12497, + 0x12498, 0x12499, 0x1249a, 0x1249b, 0x1249c, 0x1249d, 0x1249e, 0x1249f, + 0x124a0, 0x124a1, 0x124a2, 0x124a3, 0x124a4, 0x124a5, 0x124a6, 0x124a7, + 0x124a8, 0x124a9, 0x124aa, 0x124ab, 0x124ac, 0x124ad, 0x124ae, 0x124af, + 0x124b0, 0x124b1, 0x124b2, 0x124b3, 0x124b4, 0x124b5, 0x124b6, 0x124b7, + 0x124b8, 0x124b9, 0x124ba, 0x124bb, 0x124bc, 0x124bd, 0x124be, 0x124bf, + 0x124c0, 0x124c1, 0x124c2, 0x124c3, 0x124c4, 0x124c5, 0x124c6, 0x124c7, + 0x124c8, 0x124c9, 0x124ca, 0x124cb, 0x124cc, 0x124cd, 0x124ce, 0x124cf, + 0x124d0, 0x124d1, 0x124d2, 0x124d3, 0x124d4, 0x124d5, 0x124d6, 0x124d7, + 0x124d8, 0x124d9, 0x124da, 0x124db, 0x124dc, 0x124dd, 0x124de, 0x124df, + 0x124e0, 0x124e1, 0x124e2, 0x124e3, 0x124e4, 0x124e5, 0x124e6, 0x124e7, + 0x124e8, 0x124e9, 0x124ea, 0x124eb, 0x124ec, 0x124ed, 0x124ee, 0x124ef, + 0x124f0, 0x124f1, 0x124f2, 0x124f3, 0x124f4, 0x124f5, 0x124f6, 0x124f7, + 0x124f8, 0x124f9, 0x124fa, 0x124fb, 0x124fc, 0x124fd, 0x124fe, 0x124ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d1[] = { + 0x1d100, 0x1d101, 0x1d102, 0x1d103, 0x1d104, 0x1d105, 0x1d106, 0x1d107, + 0x1d108, 0x1d109, 0x1d10a, 0x1d10b, 0x1d10c, 0x1d10d, 0x1d10e, 0x1d10f, + 0x1d110, 0x1d111, 0x1d112, 0x1d113, 0x1d114, 0x1d115, 0x1d116, 0x1d117, + 0x1d118, 0x1d119, 0x1d11a, 0x1d11b, 0x1d11c, 0x1d11d, 0x1d11e, 0x1d11f, + 0x1d120, 0x1d121, 0x1d122, 0x1d123, 0x1d124, 0x1d125, 0x1d126, 0x1d127, + 0x1d128, 0x1d129, 0x1d12a, 0x1d12b, 0x1d12c, 0x1d12d, 0x1d12e, 0x1d12f, + 0x1d130, 0x1d131, 0x1d132, 0x1d133, 0x1d134, 0x1d135, 0x1d136, 0x1d137, + 0x1d138, 0x1d139, 0x1d13a, 0x1d13b, 0x1d13c, 0x1d13d, 0x1d13e, 0x1d13f, + 0x1d140, 0x1d141, 0x1d142, 0x1d143, 0x1d144, 0x1d145, 0x1d146, 0x1d147, + 0x1d148, 0x1d149, 0x1d14a, 0x1d14b, 0x1d14c, 0x1d14d, 0x1d14e, 0x1d14f, + 0x1d150, 0x1d151, 0x1d152, 0x1d153, 0x1d154, 0x1d155, 0x1d156, 0x1d157, + 0x1d158, 0x1d159, 0x1d15a, 0x1d15b, 0x1d15c, 0x1d15d, 0x1d157, 0x1d158, + 0x1d158, 0x1d158, 0x1d158, 0x1d158, 0x1d158, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x1d16a, 0x1d16b, 0x1d16c, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x1d183, 0x1d184, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x1d18c, 0x1d18d, 0x1d18e, 0x1d18f, + 0x1d190, 0x1d191, 0x1d192, 0x1d193, 0x1d194, 0x1d195, 0x1d196, 0x1d197, + 0x1d198, 0x1d199, 0x1d19a, 0x1d19b, 0x1d19c, 0x1d19d, 0x1d19e, 0x1d19f, + 0x1d1a0, 0x1d1a1, 0x1d1a2, 0x1d1a3, 0x1d1a4, 0x1d1a5, 0x1d1a6, 0x1d1a7, + 0x1d1a8, 0x1d1a9, 0x00000, 0x00000, 0x00000, 0x00000, 0x1d1ae, 0x1d1af, + 0x1d1b0, 0x1d1b1, 0x1d1b2, 0x1d1b3, 0x1d1b4, 0x1d1b5, 0x1d1b6, 0x1d1b7, + 0x1d1b8, 0x1d1b9, 0x1d1ba, 0x1d1b9, 0x1d1ba, 0x1d1b9, 0x1d1ba, 0x1d1b9, + 0x1d1ba, 0x1d1c1, 0x1d1c2, 0x1d1c3, 0x1d1c4, 0x1d1c5, 0x1d1c6, 0x1d1c7, + 0x1d1c8, 0x1d1c9, 0x1d1ca, 0x1d1cb, 0x1d1cc, 0x1d1cd, 0x1d1ce, 0x1d1cf, + 0x1d1d0, 0x1d1d1, 0x1d1d2, 0x1d1d3, 0x1d1d4, 0x1d1d5, 0x1d1d6, 0x1d1d7, + 0x1d1d8, 0x1d1d9, 0x1d1da, 0x1d1db, 0x1d1dc, 0x1d1dd, 0x1d1de, 0x1d1df, + 0x1d1e0, 0x1d1e1, 0x1d1e2, 0x1d1e3, 0x1d1e4, 0x1d1e5, 0x1d1e6, 0x1d1e7, + 0x1d1e8, 0x1d1e9, 0x1d1ea, 0x1d1eb, 0x1d1ec, 0x1d1ed, 0x1d1ee, 0x1d1ef, + 0x1d1f0, 0x1d1f1, 0x1d1f2, 0x1d1f3, 0x1d1f4, 0x1d1f5, 0x1d1f6, 0x1d1f7, + 0x1d1f8, 0x1d1f9, 0x1d1fa, 0x1d1fb, 0x1d1fc, 0x1d1fd, 0x1d1fe, 0x1d1ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d2[] = { + 0x1d200, 0x1d201, 0x1d202, 0x1d203, 0x1d204, 0x1d205, 0x1d206, 0x1d207, + 0x1d208, 0x1d209, 0x1d20a, 0x1d20b, 0x1d20c, 0x1d20d, 0x1d20e, 0x1d20f, + 0x1d210, 0x1d211, 0x1d212, 0x1d213, 0x1d214, 0x1d215, 0x1d216, 0x1d217, + 0x1d218, 0x1d219, 0x1d21a, 0x1d21b, 0x1d21c, 0x1d21d, 0x1d21e, 0x1d21f, + 0x1d220, 0x1d221, 0x1d222, 0x1d223, 0x1d224, 0x1d225, 0x1d226, 0x1d227, + 0x1d228, 0x1d229, 0x1d22a, 0x1d22b, 0x1d22c, 0x1d22d, 0x1d22e, 0x1d22f, + 0x1d230, 0x1d231, 0x1d232, 0x1d233, 0x1d234, 0x1d235, 0x1d236, 0x1d237, + 0x1d238, 0x1d239, 0x1d23a, 0x1d23b, 0x1d23c, 0x1d23d, 0x1d23e, 0x1d23f, + 0x1d240, 0x1d241, 0x00000, 0x00000, 0x00000, 0x1d245, 0x1d246, 0x1d247, + 0x1d248, 0x1d249, 0x1d24a, 0x1d24b, 0x1d24c, 0x1d24d, 0x1d24e, 0x1d24f, + 0x1d250, 0x1d251, 0x1d252, 0x1d253, 0x1d254, 0x1d255, 0x1d256, 0x1d257, + 0x1d258, 0x1d259, 0x1d25a, 0x1d25b, 0x1d25c, 0x1d25d, 0x1d25e, 0x1d25f, + 0x1d260, 0x1d261, 0x1d262, 0x1d263, 0x1d264, 0x1d265, 0x1d266, 0x1d267, + 0x1d268, 0x1d269, 0x1d26a, 0x1d26b, 0x1d26c, 0x1d26d, 0x1d26e, 0x1d26f, + 0x1d270, 0x1d271, 0x1d272, 0x1d273, 0x1d274, 0x1d275, 0x1d276, 0x1d277, + 0x1d278, 0x1d279, 0x1d27a, 0x1d27b, 0x1d27c, 0x1d27d, 0x1d27e, 0x1d27f, + 0x1d280, 0x1d281, 0x1d282, 0x1d283, 0x1d284, 0x1d285, 0x1d286, 0x1d287, + 0x1d288, 0x1d289, 0x1d28a, 0x1d28b, 0x1d28c, 0x1d28d, 0x1d28e, 0x1d28f, + 0x1d290, 0x1d291, 0x1d292, 0x1d293, 0x1d294, 0x1d295, 0x1d296, 0x1d297, + 0x1d298, 0x1d299, 0x1d29a, 0x1d29b, 0x1d29c, 0x1d29d, 0x1d29e, 0x1d29f, + 0x1d2a0, 0x1d2a1, 0x1d2a2, 0x1d2a3, 0x1d2a4, 0x1d2a5, 0x1d2a6, 0x1d2a7, + 0x1d2a8, 0x1d2a9, 0x1d2aa, 0x1d2ab, 0x1d2ac, 0x1d2ad, 0x1d2ae, 0x1d2af, + 0x1d2b0, 0x1d2b1, 0x1d2b2, 0x1d2b3, 0x1d2b4, 0x1d2b5, 0x1d2b6, 0x1d2b7, + 0x1d2b8, 0x1d2b9, 0x1d2ba, 0x1d2bb, 0x1d2bc, 0x1d2bd, 0x1d2be, 0x1d2bf, + 0x1d2c0, 0x1d2c1, 0x1d2c2, 0x1d2c3, 0x1d2c4, 0x1d2c5, 0x1d2c6, 0x1d2c7, + 0x1d2c8, 0x1d2c9, 0x1d2ca, 0x1d2cb, 0x1d2cc, 0x1d2cd, 0x1d2ce, 0x1d2cf, + 0x1d2d0, 0x1d2d1, 0x1d2d2, 0x1d2d3, 0x1d2d4, 0x1d2d5, 0x1d2d6, 0x1d2d7, + 0x1d2d8, 0x1d2d9, 0x1d2da, 0x1d2db, 0x1d2dc, 0x1d2dd, 0x1d2de, 0x1d2df, + 0x1d2e0, 0x1d2e1, 0x1d2e2, 0x1d2e3, 0x1d2e4, 0x1d2e5, 0x1d2e6, 0x1d2e7, + 0x1d2e8, 0x1d2e9, 0x1d2ea, 0x1d2eb, 0x1d2ec, 0x1d2ed, 0x1d2ee, 0x1d2ef, + 0x1d2f0, 0x1d2f1, 0x1d2f2, 0x1d2f3, 0x1d2f4, 0x1d2f5, 0x1d2f6, 0x1d2f7, + 0x1d2f8, 0x1d2f9, 0x1d2fa, 0x1d2fb, 0x1d2fc, 0x1d2fd, 0x1d2fe, 0x1d2ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d3[] = { + 0x1d300, 0x1d301, 0x1d302, 0x1d303, 0x1d304, 0x1d305, 0x1d306, 0x1d307, + 0x1d308, 0x1d309, 0x1d30a, 0x1d30b, 0x1d30c, 0x1d30d, 0x1d30e, 0x1d30f, + 0x1d310, 0x1d311, 0x1d312, 0x1d313, 0x1d314, 0x1d315, 0x1d316, 0x1d317, + 0x1d318, 0x1d319, 0x1d31a, 0x1d31b, 0x1d31c, 0x1d31d, 0x1d31e, 0x1d31f, + 0x1d320, 0x1d321, 0x1d322, 0x1d323, 0x1d324, 0x1d325, 0x1d326, 0x1d327, + 0x1d328, 0x1d329, 0x1d32a, 0x1d32b, 0x1d32c, 0x1d32d, 0x1d32e, 0x1d32f, + 0x1d330, 0x1d331, 0x1d332, 0x1d333, 0x1d334, 0x1d335, 0x1d336, 0x1d337, + 0x1d338, 0x1d339, 0x1d33a, 0x1d33b, 0x1d33c, 0x1d33d, 0x1d33e, 0x1d33f, + 0x1d340, 0x1d341, 0x1d342, 0x1d343, 0x1d344, 0x1d345, 0x1d346, 0x1d347, + 0x1d348, 0x1d349, 0x1d34a, 0x1d34b, 0x1d34c, 0x1d34d, 0x1d34e, 0x1d34f, + 0x1d350, 0x1d351, 0x1d352, 0x1d353, 0x1d354, 0x1d355, 0x1d356, 0x1d357, + 0x1d358, 0x1d359, 0x1d35a, 0x1d35b, 0x1d35c, 0x1d35d, 0x1d35e, 0x1d35f, + 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, + 0x00039, 0x1d369, 0x1d36a, 0x1d36b, 0x1d36c, 0x1d36d, 0x1d36e, 0x1d36f, + 0x1d370, 0x1d371, 0x1d372, 0x1d373, 0x1d374, 0x1d375, 0x1d376, 0x1d377, + 0x1d378, 0x1d379, 0x1d37a, 0x1d37b, 0x1d37c, 0x1d37d, 0x1d37e, 0x1d37f, + 0x1d380, 0x1d381, 0x1d382, 0x1d383, 0x1d384, 0x1d385, 0x1d386, 0x1d387, + 0x1d388, 0x1d389, 0x1d38a, 0x1d38b, 0x1d38c, 0x1d38d, 0x1d38e, 0x1d38f, + 0x1d390, 0x1d391, 0x1d392, 0x1d393, 0x1d394, 0x1d395, 0x1d396, 0x1d397, + 0x1d398, 0x1d399, 0x1d39a, 0x1d39b, 0x1d39c, 0x1d39d, 0x1d39e, 0x1d39f, + 0x1d3a0, 0x1d3a1, 0x1d3a2, 0x1d3a3, 0x1d3a4, 0x1d3a5, 0x1d3a6, 0x1d3a7, + 0x1d3a8, 0x1d3a9, 0x1d3aa, 0x1d3ab, 0x1d3ac, 0x1d3ad, 0x1d3ae, 0x1d3af, + 0x1d3b0, 0x1d3b1, 0x1d3b2, 0x1d3b3, 0x1d3b4, 0x1d3b5, 0x1d3b6, 0x1d3b7, + 0x1d3b8, 0x1d3b9, 0x1d3ba, 0x1d3bb, 0x1d3bc, 0x1d3bd, 0x1d3be, 0x1d3bf, + 0x1d3c0, 0x1d3c1, 0x1d3c2, 0x1d3c3, 0x1d3c4, 0x1d3c5, 0x1d3c6, 0x1d3c7, + 0x1d3c8, 0x1d3c9, 0x1d3ca, 0x1d3cb, 0x1d3cc, 0x1d3cd, 0x1d3ce, 0x1d3cf, + 0x1d3d0, 0x1d3d1, 0x1d3d2, 0x1d3d3, 0x1d3d4, 0x1d3d5, 0x1d3d6, 0x1d3d7, + 0x1d3d8, 0x1d3d9, 0x1d3da, 0x1d3db, 0x1d3dc, 0x1d3dd, 0x1d3de, 0x1d3df, + 0x1d3e0, 0x1d3e1, 0x1d3e2, 0x1d3e3, 0x1d3e4, 0x1d3e5, 0x1d3e6, 0x1d3e7, + 0x1d3e8, 0x1d3e9, 0x1d3ea, 0x1d3eb, 0x1d3ec, 0x1d3ed, 0x1d3ee, 0x1d3ef, + 0x1d3f0, 0x1d3f1, 0x1d3f2, 0x1d3f3, 0x1d3f4, 0x1d3f5, 0x1d3f6, 0x1d3f7, + 0x1d3f8, 0x1d3f9, 0x1d3fa, 0x1d3fb, 0x1d3fc, 0x1d3fd, 0x1d3fe, 0x1d3ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d4[] = { + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, + 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x1d455, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x1d49d, 0x00043, 0x00044, + 0x1d4a0, 0x1d4a1, 0x00047, 0x1d4a3, 0x1d4a4, 0x0004a, 0x0004b, 0x1d4a7, + 0x1d4a8, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x1d4ad, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x1d4ba, 0x00046, 0x1d4bc, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x1d4c4, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x1d4ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d5[] = { + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x1d506, 0x00044, + 0x00045, 0x00046, 0x00047, 0x1d50b, 0x1d50c, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x1d515, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x1d51d, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x1d53a, 0x00044, 0x00045, 0x00046, 0x00047, 0x1d53f, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x1d545, 0x0004f, 0x1d547, + 0x1d548, 0x1d549, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x1d551, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, + 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, + 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x1d5ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d6[] = { + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, + 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00131, 0x00237, 0x1d6a6, 0x1d6a7, + 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, + 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, + 0x003a1, 0x00398, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, + 0x003a9, 0x02207, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, + 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, + 0x0039f, 0x003a0, 0x003a1, 0x003a3, 0x003a3, 0x003a4, 0x003a5, 0x003a6, + 0x003a7, 0x003a8, 0x003a9, 0x02202, 0x00395, 0x00398, 0x0039a, 0x003a6, + 0x003a1, 0x003a0, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, + 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, + 0x0039f, 0x003a0, 0x003a1, 0x00398, 0x003a3, 0x003a4, 0x003a5, 0x003a6, + 0x003a7, 0x003a8, 0x003a9, 0x02207, 0x00391, 0x00392, 0x00393, 0x1d6ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d7[] = { + 0x00395, 0x00396, 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, + 0x0039d, 0x0039e, 0x0039f, 0x003a0, 0x003a1, 0x003a3, 0x003a3, 0x003a4, + 0x003a5, 0x003a6, 0x003a7, 0x003a8, 0x003a9, 0x02202, 0x00395, 0x00398, + 0x0039a, 0x003a6, 0x003a1, 0x003a0, 0x00391, 0x00392, 0x00393, 0x00394, + 0x00395, 0x00396, 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, + 0x0039d, 0x0039e, 0x0039f, 0x003a0, 0x003a1, 0x00398, 0x003a3, 0x003a4, + 0x003a5, 0x003a6, 0x003a7, 0x003a8, 0x003a9, 0x02207, 0x00391, 0x00392, + 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, 0x00399, 0x0039a, + 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, 0x003a1, 0x003a3, + 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, 0x003a9, 0x02202, + 0x00395, 0x00398, 0x0039a, 0x003a6, 0x003a1, 0x003a0, 0x00391, 0x00392, + 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, 0x00399, 0x0039a, + 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, 0x003a1, 0x00398, + 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, 0x003a9, 0x02207, + 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, + 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, + 0x003a1, 0x003a3, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, + 0x003a9, 0x02202, 0x00395, 0x00398, 0x0039a, 0x003a6, 0x003a1, 0x003a0, + 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, + 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, + 0x003a1, 0x00398, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, + 0x003a9, 0x02207, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, + 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, + 0x0039f, 0x003a0, 0x003a1, 0x003a3, 0x003a3, 0x003a4, 0x003a5, 0x003a6, + 0x003a7, 0x003a8, 0x003a9, 0x02202, 0x00395, 0x00398, 0x0039a, 0x003a6, + 0x003a1, 0x003a0, 0x003dc, 0x003dc, 0x1d7cc, 0x1d7cd, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, + 0x00036, 0x00037, 0x00038, 0x00039, 0x00030, 0x00031, 0x00032, 0x00033, + 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x1d7ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1f1[] = { + 0x1f100, 0x1f101, 0x1f102, 0x1f103, 0x1f104, 0x1f105, 0x1f106, 0x1f107, + 0x1f108, 0x1f109, 0x1f10a, 0x1f10b, 0x1f10c, 0x1f10d, 0x1f10e, 0x1f10f, + 0x0249c, 0x0249d, 0x0249e, 0x0249f, 0x024a0, 0x024a1, 0x024a2, 0x024a3, + 0x024a4, 0x024a5, 0x024a6, 0x024a7, 0x024a8, 0x024a9, 0x024aa, 0x024ab, + 0x024ac, 0x024ad, 0x024ae, 0x024af, 0x024b0, 0x024b1, 0x024b2, 0x024b3, + 0x024b4, 0x024b5, 0x1f12a, 0x00043, 0x00052, 0x033c5, 0x1f12e, 0x1f12f, + 0x1f130, 0x00042, 0x1f132, 0x1f133, 0x1f134, 0x1f135, 0x1f136, 0x1f137, + 0x1f138, 0x1f139, 0x1f13a, 0x1f13b, 0x1f13c, 0x0004e, 0x1f13e, 0x00050, + 0x1f140, 0x1f141, 0x00053, 0x1f143, 0x1f144, 0x1f145, 0x00057, 0x1f147, + 0x1f148, 0x1f149, 0x1f14a, 0x033b7, 0x1f14c, 0x000df, 0x1f14e, 0x1f14f, + 0x1f150, 0x1f151, 0x1f152, 0x1f153, 0x1f154, 0x1f155, 0x1f156, 0x00048, + 0x1f158, 0x1f159, 0x1f15a, 0x1f15b, 0x1f15c, 0x1f15d, 0x1f15e, 0x00050, + 0x1f160, 0x1f161, 0x1f162, 0x1f163, 0x1f164, 0x1f165, 0x1f166, 0x1f167, + 0x1f168, 0x1f169, 0x1f16a, 0x1f16b, 0x1f16c, 0x1f16d, 0x1f16e, 0x1f16f, + 0x1f170, 0x1f171, 0x1f172, 0x1f173, 0x1f174, 0x1f175, 0x1f176, 0x1f177, + 0x1f178, 0x0004a, 0x1f17a, 0x0004c, 0x0004d, 0x1f17d, 0x1f17e, 0x00050, + 0x1f180, 0x1f181, 0x1f182, 0x1f183, 0x1f184, 0x1f185, 0x1f186, 0x1f187, + 0x1f188, 0x1f189, 0x00050, 0x1f18b, 0x03380, 0x1f18d, 0x1f18e, 0x1f18f, + 0x1f190, 0x1f191, 0x1f192, 0x1f193, 0x1f194, 0x1f195, 0x1f196, 0x1f197, + 0x1f198, 0x1f199, 0x1f19a, 0x1f19b, 0x1f19c, 0x1f19d, 0x1f19e, 0x1f19f, + 0x1f1a0, 0x1f1a1, 0x1f1a2, 0x1f1a3, 0x1f1a4, 0x1f1a5, 0x1f1a6, 0x1f1a7, + 0x1f1a8, 0x1f1a9, 0x1f1aa, 0x1f1ab, 0x1f1ac, 0x1f1ad, 0x1f1ae, 0x1f1af, + 0x1f1b0, 0x1f1b1, 0x1f1b2, 0x1f1b3, 0x1f1b4, 0x1f1b5, 0x1f1b6, 0x1f1b7, + 0x1f1b8, 0x1f1b9, 0x1f1ba, 0x1f1bb, 0x1f1bc, 0x1f1bd, 0x1f1be, 0x1f1bf, + 0x1f1c0, 0x1f1c1, 0x1f1c2, 0x1f1c3, 0x1f1c4, 0x1f1c5, 0x1f1c6, 0x1f1c7, + 0x1f1c8, 0x1f1c9, 0x1f1ca, 0x1f1cb, 0x1f1cc, 0x1f1cd, 0x1f1ce, 0x1f1cf, + 0x1f1d0, 0x1f1d1, 0x1f1d2, 0x1f1d3, 0x1f1d4, 0x1f1d5, 0x1f1d6, 0x1f1d7, + 0x1f1d8, 0x1f1d9, 0x1f1da, 0x1f1db, 0x1f1dc, 0x1f1dd, 0x1f1de, 0x1f1df, + 0x1f1e0, 0x1f1e1, 0x1f1e2, 0x1f1e3, 0x1f1e4, 0x1f1e5, 0x1f1e6, 0x1f1e7, + 0x1f1e8, 0x1f1e9, 0x1f1ea, 0x1f1eb, 0x1f1ec, 0x1f1ed, 0x1f1ee, 0x1f1ef, + 0x1f1f0, 0x1f1f1, 0x1f1f2, 0x1f1f3, 0x1f1f4, 0x1f1f5, 0x1f1f6, 0x1f1f7, + 0x1f1f8, 0x1f1f9, 0x1f1fa, 0x1f1fb, 0x1f1fc, 0x1f1fd, 0x1f1fe, 0x1f1ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1f2[] = { + 0x1f200, 0x1f201, 0x1f202, 0x1f203, 0x1f204, 0x1f205, 0x1f206, 0x1f207, + 0x1f208, 0x1f209, 0x1f20a, 0x1f20b, 0x1f20c, 0x1f20d, 0x1f20e, 0x1f20f, + 0x02f3f, 0x1f211, 0x1f212, 0x03066, 0x03193, 0x1f215, 0x1f216, 0x0319d, + 0x1f218, 0x1f219, 0x1f21a, 0x0f9be, 0x1f21c, 0x1f21d, 0x1f21e, 0x1f21f, + 0x1f220, 0x1f221, 0x02f63, 0x1f223, 0x1f224, 0x1f225, 0x1f226, 0x1f227, + 0x1f228, 0x03192, 0x03194, 0x1f22b, 0x032a7, 0x03197, 0x032a8, 0x1f22f, + 0x02f9b, 0x1f231, 0x1f232, 0x1f233, 0x1f234, 0x1f235, 0x1f236, 0x1f237, + 0x1f238, 0x1f239, 0x1f23a, 0x1f23b, 0x1f23c, 0x1f23d, 0x1f23e, 0x1f23f, + 0x1f240, 0x1f241, 0x1f242, 0x1f243, 0x1f244, 0x1f245, 0x1f246, 0x1f247, + 0x1f248, 0x1f249, 0x1f24a, 0x1f24b, 0x1f24c, 0x1f24d, 0x1f24e, 0x1f24f, + 0x1f250, 0x1f251, 0x1f252, 0x1f253, 0x1f254, 0x1f255, 0x1f256, 0x1f257, + 0x1f258, 0x1f259, 0x1f25a, 0x1f25b, 0x1f25c, 0x1f25d, 0x1f25e, 0x1f25f, + 0x1f260, 0x1f261, 0x1f262, 0x1f263, 0x1f264, 0x1f265, 0x1f266, 0x1f267, + 0x1f268, 0x1f269, 0x1f26a, 0x1f26b, 0x1f26c, 0x1f26d, 0x1f26e, 0x1f26f, + 0x1f270, 0x1f271, 0x1f272, 0x1f273, 0x1f274, 0x1f275, 0x1f276, 0x1f277, + 0x1f278, 0x1f279, 0x1f27a, 0x1f27b, 0x1f27c, 0x1f27d, 0x1f27e, 0x1f27f, + 0x1f280, 0x1f281, 0x1f282, 0x1f283, 0x1f284, 0x1f285, 0x1f286, 0x1f287, + 0x1f288, 0x1f289, 0x1f28a, 0x1f28b, 0x1f28c, 0x1f28d, 0x1f28e, 0x1f28f, + 0x1f290, 0x1f291, 0x1f292, 0x1f293, 0x1f294, 0x1f295, 0x1f296, 0x1f297, + 0x1f298, 0x1f299, 0x1f29a, 0x1f29b, 0x1f29c, 0x1f29d, 0x1f29e, 0x1f29f, + 0x1f2a0, 0x1f2a1, 0x1f2a2, 0x1f2a3, 0x1f2a4, 0x1f2a5, 0x1f2a6, 0x1f2a7, + 0x1f2a8, 0x1f2a9, 0x1f2aa, 0x1f2ab, 0x1f2ac, 0x1f2ad, 0x1f2ae, 0x1f2af, + 0x1f2b0, 0x1f2b1, 0x1f2b2, 0x1f2b3, 0x1f2b4, 0x1f2b5, 0x1f2b6, 0x1f2b7, + 0x1f2b8, 0x1f2b9, 0x1f2ba, 0x1f2bb, 0x1f2bc, 0x1f2bd, 0x1f2be, 0x1f2bf, + 0x1f2c0, 0x1f2c1, 0x1f2c2, 0x1f2c3, 0x1f2c4, 0x1f2c5, 0x1f2c6, 0x1f2c7, + 0x1f2c8, 0x1f2c9, 0x1f2ca, 0x1f2cb, 0x1f2cc, 0x1f2cd, 0x1f2ce, 0x1f2cf, + 0x1f2d0, 0x1f2d1, 0x1f2d2, 0x1f2d3, 0x1f2d4, 0x1f2d5, 0x1f2d6, 0x1f2d7, + 0x1f2d8, 0x1f2d9, 0x1f2da, 0x1f2db, 0x1f2dc, 0x1f2dd, 0x1f2de, 0x1f2df, + 0x1f2e0, 0x1f2e1, 0x1f2e2, 0x1f2e3, 0x1f2e4, 0x1f2e5, 0x1f2e6, 0x1f2e7, + 0x1f2e8, 0x1f2e9, 0x1f2ea, 0x1f2eb, 0x1f2ec, 0x1f2ed, 0x1f2ee, 0x1f2ef, + 0x1f2f0, 0x1f2f1, 0x1f2f2, 0x1f2f3, 0x1f2f4, 0x1f2f5, 0x1f2f6, 0x1f2f7, + 0x1f2f8, 0x1f2f9, 0x1f2fa, 0x1f2fb, 0x1f2fc, 0x1f2fd, 0x1f2fe, 0x1f2ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2f8[] = { + 0x2f800, 0x2f801, 0x2f802, 0x2f803, 0x2f804, 0x0fa30, 0x2f806, 0x2f807, + 0x2f808, 0x2f809, 0x0fa31, 0x2f80b, 0x2f80c, 0x2f80d, 0x0fa32, 0x2f80f, + 0x2f810, 0x2f811, 0x2f812, 0x2f813, 0x2f814, 0x1f21e, 0x2f816, 0x2f817, + 0x2f818, 0x2f819, 0x2f81a, 0x0fa71, 0x2f81c, 0x02f10, 0x2f81e, 0x2f81f, + 0x2f820, 0x2f821, 0x2f822, 0x2f823, 0x2f824, 0x0fa76, 0x0fa33, 0x0fa34, + 0x0fa77, 0x2f829, 0x2f82a, 0x0f963, 0x2f82c, 0x0fa35, 0x2f82e, 0x2f82f, + 0x2f830, 0x2f831, 0x2f831, 0x2f831, 0x2f834, 0x2f835, 0x2f836, 0x2f837, + 0x2f838, 0x2f839, 0x2f83a, 0x2f83b, 0x2f83c, 0x2f83d, 0x2f83e, 0x2f83f, + 0x2f840, 0x2f841, 0x2f842, 0x2f843, 0x2f844, 0x2f845, 0x2f845, 0x0fa7a, + 0x2f848, 0x2f849, 0x2f84a, 0x2f84b, 0x0fa37, 0x2f84d, 0x2f84e, 0x2f84f, + 0x0fa00, 0x2f851, 0x2f852, 0x2f853, 0x2f854, 0x2f855, 0x2f856, 0x2f857, + 0x2f858, 0x2f859, 0x2f85a, 0x2f85b, 0x2f85c, 0x1f215, 0x2f85e, 0x2f85f, + 0x2f860, 0x2f861, 0x2f862, 0x2f863, 0x2f864, 0x2f865, 0x2f866, 0x2f867, + 0x2f868, 0x2f869, 0x2f86a, 0x2f86a, 0x2f86c, 0x2f86d, 0x2f86e, 0x0f95f, + 0x2f870, 0x2f871, 0x2f872, 0x2f873, 0x2f874, 0x02e8e, 0x2f876, 0x2f877, + 0x02f2c, 0x2f879, 0x2f87a, 0x2f87b, 0x2f87c, 0x2f87d, 0x2f87e, 0x2f87f, + 0x2f880, 0x2f881, 0x2f882, 0x2f883, 0x2f884, 0x2f885, 0x2f886, 0x2f887, + 0x2f888, 0x2f889, 0x2f88a, 0x2f88b, 0x2f88c, 0x2f88d, 0x0f928, 0x2f88f, + 0x02f36, 0x2f891, 0x2f891, 0x2f893, 0x2f894, 0x2f894, 0x2f896, 0x2f897, + 0x2f898, 0x2f899, 0x2f89a, 0x2f89b, 0x2f89c, 0x2f89d, 0x2f89e, 0x2f89f, + 0x2f8a0, 0x2f8a1, 0x2f8a2, 0x0fa3d, 0x2f8a4, 0x2f8a5, 0x2f8a6, 0x2f8a7, + 0x0fa87, 0x2f8a7, 0x2f8aa, 0x0fa3f, 0x2f8ac, 0x2f8ad, 0x2f8ae, 0x2f8af, + 0x0fa40, 0x0f90d, 0x2f8b2, 0x2f8b3, 0x2f8b4, 0x2f8b5, 0x2f8b6, 0x2f8b7, + 0x2f8b8, 0x2f8b9, 0x2f8ba, 0x2f8bb, 0x2f8bc, 0x2f8bd, 0x2f8be, 0x2f8bf, + 0x2f8c0, 0x2f8c1, 0x2f8c2, 0x2f8c3, 0x2f8c4, 0x2f8c5, 0x2f8c6, 0x2f8c7, + 0x0fa41, 0x2f8c9, 0x2f8ca, 0x2f8cb, 0x2f8cc, 0x2f8cd, 0x2f8ce, 0x0fa43, + 0x2f8d0, 0x2f8d1, 0x2f8d2, 0x2f8d3, 0x2f8d4, 0x2f8d5, 0x2f8d6, 0x2f8d7, + 0x0f929, 0x0fa93, 0x2f8da, 0x2f8db, 0x2f8dc, 0x2f8dd, 0x2f8de, 0x2f8df, + 0x2f8e0, 0x2f8e1, 0x0fa44, 0x2f8e3, 0x2f8e4, 0x2f8e5, 0x2f8e6, 0x0fad2, + 0x2f8e8, 0x2f8e9, 0x2f8ea, 0x2f8eb, 0x2f8ec, 0x2f8ed, 0x2f8ee, 0x2f8ef, + 0x2f8f0, 0x2f8f1, 0x2f8f2, 0x2f8f3, 0x2f8f4, 0x0f970, 0x2f8f6, 0x2f8f7, + 0x2f8f8, 0x2f8f9, 0x2f8fa, 0x2f8fb, 0x2f8fc, 0x2f8fd, 0x2f8fe, 0x2f8ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2f9[] = { + 0x2f900, 0x0fa45, 0x0f9ca, 0x2f903, 0x2f904, 0x2f905, 0x2f906, 0x2f907, + 0x2f908, 0x2f909, 0x2f90a, 0x0fa99, 0x2f90c, 0x2f90d, 0x2f90e, 0x2f90f, + 0x2f910, 0x2f911, 0x2f912, 0x2f913, 0x0fa9b, 0x2f915, 0x2f916, 0x2f917, + 0x2f918, 0x2f919, 0x2f91a, 0x2f91b, 0x2f91c, 0x2f91d, 0x2f91e, 0x2f91f, + 0x2f920, 0x0fa9e, 0x2f922, 0x2f923, 0x2f924, 0x2f925, 0x2f926, 0x2f927, + 0x2f928, 0x02ea9, 0x2f92a, 0x2f92b, 0x2f92c, 0x2f92c, 0x2f92e, 0x2f92f, + 0x0faa1, 0x2f931, 0x2f932, 0x2f933, 0x2f934, 0x2f935, 0x2f936, 0x2f937, + 0x0f962, 0x2f939, 0x2f93a, 0x2f93b, 0x2f93c, 0x2f93d, 0x2f93e, 0x2f93f, + 0x0faa8, 0x2f941, 0x2f942, 0x2f943, 0x2f944, 0x2f945, 0x2f946, 0x2f946, + 0x0faa9, 0x0fad4, 0x2f94a, 0x2f94b, 0x2f94c, 0x2f94d, 0x2f94e, 0x0f93b, + 0x0faab, 0x2f951, 0x2f952, 0x0fa50, 0x2f954, 0x2f955, 0x0fa1b, 0x2f957, + 0x2f958, 0x0fa54, 0x2f95a, 0x2f95b, 0x2f95c, 0x2f95d, 0x2f95d, 0x2f95f, + 0x2f960, 0x2f961, 0x2f962, 0x2f963, 0x2f964, 0x2f965, 0x2f966, 0x2f967, + 0x2f968, 0x2f969, 0x2f96a, 0x2f96b, 0x2f96c, 0x2f96d, 0x2f96e, 0x2f96f, + 0x2f970, 0x2f971, 0x2f972, 0x2f973, 0x2f974, 0x2f975, 0x2f976, 0x2f977, + 0x2f978, 0x2f979, 0x0fa5b, 0x2f97b, 0x2f97c, 0x2f97d, 0x2f97e, 0x2f97f, + 0x2f980, 0x2f981, 0x2f982, 0x2f983, 0x2f984, 0x2f985, 0x2f986, 0x2f987, + 0x2f988, 0x2f989, 0x2f98a, 0x2f893, 0x2f98c, 0x2f98d, 0x2f98e, 0x2f98f, + 0x2f990, 0x2f991, 0x2f992, 0x2f993, 0x2f994, 0x2f995, 0x2f996, 0x2f997, + 0x0f974, 0x2f999, 0x2f99a, 0x2f99b, 0x2f99c, 0x2f99d, 0x2f99e, 0x0fa5f, + 0x2f9a0, 0x2f9a1, 0x2f9a2, 0x2f9a3, 0x2f9a4, 0x2f9a5, 0x2f9a6, 0x2f9a7, + 0x2f9a8, 0x2f9a9, 0x2f9aa, 0x2f9ab, 0x2f9ac, 0x2f9ad, 0x2f9ae, 0x2f9af, + 0x2f9b0, 0x2f9b1, 0x2f9b2, 0x2f9b3, 0x0f936, 0x2f9b5, 0x2f9b6, 0x2f9b7, + 0x2f9b8, 0x2f9b9, 0x2f9ba, 0x0fab5, 0x2f9bc, 0x2f9bd, 0x2f9be, 0x2f9bf, + 0x2f9c0, 0x2f9c1, 0x2f9c2, 0x2f9c3, 0x02f90, 0x2f9c5, 0x2f9c6, 0x2f9c7, + 0x2f9c8, 0x2f9c9, 0x2f9ca, 0x2f9cb, 0x2f9cc, 0x2f9cd, 0x2f9ce, 0x2f9cf, + 0x0fabe, 0x0fac0, 0x02f97, 0x2f9d3, 0x2f9d4, 0x2f9d5, 0x2f9d6, 0x2f9d7, + 0x2f9d8, 0x2f9d9, 0x2f9da, 0x2f9db, 0x2f9dc, 0x2f9dd, 0x2f9de, 0x0fac2, + 0x2f9e0, 0x2f9e1, 0x2f9e2, 0x2f9e3, 0x2f9e4, 0x2f9e5, 0x2f9e6, 0x2f9e7, + 0x2f9e8, 0x2f9e9, 0x2f9ea, 0x2f9eb, 0x2f9ec, 0x2f9ed, 0x2f9ee, 0x2f9ef, + 0x2f9f0, 0x2f9f1, 0x2f9f2, 0x2f9f3, 0x2f9f4, 0x2f9f5, 0x2f9f6, 0x2f9f7, + 0x2f9f8, 0x2f9f9, 0x2f9fa, 0x2f9fb, 0x2f9fc, 0x2f9fd, 0x0facb, 0x2f9ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2fa[] = { + 0x2fa00, 0x2fa01, 0x2fa02, 0x2fa03, 0x2fa04, 0x2fa05, 0x2fa06, 0x2fa07, + 0x2fa08, 0x2fa09, 0x0facd, 0x2fa0b, 0x2fa0c, 0x2fa0d, 0x2fa0e, 0x2fa0f, + 0x2fa10, 0x2fa11, 0x2fa12, 0x2fa13, 0x2fa14, 0x02fc7, 0x04d56, 0x02fcb, + 0x02eea, 0x2fa19, 0x2fa1a, 0x2fa1b, 0x02fd0, 0x2fa1d, 0x2fa1e, 0x2fa1f, + 0x2fa20, 0x2fa21, 0x2fa22, 0x2fa23, 0x2fa24, 0x2fa25, 0x2fa26, 0x2fa27, + 0x2fa28, 0x2fa29, 0x2fa2a, 0x2fa2b, 0x2fa2c, 0x2fa2d, 0x2fa2e, 0x2fa2f, + 0x2fa30, 0x2fa31, 0x2fa32, 0x2fa33, 0x2fa34, 0x2fa35, 0x2fa36, 0x2fa37, + 0x2fa38, 0x2fa39, 0x2fa3a, 0x2fa3b, 0x2fa3c, 0x2fa3d, 0x2fa3e, 0x2fa3f, + 0x2fa40, 0x2fa41, 0x2fa42, 0x2fa43, 0x2fa44, 0x2fa45, 0x2fa46, 0x2fa47, + 0x2fa48, 0x2fa49, 0x2fa4a, 0x2fa4b, 0x2fa4c, 0x2fa4d, 0x2fa4e, 0x2fa4f, + 0x2fa50, 0x2fa51, 0x2fa52, 0x2fa53, 0x2fa54, 0x2fa55, 0x2fa56, 0x2fa57, + 0x2fa58, 0x2fa59, 0x2fa5a, 0x2fa5b, 0x2fa5c, 0x2fa5d, 0x2fa5e, 0x2fa5f, + 0x2fa60, 0x2fa61, 0x2fa62, 0x2fa63, 0x2fa64, 0x2fa65, 0x2fa66, 0x2fa67, + 0x2fa68, 0x2fa69, 0x2fa6a, 0x2fa6b, 0x2fa6c, 0x2fa6d, 0x2fa6e, 0x2fa6f, + 0x2fa70, 0x2fa71, 0x2fa72, 0x2fa73, 0x2fa74, 0x2fa75, 0x2fa76, 0x2fa77, + 0x2fa78, 0x2fa79, 0x2fa7a, 0x2fa7b, 0x2fa7c, 0x2fa7d, 0x2fa7e, 0x2fa7f, + 0x2fa80, 0x2fa81, 0x2fa82, 0x2fa83, 0x2fa84, 0x2fa85, 0x2fa86, 0x2fa87, + 0x2fa88, 0x2fa89, 0x2fa8a, 0x2fa8b, 0x2fa8c, 0x2fa8d, 0x2fa8e, 0x2fa8f, + 0x2fa90, 0x2fa91, 0x2fa92, 0x2fa93, 0x2fa94, 0x2fa95, 0x2fa96, 0x2fa97, + 0x2fa98, 0x2fa99, 0x2fa9a, 0x2fa9b, 0x2fa9c, 0x2fa9d, 0x2fa9e, 0x2fa9f, + 0x2faa0, 0x2faa1, 0x2faa2, 0x2faa3, 0x2faa4, 0x2faa5, 0x2faa6, 0x2faa7, + 0x2faa8, 0x2faa9, 0x2faaa, 0x2faab, 0x2faac, 0x2faad, 0x2faae, 0x2faaf, + 0x2fab0, 0x2fab1, 0x2fab2, 0x2fab3, 0x2fab4, 0x2fab5, 0x2fab6, 0x2fab7, + 0x2fab8, 0x2fab9, 0x2faba, 0x2fabb, 0x2fabc, 0x2fabd, 0x2fabe, 0x2fabf, + 0x2fac0, 0x2fac1, 0x2fac2, 0x2fac3, 0x2fac4, 0x2fac5, 0x2fac6, 0x2fac7, + 0x2fac8, 0x2fac9, 0x2faca, 0x2facb, 0x2facc, 0x2facd, 0x2face, 0x2facf, + 0x2fad0, 0x2fad1, 0x2fad2, 0x2fad3, 0x2fad4, 0x2fad5, 0x2fad6, 0x2fad7, + 0x2fad8, 0x2fad9, 0x2fada, 0x2fadb, 0x2fadc, 0x2fadd, 0x2fade, 0x2fadf, + 0x2fae0, 0x2fae1, 0x2fae2, 0x2fae3, 0x2fae4, 0x2fae5, 0x2fae6, 0x2fae7, + 0x2fae8, 0x2fae9, 0x2faea, 0x2faeb, 0x2faec, 0x2faed, 0x2faee, 0x2faef, + 0x2faf0, 0x2faf1, 0x2faf2, 0x2faf3, 0x2faf4, 0x2faf5, 0x2faf6, 0x2faf7, + 0x2faf8, 0x2faf9, 0x2fafa, 0x2fafb, 0x2fafc, 0x2fafd, 0x2fafe, 0x2faff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_e00[] = { + 0xe0000, 0x00000, 0xe0002, 0xe0003, 0xe0004, 0xe0005, 0xe0006, 0xe0007, + 0xe0008, 0xe0009, 0xe000a, 0xe000b, 0xe000c, 0xe000d, 0xe000e, 0xe000f, + 0xe0010, 0xe0011, 0xe0012, 0xe0013, 0xe0014, 0xe0015, 0xe0016, 0xe0017, + 0xe0018, 0xe0019, 0xe001a, 0xe001b, 0xe001c, 0xe001d, 0xe001e, 0xe001f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0xe0080, 0xe0081, 0xe0082, 0xe0083, 0xe0084, 0xe0085, 0xe0086, 0xe0087, + 0xe0088, 0xe0089, 0xe008a, 0xe008b, 0xe008c, 0xe008d, 0xe008e, 0xe008f, + 0xe0090, 0xe0091, 0xe0092, 0xe0093, 0xe0094, 0xe0095, 0xe0096, 0xe0097, + 0xe0098, 0xe0099, 0xe009a, 0xe009b, 0xe009c, 0xe009d, 0xe009e, 0xe009f, + 0xe00a0, 0xe00a1, 0xe00a2, 0xe00a3, 0xe00a4, 0xe00a5, 0xe00a6, 0xe00a7, + 0xe00a8, 0xe00a9, 0xe00aa, 0xe00ab, 0xe00ac, 0xe00ad, 0xe00ae, 0xe00af, + 0xe00b0, 0xe00b1, 0xe00b2, 0xe00b3, 0xe00b4, 0xe00b5, 0xe00b6, 0xe00b7, + 0xe00b8, 0xe00b9, 0xe00ba, 0xe00bb, 0xe00bc, 0xe00bd, 0xe00be, 0xe00bf, + 0xe00c0, 0xe00c1, 0xe00c2, 0xe00c3, 0xe00c4, 0xe00c5, 0xe00c6, 0xe00c7, + 0xe00c8, 0xe00c9, 0xe00ca, 0xe00cb, 0xe00cc, 0xe00cd, 0xe00ce, 0xe00cf, + 0xe00d0, 0xe00d1, 0xe00d2, 0xe00d3, 0xe00d4, 0xe00d5, 0xe00d6, 0xe00d7, + 0xe00d8, 0xe00d9, 0xe00da, 0xe00db, 0xe00dc, 0xe00dd, 0xe00de, 0xe00df, + 0xe00e0, 0xe00e1, 0xe00e2, 0xe00e3, 0xe00e4, 0xe00e5, 0xe00e6, 0xe00e7, + 0xe00e8, 0xe00e9, 0xe00ea, 0xe00eb, 0xe00ec, 0xe00ed, 0xe00ee, 0xe00ef, + 0xe00f0, 0xe00f1, 0xe00f2, 0xe00f3, 0xe00f4, 0xe00f5, 0xe00f6, 0xe00f7, + 0xe00f8, 0xe00f9, 0xe00fa, 0xe00fb, 0xe00fc, 0xe00fd, 0xe00fe, 0xe00ff +}; + +static uint32_t unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_e01[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0xe01f0, 0xe01f1, 0xe01f2, 0xe01f3, 0xe01f4, 0xe01f5, 0xe01f6, 0xe01f7, + 0xe01f8, 0xe01f9, 0xe01fa, 0xe01fb, 0xe01fc, 0xe01fd, 0xe01fe, 0xe01ff +}; + +static uint32_t *unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_table[4352] = { + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_00, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_01, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_02, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_03, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_04, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_05, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_06, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_07, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_08, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_09, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0a, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0b, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0c, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0d, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0e, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_0f, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10, NULL, + NULL, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_13, + NULL, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_16, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_17, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_18, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_19, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1a, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1b, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1c, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1e, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1f, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_20, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_21, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_22, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_24, NULL, + NULL, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_27, + NULL, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2a, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2c, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2d, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2e, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2f, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_30, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_31, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_32, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_33, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_a6, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_a7, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_a8, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_a9, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_aa, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_ab, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_f9, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fa, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fb, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fc, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fd, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_fe, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_ff, + NULL, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_101, + NULL, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_103, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_104, NULL, + NULL, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_108, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_109, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10a, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10b, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10c, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_10e, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_110, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_124, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d1, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d2, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d3, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d4, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d5, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d6, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1d7, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1f1, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_1f2, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2f8, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2f9, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_2fa, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_e00, unicode_520_ci_except_kana_ci_kana_with_voiced_sound_mark_page_e01, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL +}; + +#endif diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_520_ci_table.h b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_520_ci_table.h new file mode 100644 index 00000000000..22a56d7f300 --- /dev/null +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_520_ci_table.h @@ -0,0 +1,5028 @@ +/* + Copyright(C) 2013-2015 Kouhei Sutou <kou@clear-code.com> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; version 2 + of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + MA 02110-1301, USA + + This file uses normalization table defined in + mysql-5.6.23/strings/ctype-uca.c. + The following is the header of the file: + + Copyright (c) 2004, 2014, Oracle and/or its affiliates. All rights reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; version 2 + of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + MA 02110-1301, USA + + UCA (Unicode Collation Algorithm) support. + Written by Alexander Barkov <bar@mysql.com> +*/ + +#ifndef MYSQL_UCA_520_H +#define MYSQL_UCA_520_H + +#include <stdint.h> + +static uint32_t unicode_520_ci_page_00[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00009, 0x0000a, 0x0000b, 0x0000c, 0x0000d, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00020, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, + 0x00028, 0x00029, 0x0002a, 0x0002b, 0x0002c, 0x0002d, 0x0002e, 0x0002f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0003a, 0x0003b, 0x0003c, 0x0003d, 0x0003e, 0x0003f, + 0x00040, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, + 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, + 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, + 0x00058, 0x00059, 0x0005a, 0x0005b, 0x0005c, 0x0005d, 0x0005e, 0x0005f, + 0x00060, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, + 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, + 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, + 0x00058, 0x00059, 0x0005a, 0x0007b, 0x0007c, 0x0007d, 0x0007e, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00085, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00020, 0x000a1, 0x000a2, 0x000a3, 0x000a4, 0x000a5, 0x000a6, 0x000a7, + 0x000a8, 0x000a9, 0x00041, 0x000ab, 0x000ac, 0x000ad, 0x000ae, 0x000af, + 0x000b0, 0x000b1, 0x00032, 0x00033, 0x000b4, 0x0039c, 0x000b6, 0x000b7, + 0x000b8, 0x00031, 0x0004f, 0x000bb, 0x000bc, 0x000bd, 0x000be, 0x000bf, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x000c6, 0x00043, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00049, 0x00049, 0x00049, 0x00049, + 0x00044, 0x0004e, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x000d7, + 0x0004f, 0x00055, 0x00055, 0x00055, 0x00055, 0x00059, 0x000de, 0x000df, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x000c6, 0x00043, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00049, 0x00049, 0x00049, 0x00049, + 0x00044, 0x0004e, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x000f7, + 0x0004f, 0x00055, 0x00055, 0x00055, 0x00055, 0x00059, 0x000fe, 0x000ff +}; + +static uint32_t unicode_520_ci_page_01[] = { + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00043, 0x00043, + 0x00043, 0x00043, 0x00043, 0x00043, 0x00043, 0x00043, 0x00044, 0x00044, + 0x00044, 0x00044, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00047, 0x00047, 0x00047, 0x00047, + 0x00047, 0x00047, 0x00047, 0x00047, 0x00048, 0x00048, 0x00048, 0x00048, + 0x00049, 0x00049, 0x00049, 0x00049, 0x00049, 0x00049, 0x00049, 0x00049, + 0x00049, 0x00131, 0x00132, 0x00132, 0x0004a, 0x0004a, 0x0004b, 0x0004b, + 0x00138, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, + 0x0004c, 0x0004c, 0x0004c, 0x0004e, 0x0004e, 0x0004e, 0x0004e, 0x0004e, + 0x0004e, 0x00149, 0x0014a, 0x0014a, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x00152, 0x00152, 0x00052, 0x00052, 0x00052, 0x00052, + 0x00052, 0x00052, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, + 0x00053, 0x00053, 0x00054, 0x00054, 0x00054, 0x00054, 0x00166, 0x00166, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00057, 0x00057, 0x00059, 0x00059, + 0x00059, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x00053, + 0x00180, 0x00181, 0x00182, 0x00182, 0x00184, 0x00184, 0x00186, 0x00187, + 0x00187, 0x00189, 0x0018a, 0x0018b, 0x0018b, 0x0018d, 0x0018e, 0x0018f, + 0x00190, 0x00191, 0x00191, 0x00193, 0x00194, 0x00195, 0x00196, 0x00197, + 0x00198, 0x00198, 0x0019a, 0x0019b, 0x0019c, 0x0019d, 0x0019e, 0x0019f, + 0x0004f, 0x0004f, 0x001a2, 0x001a2, 0x001a4, 0x001a4, 0x001a6, 0x001a7, + 0x001a7, 0x001a9, 0x001aa, 0x001ab, 0x001ac, 0x001ac, 0x001ae, 0x00055, + 0x00055, 0x001b1, 0x001b2, 0x001b3, 0x001b3, 0x001b5, 0x001b5, 0x001b7, + 0x001b8, 0x001b8, 0x001ba, 0x001bb, 0x001bc, 0x001bc, 0x001be, 0x001bf, + 0x001c0, 0x001c1, 0x001c2, 0x001c3, 0x001c4, 0x001c4, 0x001c4, 0x001c7, + 0x001c7, 0x001c7, 0x001ca, 0x001ca, 0x001ca, 0x00041, 0x00041, 0x00049, + 0x00049, 0x0004f, 0x0004f, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x0018e, 0x00041, 0x00041, + 0x00041, 0x00041, 0x000c6, 0x000c6, 0x001e4, 0x001e4, 0x00047, 0x00047, + 0x0004b, 0x0004b, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x001b7, 0x001b7, + 0x0004a, 0x001c4, 0x001c4, 0x001c4, 0x00047, 0x00047, 0x00195, 0x001bf, + 0x0004e, 0x0004e, 0x00041, 0x00041, 0x000c6, 0x000c6, 0x0004f, 0x001ff +}; + +static uint32_t unicode_520_ci_page_02[] = { + 0x00041, 0x00041, 0x00041, 0x00041, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00049, 0x00049, 0x00049, 0x00049, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x00052, 0x00052, 0x00052, 0x00052, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00053, 0x00053, 0x00054, 0x00054, 0x0021c, 0x0021c, 0x00048, 0x00048, + 0x0019e, 0x00221, 0x00222, 0x00222, 0x00224, 0x00224, 0x00041, 0x00041, + 0x00045, 0x00045, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x00059, 0x00059, 0x00234, 0x00235, 0x00236, 0x00237, + 0x00238, 0x00239, 0x0023a, 0x0023b, 0x0023b, 0x0019a, 0x0023e, 0x0023f, + 0x00240, 0x00241, 0x00241, 0x00180, 0x00244, 0x00245, 0x00246, 0x00246, + 0x00248, 0x00248, 0x0024a, 0x0024a, 0x0024c, 0x0024c, 0x0024e, 0x0024e, + 0x00250, 0x00251, 0x00252, 0x00181, 0x00186, 0x00255, 0x00189, 0x0018a, + 0x00258, 0x0018f, 0x0025a, 0x00190, 0x0025c, 0x0025d, 0x0025e, 0x0025f, + 0x00193, 0x00261, 0x00262, 0x00194, 0x00264, 0x00265, 0x00266, 0x00267, + 0x00197, 0x00196, 0x0026a, 0x0026b, 0x0026c, 0x0026d, 0x0026e, 0x0019c, + 0x00270, 0x00271, 0x0019d, 0x00273, 0x00274, 0x0019f, 0x00276, 0x00277, + 0x00278, 0x00279, 0x0027a, 0x0027b, 0x0027c, 0x0027d, 0x0027e, 0x0027f, + 0x001a6, 0x00281, 0x00282, 0x001a9, 0x00284, 0x00285, 0x00286, 0x00287, + 0x001ae, 0x00244, 0x001b1, 0x001b2, 0x00245, 0x0028d, 0x0028e, 0x0028f, + 0x00290, 0x00291, 0x001b7, 0x00293, 0x00294, 0x00295, 0x00296, 0x00297, + 0x00298, 0x00299, 0x0029a, 0x0029b, 0x0029c, 0x0029d, 0x0029e, 0x0029f, + 0x002a0, 0x002a1, 0x002a2, 0x001c4, 0x002a4, 0x002a5, 0x001be, 0x002a7, + 0x002a8, 0x002a9, 0x002aa, 0x002ab, 0x002ac, 0x002ad, 0x002ae, 0x002af, + 0x00048, 0x00266, 0x0004a, 0x00052, 0x00279, 0x0027b, 0x00281, 0x00057, + 0x00059, 0x002b9, 0x002ba, 0x002bb, 0x002bc, 0x002bd, 0x002be, 0x002bf, + 0x002c0, 0x002c1, 0x002c2, 0x002c3, 0x002c4, 0x002c5, 0x002c6, 0x002c7, + 0x002c8, 0x002c9, 0x002ca, 0x002cb, 0x002cc, 0x002cd, 0x002ce, 0x002cf, + 0x002d0, 0x002d1, 0x002d2, 0x002d3, 0x002d4, 0x002d5, 0x002d6, 0x002d7, + 0x002d8, 0x002d9, 0x002da, 0x002db, 0x002dc, 0x002dd, 0x002de, 0x002df, + 0x00194, 0x0004c, 0x00053, 0x00058, 0x00295, 0x002e5, 0x002e6, 0x002e7, + 0x002e8, 0x002e9, 0x002ea, 0x002eb, 0x002ec, 0x002ed, 0x002ee, 0x002ef, + 0x002f0, 0x002f1, 0x002f2, 0x002f3, 0x002f4, 0x002f5, 0x002f6, 0x002f7, + 0x002f8, 0x002f9, 0x002fa, 0x002fb, 0x002fc, 0x002fd, 0x002fe, 0x002ff +}; + +static uint32_t unicode_520_ci_page_03[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00041, 0x00045, 0x00049, 0x0004f, 0x00055, + 0x00043, 0x00044, 0x00048, 0x0004d, 0x00052, 0x00054, 0x00056, 0x00058, + 0x00370, 0x00370, 0x00372, 0x00372, 0x002b9, 0x00375, 0x00376, 0x00376, + 0x00378, 0x00379, 0x00399, 0x0037b, 0x0037c, 0x0037d, 0x0003b, 0x0037f, + 0x00380, 0x00381, 0x00382, 0x00383, 0x000b4, 0x000a8, 0x00391, 0x000b7, + 0x00395, 0x00397, 0x00399, 0x0038b, 0x0039f, 0x0038d, 0x003a5, 0x003a9, + 0x00399, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, + 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, + 0x003a0, 0x003a1, 0x003a2, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, + 0x003a8, 0x003a9, 0x00399, 0x003a5, 0x00391, 0x00395, 0x00397, 0x00399, + 0x003a5, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, + 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, + 0x003a0, 0x003a1, 0x003a3, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, + 0x003a8, 0x003a9, 0x00399, 0x003a5, 0x0039f, 0x003a5, 0x003a9, 0x003cf, + 0x00392, 0x00398, 0x003a5, 0x003a5, 0x003a5, 0x003a6, 0x003a0, 0x003cf, + 0x003d8, 0x003d8, 0x003da, 0x003da, 0x003dc, 0x003dc, 0x003de, 0x003de, + 0x003e0, 0x003e0, 0x003e2, 0x003e2, 0x003e4, 0x003e4, 0x003e6, 0x003e6, + 0x003e8, 0x003e8, 0x003ea, 0x003ea, 0x003ec, 0x003ec, 0x003ee, 0x003ee, + 0x0039a, 0x003a1, 0x003a3, 0x003f3, 0x00398, 0x00395, 0x003f6, 0x003f7, + 0x003f7, 0x003a3, 0x003fa, 0x003fa, 0x003fc, 0x0037b, 0x003fe, 0x003ff +}; + +static uint32_t unicode_520_ci_page_04[] = { + 0x00400, 0x00400, 0x00402, 0x00403, 0x00404, 0x00405, 0x00406, 0x00407, + 0x00408, 0x00409, 0x0040a, 0x0040b, 0x0040c, 0x0040d, 0x0040e, 0x0040f, + 0x00410, 0x00411, 0x00412, 0x00413, 0x00414, 0x00400, 0x00416, 0x00417, + 0x0040d, 0x00419, 0x0041a, 0x0041b, 0x0041c, 0x0041d, 0x0041e, 0x0041f, + 0x00420, 0x00421, 0x00422, 0x00423, 0x00424, 0x00425, 0x00426, 0x00427, + 0x00428, 0x00429, 0x0042a, 0x0042b, 0x0042c, 0x0042d, 0x0042e, 0x0042f, + 0x00410, 0x00411, 0x00412, 0x00413, 0x00414, 0x00400, 0x00416, 0x00417, + 0x0040d, 0x00419, 0x0041a, 0x0041b, 0x0041c, 0x0041d, 0x0041e, 0x0041f, + 0x00420, 0x00421, 0x00422, 0x00423, 0x00424, 0x00425, 0x00426, 0x00427, + 0x00428, 0x00429, 0x0042a, 0x0042b, 0x0042c, 0x0042d, 0x0042e, 0x0042f, + 0x00400, 0x00400, 0x00402, 0x00403, 0x00404, 0x00405, 0x00406, 0x00407, + 0x00408, 0x00409, 0x0040a, 0x0040b, 0x0040c, 0x0040d, 0x0040e, 0x0040f, + 0x00460, 0x00460, 0x00462, 0x00462, 0x00464, 0x00464, 0x00466, 0x00466, + 0x00468, 0x00468, 0x0046a, 0x0046a, 0x0046c, 0x0046c, 0x0046e, 0x0046e, + 0x00470, 0x00470, 0x00472, 0x00472, 0x00474, 0x00474, 0x00476, 0x00476, + 0x00478, 0x00478, 0x0047a, 0x0047a, 0x0047c, 0x0047c, 0x0047e, 0x0047e, + 0x00480, 0x00480, 0x00482, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x0048a, 0x0048a, 0x0048c, 0x0048c, 0x0048e, 0x0048e, + 0x00413, 0x00413, 0x00492, 0x00492, 0x00494, 0x00494, 0x00496, 0x00496, + 0x00498, 0x00498, 0x0049a, 0x0049a, 0x0049c, 0x0049c, 0x0049e, 0x0049e, + 0x004a0, 0x004a0, 0x004a2, 0x004a2, 0x004a4, 0x004a4, 0x004a6, 0x004a6, + 0x004a8, 0x004a8, 0x004aa, 0x004aa, 0x004ac, 0x004ac, 0x004ae, 0x004ae, + 0x004b0, 0x004b0, 0x004b2, 0x004b2, 0x004b4, 0x004b4, 0x004b6, 0x004b6, + 0x004b8, 0x004b8, 0x004ba, 0x004ba, 0x004bc, 0x004bc, 0x004be, 0x004be, + 0x004c0, 0x00416, 0x00416, 0x004c3, 0x004c3, 0x004c5, 0x004c5, 0x004c7, + 0x004c7, 0x004c9, 0x004c9, 0x004cb, 0x004cb, 0x004cd, 0x004cd, 0x004c0, + 0x004d0, 0x004d0, 0x004d2, 0x004d2, 0x004d4, 0x004d4, 0x004d6, 0x004d6, + 0x004d8, 0x004d8, 0x004da, 0x004da, 0x004dc, 0x004dc, 0x004de, 0x004de, + 0x004e0, 0x004e0, 0x0040d, 0x0040d, 0x004e4, 0x004e4, 0x004e6, 0x004e6, + 0x004e8, 0x004e8, 0x004ea, 0x004ea, 0x004ec, 0x004ec, 0x00423, 0x00423, + 0x004f0, 0x004f0, 0x004f2, 0x004f2, 0x004f4, 0x004f4, 0x004f6, 0x004f6, + 0x004f8, 0x004f8, 0x004fa, 0x004fa, 0x004fc, 0x004fd, 0x004fe, 0x004ff +}; + +static uint32_t unicode_520_ci_page_05[] = { + 0x00500, 0x00500, 0x00502, 0x00502, 0x00504, 0x00504, 0x00506, 0x00506, + 0x00508, 0x00508, 0x0050a, 0x0050a, 0x0050c, 0x0050c, 0x0050e, 0x0050e, + 0x00510, 0x00510, 0x00512, 0x00512, 0x00514, 0x00514, 0x00516, 0x00516, + 0x00518, 0x00518, 0x0051a, 0x0051a, 0x0051c, 0x0051c, 0x0051e, 0x0051e, + 0x00520, 0x00520, 0x00522, 0x00522, 0x00524, 0x00524, 0x00526, 0x00527, + 0x00528, 0x00529, 0x0052a, 0x0052b, 0x0052c, 0x0052d, 0x0052e, 0x0052f, + 0x00530, 0x00531, 0x00532, 0x00533, 0x00534, 0x00535, 0x00536, 0x00537, + 0x00538, 0x00539, 0x0053a, 0x0053b, 0x0053c, 0x0053d, 0x0053e, 0x0053f, + 0x00540, 0x00541, 0x00542, 0x00543, 0x00544, 0x00545, 0x00546, 0x00547, + 0x00548, 0x00549, 0x0054a, 0x0054b, 0x0054c, 0x0054d, 0x0054e, 0x0054f, + 0x00550, 0x00551, 0x00552, 0x00553, 0x00554, 0x00555, 0x00556, 0x00557, + 0x00558, 0x00559, 0x0055a, 0x0055b, 0x0055c, 0x0055d, 0x0055e, 0x0055f, + 0x00560, 0x00531, 0x00532, 0x00533, 0x00534, 0x00535, 0x00536, 0x00537, + 0x00538, 0x00539, 0x0053a, 0x0053b, 0x0053c, 0x0053d, 0x0053e, 0x0053f, + 0x00540, 0x00541, 0x00542, 0x00543, 0x00544, 0x00545, 0x00546, 0x00547, + 0x00548, 0x00549, 0x0054a, 0x0054b, 0x0054c, 0x0054d, 0x0054e, 0x0054f, + 0x00550, 0x00551, 0x00552, 0x00553, 0x00554, 0x00555, 0x00556, 0x00587, + 0x00588, 0x00589, 0x0058a, 0x0058b, 0x0058c, 0x0058d, 0x0058e, 0x0058f, + 0x00590, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x005be, 0x00000, + 0x005c0, 0x00000, 0x00000, 0x005c3, 0x00000, 0x00000, 0x005c6, 0x00000, + 0x005c8, 0x005c9, 0x005ca, 0x005cb, 0x005cc, 0x005cd, 0x005ce, 0x005cf, + 0x005d0, 0x005d1, 0x005d2, 0x005d3, 0x005d4, 0x005d5, 0x005d6, 0x005d7, + 0x005d8, 0x005d9, 0x005da, 0x005da, 0x005dc, 0x005dd, 0x005dd, 0x005df, + 0x005df, 0x005e1, 0x005e2, 0x005e3, 0x005e3, 0x005e5, 0x005e5, 0x005e7, + 0x005e8, 0x005e9, 0x005ea, 0x005eb, 0x005ec, 0x005ed, 0x005ee, 0x005ef, + 0x005f0, 0x005f1, 0x005f2, 0x005f3, 0x005f4, 0x005f5, 0x005f6, 0x005f7, + 0x005f8, 0x005f9, 0x005fa, 0x005fb, 0x005fc, 0x005fd, 0x005fe, 0x005ff +}; + +static uint32_t unicode_520_ci_page_06[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00604, 0x00605, 0x00606, 0x00607, + 0x00608, 0x00609, 0x0060a, 0x0060b, 0x0060c, 0x0060d, 0x0060e, 0x0060f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x0061b, 0x0061c, 0x0061d, 0x0061e, 0x0061f, + 0x00620, 0x00621, 0x00622, 0x00623, 0x00624, 0x00625, 0x00626, 0x00627, + 0x00628, 0x00629, 0x0062a, 0x0062b, 0x0062c, 0x0062d, 0x0062e, 0x0062f, + 0x00630, 0x00631, 0x00632, 0x00633, 0x00634, 0x00635, 0x00636, 0x00637, + 0x00638, 0x00639, 0x0063a, 0x0063b, 0x0063c, 0x0063d, 0x0063e, 0x0063f, + 0x00640, 0x00641, 0x00642, 0x00643, 0x00644, 0x00645, 0x00646, 0x00647, + 0x00648, 0x00649, 0x0064a, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x0065f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0066a, 0x0066b, 0x0066c, 0x0066d, 0x0066e, 0x0066f, + 0x00000, 0x00671, 0x00672, 0x00673, 0x00621, 0x00675, 0x00676, 0x00677, + 0x00678, 0x00679, 0x0067a, 0x0067b, 0x0067c, 0x0067d, 0x0067e, 0x0067f, + 0x00680, 0x00681, 0x00682, 0x00683, 0x00684, 0x00685, 0x00686, 0x00687, + 0x00688, 0x00689, 0x0068a, 0x0068b, 0x0068c, 0x0068d, 0x0068e, 0x0068f, + 0x00690, 0x00691, 0x00692, 0x00693, 0x00694, 0x00695, 0x00696, 0x00697, + 0x00698, 0x00699, 0x0069a, 0x0069b, 0x0069c, 0x0069d, 0x0069e, 0x0069f, + 0x006a0, 0x006a1, 0x006a2, 0x006a3, 0x006a4, 0x006a5, 0x006a6, 0x006a7, + 0x006a8, 0x006a9, 0x006aa, 0x006ab, 0x006ac, 0x006ad, 0x006ae, 0x006af, + 0x006b0, 0x006b1, 0x006b2, 0x006b3, 0x006b4, 0x006b5, 0x006b6, 0x006b7, + 0x006b8, 0x006b9, 0x006ba, 0x006bb, 0x006bc, 0x006bd, 0x006be, 0x006bf, + 0x006c0, 0x006c1, 0x006c1, 0x006c3, 0x006c4, 0x006c5, 0x006c6, 0x006c7, + 0x006c8, 0x006c9, 0x006ca, 0x006cb, 0x006cc, 0x006cd, 0x006ce, 0x006cf, + 0x006d0, 0x006d1, 0x006d2, 0x006d2, 0x006d4, 0x006c0, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00648, 0x0064a, 0x00000, + 0x00000, 0x006e9, 0x00000, 0x00000, 0x00000, 0x00000, 0x006ee, 0x006ef, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x006fa, 0x006fb, 0x006fc, 0x00621, 0x00645, 0x006ff +}; + +static uint32_t unicode_520_ci_page_07[] = { + 0x00700, 0x00701, 0x00702, 0x00703, 0x00704, 0x00705, 0x00706, 0x00707, + 0x00708, 0x00709, 0x0070a, 0x0070b, 0x0070c, 0x0070d, 0x0070e, 0x00000, + 0x00710, 0x00000, 0x00712, 0x00713, 0x00713, 0x00715, 0x00716, 0x00717, + 0x00718, 0x00719, 0x0071a, 0x0071b, 0x0071b, 0x0071d, 0x0071e, 0x0071f, + 0x00720, 0x00721, 0x00722, 0x00723, 0x00723, 0x00725, 0x00726, 0x00726, + 0x00728, 0x00729, 0x0072a, 0x0072b, 0x0072c, 0x00712, 0x00713, 0x00715, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x0074b, 0x0074c, 0x0074d, 0x0074e, 0x0074f, + 0x00750, 0x00751, 0x00752, 0x00753, 0x00754, 0x00755, 0x00756, 0x00757, + 0x00758, 0x00759, 0x0075a, 0x0075b, 0x0075c, 0x0075d, 0x0075e, 0x0075f, + 0x00760, 0x00761, 0x00762, 0x00763, 0x00764, 0x00765, 0x00766, 0x00767, + 0x00768, 0x00769, 0x0076a, 0x0076b, 0x0076c, 0x0076d, 0x0076e, 0x0076f, + 0x00770, 0x00771, 0x00772, 0x00773, 0x00774, 0x00775, 0x00776, 0x00777, + 0x00778, 0x00779, 0x0077a, 0x0077b, 0x0077c, 0x0077d, 0x0077e, 0x0077f, + 0x00780, 0x00781, 0x00782, 0x00783, 0x00784, 0x00785, 0x00786, 0x00787, + 0x00788, 0x00789, 0x0078a, 0x0078b, 0x0078c, 0x0078d, 0x0078e, 0x0078f, + 0x00790, 0x00791, 0x00792, 0x00793, 0x00794, 0x00795, 0x00796, 0x00797, + 0x00798, 0x00799, 0x0079a, 0x0079b, 0x0079c, 0x0079d, 0x0079e, 0x0079f, + 0x007a0, 0x007a1, 0x007a2, 0x007a3, 0x007a4, 0x007a5, 0x007a6, 0x007a7, + 0x007a8, 0x007a9, 0x007aa, 0x007ab, 0x007ac, 0x007ad, 0x007ae, 0x007af, + 0x007b0, 0x007b1, 0x007b2, 0x007b3, 0x007b4, 0x007b5, 0x007b6, 0x007b7, + 0x007b8, 0x007b9, 0x007ba, 0x007bb, 0x007bc, 0x007bd, 0x007be, 0x007bf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x007ca, 0x007cb, 0x007cc, 0x007cd, 0x007ce, 0x007cf, + 0x007d0, 0x007d1, 0x007d2, 0x007d3, 0x007d4, 0x007d5, 0x007d6, 0x007d7, + 0x007d8, 0x007d9, 0x007da, 0x007db, 0x007dc, 0x007dd, 0x007de, 0x007df, + 0x007e0, 0x007e1, 0x007e2, 0x007e3, 0x007e4, 0x007e5, 0x007e6, 0x007e7, + 0x007d6, 0x007d7, 0x007d9, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x007f4, 0x007f5, 0x007f6, 0x007f7, + 0x007f8, 0x007f9, 0x007fa, 0x007fb, 0x007fc, 0x007fd, 0x007fe, 0x007ff +}; + +static uint32_t unicode_520_ci_page_08[] = { + 0x00800, 0x00801, 0x00802, 0x00803, 0x00804, 0x00805, 0x00806, 0x00807, + 0x00808, 0x00809, 0x0080a, 0x0080b, 0x0080c, 0x0080d, 0x0080e, 0x0080f, + 0x00810, 0x00811, 0x00812, 0x00813, 0x00814, 0x00815, 0x00816, 0x00817, + 0x00000, 0x00000, 0x0081a, 0x0081b, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x0082e, 0x0082f, + 0x00830, 0x00831, 0x00832, 0x00833, 0x00834, 0x00835, 0x00836, 0x00837, + 0x00838, 0x00839, 0x0083a, 0x0083b, 0x0083c, 0x0083d, 0x0083e, 0x0083f, + 0x00840, 0x00841, 0x00842, 0x00843, 0x00844, 0x00845, 0x00846, 0x00847, + 0x00848, 0x00849, 0x0084a, 0x0084b, 0x0084c, 0x0084d, 0x0084e, 0x0084f, + 0x00850, 0x00851, 0x00852, 0x00853, 0x00854, 0x00855, 0x00856, 0x00857, + 0x00858, 0x00859, 0x0085a, 0x0085b, 0x0085c, 0x0085d, 0x0085e, 0x0085f, + 0x00860, 0x00861, 0x00862, 0x00863, 0x00864, 0x00865, 0x00866, 0x00867, + 0x00868, 0x00869, 0x0086a, 0x0086b, 0x0086c, 0x0086d, 0x0086e, 0x0086f, + 0x00870, 0x00871, 0x00872, 0x00873, 0x00874, 0x00875, 0x00876, 0x00877, + 0x00878, 0x00879, 0x0087a, 0x0087b, 0x0087c, 0x0087d, 0x0087e, 0x0087f, + 0x00880, 0x00881, 0x00882, 0x00883, 0x00884, 0x00885, 0x00886, 0x00887, + 0x00888, 0x00889, 0x0088a, 0x0088b, 0x0088c, 0x0088d, 0x0088e, 0x0088f, + 0x00890, 0x00891, 0x00892, 0x00893, 0x00894, 0x00895, 0x00896, 0x00897, + 0x00898, 0x00899, 0x0089a, 0x0089b, 0x0089c, 0x0089d, 0x0089e, 0x0089f, + 0x008a0, 0x008a1, 0x008a2, 0x008a3, 0x008a4, 0x008a5, 0x008a6, 0x008a7, + 0x008a8, 0x008a9, 0x008aa, 0x008ab, 0x008ac, 0x008ad, 0x008ae, 0x008af, + 0x008b0, 0x008b1, 0x008b2, 0x008b3, 0x008b4, 0x008b5, 0x008b6, 0x008b7, + 0x008b8, 0x008b9, 0x008ba, 0x008bb, 0x008bc, 0x008bd, 0x008be, 0x008bf, + 0x008c0, 0x008c1, 0x008c2, 0x008c3, 0x008c4, 0x008c5, 0x008c6, 0x008c7, + 0x008c8, 0x008c9, 0x008ca, 0x008cb, 0x008cc, 0x008cd, 0x008ce, 0x008cf, + 0x008d0, 0x008d1, 0x008d2, 0x008d3, 0x008d4, 0x008d5, 0x008d6, 0x008d7, + 0x008d8, 0x008d9, 0x008da, 0x008db, 0x008dc, 0x008dd, 0x008de, 0x008df, + 0x008e0, 0x008e1, 0x008e2, 0x008e3, 0x008e4, 0x008e5, 0x008e6, 0x008e7, + 0x008e8, 0x008e9, 0x008ea, 0x008eb, 0x008ec, 0x008ed, 0x008ee, 0x008ef, + 0x008f0, 0x008f1, 0x008f2, 0x008f3, 0x008f4, 0x008f5, 0x008f6, 0x008f7, + 0x008f8, 0x008f9, 0x008fa, 0x008fb, 0x008fc, 0x008fd, 0x008fe, 0x008ff +}; + +static uint32_t unicode_520_ci_page_09[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00904, 0x00905, 0x00906, 0x00907, + 0x00908, 0x00909, 0x0090a, 0x0090b, 0x0090c, 0x0090d, 0x0090e, 0x0090f, + 0x00910, 0x00911, 0x00912, 0x00913, 0x00914, 0x00915, 0x00916, 0x00917, + 0x00918, 0x00919, 0x0091a, 0x0091b, 0x0091c, 0x0091d, 0x0091e, 0x0091f, + 0x00920, 0x00921, 0x00922, 0x00923, 0x00924, 0x00925, 0x00926, 0x00927, + 0x00928, 0x00928, 0x0092a, 0x0092b, 0x0092c, 0x0092d, 0x0092e, 0x0092f, + 0x00930, 0x00930, 0x00932, 0x00933, 0x00933, 0x00935, 0x00936, 0x00937, + 0x00938, 0x00939, 0x0093a, 0x0093b, 0x00000, 0x0093d, 0x0093e, 0x0093f, + 0x00940, 0x00941, 0x00942, 0x00943, 0x00944, 0x00945, 0x00946, 0x00947, + 0x00948, 0x00949, 0x0094a, 0x0094b, 0x0094c, 0x0094d, 0x0094e, 0x0094f, + 0x00950, 0x00000, 0x00000, 0x00000, 0x00000, 0x00955, 0x00956, 0x00957, + 0x00915, 0x00916, 0x00917, 0x0091c, 0x00921, 0x00922, 0x0092b, 0x0092f, + 0x00960, 0x00961, 0x00962, 0x00963, 0x00964, 0x00965, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00970, 0x00971, 0x00972, 0x00973, 0x00974, 0x00975, 0x00976, 0x00977, + 0x00978, 0x00979, 0x0097a, 0x0097b, 0x0097c, 0x0097d, 0x0097e, 0x0097f, + 0x00980, 0x00000, 0x00000, 0x00000, 0x00984, 0x00985, 0x00986, 0x00987, + 0x00988, 0x00989, 0x0098a, 0x0098b, 0x0098c, 0x0098d, 0x0098e, 0x0098f, + 0x00990, 0x00991, 0x00992, 0x00993, 0x00994, 0x00995, 0x00996, 0x00997, + 0x00998, 0x00999, 0x0099a, 0x0099b, 0x0099c, 0x0099d, 0x0099e, 0x0099f, + 0x009a0, 0x009a1, 0x009a2, 0x009a3, 0x009a4, 0x009a5, 0x009a6, 0x009a7, + 0x009a8, 0x009a9, 0x009aa, 0x009ab, 0x009ac, 0x009ad, 0x009ae, 0x009af, + 0x009b0, 0x009b1, 0x009b2, 0x009b3, 0x009b4, 0x009b5, 0x009b6, 0x009b7, + 0x009b8, 0x009b9, 0x009ba, 0x009bb, 0x00000, 0x009bd, 0x009be, 0x009bf, + 0x009c0, 0x009c1, 0x009c2, 0x009c3, 0x009c4, 0x009c5, 0x009c6, 0x009c7, + 0x009c8, 0x009c9, 0x009ca, 0x009cb, 0x009cc, 0x009cd, 0x009ce, 0x009cf, + 0x009d0, 0x009d1, 0x009d2, 0x009d3, 0x009d4, 0x009d5, 0x009d6, 0x009d7, + 0x009d8, 0x009d9, 0x009da, 0x009db, 0x009a1, 0x009a2, 0x009de, 0x009af, + 0x009e0, 0x009e1, 0x009e2, 0x009e3, 0x009e4, 0x009e5, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x009f0, 0x009f1, 0x009f2, 0x009f3, 0x009f4, 0x009f5, 0x009f6, 0x009f7, + 0x009f8, 0x009f9, 0x009fa, 0x009fb, 0x009fc, 0x009fd, 0x009fe, 0x009ff +}; + +static uint32_t unicode_520_ci_page_0a[] = { + 0x00a00, 0x00000, 0x00000, 0x00000, 0x00a04, 0x00a05, 0x00a06, 0x00a07, + 0x00a08, 0x00a09, 0x00a0a, 0x00a0b, 0x00a0c, 0x00a0d, 0x00a0e, 0x00a0f, + 0x00a10, 0x00a11, 0x00a12, 0x00a13, 0x00a14, 0x00a15, 0x00a16, 0x00a17, + 0x00a18, 0x00a19, 0x00a1a, 0x00a1b, 0x00a1c, 0x00a1d, 0x00a1e, 0x00a1f, + 0x00a20, 0x00a21, 0x00a22, 0x00a23, 0x00a24, 0x00a25, 0x00a26, 0x00a27, + 0x00a28, 0x00a29, 0x00a2a, 0x00a2b, 0x00a2c, 0x00a2d, 0x00a2e, 0x00a2f, + 0x00a30, 0x00a31, 0x00a32, 0x00a32, 0x00a34, 0x00a35, 0x00a36, 0x00a37, + 0x00a36, 0x00a39, 0x00a3a, 0x00a3b, 0x00000, 0x00a3d, 0x00a3e, 0x00a3f, + 0x00a40, 0x00a41, 0x00a42, 0x00a43, 0x00a44, 0x00a45, 0x00a46, 0x00a47, + 0x00a48, 0x00a49, 0x00a4a, 0x00a4b, 0x00a4c, 0x00a4d, 0x00a4e, 0x00a4f, + 0x00a50, 0x00a51, 0x00a52, 0x00a53, 0x00a54, 0x00a55, 0x00a56, 0x00a57, + 0x00a58, 0x00a16, 0x00a17, 0x00a1c, 0x00a5c, 0x00a5d, 0x00a2b, 0x00a5f, + 0x00a60, 0x00a61, 0x00a62, 0x00a63, 0x00a64, 0x00a65, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00000, 0x00000, 0x00a72, 0x00a73, 0x00a74, 0x00a75, 0x00a76, 0x00a77, + 0x00a78, 0x00a79, 0x00a7a, 0x00a7b, 0x00a7c, 0x00a7d, 0x00a7e, 0x00a7f, + 0x00a80, 0x00000, 0x00000, 0x00000, 0x00a84, 0x00a85, 0x00a86, 0x00a87, + 0x00a88, 0x00a89, 0x00a8a, 0x00a8b, 0x00a8c, 0x00a8d, 0x00a8e, 0x00a8f, + 0x00a90, 0x00a91, 0x00a92, 0x00a93, 0x00a94, 0x00a95, 0x00a96, 0x00a97, + 0x00a98, 0x00a99, 0x00a9a, 0x00a9b, 0x00a9c, 0x00a9d, 0x00a9e, 0x00a9f, + 0x00aa0, 0x00aa1, 0x00aa2, 0x00aa3, 0x00aa4, 0x00aa5, 0x00aa6, 0x00aa7, + 0x00aa8, 0x00aa9, 0x00aaa, 0x00aab, 0x00aac, 0x00aad, 0x00aae, 0x00aaf, + 0x00ab0, 0x00ab1, 0x00ab2, 0x00ab3, 0x00ab4, 0x00ab5, 0x00ab6, 0x00ab7, + 0x00ab8, 0x00ab9, 0x00aba, 0x00abb, 0x00000, 0x00abd, 0x00abe, 0x00abf, + 0x00ac0, 0x00ac1, 0x00ac2, 0x00ac3, 0x00ac4, 0x00ac5, 0x00ac6, 0x00ac7, + 0x00ac8, 0x00ac9, 0x00aca, 0x00acb, 0x00acc, 0x00acd, 0x00ace, 0x00acf, + 0x00ad0, 0x00ad1, 0x00ad2, 0x00ad3, 0x00ad4, 0x00ad5, 0x00ad6, 0x00ad7, + 0x00ad8, 0x00ad9, 0x00ada, 0x00adb, 0x00adc, 0x00add, 0x00ade, 0x00adf, + 0x00ae0, 0x00ae1, 0x00ae2, 0x00ae3, 0x00ae4, 0x00ae5, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00af0, 0x00af1, 0x00af2, 0x00af3, 0x00af4, 0x00af5, 0x00af6, 0x00af7, + 0x00af8, 0x00af9, 0x00afa, 0x00afb, 0x00afc, 0x00afd, 0x00afe, 0x00aff +}; + +static uint32_t unicode_520_ci_page_0b[] = { + 0x00b00, 0x00000, 0x00000, 0x00000, 0x00b04, 0x00b05, 0x00b06, 0x00b07, + 0x00b08, 0x00b09, 0x00b0a, 0x00b0b, 0x00b0c, 0x00b0d, 0x00b0e, 0x00b0f, + 0x00b10, 0x00b11, 0x00b12, 0x00b13, 0x00b14, 0x00b15, 0x00b16, 0x00b17, + 0x00b18, 0x00b19, 0x00b1a, 0x00b1b, 0x00b1c, 0x00b1d, 0x00b1e, 0x00b1f, + 0x00b20, 0x00b21, 0x00b22, 0x00b23, 0x00b24, 0x00b25, 0x00b26, 0x00b27, + 0x00b28, 0x00b29, 0x00b2a, 0x00b2b, 0x00b2c, 0x00b2d, 0x00b2e, 0x00b2f, + 0x00b30, 0x00b31, 0x00b32, 0x00b33, 0x00b34, 0x00b35, 0x00b36, 0x00b37, + 0x00b38, 0x00b39, 0x00b3a, 0x00b3b, 0x00000, 0x00b3d, 0x00b3e, 0x00b3f, + 0x00b40, 0x00b41, 0x00b42, 0x00b43, 0x00b44, 0x00b45, 0x00b46, 0x00b47, + 0x00b48, 0x00b49, 0x00b4a, 0x00b4b, 0x00b4c, 0x00b4d, 0x00b4e, 0x00b4f, + 0x00b50, 0x00b51, 0x00b52, 0x00b53, 0x00b54, 0x00b55, 0x00b56, 0x00b57, + 0x00b58, 0x00b59, 0x00b5a, 0x00b5b, 0x00b21, 0x00b22, 0x00b5e, 0x00b5f, + 0x00b60, 0x00b61, 0x00b62, 0x00b63, 0x00b64, 0x00b65, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00b70, 0x00b71, 0x00b72, 0x00b73, 0x00b74, 0x00b75, 0x00b76, 0x00b77, + 0x00b78, 0x00b79, 0x00b7a, 0x00b7b, 0x00b7c, 0x00b7d, 0x00b7e, 0x00b7f, + 0x00b80, 0x00b81, 0x00000, 0x00b83, 0x00b84, 0x00b85, 0x00b86, 0x00b87, + 0x00b88, 0x00b89, 0x00b8a, 0x00b8b, 0x00b8c, 0x00b8d, 0x00b8e, 0x00b8f, + 0x00b90, 0x00b91, 0x00b92, 0x00b93, 0x00b94, 0x00b95, 0x00b96, 0x00b97, + 0x00b98, 0x00b99, 0x00b9a, 0x00b9b, 0x00b9c, 0x00b9d, 0x00b9e, 0x00b9f, + 0x00ba0, 0x00ba1, 0x00ba2, 0x00ba3, 0x00ba4, 0x00ba5, 0x00ba6, 0x00ba7, + 0x00ba8, 0x00ba9, 0x00baa, 0x00bab, 0x00bac, 0x00bad, 0x00bae, 0x00baf, + 0x00bb0, 0x00bb1, 0x00bb2, 0x00bb3, 0x00bb4, 0x00bb5, 0x00bb6, 0x00bb7, + 0x00bb8, 0x00bb9, 0x00bba, 0x00bbb, 0x00bbc, 0x00bbd, 0x00bbe, 0x00bbf, + 0x00bc0, 0x00bc1, 0x00bc2, 0x00bc3, 0x00bc4, 0x00bc5, 0x00bc6, 0x00bc7, + 0x00bc8, 0x00bc9, 0x00bca, 0x00bcb, 0x00bcc, 0x00bcd, 0x00bce, 0x00bcf, + 0x00bd0, 0x00bd1, 0x00bd2, 0x00bd3, 0x00bd4, 0x00bd5, 0x00bd6, 0x00bd7, + 0x00bd8, 0x00bd9, 0x00bda, 0x00bdb, 0x00bdc, 0x00bdd, 0x00bde, 0x00bdf, + 0x00be0, 0x00be1, 0x00be2, 0x00be3, 0x00be4, 0x00be5, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00bf0, 0x00bf1, 0x00bf2, 0x00bf3, 0x00bf4, 0x00bf5, 0x00bf6, 0x00bf7, + 0x00bf8, 0x00bf9, 0x00bfa, 0x00bfb, 0x00bfc, 0x00bfd, 0x00bfe, 0x00bff +}; + +static uint32_t unicode_520_ci_page_0c[] = { + 0x00c00, 0x00000, 0x00000, 0x00000, 0x00c04, 0x00c05, 0x00c06, 0x00c07, + 0x00c08, 0x00c09, 0x00c0a, 0x00c0b, 0x00c0c, 0x00c0d, 0x00c0e, 0x00c0f, + 0x00c10, 0x00c11, 0x00c12, 0x00c13, 0x00c14, 0x00c15, 0x00c16, 0x00c17, + 0x00c18, 0x00c19, 0x00c1a, 0x00c1b, 0x00c1c, 0x00c1d, 0x00c1e, 0x00c1f, + 0x00c20, 0x00c21, 0x00c22, 0x00c23, 0x00c24, 0x00c25, 0x00c26, 0x00c27, + 0x00c28, 0x00c29, 0x00c2a, 0x00c2b, 0x00c2c, 0x00c2d, 0x00c2e, 0x00c2f, + 0x00c30, 0x00c31, 0x00c32, 0x00c33, 0x00c34, 0x00c35, 0x00c36, 0x00c37, + 0x00c38, 0x00c39, 0x00c3a, 0x00c3b, 0x00c3c, 0x00c3d, 0x00c3e, 0x00c3f, + 0x00c40, 0x00c41, 0x00c42, 0x00c43, 0x00c44, 0x00c45, 0x00c46, 0x00c47, + 0x00c48, 0x00c49, 0x00c4a, 0x00c4b, 0x00c4c, 0x00c4d, 0x00c4e, 0x00c4f, + 0x00c50, 0x00c51, 0x00c52, 0x00c53, 0x00c54, 0x00c55, 0x00c56, 0x00c57, + 0x00c58, 0x00c59, 0x00c5a, 0x00c5b, 0x00c5c, 0x00c5d, 0x00c5e, 0x00c5f, + 0x00c60, 0x00c61, 0x00c62, 0x00c63, 0x00c64, 0x00c65, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00c70, 0x00c71, 0x00c72, 0x00c73, 0x00c74, 0x00c75, 0x00c76, 0x00c77, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00031, 0x00032, 0x00033, 0x00c7f, + 0x00c80, 0x00c81, 0x00000, 0x00000, 0x00c84, 0x00c85, 0x00c86, 0x00c87, + 0x00c88, 0x00c89, 0x00c8a, 0x00c8b, 0x00c8c, 0x00c8d, 0x00c8e, 0x00c8f, + 0x00c90, 0x00c91, 0x00c92, 0x00c93, 0x00c94, 0x00c95, 0x00c96, 0x00c97, + 0x00c98, 0x00c99, 0x00c9a, 0x00c9b, 0x00c9c, 0x00c9d, 0x00c9e, 0x00c9f, + 0x00ca0, 0x00ca1, 0x00ca2, 0x00ca3, 0x00ca4, 0x00ca5, 0x00ca6, 0x00ca7, + 0x00ca8, 0x00ca9, 0x00caa, 0x00cab, 0x00cac, 0x00cad, 0x00cae, 0x00caf, + 0x00cb0, 0x00cb1, 0x00cb2, 0x00cb3, 0x00cb4, 0x00cb5, 0x00cb6, 0x00cb7, + 0x00cb8, 0x00cb9, 0x00cba, 0x00cbb, 0x00000, 0x00cbd, 0x00cbe, 0x00cbf, + 0x00cc0, 0x00cc1, 0x00cc2, 0x00cc3, 0x00cc4, 0x00cc5, 0x00cc6, 0x00cc7, + 0x00cc8, 0x00cc9, 0x00cca, 0x00ccb, 0x00ccc, 0x00ccd, 0x00cce, 0x00ccf, + 0x00cd0, 0x00cd1, 0x00cd2, 0x00cd3, 0x00cd4, 0x00cd5, 0x00cd6, 0x00cd7, + 0x00cd8, 0x00cd9, 0x00cda, 0x00cdb, 0x00cdc, 0x00cdd, 0x00cde, 0x00cdf, + 0x00ce0, 0x00ce1, 0x00ce2, 0x00ce3, 0x00ce4, 0x00ce5, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00cf0, 0x00cf1, 0x00cf2, 0x00cf3, 0x00cf4, 0x00cf5, 0x00cf6, 0x00cf7, + 0x00cf8, 0x00cf9, 0x00cfa, 0x00cfb, 0x00cfc, 0x00cfd, 0x00cfe, 0x00cff +}; + +static uint32_t unicode_520_ci_page_0d[] = { + 0x00d00, 0x00d01, 0x00000, 0x00000, 0x00d04, 0x00d05, 0x00d06, 0x00d07, + 0x00d08, 0x00d09, 0x00d0a, 0x00d0b, 0x00d0c, 0x00d0d, 0x00d0e, 0x00d0f, + 0x00d10, 0x00d11, 0x00d12, 0x00d13, 0x00d14, 0x00d15, 0x00d16, 0x00d17, + 0x00d18, 0x00d19, 0x00d1a, 0x00d1b, 0x00d1c, 0x00d1d, 0x00d1e, 0x00d1f, + 0x00d20, 0x00d21, 0x00d22, 0x00d23, 0x00d24, 0x00d25, 0x00d26, 0x00d27, + 0x00d28, 0x00d29, 0x00d2a, 0x00d2b, 0x00d2c, 0x00d2d, 0x00d2e, 0x00d2f, + 0x00d30, 0x00d31, 0x00d32, 0x00d33, 0x00d34, 0x00d35, 0x00d36, 0x00d37, + 0x00d38, 0x00d39, 0x00d3a, 0x00d3b, 0x00d3c, 0x00d3d, 0x00d3e, 0x00d3f, + 0x00d40, 0x00d41, 0x00d42, 0x00d43, 0x00d44, 0x00d45, 0x00d46, 0x00d47, + 0x00d48, 0x00d49, 0x00d4a, 0x00d4b, 0x00d4c, 0x00d4d, 0x00d4e, 0x00d4f, + 0x00d50, 0x00d51, 0x00d52, 0x00d53, 0x00d54, 0x00d55, 0x00d56, 0x00d57, + 0x00d58, 0x00d59, 0x00d5a, 0x00d5b, 0x00d5c, 0x00d5d, 0x00d5e, 0x00d5f, + 0x00d60, 0x00d61, 0x00d62, 0x00d63, 0x00d64, 0x00d65, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00d70, 0x00d71, 0x00d72, 0x00d73, 0x00d74, 0x00d75, 0x00d76, 0x00d77, + 0x00d78, 0x00d79, 0x00d7a, 0x00d7b, 0x00d7c, 0x00d7d, 0x00d7e, 0x00d7f, + 0x00d80, 0x00d81, 0x00000, 0x00000, 0x00d84, 0x00d85, 0x00d86, 0x00d87, + 0x00d88, 0x00d89, 0x00d8a, 0x00d8b, 0x00d8c, 0x00d8d, 0x00d8e, 0x00d8f, + 0x00d90, 0x00d91, 0x00d92, 0x00d93, 0x00d94, 0x00d95, 0x00d96, 0x00d97, + 0x00d98, 0x00d99, 0x00d9a, 0x00d9b, 0x00d9c, 0x00d9d, 0x00d9e, 0x00d9f, + 0x00da0, 0x00da1, 0x00da2, 0x00da3, 0x00da4, 0x00da5, 0x00da6, 0x00da7, + 0x00da8, 0x00da9, 0x00daa, 0x00dab, 0x00dac, 0x00dad, 0x00dae, 0x00daf, + 0x00db0, 0x00db1, 0x00db2, 0x00db3, 0x00db4, 0x00db5, 0x00db6, 0x00db7, + 0x00db8, 0x00db9, 0x00dba, 0x00dbb, 0x00dbc, 0x00dbd, 0x00dbe, 0x00dbf, + 0x00dc0, 0x00dc1, 0x00dc2, 0x00dc3, 0x00dc4, 0x00dc5, 0x00dc6, 0x00dc7, + 0x00dc8, 0x00dc9, 0x00dca, 0x00dcb, 0x00dcc, 0x00dcd, 0x00dce, 0x00dcf, + 0x00dd0, 0x00dd1, 0x00dd2, 0x00dd3, 0x00dd4, 0x00dd5, 0x00dd6, 0x00dd7, + 0x00dd8, 0x00dd9, 0x00dda, 0x00ddb, 0x00ddc, 0x00ddd, 0x00dde, 0x00ddf, + 0x00de0, 0x00de1, 0x00de2, 0x00de3, 0x00de4, 0x00de5, 0x00de6, 0x00de7, + 0x00de8, 0x00de9, 0x00dea, 0x00deb, 0x00dec, 0x00ded, 0x00dee, 0x00def, + 0x00df0, 0x00df1, 0x00df2, 0x00df3, 0x00df4, 0x00df5, 0x00df6, 0x00df7, + 0x00df8, 0x00df9, 0x00dfa, 0x00dfb, 0x00dfc, 0x00dfd, 0x00dfe, 0x00dff +}; + +static uint32_t unicode_520_ci_page_0e[] = { + 0x00e00, 0x00e01, 0x00e02, 0x00e03, 0x00e04, 0x00e05, 0x00e06, 0x00e07, + 0x00e08, 0x00e09, 0x00e0a, 0x00e0b, 0x00e0c, 0x00e0d, 0x00e0e, 0x00e0f, + 0x00e10, 0x00e11, 0x00e12, 0x00e13, 0x00e14, 0x00e15, 0x00e16, 0x00e17, + 0x00e18, 0x00e19, 0x00e1a, 0x00e1b, 0x00e1c, 0x00e1d, 0x00e1e, 0x00e1f, + 0x00e20, 0x00e21, 0x00e22, 0x00e23, 0x00e24, 0x00e25, 0x00e26, 0x00e27, + 0x00e28, 0x00e29, 0x00e2a, 0x00e2b, 0x00e2c, 0x00e2d, 0x00e2e, 0x00e2f, + 0x00e30, 0x00e31, 0x00e32, 0x00e33, 0x00e34, 0x00e35, 0x00e36, 0x00e37, + 0x00e38, 0x00e39, 0x00e3a, 0x00e3b, 0x00e3c, 0x00e3d, 0x00e3e, 0x00e3f, + 0x00e40, 0x00e41, 0x00e42, 0x00e43, 0x00e44, 0x00e45, 0x00e46, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00e4f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00e5a, 0x00e5b, 0x00e5c, 0x00e5d, 0x00e5e, 0x00e5f, + 0x00e60, 0x00e61, 0x00e62, 0x00e63, 0x00e64, 0x00e65, 0x00e66, 0x00e67, + 0x00e68, 0x00e69, 0x00e6a, 0x00e6b, 0x00e6c, 0x00e6d, 0x00e6e, 0x00e6f, + 0x00e70, 0x00e71, 0x00e72, 0x00e73, 0x00e74, 0x00e75, 0x00e76, 0x00e77, + 0x00e78, 0x00e79, 0x00e7a, 0x00e7b, 0x00e7c, 0x00e7d, 0x00e7e, 0x00e7f, + 0x00e80, 0x00e81, 0x00e82, 0x00e83, 0x00e84, 0x00e85, 0x00e86, 0x00e87, + 0x00e88, 0x00e89, 0x00e8a, 0x00e8b, 0x00e8c, 0x00e8d, 0x00e8e, 0x00e8f, + 0x00e90, 0x00e91, 0x00e92, 0x00e93, 0x00e94, 0x00e95, 0x00e96, 0x00e97, + 0x00e98, 0x00e99, 0x00e9a, 0x00e9b, 0x00e9c, 0x00e9d, 0x00e9e, 0x00e9f, + 0x00ea0, 0x00ea1, 0x00ea2, 0x00ea3, 0x00ea4, 0x00ea5, 0x00ea6, 0x00ea7, + 0x00ea8, 0x00ea9, 0x00eaa, 0x00eab, 0x00eac, 0x00ead, 0x00eae, 0x00eaf, + 0x00eb0, 0x00eb1, 0x00eb2, 0x00eb3, 0x00eb4, 0x00eb5, 0x00eb6, 0x00eb7, + 0x00eb8, 0x00eb9, 0x00eba, 0x00ebb, 0x00ebc, 0x00ebd, 0x00ebe, 0x00ebf, + 0x00ec0, 0x00ec1, 0x00ec2, 0x00ec3, 0x00ec4, 0x00ec5, 0x00ec6, 0x00ec7, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00ece, 0x00ecf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00eda, 0x00edb, 0x00edc, 0x00edd, 0x00ede, 0x00edf, + 0x00ee0, 0x00ee1, 0x00ee2, 0x00ee3, 0x00ee4, 0x00ee5, 0x00ee6, 0x00ee7, + 0x00ee8, 0x00ee9, 0x00eea, 0x00eeb, 0x00eec, 0x00eed, 0x00eee, 0x00eef, + 0x00ef0, 0x00ef1, 0x00ef2, 0x00ef3, 0x00ef4, 0x00ef5, 0x00ef6, 0x00ef7, + 0x00ef8, 0x00ef9, 0x00efa, 0x00efb, 0x00efc, 0x00efd, 0x00efe, 0x00eff +}; + +static uint32_t unicode_520_ci_page_0f[] = { + 0x00f00, 0x00f01, 0x00f02, 0x00f03, 0x00f04, 0x00f05, 0x00f06, 0x00f07, + 0x00f08, 0x00f09, 0x00f0a, 0x00f0b, 0x00f0b, 0x00f0d, 0x00f0e, 0x00f0f, + 0x00f10, 0x00f11, 0x00f12, 0x00f13, 0x00f14, 0x00f15, 0x00f16, 0x00f17, + 0x00000, 0x00000, 0x00f1a, 0x00f1b, 0x00f1c, 0x00f1d, 0x00f1e, 0x00f1f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, + 0x00037, 0x00038, 0x00039, 0x00030, 0x00f34, 0x00000, 0x00f36, 0x00000, + 0x00f38, 0x00000, 0x00f3a, 0x00f3b, 0x00f3c, 0x00f3d, 0x00f3e, 0x00f3f, + 0x00f40, 0x00f41, 0x00f42, 0x00f43, 0x00f44, 0x00f45, 0x00f46, 0x00f47, + 0x00f48, 0x00f49, 0x00f4a, 0x00f4b, 0x00f4c, 0x00f4d, 0x00f4e, 0x00f4f, + 0x00f50, 0x00f51, 0x00f52, 0x00f53, 0x00f54, 0x00f55, 0x00f56, 0x00f57, + 0x00f58, 0x00f59, 0x00f5a, 0x00f5b, 0x00f5c, 0x00f5d, 0x00f5e, 0x00f5f, + 0x00f60, 0x00f61, 0x00f62, 0x00f63, 0x00f64, 0x00f65, 0x00f66, 0x00f67, + 0x00f68, 0x00f69, 0x00f62, 0x00f6b, 0x00f6c, 0x00f6d, 0x00f6e, 0x00f6f, + 0x00f70, 0x00f71, 0x00f72, 0x00f73, 0x00f74, 0x00f75, 0x00f76, 0x00f77, + 0x00f78, 0x00f79, 0x00f7a, 0x00f7b, 0x00f7c, 0x00f7d, 0x00000, 0x00000, + 0x00f80, 0x00f81, 0x00000, 0x00000, 0x00f84, 0x00f85, 0x00000, 0x00000, + 0x00f88, 0x00f89, 0x00f8a, 0x00f8b, 0x00f8c, 0x00f8d, 0x00f8e, 0x00f8f, + 0x00f90, 0x00f91, 0x00f92, 0x00f93, 0x00f94, 0x00f95, 0x00f96, 0x00f97, + 0x00f98, 0x00f99, 0x00f9a, 0x00f9b, 0x00f9c, 0x00f9d, 0x00f9e, 0x00f9f, + 0x00fa0, 0x00fa1, 0x00fa2, 0x00fa3, 0x00fa4, 0x00fa5, 0x00fa6, 0x00fa7, + 0x00fa8, 0x00fa9, 0x00faa, 0x00fab, 0x00fac, 0x00fad, 0x00fae, 0x00faf, + 0x00fb0, 0x00fb1, 0x00fb2, 0x00fb3, 0x00fb4, 0x00fb5, 0x00fb6, 0x00fb7, + 0x00fb8, 0x00fb9, 0x00fad, 0x00fb1, 0x00fb2, 0x00fbd, 0x00fbe, 0x00fbf, + 0x00fc0, 0x00fc1, 0x00fc2, 0x00fc3, 0x00fc4, 0x00fc5, 0x00000, 0x00fc7, + 0x00fc8, 0x00fc9, 0x00fca, 0x00fcb, 0x00fcc, 0x00fcd, 0x00fce, 0x00fcf, + 0x00fd0, 0x00fd1, 0x00fd2, 0x00fd3, 0x00fd4, 0x00fd5, 0x00fd6, 0x00fd7, + 0x00fd8, 0x00fd9, 0x00fda, 0x00fdb, 0x00fdc, 0x00fdd, 0x00fde, 0x00fdf, + 0x00fe0, 0x00fe1, 0x00fe2, 0x00fe3, 0x00fe4, 0x00fe5, 0x00fe6, 0x00fe7, + 0x00fe8, 0x00fe9, 0x00fea, 0x00feb, 0x00fec, 0x00fed, 0x00fee, 0x00fef, + 0x00ff0, 0x00ff1, 0x00ff2, 0x00ff3, 0x00ff4, 0x00ff5, 0x00ff6, 0x00ff7, + 0x00ff8, 0x00ff9, 0x00ffa, 0x00ffb, 0x00ffc, 0x00ffd, 0x00ffe, 0x00fff +}; + +static uint32_t unicode_520_ci_page_10[] = { + 0x01000, 0x01001, 0x01002, 0x01003, 0x01004, 0x01005, 0x01006, 0x01007, + 0x01008, 0x01009, 0x0100a, 0x0100b, 0x0100c, 0x0100d, 0x0100e, 0x0100f, + 0x01010, 0x01011, 0x01012, 0x01013, 0x01014, 0x01015, 0x01016, 0x01017, + 0x01018, 0x01019, 0x0101a, 0x0101b, 0x0101c, 0x0101d, 0x0101e, 0x0101f, + 0x01020, 0x01021, 0x01022, 0x01023, 0x01024, 0x01025, 0x01026, 0x01027, + 0x01028, 0x01029, 0x0102a, 0x0102b, 0x0102b, 0x0102d, 0x0102e, 0x0102f, + 0x01030, 0x01031, 0x01032, 0x01033, 0x01034, 0x01035, 0x00000, 0x00000, + 0x00000, 0x01039, 0x0103a, 0x0103b, 0x0103c, 0x0103d, 0x0103e, 0x0103f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0104a, 0x0104b, 0x0104c, 0x0104d, 0x0104e, 0x0104f, + 0x01050, 0x01051, 0x01052, 0x01053, 0x01054, 0x01055, 0x01056, 0x01057, + 0x01058, 0x01059, 0x0105a, 0x0105b, 0x0105c, 0x0105d, 0x0105e, 0x0105f, + 0x01060, 0x01061, 0x01062, 0x01063, 0x01064, 0x01065, 0x01066, 0x01067, + 0x01068, 0x01069, 0x0106a, 0x0106b, 0x0106c, 0x0106d, 0x0106e, 0x0106f, + 0x01070, 0x01071, 0x01072, 0x01073, 0x01074, 0x01075, 0x01076, 0x01077, + 0x01078, 0x01079, 0x0107a, 0x0107b, 0x0107c, 0x0107d, 0x0107e, 0x0107f, + 0x01080, 0x01081, 0x01082, 0x01083, 0x01084, 0x01085, 0x01086, 0x01087, + 0x01088, 0x01089, 0x0108a, 0x0108b, 0x0108c, 0x00000, 0x0108e, 0x0108f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0109a, 0x0109b, 0x0109c, 0x0109d, 0x0109e, 0x0109f, + 0x010a0, 0x010a1, 0x010a2, 0x010a3, 0x010a4, 0x010a5, 0x010a6, 0x010a7, + 0x010a8, 0x010a9, 0x010aa, 0x010ab, 0x010ac, 0x010ad, 0x010ae, 0x010af, + 0x010b0, 0x010b1, 0x010b2, 0x010b3, 0x010b4, 0x010b5, 0x010b6, 0x010b7, + 0x010b8, 0x010b9, 0x010ba, 0x010bb, 0x010bc, 0x010bd, 0x010be, 0x010bf, + 0x010c0, 0x010c1, 0x010c2, 0x010c3, 0x010c4, 0x010c5, 0x010c6, 0x010c7, + 0x010c8, 0x010c9, 0x010ca, 0x010cb, 0x010cc, 0x010cd, 0x010ce, 0x010cf, + 0x010d0, 0x010d1, 0x010d2, 0x010d3, 0x010d4, 0x010d5, 0x010d6, 0x010d7, + 0x010d8, 0x010d9, 0x010da, 0x010db, 0x010dc, 0x010dd, 0x010de, 0x010df, + 0x010e0, 0x010e1, 0x010e2, 0x010e3, 0x010e4, 0x010e5, 0x010e6, 0x010e7, + 0x010e8, 0x010e9, 0x010ea, 0x010eb, 0x010ec, 0x010ed, 0x010ee, 0x010ef, + 0x010f0, 0x010f1, 0x010f2, 0x010f3, 0x010f4, 0x010f5, 0x010f6, 0x010f7, + 0x010f8, 0x010f9, 0x010fa, 0x010fb, 0x010dc, 0x010fd, 0x010fe, 0x010ff +}; + +static uint32_t unicode_520_ci_page_13[] = { + 0x01300, 0x01301, 0x01302, 0x01303, 0x01304, 0x01305, 0x01306, 0x01307, + 0x01308, 0x01309, 0x0130a, 0x0130b, 0x0130c, 0x0130d, 0x0130e, 0x0130f, + 0x01310, 0x01311, 0x01312, 0x01313, 0x01314, 0x01315, 0x01316, 0x01317, + 0x01318, 0x01319, 0x0131a, 0x0131b, 0x0131c, 0x0131d, 0x0131e, 0x0131f, + 0x01320, 0x01321, 0x01322, 0x01323, 0x01324, 0x01325, 0x01326, 0x01327, + 0x01328, 0x01329, 0x0132a, 0x0132b, 0x0132c, 0x0132d, 0x0132e, 0x0132f, + 0x01330, 0x01331, 0x01332, 0x01333, 0x01334, 0x01335, 0x01336, 0x01337, + 0x01338, 0x01339, 0x0133a, 0x0133b, 0x0133c, 0x0133d, 0x0133e, 0x0133f, + 0x01340, 0x01341, 0x01342, 0x01343, 0x01344, 0x01345, 0x01346, 0x01347, + 0x01348, 0x01349, 0x0134a, 0x0134b, 0x0134c, 0x0134d, 0x0134e, 0x0134f, + 0x01350, 0x01351, 0x01352, 0x01353, 0x01354, 0x01355, 0x01356, 0x01357, + 0x01358, 0x01359, 0x0135a, 0x0135b, 0x0135c, 0x0135d, 0x0135e, 0x00000, + 0x01360, 0x01361, 0x01362, 0x01363, 0x01364, 0x01365, 0x01366, 0x01367, + 0x01368, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01372, 0x01373, 0x01374, 0x01375, 0x01376, 0x01377, + 0x01378, 0x01379, 0x0137a, 0x0137b, 0x0137c, 0x0137d, 0x0137e, 0x0137f, + 0x01380, 0x01381, 0x01382, 0x01383, 0x01384, 0x01385, 0x01386, 0x01387, + 0x01388, 0x01389, 0x0138a, 0x0138b, 0x0138c, 0x0138d, 0x0138e, 0x0138f, + 0x01390, 0x01391, 0x01392, 0x01393, 0x01394, 0x01395, 0x01396, 0x01397, + 0x01398, 0x01399, 0x0139a, 0x0139b, 0x0139c, 0x0139d, 0x0139e, 0x0139f, + 0x013a0, 0x013a1, 0x013a2, 0x013a3, 0x013a4, 0x013a5, 0x013a6, 0x013a7, + 0x013a8, 0x013a9, 0x013aa, 0x013ab, 0x013ac, 0x013ad, 0x013ae, 0x013af, + 0x013b0, 0x013b1, 0x013b2, 0x013b3, 0x013b4, 0x013b5, 0x013b6, 0x013b7, + 0x013b8, 0x013b9, 0x013ba, 0x013bb, 0x013bc, 0x013bd, 0x013be, 0x013bf, + 0x013c0, 0x013c1, 0x013c2, 0x013c3, 0x013c4, 0x013c5, 0x013c6, 0x013c7, + 0x013c8, 0x013c9, 0x013ca, 0x013cb, 0x013cc, 0x013cd, 0x013ce, 0x013cf, + 0x013d0, 0x013d1, 0x013d2, 0x013d3, 0x013d4, 0x013d5, 0x013d6, 0x013d7, + 0x013d8, 0x013d9, 0x013da, 0x013db, 0x013dc, 0x013dd, 0x013de, 0x013df, + 0x013e0, 0x013e1, 0x013e2, 0x013e3, 0x013e4, 0x013e5, 0x013e6, 0x013e7, + 0x013e8, 0x013e9, 0x013ea, 0x013eb, 0x013ec, 0x013ed, 0x013ee, 0x013ef, + 0x013f0, 0x013f1, 0x013f2, 0x013f3, 0x013f4, 0x013f5, 0x013f6, 0x013f7, + 0x013f8, 0x013f9, 0x013fa, 0x013fb, 0x013fc, 0x013fd, 0x013fe, 0x013ff +}; + +static uint32_t unicode_520_ci_page_16[] = { + 0x01600, 0x01601, 0x01602, 0x01603, 0x01604, 0x01605, 0x01606, 0x01607, + 0x01608, 0x01609, 0x0160a, 0x0160b, 0x0160c, 0x0160d, 0x0160e, 0x0160f, + 0x01610, 0x01611, 0x01612, 0x01613, 0x01614, 0x01615, 0x01616, 0x01617, + 0x01618, 0x01619, 0x0161a, 0x0161b, 0x0161c, 0x0161d, 0x0161e, 0x0161f, + 0x01620, 0x01621, 0x01622, 0x01623, 0x01624, 0x01625, 0x01626, 0x01627, + 0x01628, 0x01629, 0x0162a, 0x0162b, 0x0162c, 0x0162d, 0x0162e, 0x0162f, + 0x01630, 0x01631, 0x01632, 0x01633, 0x01634, 0x01635, 0x01636, 0x01637, + 0x01638, 0x01639, 0x0163a, 0x0163b, 0x0163c, 0x0163d, 0x0163e, 0x0163f, + 0x01640, 0x01641, 0x01642, 0x01643, 0x01644, 0x01645, 0x01646, 0x01647, + 0x01648, 0x01649, 0x0164a, 0x0164b, 0x0164c, 0x0164d, 0x0164e, 0x0164f, + 0x01650, 0x01651, 0x01652, 0x01653, 0x01654, 0x01655, 0x01656, 0x01657, + 0x01658, 0x01659, 0x0165a, 0x0165b, 0x0165c, 0x0165d, 0x0165e, 0x0165f, + 0x01660, 0x01661, 0x01662, 0x01663, 0x01664, 0x01665, 0x01666, 0x01667, + 0x01668, 0x01669, 0x0166a, 0x0166b, 0x0166c, 0x0166d, 0x0166e, 0x0166f, + 0x01670, 0x01671, 0x01672, 0x01673, 0x01674, 0x01675, 0x01676, 0x01677, + 0x01678, 0x01679, 0x0167a, 0x0167b, 0x0167c, 0x0167d, 0x0167e, 0x0167f, + 0x01680, 0x01681, 0x01682, 0x01683, 0x01684, 0x01685, 0x01686, 0x01687, + 0x01688, 0x01689, 0x0168a, 0x0168b, 0x0168c, 0x0168d, 0x0168e, 0x0168f, + 0x01690, 0x01691, 0x01692, 0x01693, 0x01694, 0x01695, 0x01696, 0x01697, + 0x01698, 0x01699, 0x0169a, 0x0169b, 0x0169c, 0x0169d, 0x0169e, 0x0169f, + 0x016a0, 0x016a0, 0x016a2, 0x016a3, 0x016a2, 0x016a2, 0x016a6, 0x016a6, + 0x016a8, 0x016a8, 0x016aa, 0x016ab, 0x016a8, 0x016a8, 0x016a8, 0x016af, + 0x016b0, 0x016b1, 0x016b2, 0x016b2, 0x016b2, 0x016b2, 0x016b2, 0x016b7, + 0x016b8, 0x016b9, 0x016ba, 0x016ba, 0x016ba, 0x016ba, 0x016be, 0x016be, + 0x016be, 0x016c1, 0x016c1, 0x016c3, 0x016c3, 0x016c5, 0x016c5, 0x016c7, + 0x016c8, 0x016c9, 0x016ca, 0x016ca, 0x016ca, 0x016ca, 0x016ca, 0x016cf, + 0x016cf, 0x016cf, 0x016d2, 0x016d2, 0x016d2, 0x016c8, 0x016d6, 0x016d7, + 0x016d7, 0x016d7, 0x016da, 0x016da, 0x016dc, 0x016dc, 0x016de, 0x016df, + 0x016e0, 0x016e1, 0x016e2, 0x016e3, 0x016e4, 0x016e5, 0x016e6, 0x016e6, + 0x016e6, 0x016b9, 0x016ca, 0x016eb, 0x016ec, 0x016ed, 0x016ee, 0x016ef, + 0x016f0, 0x016f1, 0x016f2, 0x016f3, 0x016f4, 0x016f5, 0x016f6, 0x016f7, + 0x016f8, 0x016f9, 0x016fa, 0x016fb, 0x016fc, 0x016fd, 0x016fe, 0x016ff +}; + +static uint32_t unicode_520_ci_page_17[] = { + 0x01700, 0x01701, 0x01702, 0x01703, 0x01704, 0x01705, 0x01706, 0x01707, + 0x01708, 0x01709, 0x0170a, 0x0170b, 0x0170c, 0x0170d, 0x0170e, 0x0170f, + 0x01710, 0x01711, 0x01712, 0x01713, 0x01714, 0x01715, 0x01716, 0x01717, + 0x01718, 0x01719, 0x0171a, 0x0171b, 0x0171c, 0x0171d, 0x0171e, 0x0171f, + 0x01720, 0x01721, 0x01722, 0x01723, 0x01724, 0x01725, 0x01726, 0x01727, + 0x01728, 0x01729, 0x0172a, 0x0172b, 0x0172c, 0x0172d, 0x0172e, 0x0172f, + 0x01730, 0x01731, 0x01732, 0x01733, 0x01734, 0x01735, 0x01736, 0x01737, + 0x01738, 0x01739, 0x0173a, 0x0173b, 0x0173c, 0x0173d, 0x0173e, 0x0173f, + 0x01740, 0x01741, 0x01742, 0x01743, 0x01744, 0x01745, 0x01746, 0x01747, + 0x01748, 0x01749, 0x0174a, 0x0174b, 0x0174c, 0x0174d, 0x0174e, 0x0174f, + 0x01750, 0x01751, 0x01752, 0x01753, 0x01754, 0x01755, 0x01756, 0x01757, + 0x01758, 0x01759, 0x0175a, 0x0175b, 0x0175c, 0x0175d, 0x0175e, 0x0175f, + 0x01760, 0x01761, 0x01762, 0x01763, 0x01764, 0x01765, 0x01766, 0x01767, + 0x01768, 0x01769, 0x0176a, 0x0176b, 0x0176c, 0x0176d, 0x0176e, 0x0176f, + 0x01770, 0x01771, 0x01772, 0x01773, 0x01774, 0x01775, 0x01776, 0x01777, + 0x01778, 0x01779, 0x0177a, 0x0177b, 0x0177c, 0x0177d, 0x0177e, 0x0177f, + 0x01780, 0x01781, 0x01782, 0x01783, 0x01784, 0x01785, 0x01786, 0x01787, + 0x01788, 0x01789, 0x0178a, 0x0178b, 0x0178c, 0x0178d, 0x0178e, 0x0178f, + 0x01790, 0x01791, 0x01792, 0x01793, 0x01794, 0x01795, 0x01796, 0x01797, + 0x01798, 0x01799, 0x0179a, 0x0179b, 0x0179c, 0x0179d, 0x0179e, 0x0179f, + 0x017a0, 0x017a1, 0x017a2, 0x017a3, 0x017a4, 0x017a5, 0x017a6, 0x017a7, + 0x017a8, 0x017a9, 0x017aa, 0x017ab, 0x017ac, 0x017ad, 0x017ae, 0x017af, + 0x017b0, 0x017b1, 0x017b2, 0x017b3, 0x017b4, 0x017b5, 0x017b6, 0x017b7, + 0x017b8, 0x017b9, 0x017ba, 0x017bb, 0x017bc, 0x017bd, 0x017be, 0x017bf, + 0x017c0, 0x017c1, 0x017c2, 0x017c3, 0x017c4, 0x017c5, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x017d2, 0x00000, 0x017d4, 0x017d5, 0x017d6, 0x017d7, + 0x017d8, 0x017d9, 0x017da, 0x017db, 0x017dc, 0x00000, 0x017de, 0x017df, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x017ea, 0x017eb, 0x017ec, 0x017ed, 0x017ee, 0x017ef, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x017fa, 0x017fb, 0x017fc, 0x017fd, 0x017fe, 0x017ff +}; + +static uint32_t unicode_520_ci_page_18[] = { + 0x01800, 0x01801, 0x01802, 0x01803, 0x01804, 0x01805, 0x01806, 0x01807, + 0x01808, 0x01809, 0x0180a, 0x00000, 0x00000, 0x00000, 0x0180e, 0x0180f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0181a, 0x0181b, 0x0181c, 0x0181d, 0x0181e, 0x0181f, + 0x01820, 0x01821, 0x01822, 0x01823, 0x01824, 0x01825, 0x01826, 0x01827, + 0x01828, 0x01829, 0x0182a, 0x0182b, 0x0182c, 0x0182d, 0x0182e, 0x0182f, + 0x01830, 0x01831, 0x01832, 0x01833, 0x01834, 0x01835, 0x01836, 0x01837, + 0x01838, 0x01839, 0x0183a, 0x0183b, 0x0183c, 0x0183d, 0x0183e, 0x0183f, + 0x01840, 0x01841, 0x01842, 0x01843, 0x01844, 0x01845, 0x01846, 0x01847, + 0x01848, 0x01849, 0x0184a, 0x0184b, 0x0184c, 0x0184d, 0x0184e, 0x0184f, + 0x01850, 0x01851, 0x01852, 0x01853, 0x01854, 0x01855, 0x01856, 0x01857, + 0x01858, 0x01859, 0x0185a, 0x0185b, 0x0185c, 0x0185d, 0x0185e, 0x0185f, + 0x01860, 0x01861, 0x01862, 0x01863, 0x01864, 0x01865, 0x01866, 0x01867, + 0x01868, 0x01869, 0x0186a, 0x0186b, 0x0186c, 0x0186d, 0x0186e, 0x0186f, + 0x01870, 0x01871, 0x01872, 0x01873, 0x01874, 0x01875, 0x01876, 0x01877, + 0x01878, 0x01879, 0x0187a, 0x0187b, 0x0187c, 0x0187d, 0x0187e, 0x0187f, + 0x01880, 0x01881, 0x01882, 0x01883, 0x01884, 0x01885, 0x01886, 0x01887, + 0x01888, 0x01889, 0x0188a, 0x0188b, 0x0188c, 0x0188d, 0x0188e, 0x0188f, + 0x01890, 0x01891, 0x01892, 0x01893, 0x01894, 0x01895, 0x01896, 0x01897, + 0x01898, 0x01899, 0x0189a, 0x0189b, 0x0189c, 0x0189d, 0x0189e, 0x0189f, + 0x018a0, 0x018a1, 0x018a2, 0x018a3, 0x018a4, 0x018a5, 0x018a6, 0x018a7, + 0x018a8, 0x018a9, 0x018aa, 0x018ab, 0x018ac, 0x018ad, 0x018ae, 0x018af, + 0x018b0, 0x018b1, 0x018b2, 0x018b3, 0x018b4, 0x018b5, 0x018b6, 0x018b7, + 0x018b8, 0x018b9, 0x018ba, 0x018bb, 0x018bc, 0x018bd, 0x018be, 0x018bf, + 0x018c0, 0x018c1, 0x018c2, 0x018c3, 0x018c4, 0x018c5, 0x018c6, 0x018c7, + 0x018c8, 0x018c9, 0x018ca, 0x018cb, 0x018cc, 0x018cd, 0x018ce, 0x018cf, + 0x018d0, 0x018d1, 0x018d2, 0x018d3, 0x018d4, 0x018d5, 0x018d6, 0x018d7, + 0x018d8, 0x018d9, 0x018da, 0x018db, 0x018dc, 0x018dd, 0x018de, 0x018df, + 0x018e0, 0x018e1, 0x018e2, 0x018e3, 0x018e4, 0x018e5, 0x018e6, 0x018e7, + 0x018e8, 0x018e9, 0x018ea, 0x018eb, 0x018ec, 0x018ed, 0x018ee, 0x018ef, + 0x018f0, 0x018f1, 0x018f2, 0x018f3, 0x018f4, 0x018f5, 0x018f6, 0x018f7, + 0x018f8, 0x018f9, 0x018fa, 0x018fb, 0x018fc, 0x018fd, 0x018fe, 0x018ff +}; + +static uint32_t unicode_520_ci_page_19[] = { + 0x01900, 0x01901, 0x01902, 0x01903, 0x01904, 0x01905, 0x01906, 0x01907, + 0x01908, 0x01909, 0x0190a, 0x0190b, 0x0190c, 0x0190d, 0x0190e, 0x0190f, + 0x01910, 0x01911, 0x01912, 0x01913, 0x01914, 0x01915, 0x01916, 0x01917, + 0x01918, 0x01919, 0x0191a, 0x0191b, 0x0191c, 0x0191d, 0x0191e, 0x0191f, + 0x01920, 0x01921, 0x01922, 0x01923, 0x01924, 0x01925, 0x01926, 0x01927, + 0x01928, 0x01929, 0x0192a, 0x0192b, 0x0192c, 0x0192d, 0x0192e, 0x0192f, + 0x01930, 0x01931, 0x01932, 0x01933, 0x01934, 0x01935, 0x01936, 0x01937, + 0x01938, 0x00000, 0x00000, 0x00000, 0x0193c, 0x0193d, 0x0193e, 0x0193f, + 0x01940, 0x01941, 0x01942, 0x01943, 0x01944, 0x01945, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x01950, 0x01951, 0x01952, 0x01953, 0x01954, 0x01955, 0x01956, 0x01957, + 0x01958, 0x01959, 0x0195a, 0x0195b, 0x0195c, 0x0195d, 0x0195e, 0x0195f, + 0x01960, 0x01961, 0x01962, 0x01963, 0x01964, 0x01965, 0x01966, 0x01967, + 0x01968, 0x01969, 0x0196a, 0x0196b, 0x0196c, 0x0196d, 0x0196e, 0x0196f, + 0x01970, 0x01971, 0x01972, 0x01973, 0x01974, 0x01975, 0x01976, 0x01977, + 0x01978, 0x01979, 0x0197a, 0x0197b, 0x0197c, 0x0197d, 0x0197e, 0x0197f, + 0x01980, 0x01981, 0x01982, 0x01983, 0x01984, 0x01985, 0x01986, 0x01987, + 0x01988, 0x01989, 0x0198a, 0x0198b, 0x0198c, 0x0198d, 0x0198e, 0x0198f, + 0x01990, 0x01991, 0x01992, 0x01993, 0x01994, 0x01995, 0x01996, 0x01997, + 0x01998, 0x01999, 0x0199a, 0x0199b, 0x0199c, 0x0199d, 0x0199e, 0x0199f, + 0x019a0, 0x019a1, 0x019a2, 0x019a3, 0x019a4, 0x019a5, 0x019a6, 0x019a7, + 0x019a8, 0x019a9, 0x019aa, 0x019ab, 0x019ac, 0x019ad, 0x019ae, 0x019af, + 0x019b0, 0x019b1, 0x019b2, 0x019b3, 0x019b4, 0x019b5, 0x019b6, 0x019b7, + 0x019b8, 0x019b9, 0x019ba, 0x019bb, 0x019bc, 0x019bd, 0x019be, 0x019bf, + 0x019c0, 0x019c1, 0x019c2, 0x019c3, 0x019c4, 0x019c5, 0x019c6, 0x019c7, + 0x019c8, 0x019c9, 0x019ca, 0x019cb, 0x019cc, 0x019cd, 0x019ce, 0x019cf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00031, 0x019db, 0x019dc, 0x019dd, 0x019de, 0x019df, + 0x019e0, 0x019e1, 0x019e2, 0x019e3, 0x019e4, 0x019e5, 0x019e6, 0x019e7, + 0x019e8, 0x019e9, 0x019ea, 0x019eb, 0x019ec, 0x019ed, 0x019ee, 0x019ef, + 0x019f0, 0x019f1, 0x019f2, 0x019f3, 0x019f4, 0x019f5, 0x019f6, 0x019f7, + 0x019f8, 0x019f9, 0x019fa, 0x019fb, 0x019fc, 0x019fd, 0x019fe, 0x019ff +}; + +static uint32_t unicode_520_ci_page_1a[] = { + 0x01a00, 0x01a01, 0x01a02, 0x01a03, 0x01a04, 0x01a05, 0x01a06, 0x01a07, + 0x01a08, 0x01a09, 0x01a0a, 0x01a0b, 0x01a0c, 0x01a0d, 0x01a0e, 0x01a0f, + 0x01a10, 0x01a11, 0x01a12, 0x01a13, 0x01a14, 0x01a15, 0x01a16, 0x01a17, + 0x01a18, 0x01a19, 0x01a1a, 0x01a1b, 0x01a1c, 0x01a1d, 0x01a1e, 0x01a1f, + 0x01a20, 0x01a21, 0x01a22, 0x01a23, 0x01a24, 0x01a25, 0x01a26, 0x01a27, + 0x01a28, 0x01a29, 0x01a2a, 0x01a2b, 0x01a2c, 0x01a2d, 0x01a2e, 0x01a2f, + 0x01a30, 0x01a31, 0x01a32, 0x01a33, 0x01a34, 0x01a35, 0x01a36, 0x01a37, + 0x01a38, 0x01a39, 0x01a3a, 0x01a3b, 0x01a3c, 0x01a3d, 0x01a3e, 0x01a3f, + 0x01a40, 0x01a41, 0x01a42, 0x01a43, 0x01a44, 0x01a45, 0x01a46, 0x01a47, + 0x01a48, 0x01a49, 0x01a4a, 0x01a4b, 0x01a4c, 0x01a4d, 0x01a4e, 0x01a4f, + 0x01a50, 0x01a51, 0x01a52, 0x01a53, 0x01a54, 0x01a55, 0x01a56, 0x01a57, + 0x01a26, 0x01a26, 0x01a3b, 0x01a3b, 0x01a5c, 0x01a5d, 0x01a5e, 0x01a5f, + 0x01a60, 0x01a61, 0x01a62, 0x01a63, 0x01a63, 0x01a65, 0x01a66, 0x01a67, + 0x01a68, 0x01a69, 0x01a6a, 0x01a6b, 0x01a6c, 0x01a6d, 0x01a6e, 0x01a6f, + 0x01a70, 0x01a71, 0x01a72, 0x01a73, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x01a7d, 0x01a7e, 0x00000, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01a8a, 0x01a8b, 0x01a8c, 0x01a8d, 0x01a8e, 0x01a8f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01a9a, 0x01a9b, 0x01a9c, 0x01a9d, 0x01a9e, 0x01a9f, + 0x01aa0, 0x01aa1, 0x01aa2, 0x01aa3, 0x01aa4, 0x01aa5, 0x01aa6, 0x01aa7, + 0x01aa8, 0x01aa9, 0x01aaa, 0x01aab, 0x01aac, 0x01aad, 0x01aae, 0x01aaf, + 0x01ab0, 0x01ab1, 0x01ab2, 0x01ab3, 0x01ab4, 0x01ab5, 0x01ab6, 0x01ab7, + 0x01ab8, 0x01ab9, 0x01aba, 0x01abb, 0x01abc, 0x01abd, 0x01abe, 0x01abf, + 0x01ac0, 0x01ac1, 0x01ac2, 0x01ac3, 0x01ac4, 0x01ac5, 0x01ac6, 0x01ac7, + 0x01ac8, 0x01ac9, 0x01aca, 0x01acb, 0x01acc, 0x01acd, 0x01ace, 0x01acf, + 0x01ad0, 0x01ad1, 0x01ad2, 0x01ad3, 0x01ad4, 0x01ad5, 0x01ad6, 0x01ad7, + 0x01ad8, 0x01ad9, 0x01ada, 0x01adb, 0x01adc, 0x01add, 0x01ade, 0x01adf, + 0x01ae0, 0x01ae1, 0x01ae2, 0x01ae3, 0x01ae4, 0x01ae5, 0x01ae6, 0x01ae7, + 0x01ae8, 0x01ae9, 0x01aea, 0x01aeb, 0x01aec, 0x01aed, 0x01aee, 0x01aef, + 0x01af0, 0x01af1, 0x01af2, 0x01af3, 0x01af4, 0x01af5, 0x01af6, 0x01af7, + 0x01af8, 0x01af9, 0x01afa, 0x01afb, 0x01afc, 0x01afd, 0x01afe, 0x01aff +}; + +static uint32_t unicode_520_ci_page_1b[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x01b05, 0x01b06, 0x01b07, + 0x01b08, 0x01b09, 0x01b0a, 0x01b0b, 0x01b0c, 0x01b0d, 0x01b0e, 0x01b0f, + 0x01b10, 0x01b11, 0x01b12, 0x01b13, 0x01b14, 0x01b15, 0x01b16, 0x01b17, + 0x01b18, 0x01b19, 0x01b1a, 0x01b1b, 0x01b1c, 0x01b1d, 0x01b1e, 0x01b1f, + 0x01b20, 0x01b21, 0x01b22, 0x01b23, 0x01b24, 0x01b25, 0x01b26, 0x01b27, + 0x01b28, 0x01b29, 0x01b2a, 0x01b2b, 0x01b2c, 0x01b2d, 0x01b2e, 0x01b2f, + 0x01b30, 0x01b31, 0x01b32, 0x01b33, 0x00000, 0x01b35, 0x01b36, 0x01b37, + 0x01b38, 0x01b39, 0x01b3a, 0x01b3b, 0x01b3c, 0x01b3d, 0x01b3e, 0x01b3f, + 0x01b40, 0x01b41, 0x01b42, 0x01b43, 0x01b44, 0x01b45, 0x01b46, 0x01b47, + 0x01b48, 0x01b49, 0x01b4a, 0x01b4b, 0x01b4c, 0x01b4d, 0x01b4e, 0x01b4f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01b5a, 0x01b5b, 0x01b5c, 0x01b5d, 0x01b5e, 0x01b5f, + 0x01b60, 0x01b61, 0x01b62, 0x01b63, 0x01b64, 0x01b65, 0x01b66, 0x01b67, + 0x01b68, 0x01b69, 0x01b6a, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x01b74, 0x01b75, 0x01b76, 0x01b77, + 0x01b78, 0x01b79, 0x01b7a, 0x01b7b, 0x01b7c, 0x01b7d, 0x01b7e, 0x01b7f, + 0x00000, 0x00000, 0x00000, 0x01b83, 0x01b84, 0x01b85, 0x01b86, 0x01b87, + 0x01b88, 0x01b89, 0x01b8a, 0x01b8b, 0x01b8c, 0x01b8d, 0x01b8e, 0x01b8f, + 0x01b90, 0x01b91, 0x01b92, 0x01b93, 0x01b94, 0x01b95, 0x01b96, 0x01b97, + 0x01b98, 0x01b99, 0x01b9a, 0x01b9b, 0x01b9c, 0x01b9d, 0x01b9e, 0x01b9f, + 0x01ba0, 0x01ba1, 0x01ba2, 0x01ba3, 0x01ba4, 0x01ba5, 0x01ba6, 0x01ba7, + 0x01ba8, 0x01ba9, 0x01baa, 0x01bab, 0x01bac, 0x01bad, 0x01bae, 0x01baf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01bba, 0x01bbb, 0x01bbc, 0x01bbd, 0x01bbe, 0x01bbf, + 0x01bc0, 0x01bc1, 0x01bc2, 0x01bc3, 0x01bc4, 0x01bc5, 0x01bc6, 0x01bc7, + 0x01bc8, 0x01bc9, 0x01bca, 0x01bcb, 0x01bcc, 0x01bcd, 0x01bce, 0x01bcf, + 0x01bd0, 0x01bd1, 0x01bd2, 0x01bd3, 0x01bd4, 0x01bd5, 0x01bd6, 0x01bd7, + 0x01bd8, 0x01bd9, 0x01bda, 0x01bdb, 0x01bdc, 0x01bdd, 0x01bde, 0x01bdf, + 0x01be0, 0x01be1, 0x01be2, 0x01be3, 0x01be4, 0x01be5, 0x01be6, 0x01be7, + 0x01be8, 0x01be9, 0x01bea, 0x01beb, 0x01bec, 0x01bed, 0x01bee, 0x01bef, + 0x01bf0, 0x01bf1, 0x01bf2, 0x01bf3, 0x01bf4, 0x01bf5, 0x01bf6, 0x01bf7, + 0x01bf8, 0x01bf9, 0x01bfa, 0x01bfb, 0x01bfc, 0x01bfd, 0x01bfe, 0x01bff +}; + +static uint32_t unicode_520_ci_page_1c[] = { + 0x01c00, 0x01c01, 0x01c02, 0x01c03, 0x01c04, 0x01c05, 0x01c06, 0x01c07, + 0x01c08, 0x01c09, 0x01c0a, 0x01c0b, 0x01c0c, 0x01c0d, 0x01c0e, 0x01c0f, + 0x01c10, 0x01c11, 0x01c12, 0x01c13, 0x01c14, 0x01c15, 0x01c16, 0x01c17, + 0x01c18, 0x01c19, 0x01c1a, 0x01c1b, 0x01c1c, 0x01c1d, 0x01c1e, 0x01c1f, + 0x01c20, 0x01c21, 0x01c22, 0x01c23, 0x01c24, 0x01c25, 0x01c26, 0x01c27, + 0x01c28, 0x01c29, 0x01c2a, 0x01c2b, 0x01c2c, 0x01c2d, 0x01c2e, 0x01c2f, + 0x01c30, 0x01c31, 0x01c32, 0x01c33, 0x01c34, 0x01c35, 0x01c36, 0x00000, + 0x01c38, 0x01c39, 0x01c3a, 0x01c3b, 0x01c3c, 0x01c3d, 0x01c3e, 0x01c3f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01c4a, 0x01c4b, 0x01c4c, 0x01c4d, 0x01c4e, 0x01c4f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x01c5a, 0x01c5b, 0x01c5c, 0x01c5d, 0x01c5e, 0x01c5f, + 0x01c60, 0x01c61, 0x01c62, 0x01c63, 0x01c64, 0x01c65, 0x01c66, 0x01c67, + 0x01c68, 0x01c69, 0x01c6a, 0x01c6b, 0x01c6c, 0x01c6d, 0x01c6e, 0x01c6f, + 0x01c70, 0x01c71, 0x01c72, 0x01c73, 0x01c74, 0x01c75, 0x01c76, 0x01c77, + 0x01c78, 0x01c79, 0x01c7a, 0x01c7b, 0x01c7c, 0x01c7d, 0x01c7e, 0x01c7f, + 0x01c80, 0x01c81, 0x01c82, 0x01c83, 0x01c84, 0x01c85, 0x01c86, 0x01c87, + 0x01c88, 0x01c89, 0x01c8a, 0x01c8b, 0x01c8c, 0x01c8d, 0x01c8e, 0x01c8f, + 0x01c90, 0x01c91, 0x01c92, 0x01c93, 0x01c94, 0x01c95, 0x01c96, 0x01c97, + 0x01c98, 0x01c99, 0x01c9a, 0x01c9b, 0x01c9c, 0x01c9d, 0x01c9e, 0x01c9f, + 0x01ca0, 0x01ca1, 0x01ca2, 0x01ca3, 0x01ca4, 0x01ca5, 0x01ca6, 0x01ca7, + 0x01ca8, 0x01ca9, 0x01caa, 0x01cab, 0x01cac, 0x01cad, 0x01cae, 0x01caf, + 0x01cb0, 0x01cb1, 0x01cb2, 0x01cb3, 0x01cb4, 0x01cb5, 0x01cb6, 0x01cb7, + 0x01cb8, 0x01cb9, 0x01cba, 0x01cbb, 0x01cbc, 0x01cbd, 0x01cbe, 0x01cbf, + 0x01cc0, 0x01cc1, 0x01cc2, 0x01cc3, 0x01cc4, 0x01cc5, 0x01cc6, 0x01cc7, + 0x01cc8, 0x01cc9, 0x01cca, 0x01ccb, 0x01ccc, 0x01ccd, 0x01cce, 0x01ccf, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x01ce9, 0x01ce9, 0x01ce9, 0x01ce9, 0x00000, 0x01ce9, 0x01ce9, + 0x01ce9, 0x01ce9, 0x00000, 0x01cf3, 0x01cf4, 0x01cf5, 0x01cf6, 0x01cf7, + 0x01cf8, 0x01cf9, 0x01cfa, 0x01cfb, 0x01cfc, 0x01cfd, 0x01cfe, 0x01cff +}; + +static uint32_t unicode_520_ci_page_1d[] = { + 0x01d00, 0x01d01, 0x01d02, 0x01d03, 0x01d04, 0x01d05, 0x01d06, 0x01d07, + 0x01d08, 0x01d09, 0x01d0a, 0x01d0b, 0x01d0c, 0x01d0d, 0x01d0e, 0x01d0f, + 0x01d10, 0x01d11, 0x01d12, 0x01d13, 0x01d14, 0x01d15, 0x01d16, 0x01d17, + 0x01d18, 0x01d19, 0x01d1a, 0x01d1b, 0x01d1c, 0x01d1d, 0x01d1e, 0x01d1f, + 0x01d20, 0x01d21, 0x01d22, 0x01d23, 0x01d24, 0x01d25, 0x01d26, 0x01d27, + 0x01d28, 0x01d29, 0x01d2a, 0x01d2b, 0x00041, 0x000c6, 0x00042, 0x01d2f, + 0x00044, 0x00045, 0x0018e, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, + 0x0004c, 0x0004d, 0x0004e, 0x01d3b, 0x0004f, 0x00222, 0x00050, 0x00052, + 0x00054, 0x00055, 0x00057, 0x00041, 0x00250, 0x00251, 0x01d02, 0x00042, + 0x00044, 0x00045, 0x0018f, 0x00190, 0x01d08, 0x00047, 0x01d09, 0x0004b, + 0x0004d, 0x0014a, 0x0004f, 0x00186, 0x01d16, 0x01d17, 0x00050, 0x00054, + 0x00055, 0x01d1d, 0x0019c, 0x00056, 0x01d25, 0x00392, 0x00393, 0x00394, + 0x003a6, 0x003a7, 0x00049, 0x00052, 0x00055, 0x00056, 0x00392, 0x00393, + 0x003a1, 0x003a6, 0x003a7, 0x01d6b, 0x01d6c, 0x01d6d, 0x01d6e, 0x01d6f, + 0x01d70, 0x01d71, 0x01d72, 0x01d73, 0x01d74, 0x01d75, 0x01d76, 0x01d77, + 0x0041d, 0x00047, 0x01d7a, 0x01d7b, 0x01d7c, 0x01d7d, 0x01d7e, 0x01d7f, + 0x01d80, 0x01d81, 0x01d82, 0x01d83, 0x01d84, 0x01d85, 0x01d86, 0x01d87, + 0x01d88, 0x01d89, 0x01d8a, 0x01d8b, 0x01d8c, 0x01d8d, 0x01d8e, 0x01d8f, + 0x01d90, 0x01d91, 0x01d92, 0x01d93, 0x01d94, 0x01d95, 0x01d96, 0x01d97, + 0x01d98, 0x01d99, 0x01d9a, 0x00252, 0x00043, 0x00255, 0x00044, 0x0025c, + 0x00046, 0x0025f, 0x00261, 0x00265, 0x00197, 0x00196, 0x0026a, 0x01d7b, + 0x0029d, 0x0026d, 0x01d85, 0x0029f, 0x00271, 0x00270, 0x0019d, 0x00273, + 0x00274, 0x0019f, 0x00278, 0x00282, 0x001a9, 0x001ab, 0x00244, 0x001b1, + 0x01d1c, 0x001b2, 0x00245, 0x0005a, 0x00290, 0x00291, 0x001b7, 0x00398, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00052, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x01dd2, 0x00041, 0x000c6, 0x01dd5, 0x01dd6, 0x00043, + 0x00044, 0x00044, 0x00047, 0x00262, 0x0004b, 0x0004c, 0x0029f, 0x01d0d, + 0x0004e, 0x00274, 0x001a6, 0x01de3, 0x00053, 0x00053, 0x0005a, 0x01de7, + 0x01de8, 0x01de9, 0x01dea, 0x01deb, 0x01dec, 0x01ded, 0x01dee, 0x01def, + 0x01df0, 0x01df1, 0x01df2, 0x01df3, 0x01df4, 0x01df5, 0x01df6, 0x01df7, + 0x01df8, 0x01df9, 0x01dfa, 0x01dfb, 0x01dfc, 0x00000, 0x00000, 0x01dff +}; + +static uint32_t unicode_520_ci_page_1e[] = { + 0x00041, 0x00041, 0x00042, 0x00042, 0x00042, 0x00042, 0x00042, 0x00042, + 0x00043, 0x00043, 0x00044, 0x00044, 0x00044, 0x00044, 0x00044, 0x00044, + 0x00044, 0x00044, 0x00044, 0x00044, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00046, 0x00046, + 0x00047, 0x00047, 0x00048, 0x00048, 0x00048, 0x00048, 0x00048, 0x00048, + 0x00048, 0x00048, 0x00048, 0x00048, 0x00049, 0x00049, 0x00049, 0x00049, + 0x0004b, 0x0004b, 0x0004b, 0x0004b, 0x0004b, 0x0004b, 0x0004c, 0x0004c, + 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004c, 0x0004d, 0x0004d, + 0x0004d, 0x0004d, 0x0004d, 0x0004d, 0x0004e, 0x0004e, 0x0004e, 0x0004e, + 0x0004e, 0x0004e, 0x0004e, 0x0004e, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x00050, 0x00050, 0x00050, 0x00050, + 0x00052, 0x00052, 0x00052, 0x00052, 0x00052, 0x00052, 0x00052, 0x00052, + 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, 0x00053, + 0x00053, 0x00053, 0x00054, 0x00054, 0x00054, 0x00054, 0x00054, 0x00054, + 0x00054, 0x00054, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00056, 0x00056, 0x00056, 0x00056, + 0x00057, 0x00057, 0x00057, 0x00057, 0x00057, 0x00057, 0x00057, 0x00057, + 0x00057, 0x00057, 0x00058, 0x00058, 0x00058, 0x00058, 0x00059, 0x00059, + 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x0005a, 0x00048, 0x00054, + 0x00057, 0x00059, 0x01e9a, 0x00053, 0x01e9c, 0x01e9d, 0x000df, 0x01e9f, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, + 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, 0x00041, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, 0x00045, + 0x00049, 0x00049, 0x00049, 0x00049, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x0004f, + 0x0004f, 0x0004f, 0x0004f, 0x0004f, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, 0x00055, + 0x00055, 0x00055, 0x00059, 0x00059, 0x00059, 0x00059, 0x00059, 0x00059, + 0x00059, 0x00059, 0x01efa, 0x01efa, 0x01efc, 0x01efc, 0x01efe, 0x01eff +}; + +static uint32_t unicode_520_ci_page_1f[] = { + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, + 0x00395, 0x00395, 0x00395, 0x00395, 0x00395, 0x00395, 0x01f16, 0x01f17, + 0x00395, 0x00395, 0x00395, 0x00395, 0x00395, 0x00395, 0x01f1e, 0x01f1f, + 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, + 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, + 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, + 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, 0x00399, + 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x01f46, 0x01f47, + 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x0039f, 0x01f4e, 0x01f4f, + 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a5, + 0x01f58, 0x003a5, 0x01f5a, 0x003a5, 0x01f5c, 0x003a5, 0x01f5e, 0x003a5, + 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, + 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, + 0x00391, 0x00391, 0x00395, 0x00395, 0x00397, 0x00397, 0x00399, 0x00399, + 0x0039f, 0x0039f, 0x003a5, 0x003a5, 0x003a9, 0x003a9, 0x01f7e, 0x01f7f, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, + 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, + 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, 0x00397, + 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, + 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, 0x003a9, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x01fb5, 0x00391, 0x00391, + 0x00391, 0x00391, 0x00391, 0x00391, 0x00391, 0x01fbd, 0x00399, 0x01fbd, + 0x01fc0, 0x000a8, 0x00397, 0x00397, 0x00397, 0x01fc5, 0x00397, 0x00397, + 0x00395, 0x00395, 0x00397, 0x00397, 0x00397, 0x01fbd, 0x01fbd, 0x01fbd, + 0x00399, 0x00399, 0x00399, 0x00399, 0x01fd4, 0x01fd5, 0x00399, 0x00399, + 0x00399, 0x00399, 0x00399, 0x00399, 0x01fdc, 0x01fdd, 0x01fdd, 0x01fdd, + 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a1, 0x003a1, 0x003a5, 0x003a5, + 0x003a5, 0x003a5, 0x003a5, 0x003a5, 0x003a1, 0x000a8, 0x000a8, 0x00060, + 0x01ff0, 0x01ff1, 0x003a9, 0x003a9, 0x003a9, 0x01ff5, 0x003a9, 0x003a9, + 0x0039f, 0x0039f, 0x003a9, 0x003a9, 0x003a9, 0x000b4, 0x01fdd, 0x01fff +}; + +static uint32_t unicode_520_ci_page_20[] = { + 0x00020, 0x00020, 0x00020, 0x00020, 0x00020, 0x00020, 0x00020, 0x00020, + 0x00020, 0x00020, 0x00020, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x02010, 0x02010, 0x02012, 0x02013, 0x02014, 0x02015, 0x02016, 0x02017, + 0x02018, 0x02019, 0x0201a, 0x0201b, 0x0201c, 0x0201d, 0x0201e, 0x0201f, + 0x02020, 0x02021, 0x02022, 0x02023, 0x0002e, 0x02025, 0x02026, 0x02027, + 0x02028, 0x02029, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00020, + 0x02030, 0x02031, 0x02032, 0x02033, 0x02034, 0x02035, 0x02036, 0x02037, + 0x02038, 0x02039, 0x0203a, 0x0203b, 0x0203c, 0x0203d, 0x0203e, 0x0203f, + 0x02040, 0x02041, 0x02042, 0x02043, 0x02044, 0x02045, 0x02046, 0x02047, + 0x02048, 0x02049, 0x0204a, 0x0204b, 0x0204c, 0x0204d, 0x0204e, 0x0204f, + 0x02050, 0x02051, 0x02052, 0x02053, 0x02054, 0x02055, 0x02056, 0x02057, + 0x02058, 0x02059, 0x0205a, 0x0205b, 0x0205c, 0x0205d, 0x0205e, 0x00020, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x02065, 0x02066, 0x02067, + 0x02068, 0x02069, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00030, 0x00049, 0x02072, 0x02073, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0002b, 0x0207b, 0x0003d, 0x00028, 0x00029, 0x0004e, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0002b, 0x0207b, 0x0003d, 0x00028, 0x00029, 0x0208f, + 0x00041, 0x00045, 0x0004f, 0x00058, 0x0018f, 0x02095, 0x02096, 0x02097, + 0x02098, 0x02099, 0x0209a, 0x0209b, 0x0209c, 0x0209d, 0x0209e, 0x0209f, + 0x020a0, 0x020a1, 0x020a2, 0x020a3, 0x020a4, 0x020a5, 0x020a6, 0x020a7, + 0x020a8, 0x020a9, 0x020aa, 0x020ab, 0x020ac, 0x020ad, 0x020ae, 0x020af, + 0x020b0, 0x020b1, 0x020b2, 0x020b3, 0x020b4, 0x020b5, 0x020b6, 0x020b7, + 0x020b8, 0x020b9, 0x020ba, 0x020bb, 0x020bc, 0x020bd, 0x020be, 0x020bf, + 0x020c0, 0x020c1, 0x020c2, 0x020c3, 0x020c4, 0x020c5, 0x020c6, 0x020c7, + 0x020c8, 0x020c9, 0x020ca, 0x020cb, 0x020cc, 0x020cd, 0x020ce, 0x020cf, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x020f1, 0x020f2, 0x020f3, 0x020f4, 0x020f5, 0x020f6, 0x020f7, + 0x020f8, 0x020f9, 0x020fa, 0x020fb, 0x020fc, 0x020fd, 0x020fe, 0x020ff +}; + +static uint32_t unicode_520_ci_page_21[] = { + 0x02100, 0x02101, 0x00043, 0x02103, 0x02104, 0x02105, 0x02106, 0x00190, + 0x02108, 0x02109, 0x00047, 0x00048, 0x00048, 0x00048, 0x00048, 0x00048, + 0x00049, 0x00049, 0x0004c, 0x0004c, 0x02114, 0x0004e, 0x02116, 0x02117, + 0x02118, 0x00050, 0x00051, 0x00052, 0x00052, 0x00052, 0x0211e, 0x0211f, + 0x02120, 0x02121, 0x02122, 0x02123, 0x0005a, 0x02125, 0x003a9, 0x02127, + 0x0005a, 0x02129, 0x0004b, 0x00041, 0x00042, 0x00043, 0x0212e, 0x00045, + 0x00045, 0x00046, 0x02132, 0x0004d, 0x0004f, 0x005d0, 0x005d1, 0x005d2, + 0x005d3, 0x00049, 0x0213a, 0x0213b, 0x003a0, 0x00393, 0x00393, 0x003a0, + 0x02140, 0x02141, 0x02142, 0x02143, 0x02144, 0x00044, 0x00044, 0x00045, + 0x00049, 0x0004a, 0x0214a, 0x0214b, 0x0214c, 0x0214d, 0x02132, 0x0214f, + 0x02150, 0x02151, 0x02152, 0x02153, 0x02154, 0x02155, 0x02156, 0x02157, + 0x02158, 0x02159, 0x0215a, 0x0215b, 0x0215c, 0x0215d, 0x0215e, 0x0215f, + 0x00049, 0x02161, 0x02162, 0x02163, 0x00056, 0x02165, 0x02166, 0x02167, + 0x02168, 0x00058, 0x0216a, 0x0216b, 0x0004c, 0x00043, 0x00044, 0x0004d, + 0x00049, 0x02161, 0x02162, 0x02163, 0x00056, 0x02165, 0x02166, 0x02167, + 0x02168, 0x00058, 0x0216a, 0x0216b, 0x0004c, 0x00043, 0x00044, 0x0004d, + 0x02180, 0x02181, 0x02182, 0x02183, 0x02183, 0x00036, 0x02186, 0x02187, + 0x02188, 0x02189, 0x0218a, 0x0218b, 0x0218c, 0x0218d, 0x0218e, 0x0218f, + 0x02190, 0x02191, 0x02192, 0x02193, 0x02194, 0x02195, 0x02196, 0x02197, + 0x02198, 0x02199, 0x02190, 0x02192, 0x0219c, 0x0219d, 0x0219e, 0x0219f, + 0x021a0, 0x021a1, 0x021a2, 0x021a3, 0x021a4, 0x021a5, 0x021a6, 0x021a7, + 0x021a8, 0x021a9, 0x021aa, 0x021ab, 0x021ac, 0x021ad, 0x02194, 0x021af, + 0x021b0, 0x021b1, 0x021b2, 0x021b3, 0x021b4, 0x021b5, 0x021b6, 0x021b7, + 0x021b8, 0x021b9, 0x021ba, 0x021bb, 0x021bc, 0x021bd, 0x021be, 0x021bf, + 0x021c0, 0x021c1, 0x021c2, 0x021c3, 0x021c4, 0x021c5, 0x021c6, 0x021c7, + 0x021c8, 0x021c9, 0x021ca, 0x021cb, 0x021cc, 0x021cd, 0x021ce, 0x021cf, + 0x021cd, 0x021d1, 0x021cf, 0x021d3, 0x021ce, 0x021d5, 0x021d6, 0x021d7, + 0x021d8, 0x021d9, 0x021da, 0x021db, 0x021dc, 0x021dd, 0x021de, 0x021df, + 0x021e0, 0x021e1, 0x021e2, 0x021e3, 0x021e4, 0x021e5, 0x021e6, 0x021e7, + 0x021e8, 0x021e9, 0x021ea, 0x021eb, 0x021ec, 0x021ed, 0x021ee, 0x021ef, + 0x021f0, 0x021f1, 0x021f2, 0x021f3, 0x021f4, 0x021f5, 0x021f6, 0x021f7, + 0x021f8, 0x021f9, 0x021fa, 0x021fb, 0x021fc, 0x021fd, 0x021fe, 0x021ff +}; + +static uint32_t unicode_520_ci_page_22[] = { + 0x02200, 0x02201, 0x02202, 0x02203, 0x02203, 0x02205, 0x02206, 0x02207, + 0x02208, 0x02208, 0x0220a, 0x0220b, 0x0220b, 0x0220d, 0x0220e, 0x0220f, + 0x02210, 0x02140, 0x0207b, 0x02213, 0x02214, 0x02215, 0x02216, 0x02217, + 0x02218, 0x02219, 0x0221a, 0x0221b, 0x0221c, 0x0221d, 0x0221e, 0x0221f, + 0x02220, 0x02221, 0x02222, 0x02223, 0x02223, 0x02225, 0x02225, 0x02227, + 0x02228, 0x02229, 0x0222a, 0x0222b, 0x0222c, 0x0222d, 0x0222e, 0x0222f, + 0x02230, 0x02231, 0x02232, 0x02233, 0x02234, 0x02235, 0x02236, 0x02237, + 0x02238, 0x02239, 0x0223a, 0x0223b, 0x0223c, 0x0223d, 0x0223e, 0x0223f, + 0x02240, 0x0223c, 0x02242, 0x02243, 0x02243, 0x02245, 0x02246, 0x02245, + 0x02248, 0x02248, 0x0224a, 0x0224b, 0x0224c, 0x0224d, 0x0224e, 0x0224f, + 0x02250, 0x02251, 0x02252, 0x02253, 0x02254, 0x02255, 0x02256, 0x02257, + 0x02258, 0x02259, 0x0225a, 0x0225b, 0x0225c, 0x0225d, 0x0225e, 0x0225f, + 0x0003d, 0x02261, 0x02261, 0x02263, 0x02264, 0x02265, 0x02266, 0x02267, + 0x02268, 0x02269, 0x0226a, 0x0226b, 0x0226c, 0x0224d, 0x0003c, 0x0003e, + 0x02264, 0x02265, 0x02272, 0x02273, 0x02272, 0x02273, 0x02276, 0x02277, + 0x02276, 0x02277, 0x0227a, 0x0227b, 0x0227c, 0x0227d, 0x0227e, 0x0227f, + 0x0227a, 0x0227b, 0x02282, 0x02283, 0x02282, 0x02283, 0x02286, 0x02287, + 0x02286, 0x02287, 0x0228a, 0x0228b, 0x0228c, 0x0228d, 0x0228e, 0x0228f, + 0x02290, 0x02291, 0x02292, 0x02293, 0x02294, 0x02295, 0x02296, 0x02297, + 0x02298, 0x02299, 0x0229a, 0x0229b, 0x0229c, 0x0229d, 0x0229e, 0x0229f, + 0x022a0, 0x022a1, 0x022a2, 0x022a3, 0x022a4, 0x022a5, 0x022a6, 0x022a7, + 0x022a8, 0x022a9, 0x022aa, 0x022ab, 0x022a2, 0x022a8, 0x022a9, 0x022ab, + 0x022b0, 0x022b1, 0x022b2, 0x022b3, 0x022b4, 0x022b5, 0x022b6, 0x022b7, + 0x022b8, 0x022b9, 0x022ba, 0x022bb, 0x022bc, 0x022bd, 0x022be, 0x022bf, + 0x022c0, 0x022c1, 0x022c2, 0x022c3, 0x022c4, 0x022c5, 0x022c6, 0x022c7, + 0x022c8, 0x022c9, 0x022ca, 0x022cb, 0x022cc, 0x022cd, 0x022ce, 0x022cf, + 0x022d0, 0x022d1, 0x022d2, 0x022d3, 0x022d4, 0x022d5, 0x022d6, 0x022d7, + 0x022d8, 0x022d9, 0x022da, 0x022db, 0x022dc, 0x022dd, 0x022de, 0x022df, + 0x0227c, 0x0227d, 0x02291, 0x02292, 0x022e4, 0x022e5, 0x022e6, 0x022e7, + 0x022e8, 0x022e9, 0x022b2, 0x022b3, 0x022b4, 0x022b5, 0x022ee, 0x022ef, + 0x022f0, 0x022f1, 0x022f2, 0x022f3, 0x022f4, 0x022f5, 0x022f6, 0x022f7, + 0x022f8, 0x022f9, 0x022fa, 0x022fb, 0x022fc, 0x022fd, 0x022fe, 0x022ff +}; + +static uint32_t unicode_520_ci_page_24[] = { + 0x02400, 0x02401, 0x02402, 0x02403, 0x02404, 0x02405, 0x02406, 0x02407, + 0x02408, 0x02409, 0x0240a, 0x0240b, 0x0240c, 0x0240d, 0x0240e, 0x0240f, + 0x02410, 0x02411, 0x02412, 0x02413, 0x02414, 0x02415, 0x02416, 0x02417, + 0x02418, 0x02419, 0x0241a, 0x0241b, 0x0241c, 0x0241d, 0x0241e, 0x0241f, + 0x02420, 0x02421, 0x02422, 0x02423, 0x02424, 0x02425, 0x02426, 0x02427, + 0x02428, 0x02429, 0x0242a, 0x0242b, 0x0242c, 0x0242d, 0x0242e, 0x0242f, + 0x02430, 0x02431, 0x02432, 0x02433, 0x02434, 0x02435, 0x02436, 0x02437, + 0x02438, 0x02439, 0x0243a, 0x0243b, 0x0243c, 0x0243d, 0x0243e, 0x0243f, + 0x02440, 0x02441, 0x02442, 0x02443, 0x02444, 0x02445, 0x02446, 0x02447, + 0x02448, 0x02449, 0x0244a, 0x0244b, 0x0244c, 0x0244d, 0x0244e, 0x0244f, + 0x02450, 0x02451, 0x02452, 0x02453, 0x02454, 0x02455, 0x02456, 0x02457, + 0x02458, 0x02459, 0x0245a, 0x0245b, 0x0245c, 0x0245d, 0x0245e, 0x0245f, + 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, + 0x00039, 0x02469, 0x0246a, 0x0246b, 0x0246c, 0x0246d, 0x0246e, 0x0246f, + 0x02470, 0x02471, 0x02472, 0x02473, 0x02474, 0x02475, 0x02476, 0x02477, + 0x02478, 0x02479, 0x0247a, 0x0247b, 0x0247c, 0x0247d, 0x0247e, 0x0247f, + 0x02480, 0x02481, 0x02482, 0x02483, 0x02484, 0x02485, 0x02486, 0x02487, + 0x02488, 0x02489, 0x0248a, 0x0248b, 0x0248c, 0x0248d, 0x0248e, 0x0248f, + 0x02490, 0x02491, 0x02492, 0x02493, 0x02494, 0x02495, 0x02496, 0x02497, + 0x02498, 0x02499, 0x0249a, 0x0249b, 0x0249c, 0x0249d, 0x0249e, 0x0249f, + 0x024a0, 0x024a1, 0x024a2, 0x024a3, 0x024a4, 0x024a5, 0x024a6, 0x024a7, + 0x024a8, 0x024a9, 0x024aa, 0x024ab, 0x024ac, 0x024ad, 0x024ae, 0x024af, + 0x024b0, 0x024b1, 0x024b2, 0x024b3, 0x024b4, 0x024b5, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00030, 0x0246a, 0x0246b, 0x0246c, 0x0246d, 0x0246e, + 0x0246f, 0x02470, 0x02471, 0x02472, 0x02473, 0x00031, 0x00032, 0x00033, + 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x02469, 0x024ff +}; + +static uint32_t unicode_520_ci_page_27[] = { + 0x02700, 0x02701, 0x02702, 0x02703, 0x02704, 0x02705, 0x02706, 0x02707, + 0x02708, 0x02709, 0x0270a, 0x0270b, 0x0270c, 0x0270d, 0x0270e, 0x0270f, + 0x02710, 0x02711, 0x02712, 0x02713, 0x02714, 0x02715, 0x02716, 0x02717, + 0x02718, 0x02719, 0x0271a, 0x0271b, 0x0271c, 0x0271d, 0x0271e, 0x0271f, + 0x02720, 0x02721, 0x02722, 0x02723, 0x02724, 0x02725, 0x02726, 0x02727, + 0x02728, 0x02729, 0x0272a, 0x0272b, 0x0272c, 0x0272d, 0x0272e, 0x0272f, + 0x02730, 0x02731, 0x02732, 0x02733, 0x02734, 0x02735, 0x02736, 0x02737, + 0x02738, 0x02739, 0x0273a, 0x0273b, 0x0273c, 0x0273d, 0x0273e, 0x0273f, + 0x02740, 0x02741, 0x02742, 0x02743, 0x02744, 0x02745, 0x02746, 0x02747, + 0x02748, 0x02749, 0x0274a, 0x0274b, 0x0274c, 0x0274d, 0x0274e, 0x0274f, + 0x02750, 0x02751, 0x02752, 0x02753, 0x02754, 0x02755, 0x02756, 0x02757, + 0x02758, 0x02759, 0x0275a, 0x0275b, 0x0275c, 0x0275d, 0x0275e, 0x0275f, + 0x02760, 0x02761, 0x02762, 0x02763, 0x02764, 0x02765, 0x02766, 0x02767, + 0x02768, 0x02769, 0x0276a, 0x0276b, 0x0276c, 0x0276d, 0x0276e, 0x0276f, + 0x02770, 0x02771, 0x02772, 0x02773, 0x02774, 0x02775, 0x00031, 0x00032, + 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x02469, + 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, + 0x00039, 0x02469, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, + 0x00037, 0x00038, 0x00039, 0x02469, 0x02794, 0x02795, 0x02796, 0x02797, + 0x02798, 0x02799, 0x0279a, 0x0279b, 0x0279c, 0x0279d, 0x0279e, 0x0279f, + 0x027a0, 0x027a1, 0x027a2, 0x027a3, 0x027a4, 0x027a5, 0x027a6, 0x027a7, + 0x027a8, 0x027a9, 0x027aa, 0x027ab, 0x027ac, 0x027ad, 0x027ae, 0x027af, + 0x027b0, 0x027b1, 0x027b2, 0x027b3, 0x027b4, 0x027b5, 0x027b6, 0x027b7, + 0x027b8, 0x027b9, 0x027ba, 0x027bb, 0x027bc, 0x027bd, 0x027be, 0x027bf, + 0x027c0, 0x027c1, 0x027c2, 0x027c3, 0x027c4, 0x027c5, 0x027c6, 0x027c7, + 0x027c8, 0x027c9, 0x027ca, 0x027cb, 0x027cc, 0x027cd, 0x027ce, 0x027cf, + 0x027d0, 0x027d1, 0x027d2, 0x027d3, 0x027d4, 0x027d5, 0x027d6, 0x027d7, + 0x027d8, 0x027d9, 0x027da, 0x027db, 0x027dc, 0x027dd, 0x027de, 0x027df, + 0x027e0, 0x027e1, 0x027e2, 0x027e3, 0x027e4, 0x027e5, 0x027e6, 0x027e7, + 0x027e8, 0x027e9, 0x027ea, 0x027eb, 0x027ec, 0x027ed, 0x027ee, 0x027ef, + 0x027f0, 0x027f1, 0x027f2, 0x027f3, 0x027f4, 0x027f5, 0x027f6, 0x027f7, + 0x027f8, 0x027f9, 0x027fa, 0x027fb, 0x027fc, 0x027fd, 0x027fe, 0x027ff +}; + +static uint32_t unicode_520_ci_page_2a[] = { + 0x02a00, 0x02a01, 0x02a02, 0x02a03, 0x02a04, 0x02a05, 0x02a06, 0x02a07, + 0x02a08, 0x02a09, 0x02a0a, 0x02a0b, 0x02a0c, 0x02a0d, 0x02a0e, 0x02a0f, + 0x02a10, 0x02a11, 0x02a12, 0x02a13, 0x02a14, 0x02a15, 0x02a16, 0x02a17, + 0x02a18, 0x02a19, 0x02a1a, 0x02a1b, 0x02a1c, 0x02a1d, 0x02a1e, 0x02a1f, + 0x02a20, 0x02a21, 0x02a22, 0x02a23, 0x02a24, 0x02a25, 0x02a26, 0x02a27, + 0x02a28, 0x02a29, 0x02a2a, 0x02a2b, 0x02a2c, 0x02a2d, 0x02a2e, 0x02a2f, + 0x02a30, 0x02a31, 0x02a32, 0x02a33, 0x02a34, 0x02a35, 0x02a36, 0x02a37, + 0x02a38, 0x02a39, 0x02a3a, 0x02a3b, 0x02a3c, 0x02a3d, 0x02a3e, 0x02a3f, + 0x02a40, 0x02a41, 0x02a42, 0x02a43, 0x02a44, 0x02a45, 0x02a46, 0x02a47, + 0x02a48, 0x02a49, 0x02a4a, 0x02a4b, 0x02a4c, 0x02a4d, 0x02a4e, 0x02a4f, + 0x02a50, 0x02a51, 0x02a52, 0x02a53, 0x02a54, 0x02a55, 0x02a56, 0x02a57, + 0x02a58, 0x02a59, 0x02a5a, 0x02a5b, 0x02a5c, 0x02a5d, 0x02a5e, 0x02a5f, + 0x02a60, 0x02a61, 0x02a62, 0x02a63, 0x02a64, 0x02a65, 0x02a66, 0x02a67, + 0x02a68, 0x02a69, 0x02a6a, 0x02a6b, 0x02a6c, 0x02a6d, 0x02a6e, 0x02a6f, + 0x02a70, 0x02a71, 0x02a72, 0x02a73, 0x02a74, 0x02a75, 0x02a76, 0x02a77, + 0x02a78, 0x02a79, 0x02a7a, 0x02a7b, 0x02a7c, 0x02a7d, 0x02a7e, 0x02a7f, + 0x02a80, 0x02a81, 0x02a82, 0x02a83, 0x02a84, 0x02a85, 0x02a86, 0x02a87, + 0x02a88, 0x02a89, 0x02a8a, 0x02a8b, 0x02a8c, 0x02a8d, 0x02a8e, 0x02a8f, + 0x02a90, 0x02a91, 0x02a92, 0x02a93, 0x02a94, 0x02a95, 0x02a96, 0x02a97, + 0x02a98, 0x02a99, 0x02a9a, 0x02a9b, 0x02a9c, 0x02a9d, 0x02a9e, 0x02a9f, + 0x02aa0, 0x02aa1, 0x02aa2, 0x02aa3, 0x02aa4, 0x02aa5, 0x02aa6, 0x02aa7, + 0x02aa8, 0x02aa9, 0x02aaa, 0x02aab, 0x02aac, 0x02aad, 0x02aae, 0x02aaf, + 0x02ab0, 0x02ab1, 0x02ab2, 0x02ab3, 0x02ab4, 0x02ab5, 0x02ab6, 0x02ab7, + 0x02ab8, 0x02ab9, 0x02aba, 0x02abb, 0x02abc, 0x02abd, 0x02abe, 0x02abf, + 0x02ac0, 0x02ac1, 0x02ac2, 0x02ac3, 0x02ac4, 0x02ac5, 0x02ac6, 0x02ac7, + 0x02ac8, 0x02ac9, 0x02aca, 0x02acb, 0x02acc, 0x02acd, 0x02ace, 0x02acf, + 0x02ad0, 0x02ad1, 0x02ad2, 0x02ad3, 0x02ad4, 0x02ad5, 0x02ad6, 0x02ad7, + 0x02ad8, 0x02ad9, 0x02ada, 0x02adb, 0x02adc, 0x02adc, 0x02ade, 0x02adf, + 0x02ae0, 0x02ae1, 0x02ae2, 0x02ae3, 0x02ae4, 0x02ae5, 0x02ae6, 0x02ae7, + 0x02ae8, 0x02ae9, 0x02aea, 0x02aeb, 0x02aec, 0x02aed, 0x02aee, 0x02aef, + 0x02af0, 0x02af1, 0x02af2, 0x02af3, 0x02af4, 0x02af5, 0x02af6, 0x02af7, + 0x02af8, 0x02af9, 0x02afa, 0x02afb, 0x02afc, 0x02afd, 0x02afe, 0x02aff +}; + +static uint32_t unicode_520_ci_page_2c[] = { + 0x02c00, 0x02c01, 0x02c02, 0x02c03, 0x02c04, 0x02c05, 0x02c06, 0x02c07, + 0x02c08, 0x02c09, 0x02c0a, 0x02c0b, 0x02c0c, 0x02c0d, 0x02c0e, 0x02c0f, + 0x02c10, 0x02c11, 0x02c12, 0x02c13, 0x02c14, 0x02c15, 0x02c16, 0x02c17, + 0x02c18, 0x02c19, 0x02c1a, 0x02c1b, 0x02c1c, 0x02c1d, 0x02c1e, 0x02c1f, + 0x02c20, 0x02c21, 0x02c22, 0x02c23, 0x02c24, 0x02c25, 0x02c26, 0x02c27, + 0x02c28, 0x02c29, 0x02c2a, 0x02c2b, 0x02c2c, 0x02c2d, 0x02c2e, 0x02c2f, + 0x02c00, 0x02c01, 0x02c02, 0x02c03, 0x02c04, 0x02c05, 0x02c06, 0x02c07, + 0x02c08, 0x02c09, 0x02c0a, 0x02c0b, 0x02c0c, 0x02c0d, 0x02c0e, 0x02c0f, + 0x02c10, 0x02c11, 0x02c12, 0x02c13, 0x02c14, 0x02c15, 0x02c16, 0x02c17, + 0x02c18, 0x02c19, 0x02c1a, 0x02c1b, 0x02c1c, 0x02c1d, 0x02c1e, 0x02c1f, + 0x02c20, 0x02c21, 0x02c22, 0x02c23, 0x02c24, 0x02c25, 0x02c26, 0x02c27, + 0x02c28, 0x02c29, 0x02c2a, 0x02c2b, 0x02c2c, 0x02c2d, 0x02c2e, 0x02c5f, + 0x02c60, 0x02c60, 0x0026b, 0x01d7d, 0x0027d, 0x0023a, 0x0023e, 0x02c67, + 0x02c67, 0x02c69, 0x02c69, 0x02c6b, 0x02c6b, 0x00251, 0x00271, 0x00250, + 0x00252, 0x02c71, 0x02c72, 0x02c72, 0x02c74, 0x02c75, 0x02c75, 0x02c77, + 0x02c78, 0x02c79, 0x02c7a, 0x02c7b, 0x0004a, 0x00056, 0x0023f, 0x00240, + 0x02c80, 0x02c80, 0x02c82, 0x02c82, 0x02c84, 0x02c84, 0x02c86, 0x02c86, + 0x02c88, 0x02c88, 0x02c8a, 0x02c8a, 0x02c8c, 0x02c8c, 0x02c8e, 0x02c8e, + 0x02c90, 0x02c90, 0x02c92, 0x02c92, 0x02c94, 0x02c94, 0x02c96, 0x02c96, + 0x02c98, 0x02c98, 0x02c9a, 0x02c9a, 0x02c9c, 0x02c9c, 0x02c9e, 0x02c9e, + 0x02ca0, 0x02ca0, 0x02ca2, 0x02ca2, 0x02ca4, 0x02ca4, 0x02ca6, 0x02ca6, + 0x02ca8, 0x02ca8, 0x02caa, 0x02caa, 0x02cac, 0x02cac, 0x02cae, 0x02cae, + 0x02cb0, 0x02cb0, 0x02cb2, 0x02cb2, 0x02cb4, 0x02cb4, 0x02cb6, 0x02cb6, + 0x02cb8, 0x02cb8, 0x02cba, 0x02cba, 0x02cbc, 0x02cbc, 0x02cbe, 0x02cbe, + 0x02cc0, 0x02cc0, 0x02cc2, 0x02cc2, 0x02cc4, 0x02cc4, 0x02cc6, 0x02cc6, + 0x02cc8, 0x02cc8, 0x02cca, 0x02cca, 0x02ccc, 0x02ccc, 0x02cce, 0x02cce, + 0x02cd0, 0x02cd0, 0x02cd2, 0x02cd2, 0x02cd4, 0x02cd4, 0x02cd6, 0x02cd6, + 0x02cd8, 0x02cd8, 0x02cda, 0x02cda, 0x02cdc, 0x02cdc, 0x02cde, 0x02cde, + 0x02ce0, 0x02ce0, 0x02ce2, 0x02ce2, 0x02ce4, 0x02ce5, 0x02ce6, 0x02ce7, + 0x02ce8, 0x02ce9, 0x02cea, 0x02ceb, 0x02ceb, 0x02ced, 0x02ced, 0x00000, + 0x00000, 0x00000, 0x02cf2, 0x02cf3, 0x02cf4, 0x02cf5, 0x02cf6, 0x02cf7, + 0x02cf8, 0x02cf9, 0x02cfa, 0x02cfb, 0x02cfc, 0x02cfd, 0x02cfe, 0x02cff +}; + +static uint32_t unicode_520_ci_page_2d[] = { + 0x010a0, 0x010a1, 0x010a2, 0x010a3, 0x010a4, 0x010a5, 0x010a6, 0x010a7, + 0x010a8, 0x010a9, 0x010aa, 0x010ab, 0x010ac, 0x010ad, 0x010ae, 0x010af, + 0x010b0, 0x010b1, 0x010b2, 0x010b3, 0x010b4, 0x010b5, 0x010b6, 0x010b7, + 0x010b8, 0x010b9, 0x010ba, 0x010bb, 0x010bc, 0x010bd, 0x010be, 0x010bf, + 0x010c0, 0x010c1, 0x010c2, 0x010c3, 0x010c4, 0x010c5, 0x02d26, 0x02d27, + 0x02d28, 0x02d29, 0x02d2a, 0x02d2b, 0x02d2c, 0x02d2d, 0x02d2e, 0x02d2f, + 0x02d30, 0x02d31, 0x02d32, 0x02d33, 0x02d34, 0x02d35, 0x02d36, 0x02d37, + 0x02d38, 0x02d39, 0x02d3a, 0x02d3b, 0x02d3c, 0x02d3d, 0x02d3e, 0x02d3f, + 0x02d40, 0x02d41, 0x02d42, 0x02d43, 0x02d44, 0x02d45, 0x02d46, 0x02d47, + 0x02d48, 0x02d49, 0x02d4a, 0x02d4b, 0x02d4c, 0x02d4d, 0x02d4e, 0x02d4f, + 0x02d50, 0x02d51, 0x02d52, 0x02d53, 0x02d54, 0x02d55, 0x02d56, 0x02d57, + 0x02d58, 0x02d59, 0x02d5a, 0x02d5b, 0x02d5c, 0x02d5d, 0x02d5e, 0x02d5f, + 0x02d60, 0x02d61, 0x02d62, 0x02d63, 0x02d64, 0x02d65, 0x02d66, 0x02d67, + 0x02d68, 0x02d69, 0x02d6a, 0x02d6b, 0x02d6c, 0x02d6d, 0x02d6e, 0x02d6f, + 0x02d70, 0x02d71, 0x02d72, 0x02d73, 0x02d74, 0x02d75, 0x02d76, 0x02d77, + 0x02d78, 0x02d79, 0x02d7a, 0x02d7b, 0x02d7c, 0x02d7d, 0x02d7e, 0x02d7f, + 0x02d80, 0x02d81, 0x02d82, 0x02d83, 0x02d84, 0x02d85, 0x02d86, 0x02d87, + 0x02d88, 0x02d89, 0x02d8a, 0x02d8b, 0x02d8c, 0x02d8d, 0x02d8e, 0x02d8f, + 0x02d90, 0x02d91, 0x02d92, 0x02d93, 0x02d94, 0x02d95, 0x02d96, 0x02d97, + 0x02d98, 0x02d99, 0x02d9a, 0x02d9b, 0x02d9c, 0x02d9d, 0x02d9e, 0x02d9f, + 0x02da0, 0x02da1, 0x02da2, 0x02da3, 0x02da4, 0x02da5, 0x02da6, 0x02da7, + 0x02da8, 0x02da9, 0x02daa, 0x02dab, 0x02dac, 0x02dad, 0x02dae, 0x02daf, + 0x02db0, 0x02db1, 0x02db2, 0x02db3, 0x02db4, 0x02db5, 0x02db6, 0x02db7, + 0x02db8, 0x02db9, 0x02dba, 0x02dbb, 0x02dbc, 0x02dbd, 0x02dbe, 0x02dbf, + 0x02dc0, 0x02dc1, 0x02dc2, 0x02dc3, 0x02dc4, 0x02dc5, 0x02dc6, 0x02dc7, + 0x02dc8, 0x02dc9, 0x02dca, 0x02dcb, 0x02dcc, 0x02dcd, 0x02dce, 0x02dcf, + 0x02dd0, 0x02dd1, 0x02dd2, 0x02dd3, 0x02dd4, 0x02dd5, 0x02dd6, 0x02dd7, + 0x02dd8, 0x02dd9, 0x02dda, 0x02ddb, 0x02ddc, 0x02ddd, 0x02dde, 0x02ddf, + 0x00411, 0x00412, 0x00413, 0x00414, 0x00416, 0x00417, 0x0041a, 0x0041b, + 0x0041c, 0x0041d, 0x0041e, 0x0041f, 0x00420, 0x00421, 0x00422, 0x00425, + 0x00426, 0x00427, 0x00428, 0x00429, 0x00472, 0x02df5, 0x00410, 0x00400, + 0x02df8, 0x02df9, 0x00462, 0x0042e, 0x02dfc, 0x00466, 0x0046a, 0x02dff +}; + +static uint32_t unicode_520_ci_page_2e[] = { + 0x02e00, 0x02e01, 0x02e02, 0x02e03, 0x02e04, 0x02e05, 0x02e06, 0x02e07, + 0x02e08, 0x02e09, 0x02e0a, 0x02e0b, 0x02e0c, 0x02e0d, 0x02e0e, 0x02e0f, + 0x02e10, 0x02e11, 0x02e12, 0x02e13, 0x02e14, 0x02e15, 0x02e16, 0x02e17, + 0x02e18, 0x02e19, 0x02e1a, 0x02e1b, 0x02e1c, 0x02e1d, 0x02e1e, 0x02e1f, + 0x02e20, 0x02e21, 0x02e22, 0x02e23, 0x02e24, 0x02e25, 0x02e26, 0x02e27, + 0x02e28, 0x02e29, 0x02e2a, 0x02e2b, 0x02e2c, 0x02e2d, 0x02e2e, 0x02e2f, + 0x02e30, 0x02e31, 0x02e32, 0x02e33, 0x02e34, 0x02e35, 0x02e36, 0x02e37, + 0x02e38, 0x02e39, 0x02e3a, 0x02e3b, 0x02e3c, 0x02e3d, 0x02e3e, 0x02e3f, + 0x02e40, 0x02e41, 0x02e42, 0x02e43, 0x02e44, 0x02e45, 0x02e46, 0x02e47, + 0x02e48, 0x02e49, 0x02e4a, 0x02e4b, 0x02e4c, 0x02e4d, 0x02e4e, 0x02e4f, + 0x02e50, 0x02e51, 0x02e52, 0x02e53, 0x02e54, 0x02e55, 0x02e56, 0x02e57, + 0x02e58, 0x02e59, 0x02e5a, 0x02e5b, 0x02e5c, 0x02e5d, 0x02e5e, 0x02e5f, + 0x02e60, 0x02e61, 0x02e62, 0x02e63, 0x02e64, 0x02e65, 0x02e66, 0x02e67, + 0x02e68, 0x02e69, 0x02e6a, 0x02e6b, 0x02e6c, 0x02e6d, 0x02e6e, 0x02e6f, + 0x02e70, 0x02e71, 0x02e72, 0x02e73, 0x02e74, 0x02e75, 0x02e76, 0x02e77, + 0x02e78, 0x02e79, 0x02e7a, 0x02e7b, 0x02e7c, 0x02e7d, 0x02e7e, 0x02e7f, + 0x02e80, 0x02e81, 0x02e82, 0x02e83, 0x0319a, 0x02e85, 0x02e86, 0x02f0f, + 0x02f11, 0x02e89, 0x02f18, 0x02e8b, 0x02f29, 0x02f29, 0x02e8e, 0x02e8f, + 0x02e8e, 0x02e8f, 0x02e92, 0x02e93, 0x02e94, 0x02e95, 0x02e96, 0x02f3c, + 0x02e98, 0x02e99, 0x02e9a, 0x02e9b, 0x02f47, 0x02f49, 0x02e9e, 0x02e9f, + 0x02ea0, 0x02ea1, 0x02ea2, 0x02ea3, 0x02ea4, 0x02ea4, 0x02ea6, 0x02f5c, + 0x02ea8, 0x02ea9, 0x02eaa, 0x02f6c, 0x02f70, 0x02ead, 0x02f75, 0x02eaf, + 0x02eb0, 0x02eb1, 0x02eb2, 0x02eb1, 0x02eb1, 0x02eb2, 0x02eb7, 0x02eb7, + 0x02eb8, 0x02eb9, 0x02eba, 0x02f80, 0x02f81, 0x02f85, 0x02ebe, 0x02ebe, + 0x02ebe, 0x02ec1, 0x02ec2, 0x02ec3, 0x02ec4, 0x02ec5, 0x02f93, 0x02f93, + 0x02ec8, 0x02ec9, 0x02f9c, 0x02ecb, 0x02ecc, 0x02ecc, 0x02ecc, 0x02fa2, + 0x02ed0, 0x02fa7, 0x02ed2, 0x02ed3, 0x02ed4, 0x02fa9, 0x02ed6, 0x02fac, + 0x02ed8, 0x02ed9, 0x02eda, 0x02edb, 0x02edc, 0x02fb7, 0x02ede, 0x02ede, + 0x02ee0, 0x02fb8, 0x02ee2, 0x02fbb, 0x02fc1, 0x02ee5, 0x02ee6, 0x02ee7, + 0x02ee8, 0x02ee9, 0x02eea, 0x02eeb, 0x02eec, 0x02eed, 0x02eee, 0x02eef, + 0x02ef0, 0x02ef2, 0x02ef2, 0x02ef3, 0x02ef4, 0x02ef5, 0x02ef6, 0x02ef7, + 0x02ef8, 0x02ef9, 0x02efa, 0x02efb, 0x02efc, 0x02efd, 0x02efe, 0x02eff +}; + +static uint32_t unicode_520_ci_page_2f[] = { + 0x03192, 0x02f01, 0x02e80, 0x02f03, 0x0319a, 0x02f05, 0x03193, 0x02f07, + 0x0319f, 0x02f09, 0x02f0a, 0x02f0b, 0x02e86, 0x02f0d, 0x02f0e, 0x02f0f, + 0x02f10, 0x02f11, 0x02f12, 0x02f13, 0x02f14, 0x02f15, 0x02f16, 0x02f17, + 0x02f18, 0x02e8b, 0x02e81, 0x02f1b, 0x02f1c, 0x02f1d, 0x02f1e, 0x02f1f, + 0x02f20, 0x02f21, 0x02f22, 0x02f23, 0x02f24, 0x02f25, 0x02f26, 0x02f27, + 0x02f28, 0x02f29, 0x02e8e, 0x02f2b, 0x02f2c, 0x02f2d, 0x02f2e, 0x02f2f, + 0x02f30, 0x02f31, 0x02f32, 0x02e93, 0x02f34, 0x02f35, 0x02f36, 0x02f37, + 0x02f38, 0x02e95, 0x02f3a, 0x02f3b, 0x02f3c, 0x02f3d, 0x02f3e, 0x02f3f, + 0x02f40, 0x02f41, 0x02f42, 0x02f43, 0x02f44, 0x02f45, 0x02f46, 0x02f47, + 0x02f48, 0x02f49, 0x02f4a, 0x02f4b, 0x02f4c, 0x02f4d, 0x02f4e, 0x02f4f, + 0x02f50, 0x02f51, 0x02f52, 0x02f53, 0x02f54, 0x02f55, 0x02f56, 0x02f57, + 0x02f58, 0x02f59, 0x02f5a, 0x02f5b, 0x02f5c, 0x02f5d, 0x02f5e, 0x02f5f, + 0x02f60, 0x02f61, 0x02f62, 0x02f63, 0x02f64, 0x02f65, 0x02eaa, 0x02f67, + 0x02f68, 0x02f69, 0x02f6a, 0x02f6b, 0x02f6c, 0x02f6d, 0x02f6e, 0x02f6f, + 0x02f70, 0x02f71, 0x02f72, 0x02f73, 0x02f74, 0x02f75, 0x02f76, 0x02f77, + 0x02f78, 0x02f79, 0x02eb7, 0x02f7b, 0x02f7c, 0x02f7d, 0x02f7e, 0x02f7f, + 0x02f80, 0x02f81, 0x02f82, 0x02f83, 0x02f84, 0x02f85, 0x02f86, 0x02f87, + 0x02f88, 0x02f89, 0x02f8a, 0x02f8b, 0x02f8c, 0x02f8d, 0x02f8e, 0x02f8f, + 0x02f90, 0x02f91, 0x02f92, 0x02f93, 0x02f94, 0x02f95, 0x02f96, 0x02f97, + 0x02f98, 0x02f99, 0x02f9a, 0x02f9b, 0x02f9c, 0x02f9d, 0x02f9e, 0x02f9f, + 0x02fa0, 0x02fa1, 0x02fa2, 0x02fa3, 0x02fa4, 0x02fa5, 0x02fa6, 0x02fa7, + 0x02fa8, 0x02fa9, 0x02faa, 0x02fab, 0x02fac, 0x02fad, 0x02fae, 0x02faf, + 0x02fb0, 0x02fb1, 0x02fb2, 0x02fb3, 0x02fb4, 0x02fb5, 0x02fb6, 0x02fb7, + 0x02fb8, 0x02fb9, 0x02fba, 0x02fbb, 0x02fbc, 0x02fbd, 0x02fbe, 0x02fbf, + 0x02fc0, 0x02fc1, 0x02fc2, 0x02fc3, 0x02ee7, 0x02fc5, 0x02fc6, 0x02fc7, + 0x02fc8, 0x02fc9, 0x02fca, 0x02fcb, 0x02fcc, 0x02fcd, 0x02fce, 0x02fcf, + 0x02fd0, 0x02eeb, 0x02eed, 0x02eef, 0x02ef2, 0x02fd5, 0x02fd6, 0x02fd7, + 0x02fd8, 0x02fd9, 0x02fda, 0x02fdb, 0x02fdc, 0x02fdd, 0x02fde, 0x02fdf, + 0x02fe0, 0x02fe1, 0x02fe2, 0x02fe3, 0x02fe4, 0x02fe5, 0x02fe6, 0x02fe7, + 0x02fe8, 0x02fe9, 0x02fea, 0x02feb, 0x02fec, 0x02fed, 0x02fee, 0x02fef, + 0x02ff0, 0x02ff1, 0x02ff2, 0x02ff3, 0x02ff4, 0x02ff5, 0x02ff6, 0x02ff7, + 0x02ff8, 0x02ff9, 0x02ffa, 0x02ffb, 0x02ffc, 0x02ffd, 0x02ffe, 0x02fff +}; + +static uint32_t unicode_520_ci_page_30[] = { + 0x00020, 0x03001, 0x03002, 0x03003, 0x03004, 0x03005, 0x03006, 0x00030, + 0x02329, 0x0232a, 0x0300a, 0x0300b, 0x0300c, 0x0300d, 0x0300e, 0x0300f, + 0x03010, 0x03011, 0x03012, 0x03013, 0x03014, 0x03015, 0x03016, 0x03017, + 0x03018, 0x03019, 0x0301a, 0x0301b, 0x0301c, 0x0301d, 0x0301e, 0x0301f, + 0x03020, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x03030, 0x03031, 0x03031, 0x03033, 0x03033, 0x03035, 0x03012, 0x03037, + 0x02f17, 0x03039, 0x0303a, 0x0303b, 0x0303c, 0x0303d, 0x0303e, 0x0303f, + 0x03040, 0x03042, 0x03042, 0x03044, 0x03044, 0x03046, 0x03046, 0x03048, + 0x03048, 0x0304a, 0x0304a, 0x0304b, 0x0304b, 0x0304d, 0x0304d, 0x0304f, + 0x0304f, 0x03051, 0x03051, 0x03053, 0x03053, 0x03055, 0x03055, 0x03057, + 0x03057, 0x03059, 0x03059, 0x0305b, 0x0305b, 0x0305d, 0x0305d, 0x0305f, + 0x0305f, 0x03061, 0x03061, 0x03064, 0x03064, 0x03064, 0x03066, 0x03066, + 0x03068, 0x03068, 0x0306a, 0x0306b, 0x0306c, 0x0306d, 0x0306e, 0x0306f, + 0x0306f, 0x0306f, 0x03072, 0x03072, 0x03072, 0x03075, 0x03075, 0x03075, + 0x03078, 0x03078, 0x03078, 0x0307b, 0x0307b, 0x0307b, 0x0307e, 0x0307f, + 0x03080, 0x03081, 0x03082, 0x03084, 0x03084, 0x03086, 0x03086, 0x03088, + 0x03088, 0x03089, 0x0308a, 0x0308b, 0x0308c, 0x0308d, 0x0308f, 0x0308f, + 0x03090, 0x03091, 0x03092, 0x03093, 0x03046, 0x0304b, 0x03051, 0x03097, + 0x03098, 0x00000, 0x00000, 0x0309b, 0x0309c, 0x0309d, 0x0309d, 0x0309f, + 0x030a0, 0x03042, 0x03042, 0x03044, 0x03044, 0x03046, 0x03046, 0x03048, + 0x03048, 0x0304a, 0x0304a, 0x0304b, 0x0304b, 0x0304d, 0x0304d, 0x0304f, + 0x0304f, 0x03051, 0x03051, 0x03053, 0x03053, 0x03055, 0x03055, 0x03057, + 0x03057, 0x03059, 0x03059, 0x0305b, 0x0305b, 0x0305d, 0x0305d, 0x0305f, + 0x0305f, 0x03061, 0x03061, 0x03064, 0x03064, 0x03064, 0x03066, 0x03066, + 0x03068, 0x03068, 0x0306a, 0x0306b, 0x0306c, 0x0306d, 0x0306e, 0x0306f, + 0x0306f, 0x0306f, 0x03072, 0x03072, 0x03072, 0x03075, 0x03075, 0x03075, + 0x03078, 0x03078, 0x03078, 0x0307b, 0x0307b, 0x0307b, 0x0307e, 0x0307f, + 0x03080, 0x03081, 0x03082, 0x03084, 0x03084, 0x03086, 0x03086, 0x03088, + 0x03088, 0x03089, 0x0308a, 0x0308b, 0x0308c, 0x0308d, 0x0308f, 0x0308f, + 0x03090, 0x03091, 0x03092, 0x03093, 0x03046, 0x0304b, 0x03051, 0x0308f, + 0x03090, 0x03091, 0x03092, 0x030fb, 0x030fc, 0x030fd, 0x030fd, 0x030ff +}; + +static uint32_t unicode_520_ci_page_31[] = { + 0x03100, 0x03101, 0x03102, 0x03103, 0x03104, 0x03105, 0x03106, 0x03107, + 0x03108, 0x03109, 0x0310a, 0x0310b, 0x0310c, 0x0310d, 0x0310e, 0x0310f, + 0x03110, 0x03111, 0x03112, 0x03113, 0x03114, 0x03115, 0x03116, 0x03117, + 0x03118, 0x03119, 0x0311a, 0x0311b, 0x0311c, 0x0311d, 0x0311e, 0x0311f, + 0x03120, 0x03121, 0x03122, 0x03123, 0x03124, 0x03125, 0x03126, 0x03127, + 0x03128, 0x03129, 0x0312a, 0x0312b, 0x0312c, 0x0312d, 0x0312e, 0x0312f, + 0x03130, 0x01100, 0x01101, 0x011aa, 0x01102, 0x011ac, 0x011ad, 0x01103, + 0x01104, 0x01105, 0x011b0, 0x011b1, 0x011b2, 0x011b3, 0x011b4, 0x011b5, + 0x0111a, 0x01106, 0x01107, 0x01108, 0x01121, 0x01109, 0x0110a, 0x0110b, + 0x0110c, 0x0110d, 0x0110e, 0x0110f, 0x01110, 0x01111, 0x01112, 0x01161, + 0x01162, 0x01163, 0x01164, 0x01165, 0x01166, 0x01167, 0x01168, 0x01169, + 0x0116a, 0x0116b, 0x0116c, 0x0116d, 0x0116e, 0x0116f, 0x01170, 0x01171, + 0x01172, 0x01173, 0x01174, 0x01175, 0x01160, 0x01114, 0x01115, 0x011c7, + 0x011c8, 0x011cc, 0x011ce, 0x011d3, 0x011d7, 0x011d9, 0x0111c, 0x011dd, + 0x011df, 0x0111d, 0x0111e, 0x01120, 0x01122, 0x01123, 0x01127, 0x01129, + 0x0112b, 0x0112c, 0x0112d, 0x0112e, 0x0112f, 0x01132, 0x01136, 0x01140, + 0x01147, 0x0114c, 0x011f1, 0x011f2, 0x01157, 0x01158, 0x01159, 0x01184, + 0x01185, 0x01188, 0x01191, 0x01192, 0x01194, 0x0119e, 0x011a1, 0x0318f, + 0x03190, 0x03191, 0x03192, 0x03193, 0x03194, 0x03195, 0x03196, 0x03197, + 0x03198, 0x03199, 0x0319a, 0x0319b, 0x0319c, 0x0319d, 0x0319e, 0x0319f, + 0x03105, 0x03117, 0x03110, 0x0310d, 0x031a4, 0x031a4, 0x031a6, 0x0311b, + 0x03128, 0x0311a, 0x03127, 0x03128, 0x031ac, 0x031ad, 0x0311e, 0x03120, + 0x031b0, 0x031b1, 0x031b2, 0x03127, 0x03106, 0x0310a, 0x0310e, 0x0310f, + 0x031b8, 0x031b9, 0x031ba, 0x031bb, 0x031bc, 0x031bd, 0x031be, 0x031bf, + 0x031c0, 0x031c1, 0x031c2, 0x031c3, 0x031c4, 0x031c5, 0x031c6, 0x031c7, + 0x031c8, 0x031c9, 0x031ca, 0x031cb, 0x031cc, 0x031cd, 0x031ce, 0x031cf, + 0x031d0, 0x031d1, 0x031d2, 0x031d3, 0x031d4, 0x031d5, 0x031d6, 0x031d7, + 0x031d8, 0x031d9, 0x031da, 0x031db, 0x031dc, 0x031dd, 0x031de, 0x031df, + 0x031e0, 0x031e1, 0x031e2, 0x031e3, 0x031e4, 0x031e5, 0x031e6, 0x031e7, + 0x031e8, 0x031e9, 0x031ea, 0x031eb, 0x031ec, 0x031ed, 0x031ee, 0x031ef, + 0x0304f, 0x03057, 0x03059, 0x03068, 0x0306c, 0x0306f, 0x03072, 0x03075, + 0x03078, 0x0307b, 0x03080, 0x03089, 0x0308a, 0x0308b, 0x0308c, 0x031ff +}; + +static uint32_t unicode_520_ci_page_32[] = { + 0x03200, 0x03201, 0x03202, 0x03203, 0x03204, 0x03205, 0x03206, 0x03207, + 0x03208, 0x03209, 0x0320a, 0x0320b, 0x0320c, 0x0320d, 0x0320e, 0x0320f, + 0x03210, 0x03211, 0x03212, 0x03213, 0x03214, 0x03215, 0x03216, 0x03217, + 0x03218, 0x03219, 0x0321a, 0x0321b, 0x0321c, 0x0321d, 0x0321e, 0x0321f, + 0x03220, 0x03221, 0x03222, 0x03223, 0x03224, 0x03225, 0x03226, 0x03227, + 0x03228, 0x03229, 0x0322a, 0x0322b, 0x0322c, 0x0322d, 0x0322e, 0x0322f, + 0x03230, 0x03231, 0x03232, 0x03233, 0x03234, 0x03235, 0x03236, 0x03237, + 0x03238, 0x03239, 0x0323a, 0x0323b, 0x0323c, 0x0323d, 0x0323e, 0x0323f, + 0x03240, 0x03241, 0x03242, 0x03243, 0x03244, 0x03245, 0x02f42, 0x03247, + 0x02469, 0x02473, 0x0324a, 0x0324b, 0x0324c, 0x0324d, 0x0324e, 0x0324f, + 0x03250, 0x03251, 0x03252, 0x03253, 0x03254, 0x03255, 0x03256, 0x03257, + 0x03258, 0x03259, 0x0324a, 0x0325b, 0x0325c, 0x0325d, 0x0325e, 0x0325f, + 0x01100, 0x01102, 0x01103, 0x01105, 0x01106, 0x01107, 0x01109, 0x0110b, + 0x0110c, 0x0110e, 0x0110f, 0x01110, 0x01111, 0x01112, 0x0326e, 0x0326f, + 0x03270, 0x03271, 0x03272, 0x03273, 0x03274, 0x03275, 0x03276, 0x03277, + 0x03278, 0x03279, 0x0327a, 0x0327b, 0x0327c, 0x0327d, 0x0327e, 0x0327f, + 0x03192, 0x03193, 0x03194, 0x03195, 0x03284, 0x03285, 0x03286, 0x02f0b, + 0x03288, 0x02f17, 0x02f49, 0x02f55, 0x02f54, 0x02f4a, 0x02fa6, 0x02f1f, + 0x02f47, 0x03291, 0x03292, 0x03293, 0x03294, 0x03295, 0x03296, 0x03297, + 0x03298, 0x03299, 0x0329a, 0x02f25, 0x0329c, 0x0329d, 0x0329e, 0x0329f, + 0x032a0, 0x032a1, 0x032a2, 0x032a3, 0x03196, 0x03197, 0x03198, 0x032a7, + 0x032a8, 0x032a9, 0x032aa, 0x032ab, 0x032ac, 0x032ad, 0x032ae, 0x032af, + 0x032b0, 0x032b1, 0x032b2, 0x032b3, 0x032b4, 0x0324b, 0x032b6, 0x032b7, + 0x032b8, 0x032b9, 0x032ba, 0x032bb, 0x032bc, 0x032bd, 0x032be, 0x0324c, + 0x032c0, 0x032c1, 0x032c2, 0x032c3, 0x032c4, 0x032c5, 0x032c6, 0x032c7, + 0x032c8, 0x032c9, 0x032ca, 0x032cb, 0x032cc, 0x032cd, 0x032ce, 0x032cf, + 0x03042, 0x03044, 0x03046, 0x03048, 0x0304a, 0x0304b, 0x0304d, 0x0304f, + 0x03051, 0x03053, 0x03055, 0x03057, 0x03059, 0x0305b, 0x0305d, 0x0305f, + 0x03061, 0x03064, 0x03066, 0x03068, 0x0306a, 0x0306b, 0x0306c, 0x0306d, + 0x0306e, 0x0306f, 0x03072, 0x03075, 0x03078, 0x0307b, 0x0307e, 0x0307f, + 0x03080, 0x03081, 0x03082, 0x03084, 0x03086, 0x03088, 0x03089, 0x0308a, + 0x0308b, 0x0308c, 0x0308d, 0x0308f, 0x03090, 0x03091, 0x03092, 0x032ff +}; + +static uint32_t unicode_520_ci_page_33[] = { + 0x03300, 0x03301, 0x03302, 0x03303, 0x03304, 0x03305, 0x03306, 0x03307, + 0x03308, 0x03309, 0x0330a, 0x0330b, 0x0330c, 0x0330d, 0x0330e, 0x0330f, + 0x03310, 0x03311, 0x03312, 0x03313, 0x03314, 0x03315, 0x03316, 0x03317, + 0x03318, 0x03319, 0x0331a, 0x0331b, 0x0331c, 0x0331d, 0x0331e, 0x0331f, + 0x03320, 0x03321, 0x03322, 0x03323, 0x03324, 0x03325, 0x03326, 0x03327, + 0x03328, 0x03329, 0x0332a, 0x0332b, 0x0332c, 0x0332d, 0x0332e, 0x0332f, + 0x03330, 0x03331, 0x03332, 0x03333, 0x03334, 0x03335, 0x03336, 0x03337, + 0x03338, 0x03339, 0x0333a, 0x0333b, 0x0333c, 0x0333d, 0x0333e, 0x0333f, + 0x03340, 0x03341, 0x03342, 0x03343, 0x03344, 0x03345, 0x03346, 0x03347, + 0x03348, 0x03349, 0x0334a, 0x0334b, 0x0334c, 0x0334d, 0x0334e, 0x0334f, + 0x03350, 0x03351, 0x03352, 0x03353, 0x03354, 0x03355, 0x03356, 0x03357, + 0x03358, 0x03359, 0x0335a, 0x0335b, 0x0335c, 0x0335d, 0x0335e, 0x0335f, + 0x03360, 0x03361, 0x03362, 0x03363, 0x03364, 0x03365, 0x03366, 0x03367, + 0x03368, 0x03369, 0x0336a, 0x0336b, 0x0336c, 0x0336d, 0x0336e, 0x0336f, + 0x03370, 0x03371, 0x03372, 0x03373, 0x03374, 0x03375, 0x03376, 0x03377, + 0x03378, 0x03379, 0x0337a, 0x0337b, 0x0337c, 0x0337d, 0x0337e, 0x0337f, + 0x03380, 0x03381, 0x03382, 0x03383, 0x03384, 0x03385, 0x03386, 0x03387, + 0x03388, 0x03389, 0x0338a, 0x0338b, 0x0338c, 0x0338d, 0x0338e, 0x0338f, + 0x03390, 0x03391, 0x03392, 0x03393, 0x03394, 0x03395, 0x03396, 0x03397, + 0x03398, 0x03399, 0x0339a, 0x0339b, 0x0339c, 0x0339d, 0x0339e, 0x0339f, + 0x033a0, 0x033a1, 0x033a2, 0x033a3, 0x033a4, 0x033a5, 0x033a6, 0x033a7, + 0x033a8, 0x03380, 0x033aa, 0x033ab, 0x033ac, 0x033ad, 0x033ae, 0x033af, + 0x033b0, 0x033b1, 0x033b2, 0x033b3, 0x033b4, 0x033b5, 0x033b6, 0x033b7, + 0x033b8, 0x033b7, 0x033ba, 0x033bb, 0x033bc, 0x033bd, 0x033be, 0x033bd, + 0x033c0, 0x033c1, 0x033c2, 0x033c3, 0x033c4, 0x033c5, 0x033c6, 0x033c7, + 0x00238, 0x033c9, 0x033ca, 0x033cb, 0x033cc, 0x033cd, 0x0339e, 0x033cf, + 0x033d0, 0x033d1, 0x033d2, 0x033d3, 0x03386, 0x033d5, 0x033d6, 0x033d7, + 0x033d8, 0x033d9, 0x033da, 0x033db, 0x033dc, 0x033dd, 0x033de, 0x033df, + 0x033e0, 0x033e1, 0x033e2, 0x033e3, 0x033e4, 0x033e5, 0x033e6, 0x033e7, + 0x033e8, 0x033e9, 0x033ea, 0x033eb, 0x033ec, 0x033ed, 0x033ee, 0x033ef, + 0x033f0, 0x033f1, 0x033f2, 0x033f3, 0x033f4, 0x033f5, 0x033f6, 0x033f7, + 0x033f8, 0x033f9, 0x033fa, 0x033fb, 0x033fc, 0x033fd, 0x033fe, 0x033ff +}; + +static uint32_t unicode_520_ci_page_a6[] = { + 0x0a600, 0x0a601, 0x0a602, 0x0a603, 0x0a604, 0x0a605, 0x0a606, 0x0a607, + 0x0a608, 0x0a609, 0x0a60a, 0x0a60b, 0x0a60c, 0x0a60d, 0x0a60e, 0x0a60f, + 0x0a558, 0x0a56a, 0x0a587, 0x0a613, 0x0a614, 0x0a615, 0x0a616, 0x0a617, + 0x0a618, 0x0a619, 0x0a61a, 0x0a61b, 0x0a61c, 0x0a61d, 0x0a61e, 0x0a61f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0a56e, 0x0a5d1, 0x0a62c, 0x0a62d, 0x0a62e, 0x0a62f, + 0x0a630, 0x0a631, 0x0a632, 0x0a633, 0x0a634, 0x0a635, 0x0a636, 0x0a637, + 0x0a638, 0x0a639, 0x0a63a, 0x0a63b, 0x0a63c, 0x0a63d, 0x0a63e, 0x0a63f, + 0x0a640, 0x0a640, 0x0a642, 0x0a642, 0x0a644, 0x0a644, 0x0a646, 0x0a646, + 0x02df8, 0x02df8, 0x02df9, 0x02df9, 0x0a64c, 0x0a64c, 0x0a64e, 0x0a64e, + 0x0a650, 0x0a650, 0x0a652, 0x0a652, 0x0a654, 0x0a654, 0x02dfc, 0x02dfc, + 0x0a658, 0x0a658, 0x0a65a, 0x0a65a, 0x0a65c, 0x0a65c, 0x0a65e, 0x0a65e, + 0x0a660, 0x0a661, 0x0a662, 0x0a662, 0x0a664, 0x0a664, 0x0a666, 0x0a666, + 0x0041e, 0x0041e, 0x0041e, 0x0041e, 0x0041e, 0x0041e, 0x0041e, 0x00000, + 0x00000, 0x00000, 0x00000, 0x0a673, 0x0a674, 0x0a675, 0x0a676, 0x0a677, + 0x0a678, 0x0a679, 0x0a67a, 0x0a67b, 0x00000, 0x00000, 0x0a67e, 0x0a67f, + 0x0a680, 0x0a680, 0x0a682, 0x0a682, 0x0a684, 0x0a684, 0x0a686, 0x0a686, + 0x0a688, 0x0a688, 0x0a68a, 0x0a68a, 0x0a68c, 0x0a68c, 0x0a68e, 0x0a68e, + 0x0a690, 0x0a690, 0x0a692, 0x0a692, 0x0a694, 0x0a694, 0x0a696, 0x0a696, + 0x0a698, 0x0a699, 0x0a69a, 0x0a69b, 0x0a69c, 0x0a69d, 0x0a69e, 0x0a69f, + 0x0a6a0, 0x0a6a1, 0x0a6a2, 0x0a6a3, 0x0a6a4, 0x0a6a5, 0x0a6a6, 0x0a6a7, + 0x0a6a8, 0x0a6a9, 0x0a6aa, 0x0a6ab, 0x0a6ac, 0x0a6ad, 0x0a6ae, 0x0a6af, + 0x0a6b0, 0x0a6b1, 0x0a6b2, 0x0a6b3, 0x0a6b4, 0x0a6b5, 0x0a6b6, 0x0a6b7, + 0x0a6b8, 0x0a6b9, 0x0a6ba, 0x0a6bb, 0x0a6bc, 0x0a6bd, 0x0a6be, 0x0a6bf, + 0x0a6c0, 0x0a6c1, 0x0a6c2, 0x0a6c3, 0x0a6c4, 0x0a6c5, 0x0a6c6, 0x0a6c7, + 0x0a6c8, 0x0a6c9, 0x0a6ca, 0x0a6cb, 0x0a6cc, 0x0a6cd, 0x0a6ce, 0x0a6cf, + 0x0a6d0, 0x0a6d1, 0x0a6d2, 0x0a6d3, 0x0a6d4, 0x0a6d5, 0x0a6d6, 0x0a6d7, + 0x0a6d8, 0x0a6d9, 0x0a6da, 0x0a6db, 0x0a6dc, 0x0a6dd, 0x0a6de, 0x0a6df, + 0x0a6e0, 0x0a6e1, 0x0a6e2, 0x0a6e3, 0x0a6e4, 0x0a6e5, 0x0a6e6, 0x0a6e7, + 0x0a6e8, 0x0a6e9, 0x0a6ea, 0x0a6eb, 0x0a6ec, 0x0a6ed, 0x0a6ee, 0x0a6ef, + 0x00000, 0x00000, 0x0a6f2, 0x0a6f3, 0x0a6f4, 0x0a6f5, 0x0a6f6, 0x0a6f7, + 0x0a6f8, 0x0a6f9, 0x0a6fa, 0x0a6fb, 0x0a6fc, 0x0a6fd, 0x0a6fe, 0x0a6ff +}; + +static uint32_t unicode_520_ci_page_a7[] = { + 0x0a700, 0x0a701, 0x0a702, 0x0a703, 0x0a704, 0x0a705, 0x0a706, 0x0a707, + 0x0a708, 0x0a709, 0x0a70a, 0x0a70b, 0x0a70c, 0x0a70d, 0x0a70e, 0x0a70f, + 0x0a710, 0x0a711, 0x0a712, 0x0a713, 0x0a714, 0x0a715, 0x0a716, 0x0a717, + 0x0a718, 0x0a719, 0x0a71a, 0x0a71b, 0x0a71c, 0x0a71d, 0x0a71e, 0x0a71f, + 0x0a720, 0x0a721, 0x0a722, 0x0a722, 0x0a724, 0x0a724, 0x0a726, 0x0a726, + 0x0a728, 0x0a728, 0x0a72a, 0x0a72a, 0x0a72c, 0x0a72c, 0x0a72e, 0x0a72e, + 0x0a730, 0x0a731, 0x0a732, 0x0a732, 0x01dd5, 0x01dd5, 0x03373, 0x03373, + 0x01dd6, 0x01dd6, 0x01dd6, 0x01dd6, 0x0a73c, 0x0a73c, 0x0a73e, 0x0a73e, + 0x0a740, 0x0a740, 0x0a742, 0x0a742, 0x0a744, 0x0a744, 0x0a746, 0x0a746, + 0x0a748, 0x0a748, 0x0a74a, 0x0a74a, 0x0a74c, 0x0a74c, 0x0a74e, 0x0a74e, + 0x0a750, 0x0a750, 0x0a752, 0x0a752, 0x0a754, 0x0a754, 0x0a756, 0x0a756, + 0x0a758, 0x0a758, 0x01de3, 0x01de3, 0x0a75c, 0x0a75c, 0x0a75e, 0x0a75e, + 0x0a760, 0x0a760, 0x0a762, 0x0a762, 0x0a764, 0x0a764, 0x0a766, 0x0a766, + 0x0a768, 0x0a768, 0x0a76a, 0x0a76a, 0x0a76c, 0x0a76c, 0x01dd2, 0x01dd2, + 0x01dd2, 0x0a771, 0x0a772, 0x0a773, 0x0a774, 0x0a775, 0x0a776, 0x0a777, + 0x0a778, 0x00044, 0x00044, 0x00046, 0x00046, 0x00047, 0x0a77e, 0x0a77e, + 0x0a780, 0x0a780, 0x00052, 0x00052, 0x00053, 0x00053, 0x00054, 0x00054, + 0x0a788, 0x0a789, 0x0a78a, 0x0a78b, 0x0a78b, 0x0a78d, 0x0a78e, 0x0a78f, + 0x0a790, 0x0a791, 0x0a792, 0x0a793, 0x0a794, 0x0a795, 0x0a796, 0x0a797, + 0x0a798, 0x0a799, 0x0a79a, 0x0a79b, 0x0a79c, 0x0a79d, 0x0a79e, 0x0a79f, + 0x0a7a0, 0x0a7a1, 0x0a7a2, 0x0a7a3, 0x0a7a4, 0x0a7a5, 0x0a7a6, 0x0a7a7, + 0x0a7a8, 0x0a7a9, 0x0a7aa, 0x0a7ab, 0x0a7ac, 0x0a7ad, 0x0a7ae, 0x0a7af, + 0x0a7b0, 0x0a7b1, 0x0a7b2, 0x0a7b3, 0x0a7b4, 0x0a7b5, 0x0a7b6, 0x0a7b7, + 0x0a7b8, 0x0a7b9, 0x0a7ba, 0x0a7bb, 0x0a7bc, 0x0a7bd, 0x0a7be, 0x0a7bf, + 0x0a7c0, 0x0a7c1, 0x0a7c2, 0x0a7c3, 0x0a7c4, 0x0a7c5, 0x0a7c6, 0x0a7c7, + 0x0a7c8, 0x0a7c9, 0x0a7ca, 0x0a7cb, 0x0a7cc, 0x0a7cd, 0x0a7ce, 0x0a7cf, + 0x0a7d0, 0x0a7d1, 0x0a7d2, 0x0a7d3, 0x0a7d4, 0x0a7d5, 0x0a7d6, 0x0a7d7, + 0x0a7d8, 0x0a7d9, 0x0a7da, 0x0a7db, 0x0a7dc, 0x0a7dd, 0x0a7de, 0x0a7df, + 0x0a7e0, 0x0a7e1, 0x0a7e2, 0x0a7e3, 0x0a7e4, 0x0a7e5, 0x0a7e6, 0x0a7e7, + 0x0a7e8, 0x0a7e9, 0x0a7ea, 0x0a7eb, 0x0a7ec, 0x0a7ed, 0x0a7ee, 0x0a7ef, + 0x0a7f0, 0x0a7f1, 0x0a7f2, 0x0a7f3, 0x0a7f4, 0x0a7f5, 0x0a7f6, 0x0a7f7, + 0x0a7f8, 0x0a7f9, 0x0a7fa, 0x0a7fb, 0x0a7fc, 0x0a7fd, 0x0a7fe, 0x0a7ff +}; + +static uint32_t unicode_520_ci_page_a8[] = { + 0x0a800, 0x0a801, 0x0a802, 0x0a803, 0x0a804, 0x0a805, 0x0a806, 0x0a807, + 0x0a808, 0x0a809, 0x0a80a, 0x00000, 0x0a80c, 0x0a80d, 0x0a80e, 0x0a80f, + 0x0a810, 0x0a811, 0x0a812, 0x0a813, 0x0a814, 0x0a815, 0x0a816, 0x0a817, + 0x0a818, 0x0a819, 0x0a81a, 0x0a81b, 0x0a81c, 0x0a81d, 0x0a81e, 0x0a81f, + 0x0a820, 0x0a821, 0x0a822, 0x0a823, 0x0a824, 0x0a825, 0x0a826, 0x0a827, + 0x0a828, 0x0a829, 0x0a82a, 0x0a82b, 0x0a82c, 0x0a82d, 0x0a82e, 0x0a82f, + 0x0a830, 0x0a831, 0x0a832, 0x0a833, 0x0a834, 0x0a835, 0x0a836, 0x0a837, + 0x0a838, 0x0a839, 0x0a83a, 0x0a83b, 0x0a83c, 0x0a83d, 0x0a83e, 0x0a83f, + 0x0a840, 0x0a841, 0x0a842, 0x0a843, 0x0a844, 0x0a845, 0x0a846, 0x0a847, + 0x0a848, 0x0a849, 0x0a84a, 0x0a84b, 0x0a84c, 0x0a84d, 0x0a84e, 0x0a84f, + 0x0a850, 0x0a851, 0x0a852, 0x0a853, 0x0a854, 0x0a855, 0x0a856, 0x0a857, + 0x0a858, 0x0a859, 0x0a85a, 0x0a85b, 0x0a85c, 0x0a85d, 0x0a85e, 0x0a85f, + 0x0a860, 0x0a861, 0x0a862, 0x0a863, 0x0a864, 0x0a865, 0x0a866, 0x0a867, + 0x0a868, 0x0a869, 0x0a86a, 0x0a86b, 0x0a86c, 0x0a86d, 0x0a86e, 0x0a86f, + 0x0a870, 0x0a871, 0x0a872, 0x0a873, 0x0a874, 0x0a875, 0x0a876, 0x0a877, + 0x0a878, 0x0a879, 0x0a87a, 0x0a87b, 0x0a87c, 0x0a87d, 0x0a87e, 0x0a87f, + 0x00000, 0x00000, 0x0a882, 0x0a883, 0x0a884, 0x0a885, 0x0a886, 0x0a887, + 0x0a888, 0x0a889, 0x0a88a, 0x0a88b, 0x0a88c, 0x0a88d, 0x0a88e, 0x0a88f, + 0x0a890, 0x0a891, 0x0a892, 0x0a893, 0x0a894, 0x0a895, 0x0a896, 0x0a897, + 0x0a898, 0x0a899, 0x0a89a, 0x0a89b, 0x0a89c, 0x0a89d, 0x0a89e, 0x0a89f, + 0x0a8a0, 0x0a8a1, 0x0a8a2, 0x0a8a3, 0x0a8a4, 0x0a8a5, 0x0a8a6, 0x0a8a7, + 0x0a8a8, 0x0a8a9, 0x0a8aa, 0x0a8ab, 0x0a8ac, 0x0a8ad, 0x0a8ae, 0x0a8af, + 0x0a8b0, 0x0a8b1, 0x0a8b2, 0x0a8b3, 0x0a8b4, 0x0a8b5, 0x0a8b6, 0x0a8b7, + 0x0a8b8, 0x0a8b9, 0x0a8ba, 0x0a8bb, 0x0a8bc, 0x0a8bd, 0x0a8be, 0x0a8bf, + 0x0a8c0, 0x0a8c1, 0x0a8c2, 0x0a8c3, 0x0a8c4, 0x0a8c5, 0x0a8c6, 0x0a8c7, + 0x0a8c8, 0x0a8c9, 0x0a8ca, 0x0a8cb, 0x0a8cc, 0x0a8cd, 0x0a8ce, 0x0a8cf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0a8da, 0x0a8db, 0x0a8dc, 0x0a8dd, 0x0a8de, 0x0a8df, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x0a8f2, 0x0a8f2, 0x0a8f2, 0x0a8f2, 0x0a8f2, 0x0a8f2, + 0x0a8f8, 0x0a8f9, 0x0a8fa, 0x0a8fb, 0x0a8fc, 0x0a8fd, 0x0a8fe, 0x0a8ff +}; + +static uint32_t unicode_520_ci_page_a9[] = { + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0a90a, 0x0a90b, 0x0a90c, 0x0a90d, 0x0a90e, 0x0a90f, + 0x0a910, 0x0a911, 0x0a912, 0x0a913, 0x0a914, 0x0a915, 0x0a916, 0x0a917, + 0x0a918, 0x0a919, 0x0a91a, 0x0a91b, 0x0a91c, 0x0a91d, 0x0a91e, 0x0a91f, + 0x0a920, 0x0a921, 0x0a922, 0x0a923, 0x0a924, 0x0a925, 0x0a926, 0x0a927, + 0x0a928, 0x0a929, 0x0a92a, 0x00000, 0x00000, 0x00000, 0x0a92e, 0x0a92f, + 0x0a930, 0x0a931, 0x0a932, 0x0a933, 0x0a934, 0x0a935, 0x0a936, 0x0a937, + 0x0a938, 0x0a939, 0x0a93a, 0x0a93b, 0x0a93c, 0x0a93d, 0x0a93e, 0x0a93f, + 0x0a940, 0x0a941, 0x0a942, 0x0a943, 0x0a944, 0x0a945, 0x0a946, 0x0a947, + 0x0a948, 0x0a949, 0x0a94a, 0x0a94b, 0x0a94c, 0x0a94d, 0x0a94e, 0x0a94f, + 0x0a950, 0x0a951, 0x0a952, 0x0a953, 0x0a954, 0x0a955, 0x0a956, 0x0a957, + 0x0a958, 0x0a959, 0x0a95a, 0x0a95b, 0x0a95c, 0x0a95d, 0x0a95e, 0x0a95f, + 0x0a960, 0x0a961, 0x0a962, 0x0a963, 0x0a964, 0x0a965, 0x0a966, 0x0a967, + 0x0a968, 0x0a969, 0x0a96a, 0x0a96b, 0x0a96c, 0x0a96d, 0x0a96e, 0x0a96f, + 0x0a970, 0x0a971, 0x0a972, 0x0a973, 0x0a974, 0x0a975, 0x0a976, 0x0a977, + 0x0a978, 0x0a979, 0x0a97a, 0x0a97b, 0x0a97c, 0x0a97d, 0x0a97e, 0x0a97f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x0a984, 0x0a985, 0x0a986, 0x0a987, + 0x0a988, 0x0a989, 0x0a98a, 0x0a98b, 0x0a98c, 0x0a98d, 0x0a98e, 0x0a98f, + 0x0a990, 0x0a991, 0x0a992, 0x0a993, 0x0a994, 0x0a995, 0x0a996, 0x0a997, + 0x0a998, 0x0a999, 0x0a99a, 0x0a99b, 0x0a99c, 0x0a99d, 0x0a99e, 0x0a99f, + 0x0a9a0, 0x0a9a1, 0x0a9a2, 0x0a9a3, 0x0a9a4, 0x0a9a5, 0x0a9a6, 0x0a9a7, + 0x0a9a8, 0x0a9a9, 0x0a9aa, 0x0a9ab, 0x0a9ab, 0x0a9ad, 0x0a9ae, 0x0a9af, + 0x0a9b0, 0x0a9b1, 0x0a9b2, 0x00000, 0x0a9b4, 0x0a9b5, 0x0a9b6, 0x0a9b7, + 0x0a9b8, 0x0a9b9, 0x0a9ba, 0x0a9bb, 0x0a9bc, 0x0a9bd, 0x0a9be, 0x0a9bf, + 0x0a9c0, 0x0a9c1, 0x0a9c2, 0x0a9c3, 0x0a9c4, 0x0a9c5, 0x0a9c6, 0x0a9c7, + 0x0a9c8, 0x0a9c9, 0x0a9ca, 0x0a9cb, 0x0a9cc, 0x0a9cd, 0x0a9ce, 0x0a9cf, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0a9da, 0x0a9db, 0x0a9dc, 0x0a9dd, 0x0a9de, 0x0a9df, + 0x0a9e0, 0x0a9e1, 0x0a9e2, 0x0a9e3, 0x0a9e4, 0x0a9e5, 0x0a9e6, 0x0a9e7, + 0x0a9e8, 0x0a9e9, 0x0a9ea, 0x0a9eb, 0x0a9ec, 0x0a9ed, 0x0a9ee, 0x0a9ef, + 0x0a9f0, 0x0a9f1, 0x0a9f2, 0x0a9f3, 0x0a9f4, 0x0a9f5, 0x0a9f6, 0x0a9f7, + 0x0a9f8, 0x0a9f9, 0x0a9fa, 0x0a9fb, 0x0a9fc, 0x0a9fd, 0x0a9fe, 0x0a9ff +}; + +static uint32_t unicode_520_ci_page_aa[] = { + 0x0aa00, 0x0aa01, 0x0aa02, 0x0aa03, 0x0aa04, 0x0aa05, 0x0aa06, 0x0aa07, + 0x0aa08, 0x0aa09, 0x0aa0a, 0x0aa0b, 0x0aa0c, 0x0aa0d, 0x0aa0e, 0x0aa0f, + 0x0aa10, 0x0aa11, 0x0aa12, 0x0aa13, 0x0aa14, 0x0aa15, 0x0aa16, 0x0aa17, + 0x0aa18, 0x0aa19, 0x0aa1a, 0x0aa1b, 0x0aa1c, 0x0aa1d, 0x0aa1e, 0x0aa1f, + 0x0aa20, 0x0aa21, 0x0aa22, 0x0aa23, 0x0aa24, 0x0aa25, 0x0aa26, 0x0aa27, + 0x0aa28, 0x0aa29, 0x0aa2a, 0x0aa2b, 0x0aa2c, 0x0aa2d, 0x0aa2e, 0x0aa2f, + 0x0aa30, 0x0aa31, 0x0aa32, 0x0aa33, 0x0aa34, 0x0aa35, 0x0aa36, 0x0aa37, + 0x0aa38, 0x0aa39, 0x0aa3a, 0x0aa3b, 0x0aa3c, 0x0aa3d, 0x0aa3e, 0x0aa3f, + 0x0aa40, 0x0aa41, 0x0aa42, 0x0aa43, 0x0aa44, 0x0aa45, 0x0aa46, 0x0aa47, + 0x0aa48, 0x0aa49, 0x0aa4a, 0x0aa4b, 0x0aa4c, 0x0aa4d, 0x0aa4e, 0x0aa4f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0aa5a, 0x0aa5b, 0x0aa5c, 0x0aa5d, 0x0aa5e, 0x0aa5f, + 0x0aa60, 0x0aa61, 0x0aa62, 0x0aa63, 0x0aa64, 0x0aa65, 0x0aa66, 0x0aa67, + 0x0aa68, 0x0aa69, 0x0aa6a, 0x0aa6b, 0x0aa6c, 0x0aa6d, 0x0aa6e, 0x0aa6f, + 0x0aa70, 0x0aa71, 0x0aa72, 0x0aa73, 0x0aa74, 0x0aa75, 0x0aa76, 0x0aa77, + 0x0aa78, 0x0aa79, 0x0aa7a, 0x0aa7b, 0x0aa7c, 0x0aa7d, 0x0aa7e, 0x0aa7f, + 0x0aa80, 0x0aa81, 0x0aa82, 0x0aa83, 0x0aa84, 0x0aa85, 0x0aa86, 0x0aa87, + 0x0aa88, 0x0aa89, 0x0aa8a, 0x0aa8b, 0x0aa8c, 0x0aa8d, 0x0aa8e, 0x0aa8f, + 0x0aa90, 0x0aa91, 0x0aa92, 0x0aa93, 0x0aa94, 0x0aa95, 0x0aa96, 0x0aa97, + 0x0aa98, 0x0aa99, 0x0aa9a, 0x0aa9b, 0x0aa9c, 0x0aa9d, 0x0aa9e, 0x0aa9f, + 0x0aaa0, 0x0aaa1, 0x0aaa2, 0x0aaa3, 0x0aaa4, 0x0aaa5, 0x0aaa6, 0x0aaa7, + 0x0aaa8, 0x0aaa9, 0x0aaaa, 0x0aaab, 0x0aaac, 0x0aaad, 0x0aaae, 0x0aaaf, + 0x0aab0, 0x0aab1, 0x0aab2, 0x0aab3, 0x0aab4, 0x0aab5, 0x0aab6, 0x0aab7, + 0x0aab8, 0x0aab9, 0x0aaba, 0x0aabb, 0x0aabc, 0x0aabd, 0x0aabe, 0x00000, + 0x0aac0, 0x00000, 0x0aac2, 0x0aac3, 0x0aac4, 0x0aac5, 0x0aac6, 0x0aac7, + 0x0aac8, 0x0aac9, 0x0aaca, 0x0aacb, 0x0aacc, 0x0aacd, 0x0aace, 0x0aacf, + 0x0aad0, 0x0aad1, 0x0aad2, 0x0aad3, 0x0aad4, 0x0aad5, 0x0aad6, 0x0aad7, + 0x0aad8, 0x0aad9, 0x0aada, 0x0aadb, 0x0aadc, 0x0aadd, 0x0aade, 0x0aadf, + 0x0aae0, 0x0aae1, 0x0aae2, 0x0aae3, 0x0aae4, 0x0aae5, 0x0aae6, 0x0aae7, + 0x0aae8, 0x0aae9, 0x0aaea, 0x0aaeb, 0x0aaec, 0x0aaed, 0x0aaee, 0x0aaef, + 0x0aaf0, 0x0aaf1, 0x0aaf2, 0x0aaf3, 0x0aaf4, 0x0aaf5, 0x0aaf6, 0x0aaf7, + 0x0aaf8, 0x0aaf9, 0x0aafa, 0x0aafb, 0x0aafc, 0x0aafd, 0x0aafe, 0x0aaff +}; + +static uint32_t unicode_520_ci_page_ab[] = { + 0x0ab00, 0x0ab01, 0x0ab02, 0x0ab03, 0x0ab04, 0x0ab05, 0x0ab06, 0x0ab07, + 0x0ab08, 0x0ab09, 0x0ab0a, 0x0ab0b, 0x0ab0c, 0x0ab0d, 0x0ab0e, 0x0ab0f, + 0x0ab10, 0x0ab11, 0x0ab12, 0x0ab13, 0x0ab14, 0x0ab15, 0x0ab16, 0x0ab17, + 0x0ab18, 0x0ab19, 0x0ab1a, 0x0ab1b, 0x0ab1c, 0x0ab1d, 0x0ab1e, 0x0ab1f, + 0x0ab20, 0x0ab21, 0x0ab22, 0x0ab23, 0x0ab24, 0x0ab25, 0x0ab26, 0x0ab27, + 0x0ab28, 0x0ab29, 0x0ab2a, 0x0ab2b, 0x0ab2c, 0x0ab2d, 0x0ab2e, 0x0ab2f, + 0x0ab30, 0x0ab31, 0x0ab32, 0x0ab33, 0x0ab34, 0x0ab35, 0x0ab36, 0x0ab37, + 0x0ab38, 0x0ab39, 0x0ab3a, 0x0ab3b, 0x0ab3c, 0x0ab3d, 0x0ab3e, 0x0ab3f, + 0x0ab40, 0x0ab41, 0x0ab42, 0x0ab43, 0x0ab44, 0x0ab45, 0x0ab46, 0x0ab47, + 0x0ab48, 0x0ab49, 0x0ab4a, 0x0ab4b, 0x0ab4c, 0x0ab4d, 0x0ab4e, 0x0ab4f, + 0x0ab50, 0x0ab51, 0x0ab52, 0x0ab53, 0x0ab54, 0x0ab55, 0x0ab56, 0x0ab57, + 0x0ab58, 0x0ab59, 0x0ab5a, 0x0ab5b, 0x0ab5c, 0x0ab5d, 0x0ab5e, 0x0ab5f, + 0x0ab60, 0x0ab61, 0x0ab62, 0x0ab63, 0x0ab64, 0x0ab65, 0x0ab66, 0x0ab67, + 0x0ab68, 0x0ab69, 0x0ab6a, 0x0ab6b, 0x0ab6c, 0x0ab6d, 0x0ab6e, 0x0ab6f, + 0x0ab70, 0x0ab71, 0x0ab72, 0x0ab73, 0x0ab74, 0x0ab75, 0x0ab76, 0x0ab77, + 0x0ab78, 0x0ab79, 0x0ab7a, 0x0ab7b, 0x0ab7c, 0x0ab7d, 0x0ab7e, 0x0ab7f, + 0x0ab80, 0x0ab81, 0x0ab82, 0x0ab83, 0x0ab84, 0x0ab85, 0x0ab86, 0x0ab87, + 0x0ab88, 0x0ab89, 0x0ab8a, 0x0ab8b, 0x0ab8c, 0x0ab8d, 0x0ab8e, 0x0ab8f, + 0x0ab90, 0x0ab91, 0x0ab92, 0x0ab93, 0x0ab94, 0x0ab95, 0x0ab96, 0x0ab97, + 0x0ab98, 0x0ab99, 0x0ab9a, 0x0ab9b, 0x0ab9c, 0x0ab9d, 0x0ab9e, 0x0ab9f, + 0x0aba0, 0x0aba1, 0x0aba2, 0x0aba3, 0x0aba4, 0x0aba5, 0x0aba6, 0x0aba7, + 0x0aba8, 0x0aba9, 0x0abaa, 0x0abab, 0x0abac, 0x0abad, 0x0abae, 0x0abaf, + 0x0abb0, 0x0abb1, 0x0abb2, 0x0abb3, 0x0abb4, 0x0abb5, 0x0abb6, 0x0abb7, + 0x0abb8, 0x0abb9, 0x0abba, 0x0abbb, 0x0abbc, 0x0abbd, 0x0abbe, 0x0abbf, + 0x0abc0, 0x0abc1, 0x0abc2, 0x0abc3, 0x0abc4, 0x0abc5, 0x0abc6, 0x0abc7, + 0x0abc8, 0x0abc9, 0x0abca, 0x0abcb, 0x0abcc, 0x0abcd, 0x0abce, 0x0abcf, + 0x0abd0, 0x0abd1, 0x0abd2, 0x0abd3, 0x0abd4, 0x0abd5, 0x0abd6, 0x0abd7, + 0x0abd8, 0x0abd9, 0x0abda, 0x0abdb, 0x0abdc, 0x0abdd, 0x0abde, 0x0abdf, + 0x0abe0, 0x0abe1, 0x0abe2, 0x0abe3, 0x0abe4, 0x0abe5, 0x0abe6, 0x0abe7, + 0x0abe8, 0x0abe9, 0x0abea, 0x0abeb, 0x00000, 0x0abed, 0x0abee, 0x0abef, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0abfa, 0x0abfb, 0x0abfc, 0x0abfd, 0x0abfe, 0x0abff +}; + +static uint32_t unicode_520_ci_page_f9[] = { + 0x0f900, 0x0f901, 0x02f9e, 0x0f903, 0x0f904, 0x0f905, 0x0f906, 0x02ef2, + 0x02ef2, 0x0f909, 0x02fa6, 0x0f90b, 0x0f90c, 0x0f90d, 0x0f90e, 0x0f90f, + 0x0f910, 0x0f911, 0x0f912, 0x0f913, 0x0f914, 0x0f915, 0x0f916, 0x0f917, + 0x0f918, 0x0f919, 0x0f91a, 0x0f91b, 0x0f91c, 0x0f91d, 0x0f91e, 0x0f91f, + 0x0f920, 0x0f921, 0x0f922, 0x0f923, 0x0f924, 0x0f925, 0x0f926, 0x0f927, + 0x0f928, 0x0f929, 0x0f92a, 0x0f92b, 0x0f92c, 0x0f92d, 0x0f92e, 0x0f92f, + 0x0f930, 0x0f931, 0x0f932, 0x0f933, 0x02f7c, 0x0f935, 0x0f936, 0x0f937, + 0x0f938, 0x0f939, 0x0f93a, 0x0f93b, 0x0f93c, 0x0f93d, 0x0f93e, 0x0f93f, + 0x02fc5, 0x0f941, 0x0f942, 0x0f943, 0x0f944, 0x0f945, 0x0f946, 0x0f947, + 0x0f948, 0x0f949, 0x0f94a, 0x0f94b, 0x0f94c, 0x0f94d, 0x0f94e, 0x0f94f, + 0x0f950, 0x0f951, 0x0f952, 0x0f953, 0x0f954, 0x0f955, 0x0f956, 0x0f957, + 0x0f958, 0x0f959, 0x0f95a, 0x0f95b, 0x0f914, 0x0f95d, 0x0f95e, 0x0f95f, + 0x0f960, 0x0f961, 0x0f962, 0x0f963, 0x0f964, 0x0f965, 0x0f966, 0x0f967, + 0x0f968, 0x0f969, 0x0f96a, 0x0f96b, 0x0f96c, 0x0f96d, 0x0f96e, 0x0f96f, + 0x0f970, 0x02fa0, 0x0f972, 0x0f973, 0x0f974, 0x0f975, 0x0f976, 0x0f977, + 0x0f978, 0x0f979, 0x0f97a, 0x0f97b, 0x0f97c, 0x0f97d, 0x0f97e, 0x0f97f, + 0x0f980, 0x02f25, 0x0f982, 0x0f983, 0x0f984, 0x0f985, 0x0f986, 0x0f987, + 0x0f988, 0x0f989, 0x02f12, 0x0f98b, 0x0f98c, 0x0f98d, 0x0f98e, 0x0f98f, + 0x0f990, 0x0f991, 0x0f992, 0x0f993, 0x0f994, 0x0f995, 0x0f996, 0x0f997, + 0x0f998, 0x0f999, 0x0f99a, 0x0f99b, 0x0f99c, 0x0f99d, 0x0f99e, 0x0f99f, + 0x0f9a0, 0x0f96f, 0x0f9a2, 0x0f9a3, 0x0f9a4, 0x0f9a5, 0x0f9a6, 0x0f9a7, + 0x0f9a8, 0x0f9a9, 0x0f95f, 0x0f9ab, 0x0f9ac, 0x0f9ad, 0x0f9ae, 0x0f9af, + 0x0f9b0, 0x0f9b1, 0x0f9b2, 0x0f9b3, 0x0f9b4, 0x0f9b5, 0x0f9b6, 0x0f9b7, + 0x0f9b8, 0x0f9b9, 0x0f9ba, 0x0f9bb, 0x0f9bc, 0x0f9bd, 0x0f9be, 0x0f914, + 0x0f9c0, 0x0f9c1, 0x0f9c2, 0x0f9c3, 0x02eef, 0x0f9c5, 0x0f9c6, 0x0f9c7, + 0x0f9c8, 0x0f9c9, 0x0f9ca, 0x0f9cb, 0x0f9cc, 0x0f9cd, 0x0f9ce, 0x0f9cf, + 0x0f9d0, 0x03285, 0x0f9d2, 0x0f9d3, 0x0f9d4, 0x0f9d5, 0x0f9d6, 0x0f9d7, + 0x0f9d8, 0x0f9d9, 0x0f9da, 0x0f961, 0x0f9dc, 0x0f9dd, 0x0f9de, 0x0f9df, + 0x0f9e0, 0x0f9e1, 0x0f9e2, 0x0f9e3, 0x0f9e4, 0x0f9e5, 0x0f9e6, 0x0f9e7, + 0x0f9e8, 0x02fa5, 0x0f9ea, 0x0f9eb, 0x0f9ec, 0x0f9ed, 0x0f9ee, 0x0f9ef, + 0x0f9f0, 0x0f9f1, 0x0f9f2, 0x0f9f3, 0x0f9f4, 0x0f9f5, 0x0f9f6, 0x02f74, + 0x0f9f8, 0x0f9f9, 0x0f9fa, 0x0f9fb, 0x0f9fc, 0x0f9fd, 0x0f9fe, 0x0f9ff +}; + +static uint32_t unicode_520_ci_page_fa[] = { + 0x0fa00, 0x0fa01, 0x0fa02, 0x0fa03, 0x0fa04, 0x0fa05, 0x0fa06, 0x0fa07, + 0x02f8f, 0x0fa09, 0x02f92, 0x0fa0b, 0x0fa0c, 0x0fa0d, 0x0fa0e, 0x0fa0f, + 0x0fa10, 0x0fa11, 0x0fa12, 0x0fa13, 0x0fa14, 0x0fa15, 0x0fa16, 0x0fa17, + 0x0fa18, 0x0fa19, 0x0fa1a, 0x0fa1b, 0x0fa1c, 0x0fa1d, 0x02f7b, 0x0fa1f, + 0x0fa20, 0x0fa21, 0x0fa22, 0x0fa23, 0x0fa24, 0x0fa25, 0x0fa26, 0x0fa27, + 0x0fa28, 0x0fa29, 0x0fa2a, 0x0fa2b, 0x0fa2c, 0x0fa2d, 0x0fa2e, 0x0fa2f, + 0x0fa30, 0x0fa31, 0x0fa32, 0x0fa33, 0x0fa34, 0x0fa35, 0x0fa36, 0x0fa37, + 0x0fa38, 0x0fa39, 0x0fa3a, 0x0fa3b, 0x02f2c, 0x0fa3d, 0x0fa3e, 0x0fa3f, + 0x0fa40, 0x0fa41, 0x0fa42, 0x0fa43, 0x0fa44, 0x0fa45, 0x0fa46, 0x0fa47, + 0x0fa48, 0x02ea4, 0x0fa4a, 0x0fa4b, 0x03293, 0x0fa4d, 0x0fa4e, 0x0fa4f, + 0x0fa50, 0x03297, 0x0fa52, 0x0fa53, 0x0fa54, 0x0fa55, 0x0fa56, 0x0f996, + 0x0fa58, 0x0fa59, 0x0fa5a, 0x0fa5b, 0x0fa5c, 0x02ebe, 0x02ebe, 0x0fa5f, + 0x0fa60, 0x0fa61, 0x0fa62, 0x0fa63, 0x0fa64, 0x0fa65, 0x02ecc, 0x0fa25, + 0x0fa68, 0x0fa69, 0x0fa6a, 0x0fa6b, 0x0fa6c, 0x0fa6d, 0x0fa6e, 0x0fa6f, + 0x0fa70, 0x0fa71, 0x0fa72, 0x0fa73, 0x0fa74, 0x0fa75, 0x0fa76, 0x0fa77, + 0x0fa36, 0x0fa79, 0x0fa7a, 0x0fa7b, 0x0fa10, 0x0fa7d, 0x0fa7e, 0x0fa7f, + 0x0fa80, 0x0fa81, 0x0fa82, 0x0fa83, 0x0fa84, 0x0fa85, 0x0fa86, 0x0fa87, + 0x0fa88, 0x0fa3f, 0x0fa8a, 0x0fa40, 0x0fa8c, 0x0fa8d, 0x0fa8e, 0x0fa8f, + 0x0fa90, 0x0fa12, 0x0f929, 0x0fa93, 0x0fa94, 0x02f4d, 0x0f970, 0x0f9ca, + 0x0fa98, 0x0fa99, 0x0fa47, 0x0fa9b, 0x0fa48, 0x0fa9d, 0x0fa9e, 0x0fa9f, + 0x0fa16, 0x0faa1, 0x0faa2, 0x0faa3, 0x0faa4, 0x0faa5, 0x0fa17, 0x0faa7, + 0x0faa8, 0x0faa9, 0x0faaa, 0x0faab, 0x0faac, 0x0fa56, 0x0faae, 0x0faaf, + 0x0f996, 0x0fab1, 0x0fa5b, 0x0fab3, 0x0fab4, 0x0fab5, 0x0fab6, 0x0fab7, + 0x0fa61, 0x0fab9, 0x0fa22, 0x0fabb, 0x0fa62, 0x0f95d, 0x0fabe, 0x0fa63, + 0x0fac0, 0x0fa65, 0x0fac2, 0x0fac3, 0x0fac4, 0x0fac5, 0x0fac6, 0x0fa68, + 0x0fa1c, 0x0fac9, 0x0fa69, 0x0facb, 0x0fa6a, 0x0facd, 0x02ef2, 0x0facf, + 0x0fad0, 0x0fad1, 0x0fad2, 0x0fad3, 0x0fad4, 0x0fad5, 0x0fad6, 0x0fad7, + 0x0fad8, 0x0fad9, 0x0fada, 0x0fadb, 0x0fadc, 0x0fadd, 0x0fade, 0x0fadf, + 0x0fae0, 0x0fae1, 0x0fae2, 0x0fae3, 0x0fae4, 0x0fae5, 0x0fae6, 0x0fae7, + 0x0fae8, 0x0fae9, 0x0faea, 0x0faeb, 0x0faec, 0x0faed, 0x0faee, 0x0faef, + 0x0faf0, 0x0faf1, 0x0faf2, 0x0faf3, 0x0faf4, 0x0faf5, 0x0faf6, 0x0faf7, + 0x0faf8, 0x0faf9, 0x0fafa, 0x0fafb, 0x0fafc, 0x0fafd, 0x0fafe, 0x0faff +}; + +static uint32_t unicode_520_ci_page_fb[] = { + 0x0fb00, 0x0fb01, 0x0fb02, 0x0fb03, 0x0fb04, 0x0fb05, 0x0fb05, 0x0fb07, + 0x0fb08, 0x0fb09, 0x0fb0a, 0x0fb0b, 0x0fb0c, 0x0fb0d, 0x0fb0e, 0x0fb0f, + 0x0fb10, 0x0fb11, 0x0fb12, 0x0fb13, 0x0fb14, 0x0fb15, 0x0fb16, 0x0fb17, + 0x0fb18, 0x0fb19, 0x0fb1a, 0x0fb1b, 0x0fb1c, 0x005d9, 0x00000, 0x005f2, + 0x005e2, 0x005d0, 0x005d3, 0x005d4, 0x005da, 0x005dc, 0x005dd, 0x005e8, + 0x005ea, 0x0002b, 0x005e9, 0x005e9, 0x005e9, 0x005e9, 0x005d0, 0x005d0, + 0x005d0, 0x005d1, 0x005d2, 0x005d3, 0x005d4, 0x005d5, 0x005d6, 0x0fb37, + 0x005d8, 0x005d9, 0x005da, 0x005da, 0x005dc, 0x0fb3d, 0x005dd, 0x0fb3f, + 0x005df, 0x005e1, 0x0fb42, 0x005e3, 0x005e3, 0x0fb45, 0x005e5, 0x005e7, + 0x005e8, 0x005e9, 0x005ea, 0x005d5, 0x005d1, 0x005da, 0x005e3, 0x0fb4f, + 0x00671, 0x00671, 0x0067b, 0x0067b, 0x0067b, 0x0067b, 0x0067e, 0x0067e, + 0x0067e, 0x0067e, 0x00680, 0x00680, 0x00680, 0x00680, 0x0067a, 0x0067a, + 0x0067a, 0x0067a, 0x0067f, 0x0067f, 0x0067f, 0x0067f, 0x00679, 0x00679, + 0x00679, 0x00679, 0x006a4, 0x006a4, 0x006a4, 0x006a4, 0x006a6, 0x006a6, + 0x006a6, 0x006a6, 0x00684, 0x00684, 0x00684, 0x00684, 0x00683, 0x00683, + 0x00683, 0x00683, 0x00686, 0x00686, 0x00686, 0x00686, 0x00687, 0x00687, + 0x00687, 0x00687, 0x0068d, 0x0068d, 0x0068c, 0x0068c, 0x0068e, 0x0068e, + 0x00688, 0x00688, 0x00698, 0x00698, 0x00691, 0x00691, 0x006a9, 0x006a9, + 0x006a9, 0x006a9, 0x006af, 0x006af, 0x006af, 0x006af, 0x006b3, 0x006b3, + 0x006b3, 0x006b3, 0x006b1, 0x006b1, 0x006b1, 0x006b1, 0x006ba, 0x006ba, + 0x006bb, 0x006bb, 0x006bb, 0x006bb, 0x006c0, 0x006c0, 0x006c1, 0x006c1, + 0x006c1, 0x006c1, 0x006be, 0x006be, 0x006be, 0x006be, 0x006d2, 0x006d2, + 0x006d2, 0x006d2, 0x0fbb2, 0x0fbb3, 0x0fbb4, 0x0fbb5, 0x0fbb6, 0x0fbb7, + 0x0fbb8, 0x0fbb9, 0x0fbba, 0x0fbbb, 0x0fbbc, 0x0fbbd, 0x0fbbe, 0x0fbbf, + 0x0fbc0, 0x0fbc1, 0x0fbc2, 0x0fbc3, 0x0fbc4, 0x0fbc5, 0x0fbc6, 0x0fbc7, + 0x0fbc8, 0x0fbc9, 0x0fbca, 0x0fbcb, 0x0fbcc, 0x0fbcd, 0x0fbce, 0x0fbcf, + 0x0fbd0, 0x0fbd1, 0x0fbd2, 0x006ad, 0x006ad, 0x006ad, 0x006ad, 0x006c7, + 0x006c7, 0x006c6, 0x006c6, 0x006c8, 0x006c8, 0x00677, 0x006cb, 0x006cb, + 0x006c5, 0x006c5, 0x006c9, 0x006c9, 0x006d0, 0x006d0, 0x006d0, 0x006d0, + 0x00649, 0x00649, 0x0fbea, 0x0fbea, 0x0fbec, 0x0fbec, 0x0fbee, 0x0fbee, + 0x0fbf0, 0x0fbf0, 0x0fbf2, 0x0fbf2, 0x0fbf4, 0x0fbf4, 0x0fbf6, 0x0fbf6, + 0x0fbf6, 0x0fbf9, 0x0fbf9, 0x0fbf9, 0x006cc, 0x006cc, 0x0fbfe, 0x0fbff +}; + +static uint32_t unicode_520_ci_page_fc[] = { + 0x0fc00, 0x0fc01, 0x0fc02, 0x0fbf9, 0x0fc04, 0x0fc05, 0x0fc06, 0x0fc07, + 0x0fc08, 0x0fc09, 0x0fc0a, 0x0fc0b, 0x0fc0c, 0x0fc0d, 0x0fc0e, 0x0fc0f, + 0x0fc10, 0x0fc11, 0x0fc12, 0x0fc13, 0x0fc14, 0x0fc15, 0x0fc16, 0x0fc17, + 0x0fc18, 0x0fc19, 0x0fc1a, 0x0fc1b, 0x0fc1c, 0x0fc1d, 0x0fc1e, 0x0fc1f, + 0x0fc20, 0x0fc21, 0x0fc22, 0x0fc23, 0x0fc24, 0x0fc25, 0x0fc26, 0x0fc27, + 0x0fc28, 0x0fc29, 0x0fc2a, 0x0fc2b, 0x0fc2c, 0x0fc2d, 0x0fc2e, 0x0fc2f, + 0x0fc30, 0x0fc31, 0x0fc32, 0x0fc33, 0x0fc34, 0x0fc35, 0x0fc36, 0x0fc37, + 0x0fc38, 0x0fc39, 0x0fc3a, 0x0fc3b, 0x0fc3c, 0x0fc3d, 0x0fc3e, 0x0fc3f, + 0x0fc40, 0x0fc41, 0x0fc42, 0x0fc43, 0x0fc44, 0x0fc45, 0x0fc46, 0x0fc47, + 0x0fc48, 0x0fc49, 0x0fc4a, 0x0fc4b, 0x0fc4c, 0x0fc4d, 0x0fc4e, 0x0fc4f, + 0x0fc50, 0x0fc51, 0x0fc52, 0x0fc53, 0x0fc54, 0x0fc55, 0x0fc56, 0x0fc57, + 0x0fc58, 0x0fc59, 0x0fc5a, 0x00630, 0x00631, 0x00649, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x0fc64, 0x0fc65, 0x0fc02, 0x0fc67, + 0x0fbf9, 0x0fc04, 0x0fc6a, 0x0fc6b, 0x0fc08, 0x0fc6d, 0x0fc09, 0x0fc0a, + 0x0fc70, 0x0fc71, 0x0fc0e, 0x0fc73, 0x0fc0f, 0x0fc10, 0x0fc76, 0x0fc77, + 0x0fc12, 0x0fc79, 0x0fc13, 0x0fc14, 0x0fc31, 0x0fc32, 0x0fc35, 0x0fc36, + 0x0fc37, 0x0fc3b, 0x0fc3c, 0x0fc3d, 0x0fc3e, 0x0fc42, 0x0fc43, 0x0fc44, + 0x0fc88, 0x0fc48, 0x0fc8a, 0x0fc8b, 0x0fc4e, 0x0fc8d, 0x0fc4f, 0x0fc50, + 0x00649, 0x0fc91, 0x0fc92, 0x0fc58, 0x0fc94, 0x0fc59, 0x0fc5a, 0x0fc00, + 0x0fc01, 0x0fc99, 0x0fc02, 0x0fc9b, 0x0fc05, 0x0fc06, 0x0fc07, 0x0fc08, + 0x0fca0, 0x0fc0b, 0x0fc0c, 0x0fc0d, 0x0fc0e, 0x0fca5, 0x0fc12, 0x0fc15, + 0x0fc16, 0x0fc17, 0x0fc18, 0x0fc19, 0x0fc1b, 0x0fc1c, 0x0fc1d, 0x0fc1e, + 0x0fc1f, 0x0fc20, 0x0fcb2, 0x0fc21, 0x0fc22, 0x0fc23, 0x0fc24, 0x0fc25, + 0x0fc26, 0x0fc28, 0x0fc29, 0x0fc2a, 0x0fc2b, 0x0fc2c, 0x0fc2d, 0x0fc2e, + 0x0fc2f, 0x0fc30, 0x0fc33, 0x0fc34, 0x0fc38, 0x0fc39, 0x0fc3a, 0x0fc3b, + 0x0fc3c, 0x0fc3f, 0x0fc40, 0x0fc41, 0x0fc42, 0x0fccd, 0x0fc45, 0x0fc46, + 0x0fc47, 0x0fc48, 0x0fc4b, 0x0fc4c, 0x0fc4d, 0x0fc4e, 0x0fcd6, 0x0fc51, + 0x0fc52, 0x00647, 0x0fc55, 0x0fc56, 0x0fc57, 0x0fc58, 0x0fcde, 0x0fc02, + 0x0fc9b, 0x0fc08, 0x0fca0, 0x0fc0e, 0x0fca5, 0x0fc12, 0x0fce6, 0x0fc1f, + 0x0fce8, 0x0fce9, 0x0fcea, 0x0fc3b, 0x0fc3c, 0x0fc42, 0x0fc4e, 0x0fcd6, + 0x0fc58, 0x0fcde, 0x00000, 0x00000, 0x00000, 0x0fcf5, 0x0fcf6, 0x0fcf7, + 0x0fcf8, 0x0fcf9, 0x0fcfa, 0x0fcfb, 0x0fcfc, 0x0fcfd, 0x0fcfe, 0x0fcff +}; + +static uint32_t unicode_520_ci_page_fd[] = { + 0x0fd00, 0x0fd01, 0x0fd02, 0x0fd03, 0x0fd04, 0x0fd05, 0x0fd06, 0x0fd07, + 0x0fd08, 0x0fd09, 0x0fd0a, 0x0fd0b, 0x0fce9, 0x0fd0d, 0x0fd0e, 0x0fd0f, + 0x0fd10, 0x0fcf5, 0x0fcf6, 0x0fcf7, 0x0fcf8, 0x0fcf9, 0x0fcfa, 0x0fcfb, + 0x0fcfc, 0x0fcfd, 0x0fcfe, 0x0fd1b, 0x0fd00, 0x0fd01, 0x0fd02, 0x0fd03, + 0x0fd04, 0x0fd05, 0x0fd06, 0x0fd07, 0x0fd08, 0x0fd09, 0x0fd0a, 0x0fd0b, + 0x0fce9, 0x0fd0d, 0x0fd0e, 0x0fd0f, 0x0fd10, 0x0fd09, 0x0fd0a, 0x0fd0b, + 0x0fce9, 0x0fce8, 0x0fcea, 0x0fc27, 0x0fc1c, 0x0fc1d, 0x0fc1e, 0x0fd09, + 0x0fd0a, 0x0fd0b, 0x0fc27, 0x0fc28, 0x00627, 0x00627, 0x0fd3e, 0x0fd3f, + 0x0fd40, 0x0fd41, 0x0fd42, 0x0fd43, 0x0fd44, 0x0fd45, 0x0fd46, 0x0fd47, + 0x0fd48, 0x0fd49, 0x0fd4a, 0x0fd4b, 0x0fd4c, 0x0fd4d, 0x0fd4e, 0x0fd4f, + 0x0fd50, 0x0fd51, 0x0fd51, 0x0fd53, 0x0fd54, 0x0fd55, 0x0fd56, 0x0fd57, + 0x0fd58, 0x0fd58, 0x0fd5a, 0x0fd5b, 0x0fd5c, 0x0fd5d, 0x0fd5e, 0x0fd5f, + 0x0fd5f, 0x0fd61, 0x0fd62, 0x0fd62, 0x0fd64, 0x0fd64, 0x0fd66, 0x0fd67, + 0x0fd67, 0x0fd69, 0x0fd6a, 0x0fd6a, 0x0fd6c, 0x0fd6c, 0x0fd6e, 0x0fd6f, + 0x0fd6f, 0x0fd71, 0x0fd71, 0x0fd73, 0x0fd74, 0x0fd75, 0x0fd76, 0x0fd76, + 0x0fd78, 0x0fd79, 0x0fd7a, 0x0fd7b, 0x0fd7c, 0x0fd7c, 0x0fd7e, 0x0fd7f, + 0x0fd80, 0x0fd81, 0x0fd82, 0x0fd83, 0x0fd83, 0x0fd85, 0x0fd85, 0x0fd87, + 0x0fd87, 0x0fd89, 0x0fd8a, 0x0fd8b, 0x0fd8c, 0x0fd8d, 0x0fd8e, 0x0fd8f, + 0x0fd90, 0x0fd91, 0x0fd92, 0x0fd93, 0x0fd94, 0x0fd95, 0x0fd96, 0x0fd97, + 0x0fd97, 0x0fd99, 0x0fd9a, 0x0fd9b, 0x0fd9c, 0x0fd9c, 0x0fd9e, 0x0fd9f, + 0x0fda0, 0x0fda1, 0x0fda2, 0x0fda3, 0x0fda4, 0x0fda5, 0x0fda6, 0x0fda7, + 0x0fda8, 0x0fda9, 0x0fdaa, 0x0fdab, 0x0fdac, 0x0fdad, 0x0fdae, 0x0fdaf, + 0x0fdb0, 0x0fdb1, 0x0fdb2, 0x0fdb3, 0x0fd7e, 0x0fd80, 0x0fdb6, 0x0fdb7, + 0x0fdb8, 0x0fdb9, 0x0fdba, 0x0fdbb, 0x0fdba, 0x0fdb8, 0x0fdbe, 0x0fdbf, + 0x0fdc0, 0x0fdc1, 0x0fdc2, 0x0fdbb, 0x0fd75, 0x0fd66, 0x0fdc6, 0x0fdc7, + 0x0fdc8, 0x0fdc9, 0x0fdca, 0x0fdcb, 0x0fdcc, 0x0fdcd, 0x0fdce, 0x0fdcf, + 0x0fdd0, 0x0fdd1, 0x0fdd2, 0x0fdd3, 0x0fdd4, 0x0fdd5, 0x0fdd6, 0x0fdd7, + 0x0fdd8, 0x0fdd9, 0x0fdda, 0x0fddb, 0x0fddc, 0x0fddd, 0x0fdde, 0x0fddf, + 0x0fde0, 0x0fde1, 0x0fde2, 0x0fde3, 0x0fde4, 0x0fde5, 0x0fde6, 0x0fde7, + 0x0fde8, 0x0fde9, 0x0fdea, 0x0fdeb, 0x0fdec, 0x0fded, 0x0fdee, 0x0fdef, + 0x0fdf0, 0x0fdf1, 0x0fdf2, 0x0fdf3, 0x0fdf4, 0x0fdf5, 0x0fdf6, 0x0fdf7, + 0x0fdf8, 0x0fdf9, 0x0fdfa, 0x0fdfb, 0x0fdfc, 0x0fdfd, 0x0fdfe, 0x0fdff +}; + +static uint32_t unicode_520_ci_page_fe[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x0002c, 0x03001, 0x03002, 0x0003a, 0x0003b, 0x00021, 0x0003f, 0x03016, + 0x03017, 0x02026, 0x0fe1a, 0x0fe1b, 0x0fe1c, 0x0fe1d, 0x0fe1e, 0x0fe1f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x0fe27, + 0x0fe28, 0x0fe29, 0x0fe2a, 0x0fe2b, 0x0fe2c, 0x0fe2d, 0x0fe2e, 0x0fe2f, + 0x02025, 0x02014, 0x02013, 0x0005f, 0x0005f, 0x00028, 0x00029, 0x0007b, + 0x0007d, 0x03014, 0x03015, 0x03010, 0x03011, 0x0300a, 0x0300b, 0x02329, + 0x0232a, 0x0300c, 0x0300d, 0x0300e, 0x0300f, 0x0fe45, 0x0fe46, 0x0005b, + 0x0005d, 0x0203e, 0x0203e, 0x0203e, 0x0203e, 0x0005f, 0x0005f, 0x0005f, + 0x0002c, 0x03001, 0x0002e, 0x0fe53, 0x0003b, 0x0003a, 0x0003f, 0x00021, + 0x02014, 0x00028, 0x00029, 0x0007b, 0x0007d, 0x03014, 0x03015, 0x00023, + 0x00026, 0x0002a, 0x0002b, 0x0002d, 0x0003c, 0x0003e, 0x0003d, 0x0fe67, + 0x0005c, 0x00024, 0x00025, 0x00040, 0x0fe6c, 0x0fe6d, 0x0fe6e, 0x0fe6f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x0fe75, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00621, 0x00622, 0x00622, 0x00623, 0x00623, 0x00624, 0x00624, 0x00625, + 0x00625, 0x00626, 0x00626, 0x00626, 0x00626, 0x00627, 0x00627, 0x00628, + 0x00628, 0x00628, 0x00628, 0x00629, 0x00629, 0x0062a, 0x0062a, 0x0062a, + 0x0062a, 0x0062b, 0x0062b, 0x0062b, 0x0062b, 0x0062c, 0x0062c, 0x0062c, + 0x0062c, 0x0062d, 0x0062d, 0x0062d, 0x0062d, 0x0062e, 0x0062e, 0x0062e, + 0x0062e, 0x0062f, 0x0062f, 0x00630, 0x00630, 0x00631, 0x00631, 0x00632, + 0x00632, 0x00633, 0x00633, 0x00633, 0x00633, 0x00634, 0x00634, 0x00634, + 0x00634, 0x00635, 0x00635, 0x00635, 0x00635, 0x00636, 0x00636, 0x00636, + 0x00636, 0x00637, 0x00637, 0x00637, 0x00637, 0x00638, 0x00638, 0x00638, + 0x00638, 0x00639, 0x00639, 0x00639, 0x00639, 0x0063a, 0x0063a, 0x0063a, + 0x0063a, 0x00641, 0x00641, 0x00641, 0x00641, 0x00642, 0x00642, 0x00642, + 0x00642, 0x00643, 0x00643, 0x00643, 0x00643, 0x00644, 0x00644, 0x00644, + 0x00644, 0x00645, 0x00645, 0x00645, 0x00645, 0x00646, 0x00646, 0x00646, + 0x00646, 0x00647, 0x00647, 0x00647, 0x00647, 0x00648, 0x00648, 0x00649, + 0x00649, 0x0064a, 0x0064a, 0x0064a, 0x0064a, 0x0fef5, 0x0fef5, 0x0fef7, + 0x0fef7, 0x0fef9, 0x0fef9, 0x0fefb, 0x0fefb, 0x0fefd, 0x0fefe, 0x0feff +}; + +static uint32_t unicode_520_ci_page_ff[] = { + 0x0ff00, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, + 0x00028, 0x00029, 0x0002a, 0x0002b, 0x0002c, 0x0002d, 0x0002e, 0x0002f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x0003a, 0x0003b, 0x0003c, 0x0003d, 0x0003e, 0x0003f, + 0x00040, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, + 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, + 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, + 0x00058, 0x00059, 0x0005a, 0x0005b, 0x0005c, 0x0005d, 0x0005e, 0x0005f, + 0x00060, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, + 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, + 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, + 0x00058, 0x00059, 0x0005a, 0x0007b, 0x0007c, 0x0007d, 0x0007e, 0x02985, + 0x02986, 0x03002, 0x0300c, 0x0300d, 0x03001, 0x030fb, 0x03092, 0x03042, + 0x03044, 0x03046, 0x03048, 0x0304a, 0x03084, 0x03086, 0x03088, 0x03064, + 0x030fc, 0x03042, 0x03044, 0x03046, 0x03048, 0x0304a, 0x0304b, 0x0304d, + 0x0304f, 0x03051, 0x03053, 0x03055, 0x03057, 0x03059, 0x0305b, 0x0305d, + 0x0305f, 0x03061, 0x03064, 0x03066, 0x03068, 0x0306a, 0x0306b, 0x0306c, + 0x0306d, 0x0306e, 0x0306f, 0x03072, 0x03075, 0x03078, 0x0307b, 0x0307e, + 0x0307f, 0x03080, 0x03081, 0x03082, 0x03084, 0x03086, 0x03088, 0x03089, + 0x0308a, 0x0308b, 0x0308c, 0x0308d, 0x0308f, 0x03093, 0x00000, 0x00000, + 0x01160, 0x01100, 0x01101, 0x011aa, 0x01102, 0x011ac, 0x011ad, 0x01103, + 0x01104, 0x01105, 0x011b0, 0x011b1, 0x011b2, 0x011b3, 0x011b4, 0x011b5, + 0x0111a, 0x01106, 0x01107, 0x01108, 0x01121, 0x01109, 0x0110a, 0x0110b, + 0x0110c, 0x0110d, 0x0110e, 0x0110f, 0x01110, 0x01111, 0x01112, 0x0ffbf, + 0x0ffc0, 0x0ffc1, 0x01161, 0x01162, 0x01163, 0x01164, 0x01165, 0x01166, + 0x0ffc8, 0x0ffc9, 0x01167, 0x01168, 0x01169, 0x0116a, 0x0116b, 0x0116c, + 0x0ffd0, 0x0ffd1, 0x0116d, 0x0116e, 0x0116f, 0x01170, 0x01171, 0x01172, + 0x0ffd8, 0x0ffd9, 0x01173, 0x01174, 0x01175, 0x0ffdd, 0x0ffde, 0x0ffdf, + 0x000a2, 0x000a3, 0x000ac, 0x000af, 0x000a6, 0x000a5, 0x020a9, 0x0ffe7, + 0x02502, 0x02190, 0x02191, 0x02192, 0x02193, 0x025a0, 0x025cb, 0x0ffef, + 0x0fff0, 0x0fff1, 0x0fff2, 0x0fff3, 0x0fff4, 0x0fff5, 0x0fff6, 0x0fff7, + 0x0fff8, 0x00000, 0x00000, 0x00000, 0x0fffc, 0x0fffd, 0x0fffe, 0x0ffff +}; + +static uint32_t unicode_520_ci_page_101[] = { + 0x10100, 0x10101, 0x10102, 0x10103, 0x10104, 0x10105, 0x10106, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x10110, 0x10111, 0x10112, 0x10113, 0x10114, 0x10115, 0x10116, 0x10117, + 0x10118, 0x10119, 0x1011a, 0x1011b, 0x1011c, 0x1011d, 0x1011e, 0x1011f, + 0x10120, 0x10121, 0x10122, 0x10123, 0x10124, 0x10125, 0x10126, 0x10127, + 0x10128, 0x10129, 0x1012a, 0x1012b, 0x1012c, 0x1012d, 0x1012e, 0x1012f, + 0x10130, 0x10131, 0x10132, 0x10133, 0x10134, 0x10135, 0x10136, 0x10137, + 0x10138, 0x10139, 0x1013a, 0x1013b, 0x1013c, 0x1013d, 0x1013e, 0x1013f, + 0x10140, 0x10141, 0x00031, 0x00035, 0x10144, 0x10145, 0x10146, 0x10147, + 0x00035, 0x10149, 0x1014a, 0x1014b, 0x1014c, 0x1014d, 0x1014e, 0x00035, + 0x10150, 0x10151, 0x10152, 0x10153, 0x10154, 0x10155, 0x10156, 0x10157, + 0x00031, 0x00031, 0x00031, 0x00032, 0x00032, 0x00032, 0x00032, 0x00035, + 0x10160, 0x10161, 0x10162, 0x10163, 0x10164, 0x10165, 0x10166, 0x10167, + 0x10168, 0x10169, 0x1016a, 0x1016b, 0x1016c, 0x1016d, 0x1016e, 0x1016f, + 0x10170, 0x10171, 0x10172, 0x00035, 0x10174, 0x10175, 0x10176, 0x10177, + 0x10178, 0x10179, 0x1017a, 0x1017b, 0x1017c, 0x1017d, 0x1017e, 0x1017f, + 0x10180, 0x10181, 0x10182, 0x10183, 0x10184, 0x10185, 0x10186, 0x10187, + 0x10188, 0x10189, 0x00030, 0x1018b, 0x1018c, 0x1018d, 0x1018e, 0x1018f, + 0x10190, 0x10191, 0x10192, 0x10193, 0x10194, 0x10195, 0x10196, 0x10197, + 0x10198, 0x10199, 0x1019a, 0x1019b, 0x1019c, 0x1019d, 0x1019e, 0x1019f, + 0x101a0, 0x101a1, 0x101a2, 0x101a3, 0x101a4, 0x101a5, 0x101a6, 0x101a7, + 0x101a8, 0x101a9, 0x101aa, 0x101ab, 0x101ac, 0x101ad, 0x101ae, 0x101af, + 0x101b0, 0x101b1, 0x101b2, 0x101b3, 0x101b4, 0x101b5, 0x101b6, 0x101b7, + 0x101b8, 0x101b9, 0x101ba, 0x101bb, 0x101bc, 0x101bd, 0x101be, 0x101bf, + 0x101c0, 0x101c1, 0x101c2, 0x101c3, 0x101c4, 0x101c5, 0x101c6, 0x101c7, + 0x101c8, 0x101c9, 0x101ca, 0x101cb, 0x101cc, 0x101cd, 0x101ce, 0x101cf, + 0x101d0, 0x101d1, 0x101d2, 0x101d3, 0x101d4, 0x101d5, 0x101d6, 0x101d7, + 0x101d8, 0x101d9, 0x101da, 0x101db, 0x101dc, 0x101dd, 0x101de, 0x101df, + 0x101e0, 0x101e1, 0x101e2, 0x101e3, 0x101e4, 0x101e5, 0x101e6, 0x101e7, + 0x101e8, 0x101e9, 0x101ea, 0x101eb, 0x101ec, 0x101ed, 0x101ee, 0x101ef, + 0x101f0, 0x101f1, 0x101f2, 0x101f3, 0x101f4, 0x101f5, 0x101f6, 0x101f7, + 0x101f8, 0x101f9, 0x101fa, 0x101fb, 0x101fc, 0x00000, 0x101fe, 0x101ff +}; + +static uint32_t unicode_520_ci_page_103[] = { + 0x10300, 0x10301, 0x10302, 0x10303, 0x10304, 0x10305, 0x10306, 0x10307, + 0x10308, 0x10309, 0x1030a, 0x1030b, 0x1030c, 0x1030d, 0x1030e, 0x1030f, + 0x10310, 0x10311, 0x10312, 0x10313, 0x10314, 0x10315, 0x10316, 0x10317, + 0x10318, 0x10319, 0x1031a, 0x1031b, 0x1031c, 0x1031d, 0x1031e, 0x1031f, + 0x00031, 0x00035, 0x10322, 0x10323, 0x10324, 0x10325, 0x10326, 0x10327, + 0x10328, 0x10329, 0x1032a, 0x1032b, 0x1032c, 0x1032d, 0x1032e, 0x1032f, + 0x10330, 0x10331, 0x10332, 0x10333, 0x10334, 0x10335, 0x10336, 0x10337, + 0x10338, 0x10339, 0x1033a, 0x1033b, 0x1033c, 0x1033d, 0x1033e, 0x1033f, + 0x10340, 0x10341, 0x10342, 0x10343, 0x10344, 0x10345, 0x10346, 0x10347, + 0x10348, 0x10349, 0x1034a, 0x1034b, 0x1034c, 0x1034d, 0x1034e, 0x1034f, + 0x10350, 0x10351, 0x10352, 0x10353, 0x10354, 0x10355, 0x10356, 0x10357, + 0x10358, 0x10359, 0x1035a, 0x1035b, 0x1035c, 0x1035d, 0x1035e, 0x1035f, + 0x10360, 0x10361, 0x10362, 0x10363, 0x10364, 0x10365, 0x10366, 0x10367, + 0x10368, 0x10369, 0x1036a, 0x1036b, 0x1036c, 0x1036d, 0x1036e, 0x1036f, + 0x10370, 0x10371, 0x10372, 0x10373, 0x10374, 0x10375, 0x10376, 0x10377, + 0x10378, 0x10379, 0x1037a, 0x1037b, 0x1037c, 0x1037d, 0x1037e, 0x1037f, + 0x10380, 0x10381, 0x10382, 0x10383, 0x10384, 0x10385, 0x10386, 0x10387, + 0x10388, 0x10389, 0x1038a, 0x1038b, 0x1038c, 0x1038d, 0x1038e, 0x1038f, + 0x10390, 0x10391, 0x10392, 0x10393, 0x10394, 0x10395, 0x10396, 0x10397, + 0x10398, 0x10399, 0x1039a, 0x1039b, 0x1039c, 0x1039d, 0x1039e, 0x1039f, + 0x103a0, 0x103a1, 0x103a2, 0x103a3, 0x103a4, 0x103a5, 0x103a6, 0x103a7, + 0x103a8, 0x103a9, 0x103aa, 0x103ab, 0x103ac, 0x103ad, 0x103ae, 0x103af, + 0x103b0, 0x103b1, 0x103b2, 0x103b3, 0x103b4, 0x103b5, 0x103b6, 0x103b7, + 0x103b8, 0x103b9, 0x103ba, 0x103bb, 0x103bc, 0x103bd, 0x103be, 0x103bf, + 0x103c0, 0x103c1, 0x103c2, 0x103c3, 0x103c4, 0x103c5, 0x103c6, 0x103c7, + 0x103c8, 0x103c9, 0x103ca, 0x103cb, 0x103cc, 0x103cd, 0x103ce, 0x103cf, + 0x103d0, 0x00031, 0x00032, 0x103d3, 0x103d4, 0x103d5, 0x103d6, 0x103d7, + 0x103d8, 0x103d9, 0x103da, 0x103db, 0x103dc, 0x103dd, 0x103de, 0x103df, + 0x103e0, 0x103e1, 0x103e2, 0x103e3, 0x103e4, 0x103e5, 0x103e6, 0x103e7, + 0x103e8, 0x103e9, 0x103ea, 0x103eb, 0x103ec, 0x103ed, 0x103ee, 0x103ef, + 0x103f0, 0x103f1, 0x103f2, 0x103f3, 0x103f4, 0x103f5, 0x103f6, 0x103f7, + 0x103f8, 0x103f9, 0x103fa, 0x103fb, 0x103fc, 0x103fd, 0x103fe, 0x103ff +}; + +static uint32_t unicode_520_ci_page_104[] = { + 0x10400, 0x10401, 0x10402, 0x10403, 0x10404, 0x10405, 0x10406, 0x10407, + 0x10408, 0x10409, 0x1040a, 0x1040b, 0x1040c, 0x1040d, 0x1040e, 0x1040f, + 0x10410, 0x10411, 0x10412, 0x10413, 0x10414, 0x10415, 0x10416, 0x10417, + 0x10418, 0x10419, 0x1041a, 0x1041b, 0x1041c, 0x1041d, 0x1041e, 0x1041f, + 0x10420, 0x10421, 0x10422, 0x10423, 0x10424, 0x10425, 0x10426, 0x10427, + 0x10400, 0x10401, 0x10402, 0x10403, 0x10404, 0x10405, 0x10406, 0x10407, + 0x10408, 0x10409, 0x1040a, 0x1040b, 0x1040c, 0x1040d, 0x1040e, 0x1040f, + 0x10410, 0x10411, 0x10412, 0x10413, 0x10414, 0x10415, 0x10416, 0x10417, + 0x10418, 0x10419, 0x1041a, 0x1041b, 0x1041c, 0x1041d, 0x1041e, 0x1041f, + 0x10420, 0x10421, 0x10422, 0x10423, 0x10424, 0x10425, 0x10426, 0x10427, + 0x10450, 0x10451, 0x10452, 0x10453, 0x10454, 0x10455, 0x10456, 0x10457, + 0x10458, 0x10459, 0x1045a, 0x1045b, 0x1045c, 0x1045d, 0x1045e, 0x1045f, + 0x10460, 0x10461, 0x10462, 0x10463, 0x10464, 0x10465, 0x10466, 0x10467, + 0x10468, 0x10469, 0x1046a, 0x1046b, 0x1046c, 0x1046d, 0x1046e, 0x1046f, + 0x10470, 0x10471, 0x10472, 0x10473, 0x10474, 0x10475, 0x10476, 0x10477, + 0x10478, 0x10479, 0x1047a, 0x1047b, 0x1047c, 0x1047d, 0x1047e, 0x1047f, + 0x10480, 0x10481, 0x10482, 0x10483, 0x10484, 0x10485, 0x10486, 0x10487, + 0x10488, 0x10489, 0x1048a, 0x1048b, 0x1048c, 0x1048d, 0x1048e, 0x1048f, + 0x10490, 0x10491, 0x10492, 0x10493, 0x10494, 0x10495, 0x10496, 0x10497, + 0x10498, 0x10499, 0x1049a, 0x1049b, 0x1049c, 0x1049d, 0x1049e, 0x1049f, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x104aa, 0x104ab, 0x104ac, 0x104ad, 0x104ae, 0x104af, + 0x104b0, 0x104b1, 0x104b2, 0x104b3, 0x104b4, 0x104b5, 0x104b6, 0x104b7, + 0x104b8, 0x104b9, 0x104ba, 0x104bb, 0x104bc, 0x104bd, 0x104be, 0x104bf, + 0x104c0, 0x104c1, 0x104c2, 0x104c3, 0x104c4, 0x104c5, 0x104c6, 0x104c7, + 0x104c8, 0x104c9, 0x104ca, 0x104cb, 0x104cc, 0x104cd, 0x104ce, 0x104cf, + 0x104d0, 0x104d1, 0x104d2, 0x104d3, 0x104d4, 0x104d5, 0x104d6, 0x104d7, + 0x104d8, 0x104d9, 0x104da, 0x104db, 0x104dc, 0x104dd, 0x104de, 0x104df, + 0x104e0, 0x104e1, 0x104e2, 0x104e3, 0x104e4, 0x104e5, 0x104e6, 0x104e7, + 0x104e8, 0x104e9, 0x104ea, 0x104eb, 0x104ec, 0x104ed, 0x104ee, 0x104ef, + 0x104f0, 0x104f1, 0x104f2, 0x104f3, 0x104f4, 0x104f5, 0x104f6, 0x104f7, + 0x104f8, 0x104f9, 0x104fa, 0x104fb, 0x104fc, 0x104fd, 0x104fe, 0x104ff +}; + +static uint32_t unicode_520_ci_page_108[] = { + 0x10800, 0x10801, 0x10802, 0x10803, 0x10804, 0x10805, 0x10806, 0x10807, + 0x10808, 0x10809, 0x1080a, 0x1080b, 0x1080c, 0x1080d, 0x1080e, 0x1080f, + 0x10810, 0x10811, 0x10812, 0x10813, 0x10814, 0x10815, 0x10816, 0x10817, + 0x10818, 0x10819, 0x1081a, 0x1081b, 0x1081c, 0x1081d, 0x1081e, 0x1081f, + 0x10820, 0x10821, 0x10822, 0x10823, 0x10824, 0x10825, 0x10826, 0x10827, + 0x10828, 0x10829, 0x1082a, 0x1082b, 0x1082c, 0x1082d, 0x1082e, 0x1082f, + 0x10830, 0x10831, 0x10832, 0x10833, 0x10834, 0x10835, 0x10836, 0x10837, + 0x10838, 0x10839, 0x1083a, 0x1083b, 0x1083c, 0x1083d, 0x1083e, 0x1083f, + 0x10840, 0x10841, 0x10842, 0x10843, 0x10844, 0x10845, 0x10846, 0x10847, + 0x10848, 0x10849, 0x1084a, 0x1084b, 0x1084c, 0x1084d, 0x1084e, 0x1084f, + 0x10850, 0x10851, 0x10852, 0x10853, 0x10854, 0x10855, 0x10856, 0x10857, + 0x00031, 0x00032, 0x00033, 0x1085b, 0x1085c, 0x1085d, 0x1085e, 0x1085f, + 0x10860, 0x10861, 0x10862, 0x10863, 0x10864, 0x10865, 0x10866, 0x10867, + 0x10868, 0x10869, 0x1086a, 0x1086b, 0x1086c, 0x1086d, 0x1086e, 0x1086f, + 0x10870, 0x10871, 0x10872, 0x10873, 0x10874, 0x10875, 0x10876, 0x10877, + 0x10878, 0x10879, 0x1087a, 0x1087b, 0x1087c, 0x1087d, 0x1087e, 0x1087f, + 0x10880, 0x10881, 0x10882, 0x10883, 0x10884, 0x10885, 0x10886, 0x10887, + 0x10888, 0x10889, 0x1088a, 0x1088b, 0x1088c, 0x1088d, 0x1088e, 0x1088f, + 0x10890, 0x10891, 0x10892, 0x10893, 0x10894, 0x10895, 0x10896, 0x10897, + 0x10898, 0x10899, 0x1089a, 0x1089b, 0x1089c, 0x1089d, 0x1089e, 0x1089f, + 0x108a0, 0x108a1, 0x108a2, 0x108a3, 0x108a4, 0x108a5, 0x108a6, 0x108a7, + 0x108a8, 0x108a9, 0x108aa, 0x108ab, 0x108ac, 0x108ad, 0x108ae, 0x108af, + 0x108b0, 0x108b1, 0x108b2, 0x108b3, 0x108b4, 0x108b5, 0x108b6, 0x108b7, + 0x108b8, 0x108b9, 0x108ba, 0x108bb, 0x108bc, 0x108bd, 0x108be, 0x108bf, + 0x108c0, 0x108c1, 0x108c2, 0x108c3, 0x108c4, 0x108c5, 0x108c6, 0x108c7, + 0x108c8, 0x108c9, 0x108ca, 0x108cb, 0x108cc, 0x108cd, 0x108ce, 0x108cf, + 0x108d0, 0x108d1, 0x108d2, 0x108d3, 0x108d4, 0x108d5, 0x108d6, 0x108d7, + 0x108d8, 0x108d9, 0x108da, 0x108db, 0x108dc, 0x108dd, 0x108de, 0x108df, + 0x108e0, 0x108e1, 0x108e2, 0x108e3, 0x108e4, 0x108e5, 0x108e6, 0x108e7, + 0x108e8, 0x108e9, 0x108ea, 0x108eb, 0x108ec, 0x108ed, 0x108ee, 0x108ef, + 0x108f0, 0x108f1, 0x108f2, 0x108f3, 0x108f4, 0x108f5, 0x108f6, 0x108f7, + 0x108f8, 0x108f9, 0x108fa, 0x108fb, 0x108fc, 0x108fd, 0x108fe, 0x108ff +}; + +static uint32_t unicode_520_ci_page_109[] = { + 0x10900, 0x10901, 0x10902, 0x10903, 0x10904, 0x10905, 0x10906, 0x10907, + 0x10908, 0x10909, 0x1090a, 0x1090b, 0x1090c, 0x1090d, 0x1090e, 0x1090f, + 0x10910, 0x10911, 0x10912, 0x10913, 0x10914, 0x10915, 0x00031, 0x10917, + 0x10918, 0x10919, 0x00032, 0x00033, 0x1091c, 0x1091d, 0x1091e, 0x1091f, + 0x10920, 0x10921, 0x10922, 0x10923, 0x10924, 0x10925, 0x10926, 0x10927, + 0x10928, 0x10929, 0x1092a, 0x1092b, 0x1092c, 0x1092d, 0x1092e, 0x1092f, + 0x10930, 0x10931, 0x10932, 0x10933, 0x10934, 0x10935, 0x10936, 0x10937, + 0x10938, 0x10939, 0x1093a, 0x1093b, 0x1093c, 0x1093d, 0x1093e, 0x1093f, + 0x10940, 0x10941, 0x10942, 0x10943, 0x10944, 0x10945, 0x10946, 0x10947, + 0x10948, 0x10949, 0x1094a, 0x1094b, 0x1094c, 0x1094d, 0x1094e, 0x1094f, + 0x10950, 0x10951, 0x10952, 0x10953, 0x10954, 0x10955, 0x10956, 0x10957, + 0x10958, 0x10959, 0x1095a, 0x1095b, 0x1095c, 0x1095d, 0x1095e, 0x1095f, + 0x10960, 0x10961, 0x10962, 0x10963, 0x10964, 0x10965, 0x10966, 0x10967, + 0x10968, 0x10969, 0x1096a, 0x1096b, 0x1096c, 0x1096d, 0x1096e, 0x1096f, + 0x10970, 0x10971, 0x10972, 0x10973, 0x10974, 0x10975, 0x10976, 0x10977, + 0x10978, 0x10979, 0x1097a, 0x1097b, 0x1097c, 0x1097d, 0x1097e, 0x1097f, + 0x10980, 0x10981, 0x10982, 0x10983, 0x10984, 0x10985, 0x10986, 0x10987, + 0x10988, 0x10989, 0x1098a, 0x1098b, 0x1098c, 0x1098d, 0x1098e, 0x1098f, + 0x10990, 0x10991, 0x10992, 0x10993, 0x10994, 0x10995, 0x10996, 0x10997, + 0x10998, 0x10999, 0x1099a, 0x1099b, 0x1099c, 0x1099d, 0x1099e, 0x1099f, + 0x109a0, 0x109a1, 0x109a2, 0x109a3, 0x109a4, 0x109a5, 0x109a6, 0x109a7, + 0x109a8, 0x109a9, 0x109aa, 0x109ab, 0x109ac, 0x109ad, 0x109ae, 0x109af, + 0x109b0, 0x109b1, 0x109b2, 0x109b3, 0x109b4, 0x109b5, 0x109b6, 0x109b7, + 0x109b8, 0x109b9, 0x109ba, 0x109bb, 0x109bc, 0x109bd, 0x109be, 0x109bf, + 0x109c0, 0x109c1, 0x109c2, 0x109c3, 0x109c4, 0x109c5, 0x109c6, 0x109c7, + 0x109c8, 0x109c9, 0x109ca, 0x109cb, 0x109cc, 0x109cd, 0x109ce, 0x109cf, + 0x109d0, 0x109d1, 0x109d2, 0x109d3, 0x109d4, 0x109d5, 0x109d6, 0x109d7, + 0x109d8, 0x109d9, 0x109da, 0x109db, 0x109dc, 0x109dd, 0x109de, 0x109df, + 0x109e0, 0x109e1, 0x109e2, 0x109e3, 0x109e4, 0x109e5, 0x109e6, 0x109e7, + 0x109e8, 0x109e9, 0x109ea, 0x109eb, 0x109ec, 0x109ed, 0x109ee, 0x109ef, + 0x109f0, 0x109f1, 0x109f2, 0x109f3, 0x109f4, 0x109f5, 0x109f6, 0x109f7, + 0x109f8, 0x109f9, 0x109fa, 0x109fb, 0x109fc, 0x109fd, 0x109fe, 0x109ff +}; + +static uint32_t unicode_520_ci_page_10a[] = { + 0x10a00, 0x10a01, 0x10a02, 0x10a03, 0x10a04, 0x10a05, 0x10a06, 0x10a07, + 0x10a08, 0x10a09, 0x10a0a, 0x10a0b, 0x10a0c, 0x00000, 0x00000, 0x00000, + 0x10a10, 0x10a11, 0x10a12, 0x10a13, 0x10a14, 0x10a15, 0x10a16, 0x10a17, + 0x10a18, 0x10a19, 0x10a1a, 0x10a1b, 0x10a1c, 0x10a1d, 0x10a1e, 0x10a1f, + 0x10a20, 0x10a21, 0x10a22, 0x10a23, 0x10a24, 0x10a25, 0x10a26, 0x10a27, + 0x10a28, 0x10a29, 0x10a2a, 0x10a2b, 0x10a2c, 0x10a2d, 0x10a2e, 0x10a2f, + 0x10a30, 0x10a31, 0x10a32, 0x10a33, 0x10a34, 0x10a35, 0x10a36, 0x10a37, + 0x00000, 0x00000, 0x00000, 0x10a3b, 0x10a3c, 0x10a3d, 0x10a3e, 0x10a3f, + 0x00031, 0x00032, 0x00033, 0x00034, 0x10a44, 0x10a45, 0x10a46, 0x10a47, + 0x10a48, 0x10a49, 0x10a4a, 0x10a4b, 0x10a4c, 0x10a4d, 0x10a4e, 0x10a4f, + 0x10a50, 0x10a51, 0x10a52, 0x10a53, 0x10a54, 0x10a55, 0x10a56, 0x10a57, + 0x10a58, 0x10a59, 0x10a5a, 0x10a5b, 0x10a5c, 0x10a5d, 0x10a5e, 0x10a5f, + 0x10a60, 0x10a61, 0x10a62, 0x10a63, 0x10a64, 0x10a65, 0x10a66, 0x10a67, + 0x10a68, 0x10a69, 0x10a6a, 0x10a6b, 0x10a6c, 0x10a6d, 0x10a6e, 0x10a6f, + 0x10a70, 0x10a71, 0x10a72, 0x10a73, 0x10a74, 0x10a75, 0x10a76, 0x10a77, + 0x10a78, 0x10a79, 0x10a7a, 0x10a7b, 0x10a7c, 0x00031, 0x10a7e, 0x10a7f, + 0x10a80, 0x10a81, 0x10a82, 0x10a83, 0x10a84, 0x10a85, 0x10a86, 0x10a87, + 0x10a88, 0x10a89, 0x10a8a, 0x10a8b, 0x10a8c, 0x10a8d, 0x10a8e, 0x10a8f, + 0x10a90, 0x10a91, 0x10a92, 0x10a93, 0x10a94, 0x10a95, 0x10a96, 0x10a97, + 0x10a98, 0x10a99, 0x10a9a, 0x10a9b, 0x10a9c, 0x10a9d, 0x10a9e, 0x10a9f, + 0x10aa0, 0x10aa1, 0x10aa2, 0x10aa3, 0x10aa4, 0x10aa5, 0x10aa6, 0x10aa7, + 0x10aa8, 0x10aa9, 0x10aaa, 0x10aab, 0x10aac, 0x10aad, 0x10aae, 0x10aaf, + 0x10ab0, 0x10ab1, 0x10ab2, 0x10ab3, 0x10ab4, 0x10ab5, 0x10ab6, 0x10ab7, + 0x10ab8, 0x10ab9, 0x10aba, 0x10abb, 0x10abc, 0x10abd, 0x10abe, 0x10abf, + 0x10ac0, 0x10ac1, 0x10ac2, 0x10ac3, 0x10ac4, 0x10ac5, 0x10ac6, 0x10ac7, + 0x10ac8, 0x10ac9, 0x10aca, 0x10acb, 0x10acc, 0x10acd, 0x10ace, 0x10acf, + 0x10ad0, 0x10ad1, 0x10ad2, 0x10ad3, 0x10ad4, 0x10ad5, 0x10ad6, 0x10ad7, + 0x10ad8, 0x10ad9, 0x10ada, 0x10adb, 0x10adc, 0x10add, 0x10ade, 0x10adf, + 0x10ae0, 0x10ae1, 0x10ae2, 0x10ae3, 0x10ae4, 0x10ae5, 0x10ae6, 0x10ae7, + 0x10ae8, 0x10ae9, 0x10aea, 0x10aeb, 0x10aec, 0x10aed, 0x10aee, 0x10aef, + 0x10af0, 0x10af1, 0x10af2, 0x10af3, 0x10af4, 0x10af5, 0x10af6, 0x10af7, + 0x10af8, 0x10af9, 0x10afa, 0x10afb, 0x10afc, 0x10afd, 0x10afe, 0x10aff +}; + +static uint32_t unicode_520_ci_page_10b[] = { + 0x10b00, 0x10b01, 0x10b02, 0x10b03, 0x10b04, 0x10b05, 0x10b06, 0x10b07, + 0x10b08, 0x10b09, 0x10b0a, 0x10b0b, 0x10b0c, 0x10b0d, 0x10b0e, 0x10b0f, + 0x10b10, 0x10b11, 0x10b12, 0x10b13, 0x10b14, 0x10b15, 0x10b16, 0x10b17, + 0x10b18, 0x10b19, 0x10b1a, 0x10b1b, 0x10b1c, 0x10b1d, 0x10b1e, 0x10b1f, + 0x10b20, 0x10b21, 0x10b22, 0x10b23, 0x10b24, 0x10b25, 0x10b26, 0x10b27, + 0x10b28, 0x10b29, 0x10b2a, 0x10b2b, 0x10b2c, 0x10b2d, 0x10b2d, 0x10b2f, + 0x10b30, 0x10b31, 0x10b32, 0x10b33, 0x10b34, 0x10b35, 0x10b36, 0x10b37, + 0x10b38, 0x10b39, 0x10b3a, 0x10b3b, 0x10b3c, 0x10b3d, 0x10b3e, 0x10b3f, + 0x10b40, 0x10b41, 0x10b42, 0x10b43, 0x10b44, 0x10b45, 0x10b46, 0x10b47, + 0x10b48, 0x10b49, 0x10b4a, 0x10b4b, 0x10b4c, 0x10b4d, 0x10b4e, 0x10b4f, + 0x10b50, 0x10b51, 0x10b52, 0x10b53, 0x10b54, 0x10b55, 0x10b56, 0x10b57, + 0x00031, 0x00032, 0x00033, 0x00034, 0x10b5c, 0x10b5d, 0x10b5e, 0x10b5f, + 0x10b60, 0x10b61, 0x10b62, 0x10b63, 0x10b64, 0x10b65, 0x10b66, 0x10b67, + 0x10b68, 0x10b69, 0x10b6a, 0x10b6b, 0x10b6c, 0x10b6d, 0x10b6e, 0x10b6f, + 0x10b70, 0x10b71, 0x10b72, 0x10b73, 0x10b74, 0x10b75, 0x10b76, 0x10b77, + 0x00031, 0x00032, 0x00033, 0x00034, 0x10b7c, 0x10b7d, 0x10b7e, 0x10b7f, + 0x10b80, 0x10b81, 0x10b82, 0x10b83, 0x10b84, 0x10b85, 0x10b86, 0x10b87, + 0x10b88, 0x10b89, 0x10b8a, 0x10b8b, 0x10b8c, 0x10b8d, 0x10b8e, 0x10b8f, + 0x10b90, 0x10b91, 0x10b92, 0x10b93, 0x10b94, 0x10b95, 0x10b96, 0x10b97, + 0x10b98, 0x10b99, 0x10b9a, 0x10b9b, 0x10b9c, 0x10b9d, 0x10b9e, 0x10b9f, + 0x10ba0, 0x10ba1, 0x10ba2, 0x10ba3, 0x10ba4, 0x10ba5, 0x10ba6, 0x10ba7, + 0x10ba8, 0x10ba9, 0x10baa, 0x10bab, 0x10bac, 0x10bad, 0x10bae, 0x10baf, + 0x10bb0, 0x10bb1, 0x10bb2, 0x10bb3, 0x10bb4, 0x10bb5, 0x10bb6, 0x10bb7, + 0x10bb8, 0x10bb9, 0x10bba, 0x10bbb, 0x10bbc, 0x10bbd, 0x10bbe, 0x10bbf, + 0x10bc0, 0x10bc1, 0x10bc2, 0x10bc3, 0x10bc4, 0x10bc5, 0x10bc6, 0x10bc7, + 0x10bc8, 0x10bc9, 0x10bca, 0x10bcb, 0x10bcc, 0x10bcd, 0x10bce, 0x10bcf, + 0x10bd0, 0x10bd1, 0x10bd2, 0x10bd3, 0x10bd4, 0x10bd5, 0x10bd6, 0x10bd7, + 0x10bd8, 0x10bd9, 0x10bda, 0x10bdb, 0x10bdc, 0x10bdd, 0x10bde, 0x10bdf, + 0x10be0, 0x10be1, 0x10be2, 0x10be3, 0x10be4, 0x10be5, 0x10be6, 0x10be7, + 0x10be8, 0x10be9, 0x10bea, 0x10beb, 0x10bec, 0x10bed, 0x10bee, 0x10bef, + 0x10bf0, 0x10bf1, 0x10bf2, 0x10bf3, 0x10bf4, 0x10bf5, 0x10bf6, 0x10bf7, + 0x10bf8, 0x10bf9, 0x10bfa, 0x10bfb, 0x10bfc, 0x10bfd, 0x10bfe, 0x10bff +}; + +static uint32_t unicode_520_ci_page_10c[] = { + 0x10c00, 0x10c00, 0x10c02, 0x10c03, 0x10c03, 0x10c05, 0x10c06, 0x10c07, + 0x10c07, 0x10c09, 0x10c09, 0x10c0b, 0x10c0b, 0x10c0d, 0x10c0d, 0x10c0f, + 0x10c0f, 0x10c11, 0x10c11, 0x10c13, 0x10c14, 0x10c14, 0x10c16, 0x10c16, + 0x10c18, 0x10c18, 0x10c1a, 0x10c1a, 0x10c1c, 0x10c1c, 0x10c1e, 0x10c1e, + 0x10c20, 0x10c21, 0x10c22, 0x10c23, 0x10c24, 0x10c24, 0x10c26, 0x10c26, + 0x10c28, 0x10c28, 0x10c2a, 0x10c2a, 0x10c2c, 0x10c2d, 0x10c2d, 0x10c2f, + 0x10c30, 0x10c31, 0x10c32, 0x10c32, 0x10c34, 0x10c34, 0x10c36, 0x10c36, + 0x10c38, 0x10c38, 0x10c3a, 0x10c3a, 0x10c3c, 0x10c3d, 0x10c3e, 0x10c3f, + 0x10c3f, 0x10c41, 0x10c41, 0x10c43, 0x10c43, 0x10c45, 0x10c45, 0x10c47, + 0x10c48, 0x10c49, 0x10c4a, 0x10c4b, 0x10c4c, 0x10c4d, 0x10c4e, 0x10c4f, + 0x10c50, 0x10c51, 0x10c52, 0x10c53, 0x10c54, 0x10c55, 0x10c56, 0x10c57, + 0x10c58, 0x10c59, 0x10c5a, 0x10c5b, 0x10c5c, 0x10c5d, 0x10c5e, 0x10c5f, + 0x10c60, 0x10c61, 0x10c62, 0x10c63, 0x10c64, 0x10c65, 0x10c66, 0x10c67, + 0x10c68, 0x10c69, 0x10c6a, 0x10c6b, 0x10c6c, 0x10c6d, 0x10c6e, 0x10c6f, + 0x10c70, 0x10c71, 0x10c72, 0x10c73, 0x10c74, 0x10c75, 0x10c76, 0x10c77, + 0x10c78, 0x10c79, 0x10c7a, 0x10c7b, 0x10c7c, 0x10c7d, 0x10c7e, 0x10c7f, + 0x10c80, 0x10c81, 0x10c82, 0x10c83, 0x10c84, 0x10c85, 0x10c86, 0x10c87, + 0x10c88, 0x10c89, 0x10c8a, 0x10c8b, 0x10c8c, 0x10c8d, 0x10c8e, 0x10c8f, + 0x10c90, 0x10c91, 0x10c92, 0x10c93, 0x10c94, 0x10c95, 0x10c96, 0x10c97, + 0x10c98, 0x10c99, 0x10c9a, 0x10c9b, 0x10c9c, 0x10c9d, 0x10c9e, 0x10c9f, + 0x10ca0, 0x10ca1, 0x10ca2, 0x10ca3, 0x10ca4, 0x10ca5, 0x10ca6, 0x10ca7, + 0x10ca8, 0x10ca9, 0x10caa, 0x10cab, 0x10cac, 0x10cad, 0x10cae, 0x10caf, + 0x10cb0, 0x10cb1, 0x10cb2, 0x10cb3, 0x10cb4, 0x10cb5, 0x10cb6, 0x10cb7, + 0x10cb8, 0x10cb9, 0x10cba, 0x10cbb, 0x10cbc, 0x10cbd, 0x10cbe, 0x10cbf, + 0x10cc0, 0x10cc1, 0x10cc2, 0x10cc3, 0x10cc4, 0x10cc5, 0x10cc6, 0x10cc7, + 0x10cc8, 0x10cc9, 0x10cca, 0x10ccb, 0x10ccc, 0x10ccd, 0x10cce, 0x10ccf, + 0x10cd0, 0x10cd1, 0x10cd2, 0x10cd3, 0x10cd4, 0x10cd5, 0x10cd6, 0x10cd7, + 0x10cd8, 0x10cd9, 0x10cda, 0x10cdb, 0x10cdc, 0x10cdd, 0x10cde, 0x10cdf, + 0x10ce0, 0x10ce1, 0x10ce2, 0x10ce3, 0x10ce4, 0x10ce5, 0x10ce6, 0x10ce7, + 0x10ce8, 0x10ce9, 0x10cea, 0x10ceb, 0x10cec, 0x10ced, 0x10cee, 0x10cef, + 0x10cf0, 0x10cf1, 0x10cf2, 0x10cf3, 0x10cf4, 0x10cf5, 0x10cf6, 0x10cf7, + 0x10cf8, 0x10cf9, 0x10cfa, 0x10cfb, 0x10cfc, 0x10cfd, 0x10cfe, 0x10cff +}; + +static uint32_t unicode_520_ci_page_10e[] = { + 0x10e00, 0x10e01, 0x10e02, 0x10e03, 0x10e04, 0x10e05, 0x10e06, 0x10e07, + 0x10e08, 0x10e09, 0x10e0a, 0x10e0b, 0x10e0c, 0x10e0d, 0x10e0e, 0x10e0f, + 0x10e10, 0x10e11, 0x10e12, 0x10e13, 0x10e14, 0x10e15, 0x10e16, 0x10e17, + 0x10e18, 0x10e19, 0x10e1a, 0x10e1b, 0x10e1c, 0x10e1d, 0x10e1e, 0x10e1f, + 0x10e20, 0x10e21, 0x10e22, 0x10e23, 0x10e24, 0x10e25, 0x10e26, 0x10e27, + 0x10e28, 0x10e29, 0x10e2a, 0x10e2b, 0x10e2c, 0x10e2d, 0x10e2e, 0x10e2f, + 0x10e30, 0x10e31, 0x10e32, 0x10e33, 0x10e34, 0x10e35, 0x10e36, 0x10e37, + 0x10e38, 0x10e39, 0x10e3a, 0x10e3b, 0x10e3c, 0x10e3d, 0x10e3e, 0x10e3f, + 0x10e40, 0x10e41, 0x10e42, 0x10e43, 0x10e44, 0x10e45, 0x10e46, 0x10e47, + 0x10e48, 0x10e49, 0x10e4a, 0x10e4b, 0x10e4c, 0x10e4d, 0x10e4e, 0x10e4f, + 0x10e50, 0x10e51, 0x10e52, 0x10e53, 0x10e54, 0x10e55, 0x10e56, 0x10e57, + 0x10e58, 0x10e59, 0x10e5a, 0x10e5b, 0x10e5c, 0x10e5d, 0x10e5e, 0x10e5f, + 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, + 0x00039, 0x10e69, 0x10e6a, 0x10e6b, 0x10e6c, 0x10e6d, 0x10e6e, 0x10e6f, + 0x10e70, 0x10e71, 0x10e72, 0x10e73, 0x10e74, 0x10e75, 0x10e76, 0x10e77, + 0x10e78, 0x10e79, 0x10e7a, 0x10e7b, 0x10e7c, 0x10e7d, 0x10e7e, 0x10e7f, + 0x10e80, 0x10e81, 0x10e82, 0x10e83, 0x10e84, 0x10e85, 0x10e86, 0x10e87, + 0x10e88, 0x10e89, 0x10e8a, 0x10e8b, 0x10e8c, 0x10e8d, 0x10e8e, 0x10e8f, + 0x10e90, 0x10e91, 0x10e92, 0x10e93, 0x10e94, 0x10e95, 0x10e96, 0x10e97, + 0x10e98, 0x10e99, 0x10e9a, 0x10e9b, 0x10e9c, 0x10e9d, 0x10e9e, 0x10e9f, + 0x10ea0, 0x10ea1, 0x10ea2, 0x10ea3, 0x10ea4, 0x10ea5, 0x10ea6, 0x10ea7, + 0x10ea8, 0x10ea9, 0x10eaa, 0x10eab, 0x10eac, 0x10ead, 0x10eae, 0x10eaf, + 0x10eb0, 0x10eb1, 0x10eb2, 0x10eb3, 0x10eb4, 0x10eb5, 0x10eb6, 0x10eb7, + 0x10eb8, 0x10eb9, 0x10eba, 0x10ebb, 0x10ebc, 0x10ebd, 0x10ebe, 0x10ebf, + 0x10ec0, 0x10ec1, 0x10ec2, 0x10ec3, 0x10ec4, 0x10ec5, 0x10ec6, 0x10ec7, + 0x10ec8, 0x10ec9, 0x10eca, 0x10ecb, 0x10ecc, 0x10ecd, 0x10ece, 0x10ecf, + 0x10ed0, 0x10ed1, 0x10ed2, 0x10ed3, 0x10ed4, 0x10ed5, 0x10ed6, 0x10ed7, + 0x10ed8, 0x10ed9, 0x10eda, 0x10edb, 0x10edc, 0x10edd, 0x10ede, 0x10edf, + 0x10ee0, 0x10ee1, 0x10ee2, 0x10ee3, 0x10ee4, 0x10ee5, 0x10ee6, 0x10ee7, + 0x10ee8, 0x10ee9, 0x10eea, 0x10eeb, 0x10eec, 0x10eed, 0x10eee, 0x10eef, + 0x10ef0, 0x10ef1, 0x10ef2, 0x10ef3, 0x10ef4, 0x10ef5, 0x10ef6, 0x10ef7, + 0x10ef8, 0x10ef9, 0x10efa, 0x10efb, 0x10efc, 0x10efd, 0x10efe, 0x10eff +}; + +static uint32_t unicode_520_ci_page_110[] = { + 0x11000, 0x11001, 0x11002, 0x11003, 0x11004, 0x11005, 0x11006, 0x11007, + 0x11008, 0x11009, 0x1100a, 0x1100b, 0x1100c, 0x1100d, 0x1100e, 0x1100f, + 0x11010, 0x11011, 0x11012, 0x11013, 0x11014, 0x11015, 0x11016, 0x11017, + 0x11018, 0x11019, 0x1101a, 0x1101b, 0x1101c, 0x1101d, 0x1101e, 0x1101f, + 0x11020, 0x11021, 0x11022, 0x11023, 0x11024, 0x11025, 0x11026, 0x11027, + 0x11028, 0x11029, 0x1102a, 0x1102b, 0x1102c, 0x1102d, 0x1102e, 0x1102f, + 0x11030, 0x11031, 0x11032, 0x11033, 0x11034, 0x11035, 0x11036, 0x11037, + 0x11038, 0x11039, 0x1103a, 0x1103b, 0x1103c, 0x1103d, 0x1103e, 0x1103f, + 0x11040, 0x11041, 0x11042, 0x11043, 0x11044, 0x11045, 0x11046, 0x11047, + 0x11048, 0x11049, 0x1104a, 0x1104b, 0x1104c, 0x1104d, 0x1104e, 0x1104f, + 0x11050, 0x11051, 0x11052, 0x11053, 0x11054, 0x11055, 0x11056, 0x11057, + 0x11058, 0x11059, 0x1105a, 0x1105b, 0x1105c, 0x1105d, 0x1105e, 0x1105f, + 0x11060, 0x11061, 0x11062, 0x11063, 0x11064, 0x11065, 0x11066, 0x11067, + 0x11068, 0x11069, 0x1106a, 0x1106b, 0x1106c, 0x1106d, 0x1106e, 0x1106f, + 0x11070, 0x11071, 0x11072, 0x11073, 0x11074, 0x11075, 0x11076, 0x11077, + 0x11078, 0x11079, 0x1107a, 0x1107b, 0x1107c, 0x1107d, 0x1107e, 0x1107f, + 0x00000, 0x00000, 0x00000, 0x11083, 0x11084, 0x11085, 0x11086, 0x11087, + 0x11088, 0x11089, 0x1108a, 0x1108b, 0x1108c, 0x1108d, 0x1108e, 0x1108f, + 0x11090, 0x11091, 0x11092, 0x11093, 0x11094, 0x11095, 0x11096, 0x11097, + 0x11098, 0x11099, 0x11099, 0x1109b, 0x1109b, 0x1109d, 0x1109e, 0x1109f, + 0x110a0, 0x110a1, 0x110a2, 0x110a3, 0x110a4, 0x110a5, 0x110a6, 0x110a7, + 0x110a8, 0x110a9, 0x110aa, 0x110a5, 0x110ac, 0x110ad, 0x110ae, 0x110af, + 0x110b0, 0x110b1, 0x110b2, 0x110b3, 0x110b4, 0x110b5, 0x110b6, 0x110b7, + 0x110b8, 0x110b9, 0x00000, 0x110bb, 0x110bc, 0x00000, 0x110be, 0x110bf, + 0x110c0, 0x110c1, 0x110c2, 0x110c3, 0x110c4, 0x110c5, 0x110c6, 0x110c7, + 0x110c8, 0x110c9, 0x110ca, 0x110cb, 0x110cc, 0x110cd, 0x110ce, 0x110cf, + 0x110d0, 0x110d1, 0x110d2, 0x110d3, 0x110d4, 0x110d5, 0x110d6, 0x110d7, + 0x110d8, 0x110d9, 0x110da, 0x110db, 0x110dc, 0x110dd, 0x110de, 0x110df, + 0x110e0, 0x110e1, 0x110e2, 0x110e3, 0x110e4, 0x110e5, 0x110e6, 0x110e7, + 0x110e8, 0x110e9, 0x110ea, 0x110eb, 0x110ec, 0x110ed, 0x110ee, 0x110ef, + 0x110f0, 0x110f1, 0x110f2, 0x110f3, 0x110f4, 0x110f5, 0x110f6, 0x110f7, + 0x110f8, 0x110f9, 0x110fa, 0x110fb, 0x110fc, 0x110fd, 0x110fe, 0x110ff +}; + +static uint32_t unicode_520_ci_page_124[] = { + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x00034, + 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x00031, 0x00032, 0x00033, + 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x00031, 0x00032, + 0x00033, 0x00034, 0x00035, 0x00032, 0x00033, 0x00033, 0x00034, 0x00035, + 0x00036, 0x00037, 0x00038, 0x00039, 0x00031, 0x00032, 0x00033, 0x00033, + 0x00034, 0x00035, 0x12432, 0x12433, 0x00031, 0x00032, 0x00033, 0x00033, + 0x00034, 0x00035, 0x00033, 0x00033, 0x00034, 0x00034, 0x00034, 0x00034, + 0x00036, 0x00037, 0x00037, 0x00037, 0x00038, 0x00038, 0x00039, 0x00039, + 0x00039, 0x00039, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00034, 0x00035, 0x00035, 0x12456, 0x12457, + 0x00031, 0x00032, 0x1245a, 0x1245b, 0x1245c, 0x1245d, 0x1245e, 0x1245f, + 0x12460, 0x12461, 0x12462, 0x12463, 0x12464, 0x12465, 0x12466, 0x12467, + 0x12468, 0x12469, 0x1246a, 0x1246b, 0x1246c, 0x1246d, 0x1246e, 0x1246f, + 0x12470, 0x12471, 0x12472, 0x12473, 0x12474, 0x12475, 0x12476, 0x12477, + 0x12478, 0x12479, 0x1247a, 0x1247b, 0x1247c, 0x1247d, 0x1247e, 0x1247f, + 0x12480, 0x12481, 0x12482, 0x12483, 0x12484, 0x12485, 0x12486, 0x12487, + 0x12488, 0x12489, 0x1248a, 0x1248b, 0x1248c, 0x1248d, 0x1248e, 0x1248f, + 0x12490, 0x12491, 0x12492, 0x12493, 0x12494, 0x12495, 0x12496, 0x12497, + 0x12498, 0x12499, 0x1249a, 0x1249b, 0x1249c, 0x1249d, 0x1249e, 0x1249f, + 0x124a0, 0x124a1, 0x124a2, 0x124a3, 0x124a4, 0x124a5, 0x124a6, 0x124a7, + 0x124a8, 0x124a9, 0x124aa, 0x124ab, 0x124ac, 0x124ad, 0x124ae, 0x124af, + 0x124b0, 0x124b1, 0x124b2, 0x124b3, 0x124b4, 0x124b5, 0x124b6, 0x124b7, + 0x124b8, 0x124b9, 0x124ba, 0x124bb, 0x124bc, 0x124bd, 0x124be, 0x124bf, + 0x124c0, 0x124c1, 0x124c2, 0x124c3, 0x124c4, 0x124c5, 0x124c6, 0x124c7, + 0x124c8, 0x124c9, 0x124ca, 0x124cb, 0x124cc, 0x124cd, 0x124ce, 0x124cf, + 0x124d0, 0x124d1, 0x124d2, 0x124d3, 0x124d4, 0x124d5, 0x124d6, 0x124d7, + 0x124d8, 0x124d9, 0x124da, 0x124db, 0x124dc, 0x124dd, 0x124de, 0x124df, + 0x124e0, 0x124e1, 0x124e2, 0x124e3, 0x124e4, 0x124e5, 0x124e6, 0x124e7, + 0x124e8, 0x124e9, 0x124ea, 0x124eb, 0x124ec, 0x124ed, 0x124ee, 0x124ef, + 0x124f0, 0x124f1, 0x124f2, 0x124f3, 0x124f4, 0x124f5, 0x124f6, 0x124f7, + 0x124f8, 0x124f9, 0x124fa, 0x124fb, 0x124fc, 0x124fd, 0x124fe, 0x124ff +}; + +static uint32_t unicode_520_ci_page_1d1[] = { + 0x1d100, 0x1d101, 0x1d102, 0x1d103, 0x1d104, 0x1d105, 0x1d106, 0x1d107, + 0x1d108, 0x1d109, 0x1d10a, 0x1d10b, 0x1d10c, 0x1d10d, 0x1d10e, 0x1d10f, + 0x1d110, 0x1d111, 0x1d112, 0x1d113, 0x1d114, 0x1d115, 0x1d116, 0x1d117, + 0x1d118, 0x1d119, 0x1d11a, 0x1d11b, 0x1d11c, 0x1d11d, 0x1d11e, 0x1d11f, + 0x1d120, 0x1d121, 0x1d122, 0x1d123, 0x1d124, 0x1d125, 0x1d126, 0x1d127, + 0x1d128, 0x1d129, 0x1d12a, 0x1d12b, 0x1d12c, 0x1d12d, 0x1d12e, 0x1d12f, + 0x1d130, 0x1d131, 0x1d132, 0x1d133, 0x1d134, 0x1d135, 0x1d136, 0x1d137, + 0x1d138, 0x1d139, 0x1d13a, 0x1d13b, 0x1d13c, 0x1d13d, 0x1d13e, 0x1d13f, + 0x1d140, 0x1d141, 0x1d142, 0x1d143, 0x1d144, 0x1d145, 0x1d146, 0x1d147, + 0x1d148, 0x1d149, 0x1d14a, 0x1d14b, 0x1d14c, 0x1d14d, 0x1d14e, 0x1d14f, + 0x1d150, 0x1d151, 0x1d152, 0x1d153, 0x1d154, 0x1d155, 0x1d156, 0x1d157, + 0x1d158, 0x1d159, 0x1d15a, 0x1d15b, 0x1d15c, 0x1d15d, 0x1d157, 0x1d158, + 0x1d158, 0x1d158, 0x1d158, 0x1d158, 0x1d158, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x1d16a, 0x1d16b, 0x1d16c, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x1d183, 0x1d184, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x1d18c, 0x1d18d, 0x1d18e, 0x1d18f, + 0x1d190, 0x1d191, 0x1d192, 0x1d193, 0x1d194, 0x1d195, 0x1d196, 0x1d197, + 0x1d198, 0x1d199, 0x1d19a, 0x1d19b, 0x1d19c, 0x1d19d, 0x1d19e, 0x1d19f, + 0x1d1a0, 0x1d1a1, 0x1d1a2, 0x1d1a3, 0x1d1a4, 0x1d1a5, 0x1d1a6, 0x1d1a7, + 0x1d1a8, 0x1d1a9, 0x00000, 0x00000, 0x00000, 0x00000, 0x1d1ae, 0x1d1af, + 0x1d1b0, 0x1d1b1, 0x1d1b2, 0x1d1b3, 0x1d1b4, 0x1d1b5, 0x1d1b6, 0x1d1b7, + 0x1d1b8, 0x1d1b9, 0x1d1ba, 0x1d1b9, 0x1d1ba, 0x1d1b9, 0x1d1ba, 0x1d1b9, + 0x1d1ba, 0x1d1c1, 0x1d1c2, 0x1d1c3, 0x1d1c4, 0x1d1c5, 0x1d1c6, 0x1d1c7, + 0x1d1c8, 0x1d1c9, 0x1d1ca, 0x1d1cb, 0x1d1cc, 0x1d1cd, 0x1d1ce, 0x1d1cf, + 0x1d1d0, 0x1d1d1, 0x1d1d2, 0x1d1d3, 0x1d1d4, 0x1d1d5, 0x1d1d6, 0x1d1d7, + 0x1d1d8, 0x1d1d9, 0x1d1da, 0x1d1db, 0x1d1dc, 0x1d1dd, 0x1d1de, 0x1d1df, + 0x1d1e0, 0x1d1e1, 0x1d1e2, 0x1d1e3, 0x1d1e4, 0x1d1e5, 0x1d1e6, 0x1d1e7, + 0x1d1e8, 0x1d1e9, 0x1d1ea, 0x1d1eb, 0x1d1ec, 0x1d1ed, 0x1d1ee, 0x1d1ef, + 0x1d1f0, 0x1d1f1, 0x1d1f2, 0x1d1f3, 0x1d1f4, 0x1d1f5, 0x1d1f6, 0x1d1f7, + 0x1d1f8, 0x1d1f9, 0x1d1fa, 0x1d1fb, 0x1d1fc, 0x1d1fd, 0x1d1fe, 0x1d1ff +}; + +static uint32_t unicode_520_ci_page_1d2[] = { + 0x1d200, 0x1d201, 0x1d202, 0x1d203, 0x1d204, 0x1d205, 0x1d206, 0x1d207, + 0x1d208, 0x1d209, 0x1d20a, 0x1d20b, 0x1d20c, 0x1d20d, 0x1d20e, 0x1d20f, + 0x1d210, 0x1d211, 0x1d212, 0x1d213, 0x1d214, 0x1d215, 0x1d216, 0x1d217, + 0x1d218, 0x1d219, 0x1d21a, 0x1d21b, 0x1d21c, 0x1d21d, 0x1d21e, 0x1d21f, + 0x1d220, 0x1d221, 0x1d222, 0x1d223, 0x1d224, 0x1d225, 0x1d226, 0x1d227, + 0x1d228, 0x1d229, 0x1d22a, 0x1d22b, 0x1d22c, 0x1d22d, 0x1d22e, 0x1d22f, + 0x1d230, 0x1d231, 0x1d232, 0x1d233, 0x1d234, 0x1d235, 0x1d236, 0x1d237, + 0x1d238, 0x1d239, 0x1d23a, 0x1d23b, 0x1d23c, 0x1d23d, 0x1d23e, 0x1d23f, + 0x1d240, 0x1d241, 0x00000, 0x00000, 0x00000, 0x1d245, 0x1d246, 0x1d247, + 0x1d248, 0x1d249, 0x1d24a, 0x1d24b, 0x1d24c, 0x1d24d, 0x1d24e, 0x1d24f, + 0x1d250, 0x1d251, 0x1d252, 0x1d253, 0x1d254, 0x1d255, 0x1d256, 0x1d257, + 0x1d258, 0x1d259, 0x1d25a, 0x1d25b, 0x1d25c, 0x1d25d, 0x1d25e, 0x1d25f, + 0x1d260, 0x1d261, 0x1d262, 0x1d263, 0x1d264, 0x1d265, 0x1d266, 0x1d267, + 0x1d268, 0x1d269, 0x1d26a, 0x1d26b, 0x1d26c, 0x1d26d, 0x1d26e, 0x1d26f, + 0x1d270, 0x1d271, 0x1d272, 0x1d273, 0x1d274, 0x1d275, 0x1d276, 0x1d277, + 0x1d278, 0x1d279, 0x1d27a, 0x1d27b, 0x1d27c, 0x1d27d, 0x1d27e, 0x1d27f, + 0x1d280, 0x1d281, 0x1d282, 0x1d283, 0x1d284, 0x1d285, 0x1d286, 0x1d287, + 0x1d288, 0x1d289, 0x1d28a, 0x1d28b, 0x1d28c, 0x1d28d, 0x1d28e, 0x1d28f, + 0x1d290, 0x1d291, 0x1d292, 0x1d293, 0x1d294, 0x1d295, 0x1d296, 0x1d297, + 0x1d298, 0x1d299, 0x1d29a, 0x1d29b, 0x1d29c, 0x1d29d, 0x1d29e, 0x1d29f, + 0x1d2a0, 0x1d2a1, 0x1d2a2, 0x1d2a3, 0x1d2a4, 0x1d2a5, 0x1d2a6, 0x1d2a7, + 0x1d2a8, 0x1d2a9, 0x1d2aa, 0x1d2ab, 0x1d2ac, 0x1d2ad, 0x1d2ae, 0x1d2af, + 0x1d2b0, 0x1d2b1, 0x1d2b2, 0x1d2b3, 0x1d2b4, 0x1d2b5, 0x1d2b6, 0x1d2b7, + 0x1d2b8, 0x1d2b9, 0x1d2ba, 0x1d2bb, 0x1d2bc, 0x1d2bd, 0x1d2be, 0x1d2bf, + 0x1d2c0, 0x1d2c1, 0x1d2c2, 0x1d2c3, 0x1d2c4, 0x1d2c5, 0x1d2c6, 0x1d2c7, + 0x1d2c8, 0x1d2c9, 0x1d2ca, 0x1d2cb, 0x1d2cc, 0x1d2cd, 0x1d2ce, 0x1d2cf, + 0x1d2d0, 0x1d2d1, 0x1d2d2, 0x1d2d3, 0x1d2d4, 0x1d2d5, 0x1d2d6, 0x1d2d7, + 0x1d2d8, 0x1d2d9, 0x1d2da, 0x1d2db, 0x1d2dc, 0x1d2dd, 0x1d2de, 0x1d2df, + 0x1d2e0, 0x1d2e1, 0x1d2e2, 0x1d2e3, 0x1d2e4, 0x1d2e5, 0x1d2e6, 0x1d2e7, + 0x1d2e8, 0x1d2e9, 0x1d2ea, 0x1d2eb, 0x1d2ec, 0x1d2ed, 0x1d2ee, 0x1d2ef, + 0x1d2f0, 0x1d2f1, 0x1d2f2, 0x1d2f3, 0x1d2f4, 0x1d2f5, 0x1d2f6, 0x1d2f7, + 0x1d2f8, 0x1d2f9, 0x1d2fa, 0x1d2fb, 0x1d2fc, 0x1d2fd, 0x1d2fe, 0x1d2ff +}; + +static uint32_t unicode_520_ci_page_1d3[] = { + 0x1d300, 0x1d301, 0x1d302, 0x1d303, 0x1d304, 0x1d305, 0x1d306, 0x1d307, + 0x1d308, 0x1d309, 0x1d30a, 0x1d30b, 0x1d30c, 0x1d30d, 0x1d30e, 0x1d30f, + 0x1d310, 0x1d311, 0x1d312, 0x1d313, 0x1d314, 0x1d315, 0x1d316, 0x1d317, + 0x1d318, 0x1d319, 0x1d31a, 0x1d31b, 0x1d31c, 0x1d31d, 0x1d31e, 0x1d31f, + 0x1d320, 0x1d321, 0x1d322, 0x1d323, 0x1d324, 0x1d325, 0x1d326, 0x1d327, + 0x1d328, 0x1d329, 0x1d32a, 0x1d32b, 0x1d32c, 0x1d32d, 0x1d32e, 0x1d32f, + 0x1d330, 0x1d331, 0x1d332, 0x1d333, 0x1d334, 0x1d335, 0x1d336, 0x1d337, + 0x1d338, 0x1d339, 0x1d33a, 0x1d33b, 0x1d33c, 0x1d33d, 0x1d33e, 0x1d33f, + 0x1d340, 0x1d341, 0x1d342, 0x1d343, 0x1d344, 0x1d345, 0x1d346, 0x1d347, + 0x1d348, 0x1d349, 0x1d34a, 0x1d34b, 0x1d34c, 0x1d34d, 0x1d34e, 0x1d34f, + 0x1d350, 0x1d351, 0x1d352, 0x1d353, 0x1d354, 0x1d355, 0x1d356, 0x1d357, + 0x1d358, 0x1d359, 0x1d35a, 0x1d35b, 0x1d35c, 0x1d35d, 0x1d35e, 0x1d35f, + 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, + 0x00039, 0x1d369, 0x1d36a, 0x1d36b, 0x1d36c, 0x1d36d, 0x1d36e, 0x1d36f, + 0x1d370, 0x1d371, 0x1d372, 0x1d373, 0x1d374, 0x1d375, 0x1d376, 0x1d377, + 0x1d378, 0x1d379, 0x1d37a, 0x1d37b, 0x1d37c, 0x1d37d, 0x1d37e, 0x1d37f, + 0x1d380, 0x1d381, 0x1d382, 0x1d383, 0x1d384, 0x1d385, 0x1d386, 0x1d387, + 0x1d388, 0x1d389, 0x1d38a, 0x1d38b, 0x1d38c, 0x1d38d, 0x1d38e, 0x1d38f, + 0x1d390, 0x1d391, 0x1d392, 0x1d393, 0x1d394, 0x1d395, 0x1d396, 0x1d397, + 0x1d398, 0x1d399, 0x1d39a, 0x1d39b, 0x1d39c, 0x1d39d, 0x1d39e, 0x1d39f, + 0x1d3a0, 0x1d3a1, 0x1d3a2, 0x1d3a3, 0x1d3a4, 0x1d3a5, 0x1d3a6, 0x1d3a7, + 0x1d3a8, 0x1d3a9, 0x1d3aa, 0x1d3ab, 0x1d3ac, 0x1d3ad, 0x1d3ae, 0x1d3af, + 0x1d3b0, 0x1d3b1, 0x1d3b2, 0x1d3b3, 0x1d3b4, 0x1d3b5, 0x1d3b6, 0x1d3b7, + 0x1d3b8, 0x1d3b9, 0x1d3ba, 0x1d3bb, 0x1d3bc, 0x1d3bd, 0x1d3be, 0x1d3bf, + 0x1d3c0, 0x1d3c1, 0x1d3c2, 0x1d3c3, 0x1d3c4, 0x1d3c5, 0x1d3c6, 0x1d3c7, + 0x1d3c8, 0x1d3c9, 0x1d3ca, 0x1d3cb, 0x1d3cc, 0x1d3cd, 0x1d3ce, 0x1d3cf, + 0x1d3d0, 0x1d3d1, 0x1d3d2, 0x1d3d3, 0x1d3d4, 0x1d3d5, 0x1d3d6, 0x1d3d7, + 0x1d3d8, 0x1d3d9, 0x1d3da, 0x1d3db, 0x1d3dc, 0x1d3dd, 0x1d3de, 0x1d3df, + 0x1d3e0, 0x1d3e1, 0x1d3e2, 0x1d3e3, 0x1d3e4, 0x1d3e5, 0x1d3e6, 0x1d3e7, + 0x1d3e8, 0x1d3e9, 0x1d3ea, 0x1d3eb, 0x1d3ec, 0x1d3ed, 0x1d3ee, 0x1d3ef, + 0x1d3f0, 0x1d3f1, 0x1d3f2, 0x1d3f3, 0x1d3f4, 0x1d3f5, 0x1d3f6, 0x1d3f7, + 0x1d3f8, 0x1d3f9, 0x1d3fa, 0x1d3fb, 0x1d3fc, 0x1d3fd, 0x1d3fe, 0x1d3ff +}; + +static uint32_t unicode_520_ci_page_1d4[] = { + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, + 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x1d455, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x1d49d, 0x00043, 0x00044, + 0x1d4a0, 0x1d4a1, 0x00047, 0x1d4a3, 0x1d4a4, 0x0004a, 0x0004b, 0x1d4a7, + 0x1d4a8, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x1d4ad, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x1d4ba, 0x00046, 0x1d4bc, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x1d4c4, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x1d4ff +}; + +static uint32_t unicode_520_ci_page_1d5[] = { + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x1d506, 0x00044, + 0x00045, 0x00046, 0x00047, 0x1d50b, 0x1d50c, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x1d515, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x1d51d, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x1d53a, 0x00044, 0x00045, 0x00046, 0x00047, 0x1d53f, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x1d545, 0x0004f, 0x1d547, + 0x1d548, 0x1d549, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x1d551, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, + 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, + 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x1d5ff +}; + +static uint32_t unicode_520_ci_page_1d6[] = { + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, + 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, + 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, + 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, 0x00041, 0x00042, + 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, 0x00049, 0x0004a, + 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, 0x00051, 0x00052, + 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, 0x00059, 0x0005a, + 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, 0x00047, 0x00048, + 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, 0x0004f, 0x00050, + 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, 0x00057, 0x00058, + 0x00059, 0x0005a, 0x00041, 0x00042, 0x00043, 0x00044, 0x00045, 0x00046, + 0x00047, 0x00048, 0x00049, 0x0004a, 0x0004b, 0x0004c, 0x0004d, 0x0004e, + 0x0004f, 0x00050, 0x00051, 0x00052, 0x00053, 0x00054, 0x00055, 0x00056, + 0x00057, 0x00058, 0x00059, 0x0005a, 0x00131, 0x00237, 0x1d6a6, 0x1d6a7, + 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, + 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, + 0x003a1, 0x00398, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, + 0x003a9, 0x02207, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, + 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, + 0x0039f, 0x003a0, 0x003a1, 0x003a3, 0x003a3, 0x003a4, 0x003a5, 0x003a6, + 0x003a7, 0x003a8, 0x003a9, 0x02202, 0x00395, 0x00398, 0x0039a, 0x003a6, + 0x003a1, 0x003a0, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, + 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, + 0x0039f, 0x003a0, 0x003a1, 0x00398, 0x003a3, 0x003a4, 0x003a5, 0x003a6, + 0x003a7, 0x003a8, 0x003a9, 0x02207, 0x00391, 0x00392, 0x00393, 0x1d6ff +}; + +static uint32_t unicode_520_ci_page_1d7[] = { + 0x00395, 0x00396, 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, + 0x0039d, 0x0039e, 0x0039f, 0x003a0, 0x003a1, 0x003a3, 0x003a3, 0x003a4, + 0x003a5, 0x003a6, 0x003a7, 0x003a8, 0x003a9, 0x02202, 0x00395, 0x00398, + 0x0039a, 0x003a6, 0x003a1, 0x003a0, 0x00391, 0x00392, 0x00393, 0x00394, + 0x00395, 0x00396, 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, + 0x0039d, 0x0039e, 0x0039f, 0x003a0, 0x003a1, 0x00398, 0x003a3, 0x003a4, + 0x003a5, 0x003a6, 0x003a7, 0x003a8, 0x003a9, 0x02207, 0x00391, 0x00392, + 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, 0x00399, 0x0039a, + 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, 0x003a1, 0x003a3, + 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, 0x003a9, 0x02202, + 0x00395, 0x00398, 0x0039a, 0x003a6, 0x003a1, 0x003a0, 0x00391, 0x00392, + 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, 0x00399, 0x0039a, + 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, 0x003a1, 0x00398, + 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, 0x003a9, 0x02207, + 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, + 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, + 0x003a1, 0x003a3, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, + 0x003a9, 0x02202, 0x00395, 0x00398, 0x0039a, 0x003a6, 0x003a1, 0x003a0, + 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, 0x00397, 0x00398, + 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, 0x0039f, 0x003a0, + 0x003a1, 0x00398, 0x003a3, 0x003a4, 0x003a5, 0x003a6, 0x003a7, 0x003a8, + 0x003a9, 0x02207, 0x00391, 0x00392, 0x00393, 0x00394, 0x00395, 0x00396, + 0x00397, 0x00398, 0x00399, 0x0039a, 0x0039b, 0x0039c, 0x0039d, 0x0039e, + 0x0039f, 0x003a0, 0x003a1, 0x003a3, 0x003a3, 0x003a4, 0x003a5, 0x003a6, + 0x003a7, 0x003a8, 0x003a9, 0x02202, 0x00395, 0x00398, 0x0039a, 0x003a6, + 0x003a1, 0x003a0, 0x003dc, 0x003dc, 0x1d7cc, 0x1d7cd, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, + 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, + 0x00038, 0x00039, 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, + 0x00036, 0x00037, 0x00038, 0x00039, 0x00030, 0x00031, 0x00032, 0x00033, + 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x00039, 0x00030, 0x00031, + 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, 0x00038, 0x1d7ff +}; + +static uint32_t unicode_520_ci_page_1f1[] = { + 0x1f100, 0x1f101, 0x1f102, 0x1f103, 0x1f104, 0x1f105, 0x1f106, 0x1f107, + 0x1f108, 0x1f109, 0x1f10a, 0x1f10b, 0x1f10c, 0x1f10d, 0x1f10e, 0x1f10f, + 0x0249c, 0x0249d, 0x0249e, 0x0249f, 0x024a0, 0x024a1, 0x024a2, 0x024a3, + 0x024a4, 0x024a5, 0x024a6, 0x024a7, 0x024a8, 0x024a9, 0x024aa, 0x024ab, + 0x024ac, 0x024ad, 0x024ae, 0x024af, 0x024b0, 0x024b1, 0x024b2, 0x024b3, + 0x024b4, 0x024b5, 0x1f12a, 0x00043, 0x00052, 0x033c5, 0x1f12e, 0x1f12f, + 0x1f130, 0x00042, 0x1f132, 0x1f133, 0x1f134, 0x1f135, 0x1f136, 0x1f137, + 0x1f138, 0x1f139, 0x1f13a, 0x1f13b, 0x1f13c, 0x0004e, 0x1f13e, 0x00050, + 0x1f140, 0x1f141, 0x00053, 0x1f143, 0x1f144, 0x1f145, 0x00057, 0x1f147, + 0x1f148, 0x1f149, 0x1f14a, 0x033b7, 0x1f14c, 0x000df, 0x1f14e, 0x1f14f, + 0x1f150, 0x1f151, 0x1f152, 0x1f153, 0x1f154, 0x1f155, 0x1f156, 0x00048, + 0x1f158, 0x1f159, 0x1f15a, 0x1f15b, 0x1f15c, 0x1f15d, 0x1f15e, 0x00050, + 0x1f160, 0x1f161, 0x1f162, 0x1f163, 0x1f164, 0x1f165, 0x1f166, 0x1f167, + 0x1f168, 0x1f169, 0x1f16a, 0x1f16b, 0x1f16c, 0x1f16d, 0x1f16e, 0x1f16f, + 0x1f170, 0x1f171, 0x1f172, 0x1f173, 0x1f174, 0x1f175, 0x1f176, 0x1f177, + 0x1f178, 0x0004a, 0x1f17a, 0x0004c, 0x0004d, 0x1f17d, 0x1f17e, 0x00050, + 0x1f180, 0x1f181, 0x1f182, 0x1f183, 0x1f184, 0x1f185, 0x1f186, 0x1f187, + 0x1f188, 0x1f189, 0x00050, 0x1f18b, 0x03380, 0x1f18d, 0x1f18e, 0x1f18f, + 0x1f190, 0x1f191, 0x1f192, 0x1f193, 0x1f194, 0x1f195, 0x1f196, 0x1f197, + 0x1f198, 0x1f199, 0x1f19a, 0x1f19b, 0x1f19c, 0x1f19d, 0x1f19e, 0x1f19f, + 0x1f1a0, 0x1f1a1, 0x1f1a2, 0x1f1a3, 0x1f1a4, 0x1f1a5, 0x1f1a6, 0x1f1a7, + 0x1f1a8, 0x1f1a9, 0x1f1aa, 0x1f1ab, 0x1f1ac, 0x1f1ad, 0x1f1ae, 0x1f1af, + 0x1f1b0, 0x1f1b1, 0x1f1b2, 0x1f1b3, 0x1f1b4, 0x1f1b5, 0x1f1b6, 0x1f1b7, + 0x1f1b8, 0x1f1b9, 0x1f1ba, 0x1f1bb, 0x1f1bc, 0x1f1bd, 0x1f1be, 0x1f1bf, + 0x1f1c0, 0x1f1c1, 0x1f1c2, 0x1f1c3, 0x1f1c4, 0x1f1c5, 0x1f1c6, 0x1f1c7, + 0x1f1c8, 0x1f1c9, 0x1f1ca, 0x1f1cb, 0x1f1cc, 0x1f1cd, 0x1f1ce, 0x1f1cf, + 0x1f1d0, 0x1f1d1, 0x1f1d2, 0x1f1d3, 0x1f1d4, 0x1f1d5, 0x1f1d6, 0x1f1d7, + 0x1f1d8, 0x1f1d9, 0x1f1da, 0x1f1db, 0x1f1dc, 0x1f1dd, 0x1f1de, 0x1f1df, + 0x1f1e0, 0x1f1e1, 0x1f1e2, 0x1f1e3, 0x1f1e4, 0x1f1e5, 0x1f1e6, 0x1f1e7, + 0x1f1e8, 0x1f1e9, 0x1f1ea, 0x1f1eb, 0x1f1ec, 0x1f1ed, 0x1f1ee, 0x1f1ef, + 0x1f1f0, 0x1f1f1, 0x1f1f2, 0x1f1f3, 0x1f1f4, 0x1f1f5, 0x1f1f6, 0x1f1f7, + 0x1f1f8, 0x1f1f9, 0x1f1fa, 0x1f1fb, 0x1f1fc, 0x1f1fd, 0x1f1fe, 0x1f1ff +}; + +static uint32_t unicode_520_ci_page_1f2[] = { + 0x1f200, 0x1f201, 0x1f202, 0x1f203, 0x1f204, 0x1f205, 0x1f206, 0x1f207, + 0x1f208, 0x1f209, 0x1f20a, 0x1f20b, 0x1f20c, 0x1f20d, 0x1f20e, 0x1f20f, + 0x02f3f, 0x1f211, 0x1f212, 0x03066, 0x03193, 0x1f215, 0x1f216, 0x0319d, + 0x1f218, 0x1f219, 0x1f21a, 0x0f9be, 0x1f21c, 0x1f21d, 0x1f21e, 0x1f21f, + 0x1f220, 0x1f221, 0x02f63, 0x1f223, 0x1f224, 0x1f225, 0x1f226, 0x1f227, + 0x1f228, 0x03192, 0x03194, 0x1f22b, 0x032a7, 0x03197, 0x032a8, 0x1f22f, + 0x02f9b, 0x1f231, 0x1f232, 0x1f233, 0x1f234, 0x1f235, 0x1f236, 0x1f237, + 0x1f238, 0x1f239, 0x1f23a, 0x1f23b, 0x1f23c, 0x1f23d, 0x1f23e, 0x1f23f, + 0x1f240, 0x1f241, 0x1f242, 0x1f243, 0x1f244, 0x1f245, 0x1f246, 0x1f247, + 0x1f248, 0x1f249, 0x1f24a, 0x1f24b, 0x1f24c, 0x1f24d, 0x1f24e, 0x1f24f, + 0x1f250, 0x1f251, 0x1f252, 0x1f253, 0x1f254, 0x1f255, 0x1f256, 0x1f257, + 0x1f258, 0x1f259, 0x1f25a, 0x1f25b, 0x1f25c, 0x1f25d, 0x1f25e, 0x1f25f, + 0x1f260, 0x1f261, 0x1f262, 0x1f263, 0x1f264, 0x1f265, 0x1f266, 0x1f267, + 0x1f268, 0x1f269, 0x1f26a, 0x1f26b, 0x1f26c, 0x1f26d, 0x1f26e, 0x1f26f, + 0x1f270, 0x1f271, 0x1f272, 0x1f273, 0x1f274, 0x1f275, 0x1f276, 0x1f277, + 0x1f278, 0x1f279, 0x1f27a, 0x1f27b, 0x1f27c, 0x1f27d, 0x1f27e, 0x1f27f, + 0x1f280, 0x1f281, 0x1f282, 0x1f283, 0x1f284, 0x1f285, 0x1f286, 0x1f287, + 0x1f288, 0x1f289, 0x1f28a, 0x1f28b, 0x1f28c, 0x1f28d, 0x1f28e, 0x1f28f, + 0x1f290, 0x1f291, 0x1f292, 0x1f293, 0x1f294, 0x1f295, 0x1f296, 0x1f297, + 0x1f298, 0x1f299, 0x1f29a, 0x1f29b, 0x1f29c, 0x1f29d, 0x1f29e, 0x1f29f, + 0x1f2a0, 0x1f2a1, 0x1f2a2, 0x1f2a3, 0x1f2a4, 0x1f2a5, 0x1f2a6, 0x1f2a7, + 0x1f2a8, 0x1f2a9, 0x1f2aa, 0x1f2ab, 0x1f2ac, 0x1f2ad, 0x1f2ae, 0x1f2af, + 0x1f2b0, 0x1f2b1, 0x1f2b2, 0x1f2b3, 0x1f2b4, 0x1f2b5, 0x1f2b6, 0x1f2b7, + 0x1f2b8, 0x1f2b9, 0x1f2ba, 0x1f2bb, 0x1f2bc, 0x1f2bd, 0x1f2be, 0x1f2bf, + 0x1f2c0, 0x1f2c1, 0x1f2c2, 0x1f2c3, 0x1f2c4, 0x1f2c5, 0x1f2c6, 0x1f2c7, + 0x1f2c8, 0x1f2c9, 0x1f2ca, 0x1f2cb, 0x1f2cc, 0x1f2cd, 0x1f2ce, 0x1f2cf, + 0x1f2d0, 0x1f2d1, 0x1f2d2, 0x1f2d3, 0x1f2d4, 0x1f2d5, 0x1f2d6, 0x1f2d7, + 0x1f2d8, 0x1f2d9, 0x1f2da, 0x1f2db, 0x1f2dc, 0x1f2dd, 0x1f2de, 0x1f2df, + 0x1f2e0, 0x1f2e1, 0x1f2e2, 0x1f2e3, 0x1f2e4, 0x1f2e5, 0x1f2e6, 0x1f2e7, + 0x1f2e8, 0x1f2e9, 0x1f2ea, 0x1f2eb, 0x1f2ec, 0x1f2ed, 0x1f2ee, 0x1f2ef, + 0x1f2f0, 0x1f2f1, 0x1f2f2, 0x1f2f3, 0x1f2f4, 0x1f2f5, 0x1f2f6, 0x1f2f7, + 0x1f2f8, 0x1f2f9, 0x1f2fa, 0x1f2fb, 0x1f2fc, 0x1f2fd, 0x1f2fe, 0x1f2ff +}; + +static uint32_t unicode_520_ci_page_2f8[] = { + 0x2f800, 0x2f801, 0x2f802, 0x2f803, 0x2f804, 0x0fa30, 0x2f806, 0x2f807, + 0x2f808, 0x2f809, 0x0fa31, 0x2f80b, 0x2f80c, 0x2f80d, 0x0fa32, 0x2f80f, + 0x2f810, 0x2f811, 0x2f812, 0x2f813, 0x2f814, 0x1f21e, 0x2f816, 0x2f817, + 0x2f818, 0x2f819, 0x2f81a, 0x0fa71, 0x2f81c, 0x02f10, 0x2f81e, 0x2f81f, + 0x2f820, 0x2f821, 0x2f822, 0x2f823, 0x2f824, 0x0fa76, 0x0fa33, 0x0fa34, + 0x0fa77, 0x2f829, 0x2f82a, 0x0f963, 0x2f82c, 0x0fa35, 0x2f82e, 0x2f82f, + 0x2f830, 0x2f831, 0x2f831, 0x2f831, 0x2f834, 0x2f835, 0x2f836, 0x2f837, + 0x2f838, 0x2f839, 0x2f83a, 0x2f83b, 0x2f83c, 0x2f83d, 0x2f83e, 0x2f83f, + 0x2f840, 0x2f841, 0x2f842, 0x2f843, 0x2f844, 0x2f845, 0x2f845, 0x0fa7a, + 0x2f848, 0x2f849, 0x2f84a, 0x2f84b, 0x0fa37, 0x2f84d, 0x2f84e, 0x2f84f, + 0x0fa00, 0x2f851, 0x2f852, 0x2f853, 0x2f854, 0x2f855, 0x2f856, 0x2f857, + 0x2f858, 0x2f859, 0x2f85a, 0x2f85b, 0x2f85c, 0x1f215, 0x2f85e, 0x2f85f, + 0x2f860, 0x2f861, 0x2f862, 0x2f863, 0x2f864, 0x2f865, 0x2f866, 0x2f867, + 0x2f868, 0x2f869, 0x2f86a, 0x2f86a, 0x2f86c, 0x2f86d, 0x2f86e, 0x0f95f, + 0x2f870, 0x2f871, 0x2f872, 0x2f873, 0x2f874, 0x02e8e, 0x2f876, 0x2f877, + 0x02f2c, 0x2f879, 0x2f87a, 0x2f87b, 0x2f87c, 0x2f87d, 0x2f87e, 0x2f87f, + 0x2f880, 0x2f881, 0x2f882, 0x2f883, 0x2f884, 0x2f885, 0x2f886, 0x2f887, + 0x2f888, 0x2f889, 0x2f88a, 0x2f88b, 0x2f88c, 0x2f88d, 0x0f928, 0x2f88f, + 0x02f36, 0x2f891, 0x2f891, 0x2f893, 0x2f894, 0x2f894, 0x2f896, 0x2f897, + 0x2f898, 0x2f899, 0x2f89a, 0x2f89b, 0x2f89c, 0x2f89d, 0x2f89e, 0x2f89f, + 0x2f8a0, 0x2f8a1, 0x2f8a2, 0x0fa3d, 0x2f8a4, 0x2f8a5, 0x2f8a6, 0x2f8a7, + 0x0fa87, 0x2f8a7, 0x2f8aa, 0x0fa3f, 0x2f8ac, 0x2f8ad, 0x2f8ae, 0x2f8af, + 0x0fa40, 0x0f90d, 0x2f8b2, 0x2f8b3, 0x2f8b4, 0x2f8b5, 0x2f8b6, 0x2f8b7, + 0x2f8b8, 0x2f8b9, 0x2f8ba, 0x2f8bb, 0x2f8bc, 0x2f8bd, 0x2f8be, 0x2f8bf, + 0x2f8c0, 0x2f8c1, 0x2f8c2, 0x2f8c3, 0x2f8c4, 0x2f8c5, 0x2f8c6, 0x2f8c7, + 0x0fa41, 0x2f8c9, 0x2f8ca, 0x2f8cb, 0x2f8cc, 0x2f8cd, 0x2f8ce, 0x0fa43, + 0x2f8d0, 0x2f8d1, 0x2f8d2, 0x2f8d3, 0x2f8d4, 0x2f8d5, 0x2f8d6, 0x2f8d7, + 0x0f929, 0x0fa93, 0x2f8da, 0x2f8db, 0x2f8dc, 0x2f8dd, 0x2f8de, 0x2f8df, + 0x2f8e0, 0x2f8e1, 0x0fa44, 0x2f8e3, 0x2f8e4, 0x2f8e5, 0x2f8e6, 0x0fad2, + 0x2f8e8, 0x2f8e9, 0x2f8ea, 0x2f8eb, 0x2f8ec, 0x2f8ed, 0x2f8ee, 0x2f8ef, + 0x2f8f0, 0x2f8f1, 0x2f8f2, 0x2f8f3, 0x2f8f4, 0x0f970, 0x2f8f6, 0x2f8f7, + 0x2f8f8, 0x2f8f9, 0x2f8fa, 0x2f8fb, 0x2f8fc, 0x2f8fd, 0x2f8fe, 0x2f8ff +}; + +static uint32_t unicode_520_ci_page_2f9[] = { + 0x2f900, 0x0fa45, 0x0f9ca, 0x2f903, 0x2f904, 0x2f905, 0x2f906, 0x2f907, + 0x2f908, 0x2f909, 0x2f90a, 0x0fa99, 0x2f90c, 0x2f90d, 0x2f90e, 0x2f90f, + 0x2f910, 0x2f911, 0x2f912, 0x2f913, 0x0fa9b, 0x2f915, 0x2f916, 0x2f917, + 0x2f918, 0x2f919, 0x2f91a, 0x2f91b, 0x2f91c, 0x2f91d, 0x2f91e, 0x2f91f, + 0x2f920, 0x0fa9e, 0x2f922, 0x2f923, 0x2f924, 0x2f925, 0x2f926, 0x2f927, + 0x2f928, 0x02ea9, 0x2f92a, 0x2f92b, 0x2f92c, 0x2f92c, 0x2f92e, 0x2f92f, + 0x0faa1, 0x2f931, 0x2f932, 0x2f933, 0x2f934, 0x2f935, 0x2f936, 0x2f937, + 0x0f962, 0x2f939, 0x2f93a, 0x2f93b, 0x2f93c, 0x2f93d, 0x2f93e, 0x2f93f, + 0x0faa8, 0x2f941, 0x2f942, 0x2f943, 0x2f944, 0x2f945, 0x2f946, 0x2f946, + 0x0faa9, 0x0fad4, 0x2f94a, 0x2f94b, 0x2f94c, 0x2f94d, 0x2f94e, 0x0f93b, + 0x0faab, 0x2f951, 0x2f952, 0x0fa50, 0x2f954, 0x2f955, 0x0fa1b, 0x2f957, + 0x2f958, 0x0fa54, 0x2f95a, 0x2f95b, 0x2f95c, 0x2f95d, 0x2f95d, 0x2f95f, + 0x2f960, 0x2f961, 0x2f962, 0x2f963, 0x2f964, 0x2f965, 0x2f966, 0x2f967, + 0x2f968, 0x2f969, 0x2f96a, 0x2f96b, 0x2f96c, 0x2f96d, 0x2f96e, 0x2f96f, + 0x2f970, 0x2f971, 0x2f972, 0x2f973, 0x2f974, 0x2f975, 0x2f976, 0x2f977, + 0x2f978, 0x2f979, 0x0fa5b, 0x2f97b, 0x2f97c, 0x2f97d, 0x2f97e, 0x2f97f, + 0x2f980, 0x2f981, 0x2f982, 0x2f983, 0x2f984, 0x2f985, 0x2f986, 0x2f987, + 0x2f988, 0x2f989, 0x2f98a, 0x2f893, 0x2f98c, 0x2f98d, 0x2f98e, 0x2f98f, + 0x2f990, 0x2f991, 0x2f992, 0x2f993, 0x2f994, 0x2f995, 0x2f996, 0x2f997, + 0x0f974, 0x2f999, 0x2f99a, 0x2f99b, 0x2f99c, 0x2f99d, 0x2f99e, 0x0fa5f, + 0x2f9a0, 0x2f9a1, 0x2f9a2, 0x2f9a3, 0x2f9a4, 0x2f9a5, 0x2f9a6, 0x2f9a7, + 0x2f9a8, 0x2f9a9, 0x2f9aa, 0x2f9ab, 0x2f9ac, 0x2f9ad, 0x2f9ae, 0x2f9af, + 0x2f9b0, 0x2f9b1, 0x2f9b2, 0x2f9b3, 0x0f936, 0x2f9b5, 0x2f9b6, 0x2f9b7, + 0x2f9b8, 0x2f9b9, 0x2f9ba, 0x0fab5, 0x2f9bc, 0x2f9bd, 0x2f9be, 0x2f9bf, + 0x2f9c0, 0x2f9c1, 0x2f9c2, 0x2f9c3, 0x02f90, 0x2f9c5, 0x2f9c6, 0x2f9c7, + 0x2f9c8, 0x2f9c9, 0x2f9ca, 0x2f9cb, 0x2f9cc, 0x2f9cd, 0x2f9ce, 0x2f9cf, + 0x0fabe, 0x0fac0, 0x02f97, 0x2f9d3, 0x2f9d4, 0x2f9d5, 0x2f9d6, 0x2f9d7, + 0x2f9d8, 0x2f9d9, 0x2f9da, 0x2f9db, 0x2f9dc, 0x2f9dd, 0x2f9de, 0x0fac2, + 0x2f9e0, 0x2f9e1, 0x2f9e2, 0x2f9e3, 0x2f9e4, 0x2f9e5, 0x2f9e6, 0x2f9e7, + 0x2f9e8, 0x2f9e9, 0x2f9ea, 0x2f9eb, 0x2f9ec, 0x2f9ed, 0x2f9ee, 0x2f9ef, + 0x2f9f0, 0x2f9f1, 0x2f9f2, 0x2f9f3, 0x2f9f4, 0x2f9f5, 0x2f9f6, 0x2f9f7, + 0x2f9f8, 0x2f9f9, 0x2f9fa, 0x2f9fb, 0x2f9fc, 0x2f9fd, 0x0facb, 0x2f9ff +}; + +static uint32_t unicode_520_ci_page_2fa[] = { + 0x2fa00, 0x2fa01, 0x2fa02, 0x2fa03, 0x2fa04, 0x2fa05, 0x2fa06, 0x2fa07, + 0x2fa08, 0x2fa09, 0x0facd, 0x2fa0b, 0x2fa0c, 0x2fa0d, 0x2fa0e, 0x2fa0f, + 0x2fa10, 0x2fa11, 0x2fa12, 0x2fa13, 0x2fa14, 0x02fc7, 0x04d56, 0x02fcb, + 0x02eea, 0x2fa19, 0x2fa1a, 0x2fa1b, 0x02fd0, 0x2fa1d, 0x2fa1e, 0x2fa1f, + 0x2fa20, 0x2fa21, 0x2fa22, 0x2fa23, 0x2fa24, 0x2fa25, 0x2fa26, 0x2fa27, + 0x2fa28, 0x2fa29, 0x2fa2a, 0x2fa2b, 0x2fa2c, 0x2fa2d, 0x2fa2e, 0x2fa2f, + 0x2fa30, 0x2fa31, 0x2fa32, 0x2fa33, 0x2fa34, 0x2fa35, 0x2fa36, 0x2fa37, + 0x2fa38, 0x2fa39, 0x2fa3a, 0x2fa3b, 0x2fa3c, 0x2fa3d, 0x2fa3e, 0x2fa3f, + 0x2fa40, 0x2fa41, 0x2fa42, 0x2fa43, 0x2fa44, 0x2fa45, 0x2fa46, 0x2fa47, + 0x2fa48, 0x2fa49, 0x2fa4a, 0x2fa4b, 0x2fa4c, 0x2fa4d, 0x2fa4e, 0x2fa4f, + 0x2fa50, 0x2fa51, 0x2fa52, 0x2fa53, 0x2fa54, 0x2fa55, 0x2fa56, 0x2fa57, + 0x2fa58, 0x2fa59, 0x2fa5a, 0x2fa5b, 0x2fa5c, 0x2fa5d, 0x2fa5e, 0x2fa5f, + 0x2fa60, 0x2fa61, 0x2fa62, 0x2fa63, 0x2fa64, 0x2fa65, 0x2fa66, 0x2fa67, + 0x2fa68, 0x2fa69, 0x2fa6a, 0x2fa6b, 0x2fa6c, 0x2fa6d, 0x2fa6e, 0x2fa6f, + 0x2fa70, 0x2fa71, 0x2fa72, 0x2fa73, 0x2fa74, 0x2fa75, 0x2fa76, 0x2fa77, + 0x2fa78, 0x2fa79, 0x2fa7a, 0x2fa7b, 0x2fa7c, 0x2fa7d, 0x2fa7e, 0x2fa7f, + 0x2fa80, 0x2fa81, 0x2fa82, 0x2fa83, 0x2fa84, 0x2fa85, 0x2fa86, 0x2fa87, + 0x2fa88, 0x2fa89, 0x2fa8a, 0x2fa8b, 0x2fa8c, 0x2fa8d, 0x2fa8e, 0x2fa8f, + 0x2fa90, 0x2fa91, 0x2fa92, 0x2fa93, 0x2fa94, 0x2fa95, 0x2fa96, 0x2fa97, + 0x2fa98, 0x2fa99, 0x2fa9a, 0x2fa9b, 0x2fa9c, 0x2fa9d, 0x2fa9e, 0x2fa9f, + 0x2faa0, 0x2faa1, 0x2faa2, 0x2faa3, 0x2faa4, 0x2faa5, 0x2faa6, 0x2faa7, + 0x2faa8, 0x2faa9, 0x2faaa, 0x2faab, 0x2faac, 0x2faad, 0x2faae, 0x2faaf, + 0x2fab0, 0x2fab1, 0x2fab2, 0x2fab3, 0x2fab4, 0x2fab5, 0x2fab6, 0x2fab7, + 0x2fab8, 0x2fab9, 0x2faba, 0x2fabb, 0x2fabc, 0x2fabd, 0x2fabe, 0x2fabf, + 0x2fac0, 0x2fac1, 0x2fac2, 0x2fac3, 0x2fac4, 0x2fac5, 0x2fac6, 0x2fac7, + 0x2fac8, 0x2fac9, 0x2faca, 0x2facb, 0x2facc, 0x2facd, 0x2face, 0x2facf, + 0x2fad0, 0x2fad1, 0x2fad2, 0x2fad3, 0x2fad4, 0x2fad5, 0x2fad6, 0x2fad7, + 0x2fad8, 0x2fad9, 0x2fada, 0x2fadb, 0x2fadc, 0x2fadd, 0x2fade, 0x2fadf, + 0x2fae0, 0x2fae1, 0x2fae2, 0x2fae3, 0x2fae4, 0x2fae5, 0x2fae6, 0x2fae7, + 0x2fae8, 0x2fae9, 0x2faea, 0x2faeb, 0x2faec, 0x2faed, 0x2faee, 0x2faef, + 0x2faf0, 0x2faf1, 0x2faf2, 0x2faf3, 0x2faf4, 0x2faf5, 0x2faf6, 0x2faf7, + 0x2faf8, 0x2faf9, 0x2fafa, 0x2fafb, 0x2fafc, 0x2fafd, 0x2fafe, 0x2faff +}; + +static uint32_t unicode_520_ci_page_e00[] = { + 0xe0000, 0x00000, 0xe0002, 0xe0003, 0xe0004, 0xe0005, 0xe0006, 0xe0007, + 0xe0008, 0xe0009, 0xe000a, 0xe000b, 0xe000c, 0xe000d, 0xe000e, 0xe000f, + 0xe0010, 0xe0011, 0xe0012, 0xe0013, 0xe0014, 0xe0015, 0xe0016, 0xe0017, + 0xe0018, 0xe0019, 0xe001a, 0xe001b, 0xe001c, 0xe001d, 0xe001e, 0xe001f, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0xe0080, 0xe0081, 0xe0082, 0xe0083, 0xe0084, 0xe0085, 0xe0086, 0xe0087, + 0xe0088, 0xe0089, 0xe008a, 0xe008b, 0xe008c, 0xe008d, 0xe008e, 0xe008f, + 0xe0090, 0xe0091, 0xe0092, 0xe0093, 0xe0094, 0xe0095, 0xe0096, 0xe0097, + 0xe0098, 0xe0099, 0xe009a, 0xe009b, 0xe009c, 0xe009d, 0xe009e, 0xe009f, + 0xe00a0, 0xe00a1, 0xe00a2, 0xe00a3, 0xe00a4, 0xe00a5, 0xe00a6, 0xe00a7, + 0xe00a8, 0xe00a9, 0xe00aa, 0xe00ab, 0xe00ac, 0xe00ad, 0xe00ae, 0xe00af, + 0xe00b0, 0xe00b1, 0xe00b2, 0xe00b3, 0xe00b4, 0xe00b5, 0xe00b6, 0xe00b7, + 0xe00b8, 0xe00b9, 0xe00ba, 0xe00bb, 0xe00bc, 0xe00bd, 0xe00be, 0xe00bf, + 0xe00c0, 0xe00c1, 0xe00c2, 0xe00c3, 0xe00c4, 0xe00c5, 0xe00c6, 0xe00c7, + 0xe00c8, 0xe00c9, 0xe00ca, 0xe00cb, 0xe00cc, 0xe00cd, 0xe00ce, 0xe00cf, + 0xe00d0, 0xe00d1, 0xe00d2, 0xe00d3, 0xe00d4, 0xe00d5, 0xe00d6, 0xe00d7, + 0xe00d8, 0xe00d9, 0xe00da, 0xe00db, 0xe00dc, 0xe00dd, 0xe00de, 0xe00df, + 0xe00e0, 0xe00e1, 0xe00e2, 0xe00e3, 0xe00e4, 0xe00e5, 0xe00e6, 0xe00e7, + 0xe00e8, 0xe00e9, 0xe00ea, 0xe00eb, 0xe00ec, 0xe00ed, 0xe00ee, 0xe00ef, + 0xe00f0, 0xe00f1, 0xe00f2, 0xe00f3, 0xe00f4, 0xe00f5, 0xe00f6, 0xe00f7, + 0xe00f8, 0xe00f9, 0xe00fa, 0xe00fb, 0xe00fc, 0xe00fd, 0xe00fe, 0xe00ff +}; + +static uint32_t unicode_520_ci_page_e01[] = { + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, 0x00000, + 0xe01f0, 0xe01f1, 0xe01f2, 0xe01f3, 0xe01f4, 0xe01f5, 0xe01f6, 0xe01f7, + 0xe01f8, 0xe01f9, 0xe01fa, 0xe01fb, 0xe01fc, 0xe01fd, 0xe01fe, 0xe01ff +}; + +static uint32_t *unicode_520_ci_table[4352] = { + unicode_520_ci_page_00, unicode_520_ci_page_01, + unicode_520_ci_page_02, unicode_520_ci_page_03, + unicode_520_ci_page_04, unicode_520_ci_page_05, + unicode_520_ci_page_06, unicode_520_ci_page_07, + unicode_520_ci_page_08, unicode_520_ci_page_09, + unicode_520_ci_page_0a, unicode_520_ci_page_0b, + unicode_520_ci_page_0c, unicode_520_ci_page_0d, + unicode_520_ci_page_0e, unicode_520_ci_page_0f, + unicode_520_ci_page_10, NULL, + NULL, unicode_520_ci_page_13, + NULL, NULL, + unicode_520_ci_page_16, unicode_520_ci_page_17, + unicode_520_ci_page_18, unicode_520_ci_page_19, + unicode_520_ci_page_1a, unicode_520_ci_page_1b, + unicode_520_ci_page_1c, unicode_520_ci_page_1d, + unicode_520_ci_page_1e, unicode_520_ci_page_1f, + unicode_520_ci_page_20, unicode_520_ci_page_21, + unicode_520_ci_page_22, NULL, + unicode_520_ci_page_24, NULL, + NULL, unicode_520_ci_page_27, + NULL, NULL, + unicode_520_ci_page_2a, NULL, + unicode_520_ci_page_2c, unicode_520_ci_page_2d, + unicode_520_ci_page_2e, unicode_520_ci_page_2f, + unicode_520_ci_page_30, unicode_520_ci_page_31, + unicode_520_ci_page_32, unicode_520_ci_page_33, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + unicode_520_ci_page_a6, unicode_520_ci_page_a7, + unicode_520_ci_page_a8, unicode_520_ci_page_a9, + unicode_520_ci_page_aa, unicode_520_ci_page_ab, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, unicode_520_ci_page_f9, + unicode_520_ci_page_fa, unicode_520_ci_page_fb, + unicode_520_ci_page_fc, unicode_520_ci_page_fd, + unicode_520_ci_page_fe, unicode_520_ci_page_ff, + NULL, unicode_520_ci_page_101, + NULL, unicode_520_ci_page_103, + unicode_520_ci_page_104, NULL, + NULL, NULL, + unicode_520_ci_page_108, unicode_520_ci_page_109, + unicode_520_ci_page_10a, unicode_520_ci_page_10b, + unicode_520_ci_page_10c, NULL, + unicode_520_ci_page_10e, NULL, + unicode_520_ci_page_110, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + unicode_520_ci_page_124, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, unicode_520_ci_page_1d1, + unicode_520_ci_page_1d2, unicode_520_ci_page_1d3, + unicode_520_ci_page_1d4, unicode_520_ci_page_1d5, + unicode_520_ci_page_1d6, unicode_520_ci_page_1d7, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, unicode_520_ci_page_1f1, + unicode_520_ci_page_1f2, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + unicode_520_ci_page_2f8, unicode_520_ci_page_2f9, + unicode_520_ci_page_2fa, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + unicode_520_ci_page_e00, unicode_520_ci_page_e01, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL, + NULL, NULL +}; + +#endif diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h index 4e6b6c539f4..7153cb1872e 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_ci_except_kana_ci_kana_with_voiced_sound_mark_table.h @@ -1,5 +1,5 @@ /* - Copyright(C) 2013 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2013-2015 Kouhei Sutou <kou@clear-code.com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -17,10 +17,10 @@ MA 02110-1301, USA This file uses normalization table defined in - mysql-5.5.29/strings/ctype-uca.c. + mysql-5.6.23/strings/ctype-uca.c. The following is the header of the file: - Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2004, 2014, Oracle and/or its affiliates. All rights reserved. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_ci_table.h b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_ci_table.h index 861c2029e68..e3a6b4bcc4d 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_ci_table.h +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/normalizers/mysql_unicode_ci_table.h @@ -1,5 +1,5 @@ /* - Copyright(C) 2013 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2013-2015 Kouhei Sutou <kou@clear-code.com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -17,10 +17,10 @@ MA 02110-1301, USA This file uses normalization table defined in - mysql-5.5.29/strings/ctype-uca.c. + mysql-5.6.23/strings/ctype-uca.c. The following is the header of the file: - Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2004, 2014, Oracle and/or its affiliates. All rights reserved. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/Makefile.am b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/Makefile.am index cb257b9293b..9808a7f9e35 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/Makefile.am +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/Makefile.am @@ -1,7 +1,7 @@ REPOSITORIES_PATH = repositories DISTRIBUTIONS = debian ARCHITECTURES = i386 amd64 -CODE_NAMES = wheezy +CODE_NAMES = wheezy jessie all: diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/Vagrantfile b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/Vagrantfile index a7e11791feb..7115ac3f6c0 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/Vagrantfile +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/Vagrantfile @@ -14,6 +14,14 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| :id => "debian-wheezy-amd64", :box_url => "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_debian-7.8_chef-provisionerless.box", }, + { + :id => "debian-jessie-i386", + :box_url => "http://packages.groonga.org/tmp/opscode_debian-8.0-i386_chef-provisionerless.box", + }, + { + :id => "debian-jessie-amd64", + :box_url => "http://packages.groonga.org/tmp/opscode_debian-8.0_chef-provisionerless.box", + }, ] vms.each do |vm| diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/build-deb.sh b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/build-deb.sh index 21bf8af6b35..fdfc8f52f03 100755 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/build-deb.sh +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/apt/build-deb.sh @@ -54,4 +54,4 @@ run cd - package_initial=$(echo "${PACKAGE}" | sed -e 's/\(.\).*/\1/') pool_dir="/vagrant/repositories/${distribution}/pool/${code_name}/${component}/${package_initial}/${PACKAGE}" run mkdir -p "${pool_dir}/" -run cp *.tar.gz *.diff.gz *.dsc *.deb "${pool_dir}/" +run cp *.tar.gz *.dsc *.deb "${pool_dir}/" diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/debian/changelog b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/debian/changelog index a02be693955..7fa8d1e17f0 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/debian/changelog +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/debian/changelog @@ -1,3 +1,9 @@ +groonga-normalizer-mysql (1.0.9-1) unstable; urgency=low + + * New upstream release. + + -- HAYASHI Kentaro <hayashi@clear-code.com> Sun, 29 Mar 2015 00:00:00 +0900 + groonga-normalizer-mysql (1.0.8-1) unstable; urgency=low * New upstream release. diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/rpm/centos/groonga-normalizer-mysql.spec.in b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/rpm/centos/groonga-normalizer-mysql.spec.in index b4751c43762..4084576a704 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/rpm/centos/groonga-normalizer-mysql.spec.in +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/rpm/centos/groonga-normalizer-mysql.spec.in @@ -52,6 +52,9 @@ rm $RPM_BUILD_ROOT%{_libdir}/groonga/plugins/*/*.la %{_libdir}/pkgconfig/groonga-normalizer-mysql.pc %changelog +* Sun Mar 29 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 1.0.9-1 +- new upstream release. + * Tue Feb 10 2015 Kouhei Sutou <kou@clear-code.com> - 1.0.8-1 - new upstream release. diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/rpm/fedora/groonga-normalizer-mysql.spec.in b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/rpm/fedora/groonga-normalizer-mysql.spec.in index fbde53c117f..974b7c31649 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/rpm/fedora/groonga-normalizer-mysql.spec.in +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/rpm/fedora/groonga-normalizer-mysql.spec.in @@ -51,6 +51,9 @@ rm $RPM_BUILD_ROOT%{_libdir}/groonga/plugins/*/*.la %{_libdir}/pkgconfig/groonga-normalizer-mysql.pc %changelog +* Sun Mar 29 2015 HAYASHI Kentaro <hayashi@clear-code.com> - 1.0.9-1 +- new upstream release. + * Tue Feb 10 2015 Kouhei Sutou <kou@clear-code.com> - 1.0.8-1 - new upstream release. diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/ubuntu/Makefile.am b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/ubuntu/Makefile.am index 4afc4a982f0..73dbef7791b 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/ubuntu/Makefile.am +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/packages/ubuntu/Makefile.am @@ -1,4 +1,4 @@ -CODE_NAMES = precise,trusty,utopic +CODE_NAMES = precise,trusty,utopic,vivid SOURCE = ../$(PACKAGE)-$(VERSION).tar.gz all: diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/generate_uca_table.rb b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/generate_uca_table.rb index 57dba4e0477..4dcb14c324e 100755 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/generate_uca_table.rb +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/generate_uca_table.rb @@ -1,7 +1,7 @@ #!/usr/bin/env ruby # -*- coding: utf-8 -*- # -# Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com> +# Copyright (C) 2013-2015 Kouhei Sutou <kou@clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Library General Public @@ -22,6 +22,7 @@ require "optparse" $LOAD_PATH.unshift(File.dirname(__FILE__)) require "parser" +@version = nil @suffix = "" @split_small_kana_p = false @split_kana_with_voiced_sound_mark_p = false @@ -30,6 +31,10 @@ require "parser" option_parser = OptionParser.new option_parser.banner += " MYSQL_SOURCE/strings/ctype-uca.c" +option_parser.on("--version=VERSION", "Use VERSION as UCA version") do |version| + @version = version +end + option_parser.on("--suffix=SUFFIX", "Add SUFFIX to names") do |suffix| @suffix = suffix end @@ -67,7 +72,7 @@ end ctype_uca_c_path = ARGV[0] -parser = CTypeUCAParser.new +parser = CTypeUCAParser.new(@version) File.open(ctype_uca_c_path) do |ctype_uca_c| parser.parse(ctype_uca_c) end @@ -185,11 +190,15 @@ end normalized_ctype_uca_c_path = ctype_uca_c_path.sub(/\A.*\/([^\/]+\/strings\/ctype-uca\.c)\z/, "\\1") -@suffix_upper_case = @suffix.upcase +header_guard_id = "MYSQL_UCA" +if @version + header_guard_id << "_#{@version}" +end +header_guard_id << "#{@suffix.upcase}_H" puts(<<-HEADER) /* - Copyright(C) 2013 Kouhei Sutou <kou@clear-code.com> + Copyright(C) 2013-2015 Kouhei Sutou <kou@clear-code.com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -210,7 +219,7 @@ puts(<<-HEADER) #{normalized_ctype_uca_c_path}. The following is the header of the file: - Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2004, 2014, Oracle and/or its affiliates. All rights reserved. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -231,14 +240,23 @@ puts(<<-HEADER) Written by Alexander Barkov <bar@mysql.com> */ -#ifndef MYSQL_UCA#{@suffix_upper_case}_H -#define MYSQL_UCA#{@suffix_upper_case}_H +#ifndef #{header_guard_id} +#define #{header_guard_id} #include <stdint.h> HEADER +def variable_name_prefix + prefix = "unicode" + if @version + prefix << "_#{@version}" + end + prefix << "_ci#{@suffix}" + prefix +end + def page_name(page) - "unicode_ci#{@suffix}_page_%02x" % page + "#{variable_name_prefix}_page_%02x" % page end sorted_target_pages.each do |page, characters| @@ -261,10 +279,10 @@ end puts(<<-PAGES_HEADER) -static uint32_t *unicode_ci#{@suffix}_table[256] = { +static uint32_t *#{variable_name_prefix}_table[#{parser.n_pages}] = { PAGES_HEADER -pages = ["NULL"] * 256 +pages = ["NULL"] * parser.n_pages sorted_target_pages.each do |page, characters| pages[page] = page_name(page) end diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/parser.rb b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/parser.rb index b8652825f80..cc0b2f01606 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/parser.rb +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/parser.rb @@ -80,7 +80,8 @@ end class CTypeUCAParser attr_reader :pages - def initialize + def initialize(version=nil) + @version = version @pages = {} @lengths = [] end @@ -108,22 +109,43 @@ class CTypeUCAParser end end + def n_pages + @lengths.size + end + private + def page_data_pattern + if @version == "520" + / uca520_p([\da-fA-F]{3})\[\]=/ + else + / page([\da-fA-F]{3})data\[\]=/ + end + end + + def length_pattern + if @version == "520" + / uca520_length\[4352\]=/ + else + / uca_length\[256\]=/ + end + end + def parse_ctype_uca(input) current_page = nil in_length = false input.each_line do |line| case line - when / page([\da-fA-F]{3})data\[\]=/ + when page_data_pattern current_page = $1.to_i(16) @pages[current_page] = [] - when /^\s*0x(?:[\da-z]+)(?:,\s*0x(?:[\da-z]+))*,?$/i + when /^\s*(0x(?:[\da-z]+)(?:,\s*0x(?:[\da-z]+))*),?(?: \/\*.+\*\/)?$/i + weight_values = $1 next if current_page.nil? - weights = line.chomp.split(/,\s*/).collect do |component| + weights = weight_values.split(/,\s*/).collect do |component| Integer(component) end @pages[current_page].concat(weights) - when / uca_length\[256\]=/ + when length_pattern in_length = true when /^\d+(?:,\d+)*,?$/ next unless in_length diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/version b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/version index 337a6a8f189..e5a4a5e7d84 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/version +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/version @@ -1 +1 @@ -1.0.8
\ No newline at end of file +1.0.9
\ No newline at end of file diff --git a/storage/mroonga/version b/storage/mroonga/version index 2defa1a3bb8..760b908bf87 100644 --- a/storage/mroonga/version +++ b/storage/mroonga/version @@ -1 +1 @@ -5.00
\ No newline at end of file +5.02
\ No newline at end of file diff --git a/storage/mroonga/version_in_hex b/storage/mroonga/version_in_hex index 960c1688afd..7716095f96c 100644 --- a/storage/mroonga/version_in_hex +++ b/storage/mroonga/version_in_hex @@ -1 +1 @@ -0x0500
\ No newline at end of file +0x0502
\ No newline at end of file diff --git a/storage/mroonga/version_micro b/storage/mroonga/version_micro index c227083464f..d8263ee9860 100644 --- a/storage/mroonga/version_micro +++ b/storage/mroonga/version_micro @@ -1 +1 @@ -0
\ No newline at end of file +2
\ No newline at end of file |