SERVER-26072 add collation support for 59 new locales

We now support all collations that are supported by ICU
57.1, except for aliases. See the SERVER ticket for a list
of the new locales. The exact ICU data files which we now
package into MongoDB is given by COLLATION_SOURCE in ICU
57.1's colfiles.mk.

4 files changed, 75 insertions, 2 deletions

diff --git a/src/third_party/icu4c-57.1/source/mongo_sources/icudt57b.dat b/src/third_party/icu4c-57.1/source/mongo_sources/icudt57b.dat
index 74799950bb5..93edde31ef2 100644
--- a/src/third_party/icu4c-57.1/source/mongo_sources/icudt57b.dat
+++ b/src/third_party/icu4c-57.1/source/mongo_sources/icudt57b.dat
diff --git a/src/third_party/icu4c-57.1/source/mongo_sources/icudt57l.dat b/src/third_party/icu4c-57.1/source/mongo_sources/icudt57l.dat
index da96c94e4a7..26546ff3299 100644
--- a/src/third_party/icu4c-57.1/source/mongo_sources/icudt57l.dat
+++ b/src/third_party/icu4c-57.1/source/mongo_sources/icudt57l.dat
diff --git a/src/third_party/icu4c-57.1/source/mongo_sources/languages.txt b/src/third_party/icu4c-57.1/source/mongo_sources/languages.txt
index 05911c75573..b44e5904e0f 100644
--- a/src/third_party/icu4c-57.1/source/mongo_sources/languages.txt
+++ b/src/third_party/icu4c-57.1/source/mongo_sources/languages.txt
@@ -1,52 +1,111 @@
 # This file is taken as input to the ICU data packaging process, and contains the base names of all
 # ICU locales that have collation data included in MongoDB's custom data file for ICU.
+af
+am
 ar
+as
+az
+be
+bg
 bn
+bo
+bs
+bs_Cyrl
 ca
+chr
 cs
+cy
 da
 de
 de_AT
+dsb
+dz
+ee
 el
 en
 en_US
 en_US_POSIX
+eo
 es
 et
 fa
 fa_AF
 fi
 fil
+fo
 fr
 fr_CA
 ga
+gl
+gu
+ha
+haw
 he
 hi
 hr
+hsb
 hu
 hy
 id
+ig
 is
 it
 ja
 ka
+kk
+kl
+km
+kn
 ko
+kok
+ky
+lb
+lkt
+ln
+lo
 lt
 lv
+mk
+ml
+mn
+mr
+ms
+mt
+my
+nb
+ne
+nl
 nn
+om
+or
 pa
 pl
 ps
 pt
 ro
 ru
+se
+si
 sk
 sl
+smn
+sq
+sr
+sr_Latn
 sv
+sw
+ta
+te
 th
+to
 tr
+ug
 uk
 ur
 vi
+wae
+yi
+yo
 zh
 zh_Hant
+zu
diff --git a/src/third_party/scripts/icu_get_sources.sh b/src/third_party/scripts/icu_get_sources.sh
index e19cfd464db..43aba3da9a8 100755
--- a/src/third_party/scripts/icu_get_sources.sh
+++ b/src/third_party/scripts/icu_get_sources.sh
@@ -9,16 +9,26 @@
 # list each locale for which collation data should be packaged as part of the generated custom data
 # file.
 #
+# The script accepts a single optional argument, which is the path to the .dat archive to use as the
+# source of the trimmed-down ICU .dat files generated as output. If omitted, the .dat archive which
+# is included in the ICU source code is used by default.
+#
 # This script returns a zero exit code on success.
 
 set -euo pipefail
 IFS=$'\n\t'
 
-if [ "$#" -ne 0 ]; then
+# Expect 0 or 1 arguments.
+if [ "$#" -gt 1 ]; then
     echo "$0: too many arguments" >&2
     exit 1
 fi
 
+# Set value of original .dat archive from the first argument.
+if [ "$#" -eq 1 ]; then
+    ORIGINAL_DATA_FILE=${1}
+fi
+
 KERNEL="$(uname)"
 if [ "$KERNEL" != Linux ]; then
     echo "$0: kernel '$KERNEL' not supported" >&2
@@ -72,7 +82,11 @@ make install
 # Generate trimmed-down list of data to include in custom data files.
 #
 
-ORIGINAL_DATA_FILE="${TARBALL_DIR}/source/data/in/icudt${MAJOR_VERSION}l.dat"
+# If the original data file wasn't passed as an argument, use the one from the ICU source tree.
+if [ -z ${ORIGINAL_DATA_FILE+x} ]; then
+    ORIGINAL_DATA_FILE="${TARBALL_DIR}/source/data/in/icudt${MAJOR_VERSION}l.dat"
+fi
+
 ORIGINAL_DATA_LIST="${DATA_DIR}/icudt${MAJOR_VERSION}l.lst.orig"
 NEW_DATA_LIST="${DATA_DIR}/icudt${MAJOR_VERSION}l.lst"