Allow multiple NLS domains to be specified when searching

Also, use the new functionality in the existing gettext builder plugin.

6 files changed, 75 insertions, 236 deletions

diff --git a/data/tests/locale/en_GB/LC_MESSAGES/app.mo b/data/tests/locale/en_GB/LC_MESSAGES/app.mo
new file mode 100755
index 0000000..985a1ce
--- /dev/null
+++ b/data/tests/locale/en_GB/LC_MESSAGES/app.mo
diff --git a/data/tests/locale/ru/LC_MESSAGES/app.mo b/data/tests/locale/ru/LC_MESSAGES/app.mo
new file mode 100755
index 0000000..e70e55a
--- /dev/null
+++ b/data/tests/locale/ru/LC_MESSAGES/app.mo
diff --git a/libappstream-builder/plugins/asb-plugin-gettext.c b/libappstream-builder/plugins/asb-plugin-gettext.c
index e7662ce..6ca0a8a 100644
--- a/libappstream-builder/plugins/asb-plugin-gettext.c
+++ b/libappstream-builder/plugins/asb-plugin-gettext.c
@@ -1,6 +1,6 @@
 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
  *
- * Copyright (C) 2014 Richard Hughes <richard@hughsie.com>
+ * Copyright (C) 2014-2016 Richard Hughes <richard@hughsie.com>
  *
  * Licensed under the GNU Lesser General Public License Version 2.1
  *
@@ -20,37 +20,9 @@
  */
 
 #include <config.h>
-#include <fnmatch.h>
 
 #include <asb-plugin.h>
 
-typedef struct {
-	guint32		 magic;
-	guint32		 revision;
-	guint32		 nstrings;
-	guint32		 orig_tab_offset;
-	guint32		 trans_tab_offset;
-	guint32		 hash_tab_size;
-	guint32		 hash_tab_offset;
-	guint32		 n_sysdep_segments;
-	guint32		 sysdep_segments_offset;
-	guint32		 n_sysdep_strings;
-	guint32		 orig_sysdep_tab_offset;
-	guint32		 trans_sysdep_tab_offset;
-} AsbGettextHeader;
-
-typedef struct {
-	gchar		*locale;
-	guint		 nstrings;
-	guint		 percentage;
-} AsbGettextEntry;
-
-typedef struct {
-	guint		 max_nstrings;
-	GList		*data;
-	gchar		*prefered_mo_filename;
-} AsbGettextContext;
-
 /**
  * asb_plugin_get_name:
  */
@@ -70,181 +42,6 @@ asb_plugin_add_globs (AsbPlugin *plugin, GPtrArray *globs)
 }
 
 /**
- * asb_gettext_entry_new:
- **/
-static AsbGettextEntry *
-asb_gettext_entry_new (void)
-{
-	AsbGettextEntry *entry;
-	entry = g_slice_new0 (AsbGettextEntry);
-	return entry;
-}
-
-/**
- * asb_gettext_entry_free:
- **/
-static void
-asb_gettext_entry_free (AsbGettextEntry *entry)
-{
-	g_free (entry->locale);
-	g_slice_free (AsbGettextEntry, entry);
-}
-
-/**
- * asb_gettext_ctx_new:
- **/
-static AsbGettextContext *
-asb_gettext_ctx_new (void)
-{
-	AsbGettextContext *ctx;
-	ctx = g_new0 (AsbGettextContext, 1);
-	return ctx;
-}
-
-/**
- * asb_gettext_ctx_free:
- **/
-static void
-asb_gettext_ctx_free (AsbGettextContext *ctx)
-{
-	g_list_free_full (ctx->data, (GDestroyNotify) asb_gettext_entry_free);
-	g_free (ctx->prefered_mo_filename);
-	g_free (ctx);
-}
-
-/**
- * asb_gettext_parse_file:
- **/
-static gboolean
-asb_gettext_parse_file (AsbGettextContext *ctx,
-			const gchar *locale,
-			const gchar *filename,
-			GError **error)
-{
-	AsbGettextEntry *entry;
-	AsbGettextHeader *h;
-	g_autofree gchar *data = NULL;
-	gboolean swapped;
-
-	/* read data, although we only strictly need the header */
-	if (!g_file_get_contents (filename, &data, NULL, error))
-		return FALSE;
-
-	h = (AsbGettextHeader *) data;
-	if (h->magic == 0x950412de)
-		swapped = FALSE;
-	else if (h->magic == 0xde120495)
-		swapped = TRUE;
-	else
-		return FALSE;
-	entry = asb_gettext_entry_new ();
-	entry->locale = g_strdup (locale);
-	if (swapped)
-		entry->nstrings = GUINT32_SWAP_LE_BE (h->nstrings);
-	else
-		entry->nstrings = h->nstrings;
-	if (entry->nstrings > ctx->max_nstrings)
-		ctx->max_nstrings = entry->nstrings;
-	ctx->data = g_list_prepend (ctx->data, entry);
-	return TRUE;
-}
-
-/**
- * asb_gettext_ctx_search_locale:
- **/
-static gboolean
-asb_gettext_ctx_search_locale (AsbGettextContext *ctx,
-			       const gchar *locale,
-			       const gchar *messages_path,
-			       GError **error)
-{
-	const gchar *filename;
-	guint i;
-	g_autoptr(GDir) dir = NULL;
-	g_autoptr(GPtrArray) mo_paths = NULL;
-
-	dir = g_dir_open (messages_path, 0, error);
-	if (dir == NULL)
-		return FALSE;
-
-	/* do a first pass at this, trying to find the prefered .mo */
-	mo_paths = g_ptr_array_new_with_free_func (g_free);
-	while ((filename = g_dir_read_name (dir)) != NULL) {
-		g_autofree gchar *path = NULL;
-		path = g_build_filename (messages_path, filename, NULL);
-		if (!g_file_test (path, G_FILE_TEST_EXISTS))
-			continue;
-		if (g_strcmp0 (filename, ctx->prefered_mo_filename) == 0) {
-			if (!asb_gettext_parse_file (ctx, locale, path, error))
-				return FALSE;
-			return TRUE;
-		}
-		g_ptr_array_add (mo_paths, g_strdup (path));
-	}
-
-	/* fall back to parsing *everything*, which might give us more
-	 * language results than is actually true */
-	for (i = 0; i < mo_paths->len; i++) {
-		filename = g_ptr_array_index (mo_paths, i);
-		if (!asb_gettext_parse_file (ctx, locale, filename, error))
-			return FALSE;
-	}
-
-	return TRUE;
-}
-
-static gint
-asb_gettext_entry_sort_cb (gconstpointer a, gconstpointer b)
-{
-	return g_strcmp0 (((AsbGettextEntry *) a)->locale, ((AsbGettextEntry *) b)->locale);
-}
-
-/**
- * asb_gettext_ctx_search_path:
- **/
-static gboolean
-asb_gettext_ctx_search_path (AsbGettextContext *ctx,
-			     const gchar *prefix,
-			     GError **error)
-{
-	const gchar *filename;
-	AsbGettextEntry *e;
-	GList *l;
-	g_autoptr(GDir) dir = NULL;
-	g_autofree gchar *root = NULL;
-
-	/* search for .mo files in the prefix */
-	root = g_build_filename (prefix, "/usr/share/locale", NULL);
-	if (!g_file_test (root, G_FILE_TEST_EXISTS)) {
-		g_free (root);
-		root = g_build_filename (prefix, "/files/share/locale", NULL);
-	}
-	if (!g_file_test (root, G_FILE_TEST_EXISTS))
-		return TRUE;
-	dir = g_dir_open (root, 0, error);
-	if (dir == NULL)
-		return FALSE;
-	while ((filename = g_dir_read_name (dir)) != NULL) {
-		g_autofree gchar *path = NULL;
-		path = g_build_filename (root, filename, "LC_MESSAGES", NULL);
-		if (g_file_test (path, G_FILE_TEST_EXISTS)) {
-			if (!asb_gettext_ctx_search_locale (ctx, filename, path, error))
-				return FALSE;
-		}
-	}
-
-	/* calculate percentages */
-	for (l = ctx->data; l != NULL; l = l->next) {
-		e = l->data;
-		e->percentage = MIN (e->nstrings * 100 / ctx->max_nstrings, 100);
-	}
-
-	/* sort */
-	ctx->data = g_list_sort (ctx->data, asb_gettext_entry_sort_cb);
-	return TRUE;
-}
-
-/**
  * asb_plugin_process_app:
  */
 gboolean
@@ -254,26 +51,24 @@ asb_plugin_process_app (AsbPlugin *plugin,
 			const gchar *tmpdir,
 			GError **error)
 {
-	AsbGettextContext *ctx = NULL;
-	AsbGettextEntry *e;
-	gboolean ret;
-	GList *l;
-
-	/* search */
-	ctx = asb_gettext_ctx_new ();
-	ctx->prefered_mo_filename = g_strdup_printf ("%s.mo", asb_package_get_name (pkg));
-	ret = asb_gettext_ctx_search_path (ctx, tmpdir, error);
-	if (!ret)
-		goto out;
+	g_autofree gchar *root = NULL;
+	g_auto(GStrv) intl_domains = NULL;
 
-	/* print results */
-	for (l = ctx->data; l != NULL; l = l->next) {
-		e = l->data;
-		if (e->percentage < 25)
-			continue;
-		as_app_add_language (AS_APP (app), e->percentage, e->locale);
+	/* search for .mo files in the prefix */
+	root = g_build_filename (tmpdir, "/usr/share/locale", NULL);
+	if (!g_file_test (root, G_FILE_TEST_EXISTS)) {
+		g_free (root);
+		root = g_build_filename (tmpdir, "/files/share/locale", NULL);
 	}
-out:
-	asb_gettext_ctx_free (ctx);
-	return ret;
+	if (!g_file_test (root, G_FILE_TEST_EXISTS))
+		return TRUE;
+
+	/* generate */
+	intl_domains = g_strsplit (asb_package_get_name (pkg), ",", -1);
+	return as_app_gettext_search_path (AS_APP (app),
+					   root,
+					   intl_domains,
+					   25,
+					   NULL,
+					   error);
 }
diff --git a/libappstream-glib/as-app-gettext.c b/libappstream-glib/as-app-gettext.c
index 402894f..8283bef 100644
--- a/libappstream-glib/as-app-gettext.c
+++ b/libappstream-glib/as-app-gettext.c
@@ -64,7 +64,7 @@ typedef struct {
 typedef struct {
 	guint		 max_nstrings;
 	GList		*data;
-	gchar		*base_filename;
+	gchar		**intl_domains;
 	guint		 min_percentage;
 } AsGettextContext;
 
@@ -107,7 +107,7 @@ static void
 as_gettext_ctx_free (AsGettextContext *ctx)
 {
 	g_list_free_full (ctx->data, (GDestroyNotify) as_gettext_entry_free);
-	g_free (ctx->base_filename);
+	g_strfreev (ctx->intl_domains);
 	g_free (ctx);
 }
 
@@ -158,6 +158,7 @@ as_gettext_ctx_search_locale (AsGettextContext *ctx,
 			      GError **error)
 {
 	const gchar *filename;
+	gboolean found_anything = FALSE;
 	guint i;
 	g_autoptr(GDir) dir = NULL;
 	g_autoptr(GPtrArray) mo_paths = NULL;
@@ -174,14 +175,23 @@ as_gettext_ctx_search_locale (AsGettextContext *ctx,
 		path = g_build_filename (messages_path, filename, NULL);
 		if (!g_file_test (path, G_FILE_TEST_EXISTS))
 			continue;
-		if (g_strcmp0 (filename, ctx->base_filename) == 0) {
-			if (!as_gettext_parse_file (ctx, locale, path, error))
-				return FALSE;
-			return TRUE;
+		for (i = 0; ctx->intl_domains != NULL &&
+			    ctx->intl_domains[i] != NULL; i++) {
+			g_autofree gchar *fn = NULL;
+			fn = g_strdup_printf ("%s.mo", ctx->intl_domains[i]);
+			if (g_strcmp0 (filename, fn) == 0) {
+				if (!as_gettext_parse_file (ctx, locale, path, error))
+					return FALSE;
+				found_anything = TRUE;
+			}
 		}
 		g_ptr_array_add (mo_paths, g_strdup (path));
 	}
 
+	/* we got data from one or more of the intl_domains */
+	if (found_anything == TRUE)
+		return TRUE;
+
 	/* fall back to parsing *everything*, which might give us more
 	 * language results than is actually true */
 	for (i = 0; i < mo_paths->len; i++) {
@@ -209,7 +219,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(AsGettextContext, as_gettext_ctx_free)
  * as_app_gettext_search_path:
  * @app: an #AsApp
  * @path: a path to search, e.g. "/usr/share/locale"
- * @base_filename: a filename, e.g. "gnome-software.mo"
+ * @intl_domains: (allow-none): gettext domains, e.g. ["gnome-software"]
  * @min_percentage: minimum percentage to add language
  * @cancellable: a #GCancellable or %NULL
  * @error: a #GError or %NULL
@@ -217,7 +227,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(AsGettextContext, as_gettext_ctx_free)
  * Searches a gettext catalog path for languages, and using a heuristic
  * adds <language> tags to the specified application.
  *
- * If @base_filename is not set then any filename is matched, which may
+ * If @intl_domains is not set then all domains are matched, which may
  * include more languages than you intended to.
  *
  * @min_percentage sets the minimum percentage to add a language tag.
@@ -235,7 +245,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(AsGettextContext, as_gettext_ctx_free)
 gboolean
 as_app_gettext_search_path (AsApp *app,
 			    const gchar *path,
-			    const gchar *base_filename,
+			    gchar **intl_domains,
 			    guint min_percentage,
 			    GCancellable *cancellable,
 			    GError **error)
@@ -251,7 +261,7 @@ as_app_gettext_search_path (AsApp *app,
 		return FALSE;
 	ctx = as_gettext_ctx_new ();
 	ctx->min_percentage = min_percentage;
-	ctx->base_filename = g_strdup (base_filename);
+	ctx->intl_domains = g_strdupv (intl_domains);
 	while ((filename = g_dir_read_name (dir)) != NULL) {
 		g_autofree gchar *fn = NULL;
 		fn = g_build_filename (path, filename, "LC_MESSAGES", NULL);
diff --git a/libappstream-glib/as-app-gettext.h b/libappstream-glib/as-app-gettext.h
index 1645696..432e1be 100644
--- a/libappstream-glib/as-app-gettext.h
+++ b/libappstream-glib/as-app-gettext.h
@@ -35,7 +35,7 @@ G_BEGIN_DECLS
 
 gboolean	 as_app_gettext_search_path	(AsApp		*app,
 						 const gchar	*path,
-						 const gchar	*base_filename,
+						 gchar		**intl_domains,
 						 guint		 min_percentage,
 						 GCancellable	*cancellable,
 						 GError		**error);
diff --git a/libappstream-glib/as-self-test.c b/libappstream-glib/as-self-test.c
index 24275cc..2c7b888 100644
--- a/libappstream-glib/as-self-test.c
+++ b/libappstream-glib/as-self-test.c
@@ -333,13 +333,15 @@ as_test_app_gettext_func (void)
 	g_autofree gchar *fn = NULL;
 	g_autoptr(AsApp) app = NULL;
 	g_autoptr(GList) list = NULL;
+	g_auto(GStrv) intl_domains = NULL;
 
 	app = as_app_new ();
 	fn = as_test_get_filename ("locale");
 	g_assert (fn != NULL);
+	intl_domains = g_strsplit ("app,notgoingtoexist", ",", -1);
 	ret = as_app_gettext_search_path (app,
 					  fn,
-					  "app.mo",
+					  intl_domains,
 					  25,
 					  NULL,
 					  &error);
@@ -357,6 +359,37 @@ as_test_app_gettext_func (void)
 }
 
 static void
+as_test_app_gettext_nodomain_func (void)
+{
+	GError *error = NULL;
+	gboolean ret;
+	g_autofree gchar *fn = NULL;
+	g_autoptr(AsApp) app = NULL;
+	g_autoptr(GList) list = NULL;
+	g_auto(GStrv) intl_domains = NULL;
+
+	app = as_app_new ();
+	fn = as_test_get_filename ("locale");
+	ret = as_app_gettext_search_path (app,
+					  fn,
+					  NULL,
+					  50,
+					  NULL,
+					  &error);
+	g_assert_no_error (error);
+	g_assert (ret);
+
+	/* check langs */
+	g_assert_cmpint (as_app_get_language (app, "en_GB"), ==, 100);
+	g_assert_cmpint (as_app_get_language (app, "ru"), ==, -1);
+	g_assert_cmpint (as_app_get_language (app, "fr_FR"), ==, -1);
+
+	/* check size */
+	list = as_app_get_languages (app);
+	g_assert_cmpint (g_list_length (list), ==, 1);
+}
+
+static void
 as_test_tag_func (void)
 {
 	guint i;
@@ -4447,6 +4480,7 @@ main (int argc, char **argv)
 	g_test_add_func ("/AppStream/screenshot", as_test_screenshot_func);
 	g_test_add_func ("/AppStream/app", as_test_app_func);
 	g_test_add_func ("/AppStream/app{gettext}", as_test_app_gettext_func);
+	g_test_add_func ("/AppStream/app{gettext-nodomain}", as_test_app_gettext_nodomain_func);
 	g_test_add_func ("/AppStream/app{translated}", as_test_app_translated_func);
 	g_test_add_func ("/AppStream/app{validate-style}", as_test_app_validate_style_func);
 	g_test_add_func ("/AppStream/app{validate-appdata-good}", as_test_app_validate_appdata_good_func);