Move the gettext parsing to libappstream-glib

5 files changed, 362 insertions, 0 deletions

diff --git a/libappstream-glib/Makefile.am b/libappstream-glib/Makefile.am
index feb7d21..f017040 100644
--- a/libappstream-glib/Makefile.am
+++ b/libappstream-glib/Makefile.am
@@ -69,6 +69,7 @@ libappstream_glib_includedir = $(includedir)/libappstream-glib
 libappstream_glib_include_HEADERS =				\
 	appstream-glib.h					\
 	as-app.h						\
+	as-app-gettext.h					\
 	as-bundle.h						\
 	as-checksum.h						\
 	as-enums.h						\
@@ -89,6 +90,7 @@ libappstream_glib_include_HEADERS =				\
 libappstream_glib_la_SOURCES =					\
 	as-app.c						\
 	as-app-desktop.c					\
+	as-app-gettext.c					\
 	as-app-inf.c						\
 	as-app-private.h					\
 	as-app-validate.c					\
diff --git a/libappstream-glib/appstream-glib.h b/libappstream-glib/appstream-glib.h
index 4c5945c..ad5fab8 100644
--- a/libappstream-glib/appstream-glib.h
+++ b/libappstream-glib/appstream-glib.h
@@ -25,6 +25,7 @@
 #define __APPSTREAM_GLIB_H_INSIDE__
 
 #include <as-app.h>
+#include <as-app-gettext.h>
 #include <as-bundle.h>
 #include <as-checksum.h>
 #include <as-enums.h>
diff --git a/libappstream-glib/as-app-gettext.c b/libappstream-glib/as-app-gettext.c
new file mode 100644
index 0000000..402894f
--- /dev/null
+++ b/libappstream-glib/as-app-gettext.c
@@ -0,0 +1,281 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ *
+ * Copyright (C) 2015 Richard Hughes <richard@hughsie.com>
+ *
+ * Licensed under the GNU Lesser General Public License Version 2.1
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+/**
+ * SECTION:as-gettext
+ * @short_description: a hashed array gettext of applications
+ * @include: appstream-glib.h
+ * @stability: Stable
+ *
+ * This object will parse a gettext catalog directory and calculate the
+ * language stats for an application.
+ *
+ * See also: #AsApp
+ */
+
+#include "config.h"
+
+#include <fnmatch.h>
+
+#include "as-app-gettext.h"
+
+typedef struct {
+} AsGettextPrivate;
+
+typedef struct {
+	guint32		 magic;
+	guint32		 revision;
+	guint32		 nstrings;
+	guint32		 orig_tab_offset;
+	guint32		 trans_tab_offset;
+	guint32		 hash_tab_size;
+	guint32		 hash_tab_offset;
+	guint32		 n_sysdep_segments;
+	guint32		 sysdep_segments_offset;
+	guint32		 n_sysdep_strings;
+	guint32		 orig_sysdep_tab_offset;
+	guint32		 trans_sysdep_tab_offset;
+} AsGettextHeader;
+
+typedef struct {
+	gchar		*locale;
+	guint		 nstrings;
+	guint		 percentage;
+} AsGettextEntry;
+
+typedef struct {
+	guint		 max_nstrings;
+	GList		*data;
+	gchar		*base_filename;
+	guint		 min_percentage;
+} AsGettextContext;
+
+/**
+ * as_gettext_entry_new:
+ **/
+static AsGettextEntry *
+as_gettext_entry_new (void)
+{
+	AsGettextEntry *entry;
+	entry = g_slice_new0 (AsGettextEntry);
+	return entry;
+}
+
+/**
+ * as_gettext_entry_free:
+ **/
+static void
+as_gettext_entry_free (AsGettextEntry *entry)
+{
+	g_free (entry->locale);
+	g_slice_free (AsGettextEntry, entry);
+}
+
+/**
+ * as_gettext_ctx_new:
+ **/
+static AsGettextContext *
+as_gettext_ctx_new (void)
+{
+	AsGettextContext *ctx;
+	ctx = g_new0 (AsGettextContext, 1);
+	return ctx;
+}
+
+/**
+ * as_gettext_ctx_free:
+ **/
+static void
+as_gettext_ctx_free (AsGettextContext *ctx)
+{
+	g_list_free_full (ctx->data, (GDestroyNotify) as_gettext_entry_free);
+	g_free (ctx->base_filename);
+	g_free (ctx);
+}
+
+/**
+ * as_gettext_parse_file:
+ **/
+static gboolean
+as_gettext_parse_file (AsGettextContext *ctx,
+			const gchar *locale,
+			const gchar *filename,
+			GError **error)
+{
+	AsGettextEntry *entry;
+	AsGettextHeader *h;
+	g_autofree gchar *data = NULL;
+	gboolean swapped;
+
+	/* read data, although we only strictly need the header */
+	if (!g_file_get_contents (filename, &data, NULL, error))
+		return FALSE;
+
+	h = (AsGettextHeader *) data;
+	if (h->magic == 0x950412de)
+		swapped = FALSE;
+	else if (h->magic == 0xde120495)
+		swapped = TRUE;
+	else
+		return FALSE;
+	entry = as_gettext_entry_new ();
+	entry->locale = g_strdup (locale);
+	if (swapped)
+		entry->nstrings = GUINT32_SWAP_LE_BE (h->nstrings);
+	else
+		entry->nstrings = h->nstrings;
+	if (entry->nstrings > ctx->max_nstrings)
+		ctx->max_nstrings = entry->nstrings;
+	ctx->data = g_list_prepend (ctx->data, entry);
+	return TRUE;
+}
+
+/**
+ * as_gettext_ctx_search_locale:
+ **/
+static gboolean
+as_gettext_ctx_search_locale (AsGettextContext *ctx,
+			      const gchar *locale,
+			      const gchar *messages_path,
+			      GError **error)
+{
+	const gchar *filename;
+	guint i;
+	g_autoptr(GDir) dir = NULL;
+	g_autoptr(GPtrArray) mo_paths = NULL;
+
+	/* list files */
+	dir = g_dir_open (messages_path, 0, error);
+	if (dir == NULL)
+		return FALSE;
+
+	/* do a first pass at this, trying to find the prefered .mo */
+	mo_paths = g_ptr_array_new_with_free_func (g_free);
+	while ((filename = g_dir_read_name (dir)) != NULL) {
+		g_autofree gchar *path = NULL;
+		path = g_build_filename (messages_path, filename, NULL);
+		if (!g_file_test (path, G_FILE_TEST_EXISTS))
+			continue;
+		if (g_strcmp0 (filename, ctx->base_filename) == 0) {
+			if (!as_gettext_parse_file (ctx, locale, path, error))
+				return FALSE;
+			return TRUE;
+		}
+		g_ptr_array_add (mo_paths, g_strdup (path));
+	}
+
+	/* fall back to parsing *everything*, which might give us more
+	 * language results than is actually true */
+	for (i = 0; i < mo_paths->len; i++) {
+		filename = g_ptr_array_index (mo_paths, i);
+		if (!as_gettext_parse_file (ctx, locale, filename, error))
+			return FALSE;
+	}
+
+	return TRUE;
+}
+
+/**
+ * as_gettext_entry_sort_cb:
+ **/
+static gint
+as_gettext_entry_sort_cb (gconstpointer a, gconstpointer b)
+{
+	return g_strcmp0 (((AsGettextEntry *) a)->locale,
+			  ((AsGettextEntry *) b)->locale);
+}
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(AsGettextContext, as_gettext_ctx_free)
+
+/**
+ * as_app_gettext_search_path:
+ * @app: an #AsApp
+ * @path: a path to search, e.g. "/usr/share/locale"
+ * @base_filename: a filename, e.g. "gnome-software.mo"
+ * @min_percentage: minimum percentage to add language
+ * @cancellable: a #GCancellable or %NULL
+ * @error: a #GError or %NULL
+ *
+ * Searches a gettext catalog path for languages, and using a heuristic
+ * adds <language> tags to the specified application.
+ *
+ * If @base_filename is not set then any filename is matched, which may
+ * include more languages than you intended to.
+ *
+ * @min_percentage sets the minimum percentage to add a language tag.
+ * The usual value would be 25% and any language less complete than
+ * this will not be added.
+ *
+ * The purpose of this functionality is to avoid blowing up the size
+ * of the AppStream metadata with a lot of extra data detailing
+ * languages with very few translated strings.
+ *
+ * Returns: %TRUE for success
+ *
+ * Since: 0.5.6
+ **/
+gboolean
+as_app_gettext_search_path (AsApp *app,
+			    const gchar *path,
+			    const gchar *base_filename,
+			    guint min_percentage,
+			    GCancellable *cancellable,
+			    GError **error)
+{
+	AsGettextEntry *e;
+	GList *l;
+	const gchar *filename;
+	g_autoptr(AsGettextContext) ctx = NULL;
+	g_autoptr(GDir) dir = NULL;
+
+	dir = g_dir_open (path, 0, error);
+	if (dir == NULL)
+		return FALSE;
+	ctx = as_gettext_ctx_new ();
+	ctx->min_percentage = min_percentage;
+	ctx->base_filename = g_strdup (base_filename);
+	while ((filename = g_dir_read_name (dir)) != NULL) {
+		g_autofree gchar *fn = NULL;
+		fn = g_build_filename (path, filename, "LC_MESSAGES", NULL);
+		if (g_file_test (fn, G_FILE_TEST_EXISTS)) {
+			if (!as_gettext_ctx_search_locale (ctx, filename, fn, error))
+				return FALSE;
+		}
+	}
+
+	/* calculate percentages */
+	for (l = ctx->data; l != NULL; l = l->next) {
+		e = l->data;
+		e->percentage = MIN (e->nstrings * 100 / ctx->max_nstrings, 100);
+	}
+
+	/* sort */
+	ctx->data = g_list_sort (ctx->data, as_gettext_entry_sort_cb);
+
+	/* add results */
+	for (l = ctx->data; l != NULL; l = l->next) {
+		e = l->data;
+		if (e->percentage < ctx->min_percentage)
+			continue;
+		as_app_add_language (app, e->percentage, e->locale);
+	}
+	return TRUE;
+}
diff --git a/libappstream-glib/as-app-gettext.h b/libappstream-glib/as-app-gettext.h
new file mode 100644
index 0000000..1645696
--- /dev/null
+++ b/libappstream-glib/as-app-gettext.h
@@ -0,0 +1,45 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ *
+ * Copyright (C) 2015 Richard Hughes <richard@hughsie.com>
+ *
+ * Licensed under the GNU Lesser General Public License Version 2.1
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+#if !defined (__APPSTREAM_GLIB_H) && !defined (AS_COMPILATION)
+#error "Only <appstream-glib.h> can be included directly."
+#endif
+
+#ifndef __AS_APP_GETTEXT_H
+#define __AS_APP_GETTEXT_H
+
+#include <glib.h>
+#include <gio/gio.h>
+
+#include "as-app.h"
+
+G_BEGIN_DECLS
+
+gboolean	 as_app_gettext_search_path	(AsApp		*app,
+						 const gchar	*path,
+						 const gchar	*base_filename,
+						 guint		 min_percentage,
+						 GCancellable	*cancellable,
+						 GError		**error);
+
+G_END_DECLS
+
+#endif /* __AS_APP_GETTEXT_H */
diff --git a/libappstream-glib/as-self-test.c b/libappstream-glib/as-self-test.c
index 4279878..53d3b29 100644
--- a/libappstream-glib/as-self-test.c
+++ b/libappstream-glib/as-self-test.c
@@ -27,6 +27,7 @@
 #include <fnmatch.h>
 
 #include "as-app-private.h"
+#include "as-app-gettext.h"
 #include "as-bundle-private.h"
 #include "as-checksum-private.h"
 #include "as-enums.h"
@@ -325,6 +326,37 @@ as_test_monitor_file_func (void)
 }
 
 static void
+as_test_app_gettext_func (void)
+{
+	GError *error = NULL;
+	gboolean ret;
+	g_autofree gchar *fn = NULL;
+	g_autoptr(AsApp) app = NULL;
+	g_autoptr(GList) list = NULL;
+
+	app = as_app_new ();
+	fn = as_test_get_filename ("locale");
+	g_assert (fn != NULL);
+	ret = as_app_gettext_search_path (app,
+					  fn,
+					  "app.mo",
+					  25,
+					  NULL,
+					  &error);
+	g_assert_no_error (error);
+	g_assert (ret);
+
+	/* check langs */
+	g_assert_cmpint (as_app_get_language (app, "en_GB"), ==, 100);
+	g_assert_cmpint (as_app_get_language (app, "ru"), ==, 33);
+	g_assert_cmpint (as_app_get_language (app, "fr_FR"), ==, -1);
+
+	/* check size */
+	list = as_app_get_languages (app);
+	g_assert_cmpint (g_list_length (list), ==, 2);
+}
+
+static void
 as_test_tag_func (void)
 {
 	guint i;
@@ -4412,6 +4444,7 @@ main (int argc, char **argv)
 	g_test_add_func ("/AppStream/image{alpha}", as_test_image_alpha_func);
 	g_test_add_func ("/AppStream/screenshot", as_test_screenshot_func);
 	g_test_add_func ("/AppStream/app", as_test_app_func);
+	g_test_add_func ("/AppStream/app{gettext}", as_test_app_gettext_func);
 	g_test_add_func ("/AppStream/app{translated}", as_test_app_translated_func);
 	g_test_add_func ("/AppStream/app{validate-style}", as_test_app_validate_style_func);
 	g_test_add_func ("/AppStream/app{validate-appdata-good}", as_test_app_validate_appdata_good_func);