From b24b77b028093bef5ebaf1fbdfb6b545f823486d Mon Sep 17 00:00:00 2001 From: Richard Hughes Date: Thu, 21 Jan 2016 11:32:02 +0000 Subject: Move the gettext parsing to libappstream-glib --- libappstream-glib/Makefile.am | 2 + libappstream-glib/appstream-glib.h | 1 + libappstream-glib/as-app-gettext.c | 281 +++++++++++++++++++++++++++++++++++++ libappstream-glib/as-app-gettext.h | 45 ++++++ libappstream-glib/as-self-test.c | 33 +++++ 5 files changed, 362 insertions(+) create mode 100644 libappstream-glib/as-app-gettext.c create mode 100644 libappstream-glib/as-app-gettext.h diff --git a/libappstream-glib/Makefile.am b/libappstream-glib/Makefile.am index feb7d21..f017040 100644 --- a/libappstream-glib/Makefile.am +++ b/libappstream-glib/Makefile.am @@ -69,6 +69,7 @@ libappstream_glib_includedir = $(includedir)/libappstream-glib libappstream_glib_include_HEADERS = \ appstream-glib.h \ as-app.h \ + as-app-gettext.h \ as-bundle.h \ as-checksum.h \ as-enums.h \ @@ -89,6 +90,7 @@ libappstream_glib_include_HEADERS = \ libappstream_glib_la_SOURCES = \ as-app.c \ as-app-desktop.c \ + as-app-gettext.c \ as-app-inf.c \ as-app-private.h \ as-app-validate.c \ diff --git a/libappstream-glib/appstream-glib.h b/libappstream-glib/appstream-glib.h index 4c5945c..ad5fab8 100644 --- a/libappstream-glib/appstream-glib.h +++ b/libappstream-glib/appstream-glib.h @@ -25,6 +25,7 @@ #define __APPSTREAM_GLIB_H_INSIDE__ #include +#include #include #include #include diff --git a/libappstream-glib/as-app-gettext.c b/libappstream-glib/as-app-gettext.c new file mode 100644 index 0000000..402894f --- /dev/null +++ b/libappstream-glib/as-app-gettext.c @@ -0,0 +1,281 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- + * + * Copyright (C) 2015 Richard Hughes + * + * Licensed under the GNU Lesser General Public License Version 2.1 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * SECTION:as-gettext + * @short_description: a hashed array gettext of applications + * @include: appstream-glib.h + * @stability: Stable + * + * This object will parse a gettext catalog directory and calculate the + * language stats for an application. + * + * See also: #AsApp + */ + +#include "config.h" + +#include + +#include "as-app-gettext.h" + +typedef struct { +} AsGettextPrivate; + +typedef struct { + guint32 magic; + guint32 revision; + guint32 nstrings; + guint32 orig_tab_offset; + guint32 trans_tab_offset; + guint32 hash_tab_size; + guint32 hash_tab_offset; + guint32 n_sysdep_segments; + guint32 sysdep_segments_offset; + guint32 n_sysdep_strings; + guint32 orig_sysdep_tab_offset; + guint32 trans_sysdep_tab_offset; +} AsGettextHeader; + +typedef struct { + gchar *locale; + guint nstrings; + guint percentage; +} AsGettextEntry; + +typedef struct { + guint max_nstrings; + GList *data; + gchar *base_filename; + guint min_percentage; +} AsGettextContext; + +/** + * as_gettext_entry_new: + **/ +static AsGettextEntry * +as_gettext_entry_new (void) +{ + AsGettextEntry *entry; + entry = g_slice_new0 (AsGettextEntry); + return entry; +} + +/** + * as_gettext_entry_free: + **/ +static void +as_gettext_entry_free (AsGettextEntry *entry) +{ + g_free (entry->locale); + g_slice_free (AsGettextEntry, entry); +} + +/** + * as_gettext_ctx_new: + **/ +static AsGettextContext * +as_gettext_ctx_new (void) +{ + AsGettextContext *ctx; + ctx = g_new0 (AsGettextContext, 1); + return ctx; +} + +/** + * as_gettext_ctx_free: + **/ +static void +as_gettext_ctx_free (AsGettextContext *ctx) +{ + g_list_free_full (ctx->data, (GDestroyNotify) as_gettext_entry_free); + g_free (ctx->base_filename); + g_free (ctx); +} + +/** + * as_gettext_parse_file: + **/ +static gboolean +as_gettext_parse_file (AsGettextContext *ctx, + const gchar *locale, + const gchar *filename, + GError **error) +{ + AsGettextEntry *entry; + AsGettextHeader *h; + g_autofree gchar *data = NULL; + gboolean swapped; + + /* read data, although we only strictly need the header */ + if (!g_file_get_contents (filename, &data, NULL, error)) + return FALSE; + + h = (AsGettextHeader *) data; + if (h->magic == 0x950412de) + swapped = FALSE; + else if (h->magic == 0xde120495) + swapped = TRUE; + else + return FALSE; + entry = as_gettext_entry_new (); + entry->locale = g_strdup (locale); + if (swapped) + entry->nstrings = GUINT32_SWAP_LE_BE (h->nstrings); + else + entry->nstrings = h->nstrings; + if (entry->nstrings > ctx->max_nstrings) + ctx->max_nstrings = entry->nstrings; + ctx->data = g_list_prepend (ctx->data, entry); + return TRUE; +} + +/** + * as_gettext_ctx_search_locale: + **/ +static gboolean +as_gettext_ctx_search_locale (AsGettextContext *ctx, + const gchar *locale, + const gchar *messages_path, + GError **error) +{ + const gchar *filename; + guint i; + g_autoptr(GDir) dir = NULL; + g_autoptr(GPtrArray) mo_paths = NULL; + + /* list files */ + dir = g_dir_open (messages_path, 0, error); + if (dir == NULL) + return FALSE; + + /* do a first pass at this, trying to find the prefered .mo */ + mo_paths = g_ptr_array_new_with_free_func (g_free); + while ((filename = g_dir_read_name (dir)) != NULL) { + g_autofree gchar *path = NULL; + path = g_build_filename (messages_path, filename, NULL); + if (!g_file_test (path, G_FILE_TEST_EXISTS)) + continue; + if (g_strcmp0 (filename, ctx->base_filename) == 0) { + if (!as_gettext_parse_file (ctx, locale, path, error)) + return FALSE; + return TRUE; + } + g_ptr_array_add (mo_paths, g_strdup (path)); + } + + /* fall back to parsing *everything*, which might give us more + * language results than is actually true */ + for (i = 0; i < mo_paths->len; i++) { + filename = g_ptr_array_index (mo_paths, i); + if (!as_gettext_parse_file (ctx, locale, filename, error)) + return FALSE; + } + + return TRUE; +} + +/** + * as_gettext_entry_sort_cb: + **/ +static gint +as_gettext_entry_sort_cb (gconstpointer a, gconstpointer b) +{ + return g_strcmp0 (((AsGettextEntry *) a)->locale, + ((AsGettextEntry *) b)->locale); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(AsGettextContext, as_gettext_ctx_free) + +/** + * as_app_gettext_search_path: + * @app: an #AsApp + * @path: a path to search, e.g. "/usr/share/locale" + * @base_filename: a filename, e.g. "gnome-software.mo" + * @min_percentage: minimum percentage to add language + * @cancellable: a #GCancellable or %NULL + * @error: a #GError or %NULL + * + * Searches a gettext catalog path for languages, and using a heuristic + * adds tags to the specified application. + * + * If @base_filename is not set then any filename is matched, which may + * include more languages than you intended to. + * + * @min_percentage sets the minimum percentage to add a language tag. + * The usual value would be 25% and any language less complete than + * this will not be added. + * + * The purpose of this functionality is to avoid blowing up the size + * of the AppStream metadata with a lot of extra data detailing + * languages with very few translated strings. + * + * Returns: %TRUE for success + * + * Since: 0.5.6 + **/ +gboolean +as_app_gettext_search_path (AsApp *app, + const gchar *path, + const gchar *base_filename, + guint min_percentage, + GCancellable *cancellable, + GError **error) +{ + AsGettextEntry *e; + GList *l; + const gchar *filename; + g_autoptr(AsGettextContext) ctx = NULL; + g_autoptr(GDir) dir = NULL; + + dir = g_dir_open (path, 0, error); + if (dir == NULL) + return FALSE; + ctx = as_gettext_ctx_new (); + ctx->min_percentage = min_percentage; + ctx->base_filename = g_strdup (base_filename); + while ((filename = g_dir_read_name (dir)) != NULL) { + g_autofree gchar *fn = NULL; + fn = g_build_filename (path, filename, "LC_MESSAGES", NULL); + if (g_file_test (fn, G_FILE_TEST_EXISTS)) { + if (!as_gettext_ctx_search_locale (ctx, filename, fn, error)) + return FALSE; + } + } + + /* calculate percentages */ + for (l = ctx->data; l != NULL; l = l->next) { + e = l->data; + e->percentage = MIN (e->nstrings * 100 / ctx->max_nstrings, 100); + } + + /* sort */ + ctx->data = g_list_sort (ctx->data, as_gettext_entry_sort_cb); + + /* add results */ + for (l = ctx->data; l != NULL; l = l->next) { + e = l->data; + if (e->percentage < ctx->min_percentage) + continue; + as_app_add_language (app, e->percentage, e->locale); + } + return TRUE; +} diff --git a/libappstream-glib/as-app-gettext.h b/libappstream-glib/as-app-gettext.h new file mode 100644 index 0000000..1645696 --- /dev/null +++ b/libappstream-glib/as-app-gettext.h @@ -0,0 +1,45 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- + * + * Copyright (C) 2015 Richard Hughes + * + * Licensed under the GNU Lesser General Public License Version 2.1 + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#if !defined (__APPSTREAM_GLIB_H) && !defined (AS_COMPILATION) +#error "Only can be included directly." +#endif + +#ifndef __AS_APP_GETTEXT_H +#define __AS_APP_GETTEXT_H + +#include +#include + +#include "as-app.h" + +G_BEGIN_DECLS + +gboolean as_app_gettext_search_path (AsApp *app, + const gchar *path, + const gchar *base_filename, + guint min_percentage, + GCancellable *cancellable, + GError **error); + +G_END_DECLS + +#endif /* __AS_APP_GETTEXT_H */ diff --git a/libappstream-glib/as-self-test.c b/libappstream-glib/as-self-test.c index 4279878..53d3b29 100644 --- a/libappstream-glib/as-self-test.c +++ b/libappstream-glib/as-self-test.c @@ -27,6 +27,7 @@ #include #include "as-app-private.h" +#include "as-app-gettext.h" #include "as-bundle-private.h" #include "as-checksum-private.h" #include "as-enums.h" @@ -324,6 +325,37 @@ as_test_monitor_file_func (void) g_assert_cmpint (cnt_changed, ==, 1); } +static void +as_test_app_gettext_func (void) +{ + GError *error = NULL; + gboolean ret; + g_autofree gchar *fn = NULL; + g_autoptr(AsApp) app = NULL; + g_autoptr(GList) list = NULL; + + app = as_app_new (); + fn = as_test_get_filename ("locale"); + g_assert (fn != NULL); + ret = as_app_gettext_search_path (app, + fn, + "app.mo", + 25, + NULL, + &error); + g_assert_no_error (error); + g_assert (ret); + + /* check langs */ + g_assert_cmpint (as_app_get_language (app, "en_GB"), ==, 100); + g_assert_cmpint (as_app_get_language (app, "ru"), ==, 33); + g_assert_cmpint (as_app_get_language (app, "fr_FR"), ==, -1); + + /* check size */ + list = as_app_get_languages (app); + g_assert_cmpint (g_list_length (list), ==, 2); +} + static void as_test_tag_func (void) { @@ -4412,6 +4444,7 @@ main (int argc, char **argv) g_test_add_func ("/AppStream/image{alpha}", as_test_image_alpha_func); g_test_add_func ("/AppStream/screenshot", as_test_screenshot_func); g_test_add_func ("/AppStream/app", as_test_app_func); + g_test_add_func ("/AppStream/app{gettext}", as_test_app_gettext_func); g_test_add_func ("/AppStream/app{translated}", as_test_app_translated_func); g_test_add_func ("/AppStream/app{validate-style}", as_test_app_validate_style_func); g_test_add_func ("/AppStream/app{validate-appdata-good}", as_test_app_validate_appdata_good_func); -- cgit v1.2.1