From 75db3a8f67404bd670433115e1d5a44536bd508e Mon Sep 17 00:00:00 2001 From: Richard Hughes Date: Fri, 18 Nov 2016 14:54:06 +0000 Subject: Add support for ONLY_NATIVE_LANGS when parsing yaml files This saves ~20Mb of RSS on a typical desktop Debian install. --- libappstream-glib/as-app.c | 8 ++++ libappstream-glib/as-self-test.c | 18 ++++++--- libappstream-glib/as-store.c | 5 ++- libappstream-glib/as-yaml.c | 86 +++++++++++++++++++++++++++++++--------- libappstream-glib/as-yaml.h | 17 ++++++++ 5 files changed, 110 insertions(+), 24 deletions(-) diff --git a/libappstream-glib/as-app.c b/libappstream-glib/as-app.c index 3aed890..6103127 100644 --- a/libappstream-glib/as-app.c +++ b/libappstream-glib/as-app.c @@ -4944,6 +4944,8 @@ as_app_node_parse_dep11 (AsApp *app, GNode *node, } if (g_strcmp0 (tmp, "Name") == 0) { for (c = n->children; c != NULL; c = c->next) { + if (as_yaml_node_get_key (c) == NULL) + continue; as_app_set_name (app, as_yaml_node_get_key (c), as_yaml_node_get_value (c)); @@ -4952,6 +4954,8 @@ as_app_node_parse_dep11 (AsApp *app, GNode *node, } if (g_strcmp0 (tmp, "Summary") == 0) { for (c = n->children; c != NULL; c = c->next) { + if (as_yaml_node_get_key (c) == NULL) + continue; as_app_set_comment (app, as_yaml_node_get_key (c), as_yaml_node_get_value (c)); @@ -4960,6 +4964,8 @@ as_app_node_parse_dep11 (AsApp *app, GNode *node, } if (g_strcmp0 (tmp, "Description") == 0) { for (c = n->children; c != NULL; c = c->next) { + if (as_yaml_node_get_key (c) == NULL) + continue; as_app_set_description (app, as_yaml_node_get_key (c), as_yaml_node_get_value (c)); @@ -4971,6 +4977,8 @@ as_app_node_parse_dep11 (AsApp *app, GNode *node, for (c2 = c->children; c2 != NULL; c2 = c2->next) { if (as_yaml_node_get_key (c2) == NULL) continue; + if (as_yaml_node_get_key (c) == NULL) + continue; as_app_add_keyword (app, as_yaml_node_get_key (c), as_yaml_node_get_key (c2)); diff --git a/libappstream-glib/as-self-test.c b/libappstream-glib/as-self-test.c index 49be359..3aefcbf 100644 --- a/libappstream-glib/as-self-test.c +++ b/libappstream-glib/as-self-test.c @@ -4389,12 +4389,16 @@ as_test_yaml_broken_func (void) g_autoptr(GError) error2 = NULL; node = as_yaml_from_data ("s---\n" "File: DEP-11\n", - -1, &error1); + -1, + AS_YAML_FROM_FLAG_NONE, + &error1); g_assert_error (error1, AS_NODE_ERROR, AS_NODE_ERROR_INVALID_MARKUP); g_assert (node == NULL); node = as_yaml_from_data ("---\n" "%File: DEP-11\n", - -1, &error2); + -1, + AS_YAML_FROM_FLAG_NONE, + &error2); g_assert_error (error2, AS_NODE_ERROR, AS_NODE_ERROR_INVALID_MARKUP); g_assert_cmpstr (error2->message, ==, "scanner error: while scanning a directive at ln:2 col:1, " @@ -4422,7 +4426,9 @@ as_test_yaml_func (void) "File: DEP-11\n" "Origin: aequorea\n" "Version: '0.6'\n", - -1, &error); + -1, + AS_YAML_FROM_FLAG_NONE, + &error); g_assert_no_error (error); g_assert (node != NULL); str = as_yaml_to_string (node); @@ -4448,7 +4454,9 @@ as_test_yaml_func (void) " - AppMenu\n" " - SearchProvider\n" " - Notifications\n", - -1, &error); + -1, + AS_YAML_FROM_FLAG_NONE, + &error); g_assert_no_error (error); g_assert (node != NULL); str = as_yaml_to_string (node); @@ -4472,7 +4480,7 @@ as_test_yaml_func (void) filename = as_test_get_filename ("usr/share/app-info/yaml/aequorea.yml"); g_assert (filename != NULL); file = g_file_new_for_path (filename); - node = as_yaml_from_file (file, NULL, &error); + node = as_yaml_from_file (file, AS_YAML_FROM_FLAG_NONE, NULL, &error); g_assert_no_error (error); g_assert (node != NULL); str = as_yaml_to_string (node); diff --git a/libappstream-glib/as-store.c b/libappstream-glib/as-store.c index 1a4a594..a751893 100644 --- a/libappstream-glib/as-store.c +++ b/libappstream-glib/as-store.c @@ -1529,6 +1529,7 @@ as_store_load_yaml_file (AsStore *store, AsStorePrivate *priv = GET_PRIVATE (store); AsNode *app_n; AsNode *n; + AsYamlFromFlags flags = AS_YAML_FROM_FLAG_NONE; const gchar *tmp; g_autoptr(AsNodeContext) ctx = NULL; g_autofree gchar *icon_path = NULL; @@ -1536,7 +1537,9 @@ as_store_load_yaml_file (AsStore *store, _cleanup_uninhibit_ guint32 *tok = NULL; /* load file */ - root = as_yaml_from_file (file, cancellable, error); + if (priv->add_flags & AS_STORE_ADD_FLAG_ONLY_NATIVE_LANGS) + flags |= AS_YAML_FROM_FLAG_ONLY_NATIVE_LANGS; + root = as_yaml_from_file (file, flags, cancellable, error); if (root == NULL) return FALSE; diff --git a/libappstream-glib/as-yaml.c b/libappstream-glib/as-yaml.c index 76cab57..579695a 100644 --- a/libappstream-glib/as-yaml.c +++ b/libappstream-glib/as-yaml.c @@ -26,6 +26,7 @@ #endif #include "as-node.h" +#include "as-ref-string.h" #include "as-yaml.h" typedef enum { @@ -38,8 +39,8 @@ typedef enum { } AsYamlNodeKind; typedef struct { - gchar *key; - gchar *value; + AsRefString *key; + AsRefString *value; AsYamlNodeKind kind; } AsYamlNode; @@ -113,8 +114,10 @@ as_node_yaml_destroy_node_cb (AsNode *node, gpointer data) AsYamlNode *ym = node->data; if (ym == NULL) return FALSE; - g_free (ym->key); - g_free (ym->value); + if (ym->key != NULL) + as_ref_string_unref (ym->key); + if (ym->value != NULL) + as_ref_string_unref (ym->value); g_slice_free (AsYamlNode, ym); return FALSE; } @@ -197,7 +200,8 @@ as_yaml_node_new (AsYamlNodeKind kind, const gchar *id) AsYamlNode *ym; ym = g_slice_new0 (AsYamlNode); ym->kind = kind; - ym->key = g_strdup (id); + if (id != NULL) + ym->key = as_ref_string_ref (id); return ym; } @@ -315,8 +319,35 @@ as_yaml_parser_error_to_gerror (yaml_parser_t *parser, GError **error) return FALSE; } +typedef struct { + AsYamlFromFlags flags; + const gchar * const *locales; + yaml_parser_t *parser; +} AsYamlContext; + +static gboolean +as_yaml_node_valid (AsYamlContext *ctx, AsNode *parent, const gchar *key) +{ + const gchar *sections[] = { "Name", "Summary", "Description", NULL }; + AsYamlNode *ym = parent->data; + + /* if native filtering enabled */ + if ((ctx->flags & AS_YAML_FROM_FLAG_ONLY_NATIVE_LANGS) == 0) + return TRUE; + + /* filter by translatable sections */ + if (!g_strv_contains (sections, ym->key)) + return TRUE; + + /* non-native language */ + if (g_strv_contains (ctx->locales, key)) + return TRUE; + + return FALSE; +} + static gboolean -as_node_yaml_process_layer (yaml_parser_t *parser, AsNode *parent, GError **error) +as_node_yaml_process_layer (AsYamlContext *ctx, AsNode *parent, GError **error) { AsYamlNode *ym; AsNode *last_scalar = NULL; @@ -324,12 +355,14 @@ as_node_yaml_process_layer (yaml_parser_t *parser, AsNode *parent, GError **erro const gchar *tmp; gboolean valid = TRUE; yaml_event_t event; + AsRefString *map_str = as_ref_string_new_static ("{"); + AsRefString *seq_str = as_ref_string_new_static ("["); while (valid) { /* process event */ - if (!yaml_parser_parse (parser, &event)) - return as_yaml_parser_error_to_gerror (parser, error); + if (!yaml_parser_parse (ctx->parser, &event)) + return as_yaml_parser_error_to_gerror (ctx->parser, error); switch (event.type) { case YAML_SCALAR_EVENT: @@ -337,37 +370,43 @@ as_node_yaml_process_layer (yaml_parser_t *parser, AsNode *parent, GError **erro if (as_yaml_node_get_kind (parent) != AS_YAML_NODE_KIND_SEQ && last_scalar != NULL) { ym = last_scalar->data; - ym->value = g_strdup (tmp); + if (ym->key != NULL) + ym->value = as_ref_string_new_copy (tmp); ym->kind = AS_YAML_NODE_KIND_KEY_VALUE; last_scalar = NULL; } else { - ym = as_yaml_node_new (AS_YAML_NODE_KIND_KEY, tmp); + if (as_yaml_node_valid (ctx, parent, tmp)) { + g_autoptr(AsRefString) rstr = as_ref_string_new_copy (tmp); + ym = as_yaml_node_new (AS_YAML_NODE_KIND_KEY, rstr); + } else { + ym = as_yaml_node_new (AS_YAML_NODE_KIND_KEY, NULL); + } last_scalar = g_node_append_data (parent, ym); } break; case YAML_MAPPING_START_EVENT: if (last_scalar == NULL) { - ym = as_yaml_node_new (AS_YAML_NODE_KIND_MAP, "{"); + ym = as_yaml_node_new (AS_YAML_NODE_KIND_MAP, map_str); new = g_node_append_data (parent, ym); } else { ym = last_scalar->data; ym->kind = AS_YAML_NODE_KIND_MAP; new = last_scalar; } - if (!as_node_yaml_process_layer (parser, new, error)) + if (!as_node_yaml_process_layer (ctx, new, error)) return FALSE; last_scalar = NULL; break; case YAML_SEQUENCE_START_EVENT: if (last_scalar == NULL) { - ym = as_yaml_node_new (AS_YAML_NODE_KIND_SEQ, "["); + ym = as_yaml_node_new (AS_YAML_NODE_KIND_SEQ, seq_str); new = g_node_append_data (parent, ym); } else { ym = last_scalar->data; ym->kind = AS_YAML_NODE_KIND_SEQ; new = last_scalar; } - if (!as_node_yaml_process_layer (parser, new, error)) + if (!as_node_yaml_process_layer (ctx, new, error)) return FALSE; last_scalar = NULL; break; @@ -392,10 +431,14 @@ G_DEFINE_AUTO_CLEANUP_FREE_FUNC(AsYamlParser, yaml_parser_delete, NULL); #endif AsNode * -as_yaml_from_data (const gchar *data, gssize data_len, GError **error) +as_yaml_from_data (const gchar *data, + gssize data_len, + AsYamlFromFlags flags, + GError **error) { g_autoptr(AsYaml) node = NULL; #ifdef AS_BUILD_DEP11 + AsYamlContext ctx; yaml_parser_t parser; g_auto(AsYamlParser) parser_cleanup = NULL; @@ -409,7 +452,10 @@ as_yaml_from_data (const gchar *data, gssize data_len, GError **error) data_len = (gssize) strlen (data); yaml_parser_set_input_string (&parser, (guchar *) data, (gsize) data_len); node = g_node_new (NULL); - if (!as_node_yaml_process_layer (&parser, node, error)) + ctx.parser = &parser; + ctx.flags = flags; + ctx.locales = g_get_language_names (); + if (!as_node_yaml_process_layer (&ctx, node, error)) return NULL; #else g_set_error_literal (error, @@ -441,7 +487,7 @@ as_yaml_read_handler_cb (void *data, #endif AsNode * -as_yaml_from_file (GFile *file, GCancellable *cancellable, GError **error) +as_yaml_from_file (GFile *file, AsYamlFromFlags flags, GCancellable *cancellable, GError **error) { g_autoptr(AsYaml) node = NULL; #ifdef AS_BUILD_DEP11 @@ -453,6 +499,7 @@ as_yaml_from_file (GFile *file, GCancellable *cancellable, GError **error) g_autoptr(GFileInfo) info = NULL; g_autoptr(GInputStream) file_stream = NULL; g_autoptr(GInputStream) stream_data = NULL; + AsYamlContext ctx; /* what kind of file is this */ info = g_file_query_info (file, @@ -491,7 +538,10 @@ as_yaml_from_file (GFile *file, GCancellable *cancellable, GError **error) parser_cleanup = &parser; yaml_parser_set_input (&parser, as_yaml_read_handler_cb, stream_data); node = g_node_new (NULL); - if (!as_node_yaml_process_layer (&parser, node, error)) + ctx.parser = &parser; + ctx.flags = flags; + ctx.locales = g_get_language_names (); + if (!as_node_yaml_process_layer (&ctx, node, error)) return NULL; #else g_set_error_literal (error, diff --git a/libappstream-glib/as-yaml.h b/libappstream-glib/as-yaml.h index 99c7975..cf9cc3c 100644 --- a/libappstream-glib/as-yaml.h +++ b/libappstream-glib/as-yaml.h @@ -31,14 +31,31 @@ G_BEGIN_DECLS + +/** + * AsYamlFromFlags: + * @AS_YAML_FROM_FLAG_NONE: No extra flags to use + * @AS_YAML_FROM_FLAG_ONLY_NATIVE_LANGS: Only load native languages + * + * The flags for converting from XML. + **/ +typedef enum { + AS_YAML_FROM_FLAG_NONE = 0, /* Since: 0.1.0 */ + AS_YAML_FROM_FLAG_ONLY_NATIVE_LANGS = 1 << 0, /* Since: 0.6.6 */ + /*< private >*/ + AS_YAML_FROM_FLAG_LAST +} AsYamlFromFlags; + typedef GNode AsYaml; void as_yaml_unref (AsYaml *node); GString *as_yaml_to_string (AsYaml *node); AsYaml *as_yaml_from_data (const gchar *data, gssize data_len, + AsYamlFromFlags flags, GError **error); AsYaml *as_yaml_from_file (GFile *file, + AsYamlFromFlags flags, GCancellable *cancellable, GError **error); const gchar *as_yaml_node_get_key (const AsYaml *node); -- cgit v1.2.1