Add support for ONLY_NATIVE_LANGS when parsing yaml fileswip/hughsie/AsRefString

This saves ~20Mb of RSS on a typical desktop Debian install.

5 files changed, 110 insertions, 24 deletions

diff --git a/libappstream-glib/as-app.c b/libappstream-glib/as-app.c
index 3aed890..6103127 100644
--- a/libappstream-glib/as-app.c
+++ b/libappstream-glib/as-app.c
@@ -4944,6 +4944,8 @@ as_app_node_parse_dep11 (AsApp *app, GNode *node,
 		}
 		if (g_strcmp0 (tmp, "Name") == 0) {
 			for (c = n->children; c != NULL; c = c->next) {
+				if (as_yaml_node_get_key (c) == NULL)
+					continue;
 				as_app_set_name (app,
 						 as_yaml_node_get_key (c),
 						 as_yaml_node_get_value (c));
@@ -4952,6 +4954,8 @@ as_app_node_parse_dep11 (AsApp *app, GNode *node,
 		}
 		if (g_strcmp0 (tmp, "Summary") == 0) {
 			for (c = n->children; c != NULL; c = c->next) {
+				if (as_yaml_node_get_key (c) == NULL)
+					continue;
 				as_app_set_comment (app,
 						    as_yaml_node_get_key (c),
 						    as_yaml_node_get_value (c));
@@ -4960,6 +4964,8 @@ as_app_node_parse_dep11 (AsApp *app, GNode *node,
 		}
 		if (g_strcmp0 (tmp, "Description") == 0) {
 			for (c = n->children; c != NULL; c = c->next) {
+				if (as_yaml_node_get_key (c) == NULL)
+					continue;
 				as_app_set_description (app,
 							as_yaml_node_get_key (c),
 							as_yaml_node_get_value (c));
@@ -4971,6 +4977,8 @@ as_app_node_parse_dep11 (AsApp *app, GNode *node,
 				for (c2 = c->children; c2 != NULL; c2 = c2->next) {
 					if (as_yaml_node_get_key (c2) == NULL)
 						continue;
+					if (as_yaml_node_get_key (c) == NULL)
+						continue;
 					as_app_add_keyword (app,
 							   as_yaml_node_get_key (c),
 							   as_yaml_node_get_key (c2));
diff --git a/libappstream-glib/as-self-test.c b/libappstream-glib/as-self-test.c
index 49be359..3aefcbf 100644
--- a/libappstream-glib/as-self-test.c
+++ b/libappstream-glib/as-self-test.c
@@ -4389,12 +4389,16 @@ as_test_yaml_broken_func (void)
 	g_autoptr(GError) error2 = NULL;
 	node = as_yaml_from_data ("s---\n"
 				  "File: DEP-11\n",
-				  -1, &error1);
+				  -1,
+				  AS_YAML_FROM_FLAG_NONE,
+				  &error1);
 	g_assert_error (error1, AS_NODE_ERROR, AS_NODE_ERROR_INVALID_MARKUP);
 	g_assert (node == NULL);
 	node = as_yaml_from_data ("---\n"
 				  "%File: DEP-11\n",
-				  -1, &error2);
+				  -1,
+				  AS_YAML_FROM_FLAG_NONE,
+				  &error2);
 	g_assert_error (error2, AS_NODE_ERROR, AS_NODE_ERROR_INVALID_MARKUP);
 	g_assert_cmpstr (error2->message, ==,
 			 "scanner error: while scanning a directive at ln:2 col:1, "
@@ -4422,7 +4426,9 @@ as_test_yaml_func (void)
 		"File: DEP-11\n"
 		"Origin: aequorea\n"
 		"Version: '0.6'\n",
-		-1, &error);
+		-1,
+		AS_YAML_FROM_FLAG_NONE,
+		&error);
 	g_assert_no_error (error);
 	g_assert (node != NULL);
 	str = as_yaml_to_string (node);
@@ -4448,7 +4454,9 @@ as_test_yaml_func (void)
 		"  - AppMenu\n"
 		"  - SearchProvider\n"
 		"  - Notifications\n",
-		-1, &error);
+		-1,
+		AS_YAML_FROM_FLAG_NONE,
+		&error);
 	g_assert_no_error (error);
 	g_assert (node != NULL);
 	str = as_yaml_to_string (node);
@@ -4472,7 +4480,7 @@ as_test_yaml_func (void)
 	filename = as_test_get_filename ("usr/share/app-info/yaml/aequorea.yml");
 	g_assert (filename != NULL);
 	file = g_file_new_for_path (filename);
-	node = as_yaml_from_file (file, NULL, &error);
+	node = as_yaml_from_file (file, AS_YAML_FROM_FLAG_NONE, NULL, &error);
 	g_assert_no_error (error);
 	g_assert (node != NULL);
 	str = as_yaml_to_string (node);
diff --git a/libappstream-glib/as-store.c b/libappstream-glib/as-store.c
index 1a4a594..a751893 100644
--- a/libappstream-glib/as-store.c
+++ b/libappstream-glib/as-store.c
@@ -1529,6 +1529,7 @@ as_store_load_yaml_file (AsStore *store,
 	AsStorePrivate *priv = GET_PRIVATE (store);
 	AsNode *app_n;
 	AsNode *n;
+	AsYamlFromFlags flags = AS_YAML_FROM_FLAG_NONE;
 	const gchar *tmp;
 	g_autoptr(AsNodeContext) ctx = NULL;
 	g_autofree gchar *icon_path = NULL;
@@ -1536,7 +1537,9 @@ as_store_load_yaml_file (AsStore *store,
 	_cleanup_uninhibit_ guint32 *tok = NULL;
 
 	/* load file */
-	root = as_yaml_from_file (file, cancellable, error);
+	if (priv->add_flags & AS_STORE_ADD_FLAG_ONLY_NATIVE_LANGS)
+		flags |= AS_YAML_FROM_FLAG_ONLY_NATIVE_LANGS;
+	root = as_yaml_from_file (file, flags, cancellable, error);
 	if (root == NULL)
 		return FALSE;
 
diff --git a/libappstream-glib/as-yaml.c b/libappstream-glib/as-yaml.c
index 76cab57..579695a 100644
--- a/libappstream-glib/as-yaml.c
+++ b/libappstream-glib/as-yaml.c
@@ -26,6 +26,7 @@
 #endif
 
 #include "as-node.h"
+#include "as-ref-string.h"
 #include "as-yaml.h"
 
 typedef enum {
@@ -38,8 +39,8 @@ typedef enum {
 } AsYamlNodeKind;
 
 typedef struct {
-	gchar		*key;
-	gchar		*value;
+	AsRefString	*key;
+	AsRefString	*value;
 	AsYamlNodeKind	 kind;
 } AsYamlNode;
 
@@ -113,8 +114,10 @@ as_node_yaml_destroy_node_cb (AsNode *node, gpointer data)
 	AsYamlNode *ym = node->data;
 	if (ym == NULL)
 		return FALSE;
-	g_free (ym->key);
-	g_free (ym->value);
+	if (ym->key != NULL)
+		as_ref_string_unref (ym->key);
+	if (ym->value != NULL)
+		as_ref_string_unref (ym->value);
 	g_slice_free (AsYamlNode, ym);
 	return FALSE;
 }
@@ -197,7 +200,8 @@ as_yaml_node_new (AsYamlNodeKind kind, const gchar *id)
 	AsYamlNode *ym;
 	ym = g_slice_new0 (AsYamlNode);
 	ym->kind = kind;
-	ym->key = g_strdup (id);
+	if (id != NULL)
+		ym->key = as_ref_string_ref (id);
 	return ym;
 }
 
@@ -315,8 +319,35 @@ as_yaml_parser_error_to_gerror (yaml_parser_t *parser, GError **error)
 	return FALSE;
 }
 
+typedef struct {
+	AsYamlFromFlags		 flags;
+	const gchar * const	*locales;
+	yaml_parser_t		*parser;
+} AsYamlContext;
+
+static gboolean
+as_yaml_node_valid (AsYamlContext *ctx, AsNode *parent, const gchar *key)
+{
+	const gchar *sections[] = { "Name", "Summary", "Description", NULL };
+	AsYamlNode *ym = parent->data;
+
+	/* if native filtering enabled */
+	if ((ctx->flags & AS_YAML_FROM_FLAG_ONLY_NATIVE_LANGS) == 0)
+		return TRUE;
+
+	/* filter by translatable sections */
+	if (!g_strv_contains (sections, ym->key))
+		return TRUE;
+
+	/* non-native language */
+	if (g_strv_contains (ctx->locales, key))
+		return TRUE;
+
+	return FALSE;
+}
+
 static gboolean
-as_node_yaml_process_layer (yaml_parser_t *parser, AsNode *parent, GError **error)
+as_node_yaml_process_layer (AsYamlContext *ctx, AsNode *parent, GError **error)
 {
 	AsYamlNode *ym;
 	AsNode *last_scalar = NULL;
@@ -324,12 +355,14 @@ as_node_yaml_process_layer (yaml_parser_t *parser, AsNode *parent, GError **erro
 	const gchar *tmp;
 	gboolean valid = TRUE;
 	yaml_event_t event;
+	AsRefString *map_str = as_ref_string_new_static ("{");
+	AsRefString *seq_str = as_ref_string_new_static ("[");
 
 	while (valid) {
 
 		/* process event */
-		if (!yaml_parser_parse (parser, &event))
-			return as_yaml_parser_error_to_gerror (parser, error);
+		if (!yaml_parser_parse (ctx->parser, &event))
+			return as_yaml_parser_error_to_gerror (ctx->parser, error);
 
 		switch (event.type) {
 		case YAML_SCALAR_EVENT:
@@ -337,37 +370,43 @@ as_node_yaml_process_layer (yaml_parser_t *parser, AsNode *parent, GError **erro
 			if (as_yaml_node_get_kind (parent) != AS_YAML_NODE_KIND_SEQ &&
 			    last_scalar != NULL) {
 				ym = last_scalar->data;
-				ym->value = g_strdup (tmp);
+				if (ym->key != NULL)
+					ym->value = as_ref_string_new_copy (tmp);
 				ym->kind = AS_YAML_NODE_KIND_KEY_VALUE;
 				last_scalar = NULL;
 			} else {
-				ym = as_yaml_node_new (AS_YAML_NODE_KIND_KEY, tmp);
+				if (as_yaml_node_valid (ctx, parent, tmp)) {
+					g_autoptr(AsRefString) rstr = as_ref_string_new_copy (tmp);
+					ym = as_yaml_node_new (AS_YAML_NODE_KIND_KEY, rstr);
+				} else {
+					ym = as_yaml_node_new (AS_YAML_NODE_KIND_KEY, NULL);
+				}
 				last_scalar = g_node_append_data (parent, ym);
 			}
 			break;
 		case YAML_MAPPING_START_EVENT:
 			if (last_scalar == NULL) {
-				ym = as_yaml_node_new (AS_YAML_NODE_KIND_MAP, "{");
+				ym = as_yaml_node_new (AS_YAML_NODE_KIND_MAP, map_str);
 				new = g_node_append_data (parent, ym);
 			} else {
 				ym = last_scalar->data;
 				ym->kind = AS_YAML_NODE_KIND_MAP;
 				new = last_scalar;
 			}
-			if (!as_node_yaml_process_layer (parser, new, error))
+			if (!as_node_yaml_process_layer (ctx, new, error))
 				return FALSE;
 			last_scalar = NULL;
 			break;
 		case YAML_SEQUENCE_START_EVENT:
 			if (last_scalar == NULL) {
-				ym = as_yaml_node_new (AS_YAML_NODE_KIND_SEQ, "[");
+				ym = as_yaml_node_new (AS_YAML_NODE_KIND_SEQ, seq_str);
 				new = g_node_append_data (parent, ym);
 			} else {
 				ym = last_scalar->data;
 				ym->kind = AS_YAML_NODE_KIND_SEQ;
 				new = last_scalar;
 			}
-			if (!as_node_yaml_process_layer (parser, new, error))
+			if (!as_node_yaml_process_layer (ctx, new, error))
 				return FALSE;
 			last_scalar = NULL;
 			break;
@@ -392,10 +431,14 @@ G_DEFINE_AUTO_CLEANUP_FREE_FUNC(AsYamlParser, yaml_parser_delete, NULL);
 #endif
 
 AsNode *
-as_yaml_from_data (const gchar *data, gssize data_len, GError **error)
+as_yaml_from_data (const gchar *data,
+		   gssize data_len,
+		   AsYamlFromFlags flags,
+		   GError **error)
 {
 	g_autoptr(AsYaml) node = NULL;
 #ifdef AS_BUILD_DEP11
+	AsYamlContext ctx;
 	yaml_parser_t parser;
 	g_auto(AsYamlParser) parser_cleanup = NULL;
 
@@ -409,7 +452,10 @@ as_yaml_from_data (const gchar *data, gssize data_len, GError **error)
 		data_len = (gssize) strlen (data);
 	yaml_parser_set_input_string (&parser, (guchar *) data, (gsize) data_len);
 	node = g_node_new (NULL);
-	if (!as_node_yaml_process_layer (&parser, node, error))
+	ctx.parser = &parser;
+	ctx.flags = flags;
+	ctx.locales = g_get_language_names ();
+	if (!as_node_yaml_process_layer (&ctx, node, error))
 		return NULL;
 #else
 	g_set_error_literal (error,
@@ -441,7 +487,7 @@ as_yaml_read_handler_cb (void *data,
 #endif
 
 AsNode *
-as_yaml_from_file (GFile *file, GCancellable *cancellable, GError **error)
+as_yaml_from_file (GFile *file, AsYamlFromFlags flags, GCancellable *cancellable, GError **error)
 {
 	g_autoptr(AsYaml) node = NULL;
 #ifdef AS_BUILD_DEP11
@@ -453,6 +499,7 @@ as_yaml_from_file (GFile *file, GCancellable *cancellable, GError **error)
 	g_autoptr(GFileInfo) info = NULL;
 	g_autoptr(GInputStream) file_stream = NULL;
 	g_autoptr(GInputStream) stream_data = NULL;
+	AsYamlContext ctx;
 
 	/* what kind of file is this */
 	info = g_file_query_info (file,
@@ -491,7 +538,10 @@ as_yaml_from_file (GFile *file, GCancellable *cancellable, GError **error)
 	parser_cleanup = &parser;
 	yaml_parser_set_input (&parser, as_yaml_read_handler_cb, stream_data);
 	node = g_node_new (NULL);
-	if (!as_node_yaml_process_layer (&parser, node, error))
+	ctx.parser = &parser;
+	ctx.flags = flags;
+	ctx.locales = g_get_language_names ();
+	if (!as_node_yaml_process_layer (&ctx, node, error))
 		return NULL;
 #else
 	g_set_error_literal (error,
diff --git a/libappstream-glib/as-yaml.h b/libappstream-glib/as-yaml.h
index 99c7975..cf9cc3c 100644
--- a/libappstream-glib/as-yaml.h
+++ b/libappstream-glib/as-yaml.h
@@ -31,14 +31,31 @@
 
 G_BEGIN_DECLS
 
+
+/**
+ * AsYamlFromFlags:
+ * @AS_YAML_FROM_FLAG_NONE:			No extra flags to use
+ * @AS_YAML_FROM_FLAG_ONLY_NATIVE_LANGS:	Only load native languages
+ *
+ * The flags for converting from XML.
+ **/
+typedef enum {
+	AS_YAML_FROM_FLAG_NONE			= 0,		/* Since: 0.1.0 */
+	AS_YAML_FROM_FLAG_ONLY_NATIVE_LANGS	= 1 << 0,	/* Since: 0.6.6 */
+	/*< private >*/
+	AS_YAML_FROM_FLAG_LAST
+} AsYamlFromFlags;
+
 typedef GNode AsYaml;
 
 void		 as_yaml_unref			(AsYaml		*node);
 GString		*as_yaml_to_string		(AsYaml		*node);
 AsYaml		*as_yaml_from_data		(const gchar	*data,
 						 gssize		 data_len,
+						 AsYamlFromFlags flags,
 						 GError		**error);
 AsYaml		*as_yaml_from_file		(GFile		*file,
+						 AsYamlFromFlags flags,
 						 GCancellable	*cancellable,
 						 GError		**error);
 const gchar	*as_yaml_node_get_key		(const AsYaml	*node);