Only load native languages when parsing AppStream data

This adds an optional flag that allows the node parser to ignore any nodes that
do not match the native languages of the user. This increases startup speed of
gnome-software by ~40ms but also drops the RSS by a massive 12Mb for the Fedora
AppStream file.

6 files changed, 43 insertions, 12 deletions

diff --git a/client/as-util.c b/client/as-util.c
index dee4934..991ae64 100644
--- a/client/as-util.c
+++ b/client/as-util.c
@@ -1291,6 +1291,7 @@ as_util_search (AsUtilPrivate *priv, gchar **values, GError **error)
 	store = as_store_new ();
 	as_store_set_add_flags (store,
 				AS_STORE_ADD_FLAG_USE_UNIQUE_ID |
+				AS_STORE_ADD_FLAG_ONLY_NATIVE_LANGS |
 				AS_STORE_ADD_FLAG_USE_MERGE_HEURISTIC);
 	if (!as_store_load (store,
 			    AS_STORE_LOAD_FLAG_IGNORE_INVALID |
diff --git a/libappstream-glib/as-node.c b/libappstream-glib/as-node.c
index 9579bea..f793274 100644
--- a/libappstream-glib/as-node.c
+++ b/libappstream-glib/as-node.c
@@ -46,7 +46,8 @@ typedef struct
 	GList		*attrs;
 	gchar		*name;		/* only used if tag == AS_TAG_UNKNOWN */
 	gchar		*cdata;
-	gboolean	 cdata_escaped;
+	guint8		 cdata_escaped:1;
+	guint8		 cdata_ignore:1;
 	AsTag		 tag;
 } AsNodeData;
 
@@ -443,6 +444,7 @@ as_node_reflow_text (const gchar *text, gssize text_len)
 typedef struct {
 	AsNode			*current;
 	AsNodeFromXmlFlags	 flags;
+	const gchar * const	*locales;
 } AsNodeToXmlHelper;
 
 /**
@@ -488,6 +490,7 @@ as_node_start_element_cb (GMarkupParseContext *context,
 {
 	AsNodeToXmlHelper *helper = (AsNodeToXmlHelper *) user_data;
 	AsNodeData *data;
+	AsNodeData *data_parent;
 	AsNode *current;
 	gchar *tmp;
 	guint i;
@@ -501,6 +504,19 @@ as_node_start_element_cb (GMarkupParseContext *context,
 				     attribute_values[i]);
 	}
 
+	/* parent node is being ignored */
+	data_parent = helper->current->data;
+	if (data_parent->cdata_ignore)
+		data->cdata_ignore = TRUE;
+
+	/* check if we should ignore the locale */
+	if (helper->flags & AS_NODE_FROM_XML_FLAG_ONLY_NATIVE_LANGS) {
+		const gchar *lang = as_node_attr_lookup (data, "xml:lang");
+		if (lang != NULL && !g_strv_contains (helper->locales, lang))
+			data->cdata_ignore = TRUE;
+
+	}
+
 	/* add the node to the DOM */
 	current = g_node_append_data (helper->current, data);
 
@@ -540,6 +556,11 @@ as_node_text_cb (GMarkupParseContext *context,
 	if (text_len == 0)
 		return;
 
+	/* ignoring */
+	data = helper->current->data;
+	if (data->cdata_ignore)
+		return;
+
 	/* all whitespace? */
 	for (i = 0; i < text_len; i++) {
 		if (!g_ascii_isspace(text[i]))
@@ -549,7 +570,6 @@ as_node_text_cb (GMarkupParseContext *context,
 		return;
 
 	/* split up into lines and add each with spaces stripped */
-	data = helper->current->data;
 	if (data->cdata != NULL) {
 		g_set_error (error,
 			     AS_NODE_ERROR,
@@ -649,6 +669,7 @@ as_node_from_xml (const gchar *data,
 	root = as_node_new ();
 	helper.flags = flags;
 	helper.current = root;
+	helper.locales = g_get_language_names ();
 	ctx = g_markup_parse_context_new (&parser,
 					  G_MARKUP_PREFIX_ERROR_POSITION,
 					  &helper,
@@ -784,6 +805,7 @@ as_node_from_file (GFile *file,
 	root = as_node_new ();
 	helper.flags = flags;
 	helper.current = root;
+	helper.locales = g_get_language_names ();
 	ctx = g_markup_parse_context_new (&parser,
 					  G_MARKUP_PREFIX_ERROR_POSITION,
 					  &helper,
diff --git a/libappstream-glib/as-node.h b/libappstream-glib/as-node.h
index d18c924..e3fe936 100644
--- a/libappstream-glib/as-node.h
+++ b/libappstream-glib/as-node.h
@@ -61,13 +61,15 @@ typedef enum {
  * @AS_NODE_FROM_XML_FLAG_NONE:			No extra flags to use
  * @AS_NODE_FROM_XML_FLAG_LITERAL_TEXT:		Treat the text as an exact string
  * @AS_NODE_FROM_XML_FLAG_KEEP_COMMENTS:	Retain comments in the XML file
+ * @AS_NODE_FROM_XML_FLAG_ONLY_NATIVE_LANGS:	Only load native languages
  *
  * The flags for converting from XML.
  **/
 typedef enum {
-	AS_NODE_FROM_XML_FLAG_NONE		= 0,	/* Since: 0.1.0 */
-	AS_NODE_FROM_XML_FLAG_LITERAL_TEXT	= 1,	/* Since: 0.1.3 */
-	AS_NODE_FROM_XML_FLAG_KEEP_COMMENTS	= 2,	/* Since: 0.1.6 */
+	AS_NODE_FROM_XML_FLAG_NONE		= 0,		/* Since: 0.1.0 */
+	AS_NODE_FROM_XML_FLAG_LITERAL_TEXT	= 1 << 0,	/* Since: 0.1.3 */
+	AS_NODE_FROM_XML_FLAG_KEEP_COMMENTS	= 1 << 1,	/* Since: 0.1.6 */
+	AS_NODE_FROM_XML_FLAG_ONLY_NATIVE_LANGS	= 1 << 2,	/* Since: 0.6.5 */
 	/*< private >*/
 	AS_NODE_FROM_XML_FLAG_LAST
 } AsNodeFromXmlFlags;
diff --git a/libappstream-glib/as-self-test.c b/libappstream-glib/as-self-test.c
index 954947a..b5511d5 100644
--- a/libappstream-glib/as-self-test.c
+++ b/libappstream-glib/as-self-test.c
@@ -3814,6 +3814,7 @@ as_test_store_speed_appstream_func (void)
 	for (i = 0; i < loops; i++) {
 		g_autoptr(AsStore) store = NULL;
 		store = as_store_new ();
+		as_store_set_add_flags (store, AS_STORE_ADD_FLAG_ONLY_NATIVE_LANGS);
 		ret = as_store_from_file (store, file, NULL, NULL, &error);
 		g_assert_no_error (error);
 		g_assert (ret);
diff --git a/libappstream-glib/as-store.c b/libappstream-glib/as-store.c
index 796de5e..158b727 100644
--- a/libappstream-glib/as-store.c
+++ b/libappstream-glib/as-store.c
@@ -1777,6 +1777,7 @@ as_store_from_file_internal (AsStore *store,
 			     GError **error)
 {
 	AsStorePrivate *priv = GET_PRIVATE (store);
+	AsNodeFromXmlFlags flags = AS_NODE_FROM_XML_FLAG_LITERAL_TEXT;
 	g_autofree gchar *filename = NULL;
 	g_autofree gchar *icon_prefix = NULL;
 	g_autoptr(GError) error_local = NULL;
@@ -1806,10 +1807,9 @@ as_store_from_file_internal (AsStore *store,
 #endif
 
 	/* an AppStream XML file */
-	root = as_node_from_file (file,
-				  AS_NODE_FROM_XML_FLAG_LITERAL_TEXT,
-				  cancellable,
-				  &error_local);
+	if (priv->add_flags & AS_STORE_ADD_FLAG_ONLY_NATIVE_LANGS)
+		flags |= AS_NODE_FROM_XML_FLAG_ONLY_NATIVE_LANGS;
+	root = as_node_from_file (file, flags, cancellable, &error_local);
 	if (root == NULL) {
 		g_set_error (error,
 			     AS_STORE_ERROR,
@@ -1951,6 +1951,8 @@ as_store_from_xml (AsStore *store,
 		   const gchar *icon_root,
 		   GError **error)
 {
+	AsStorePrivate *priv = GET_PRIVATE (store);
+	AsNodeFromXmlFlags flags = AS_NODE_FROM_XML_FLAG_LITERAL_TEXT;
 	g_autoptr(GError) error_local = NULL;
 	g_autoptr(AsNode) root = NULL;
 
@@ -1961,9 +1963,10 @@ as_store_from_xml (AsStore *store,
 	if (data[0] == '\0')
 		return TRUE;
 
-	root = as_node_from_xml (data,
-				 AS_NODE_FROM_XML_FLAG_LITERAL_TEXT,
-				 &error_local);
+	/* load XML data */
+	if (priv->add_flags & AS_STORE_ADD_FLAG_ONLY_NATIVE_LANGS)
+		flags |= AS_NODE_FROM_XML_FLAG_ONLY_NATIVE_LANGS;
+	root = as_node_from_xml (data, flags, &error_local);
 	if (root == NULL) {
 		g_set_error (error,
 			     AS_STORE_ERROR,
diff --git a/libappstream-glib/as-store.h b/libappstream-glib/as-store.h
index 3d4a34a..e2b9a56 100644
--- a/libappstream-glib/as-store.h
+++ b/libappstream-glib/as-store.h
@@ -98,6 +98,7 @@ typedef enum {
  * @AS_STORE_ADD_FLAG_PREFER_LOCAL:			Local files will be used by default
  * @AS_STORE_ADD_FLAG_USE_UNIQUE_ID:			Allow multiple apps with the same AppStream ID
  * @AS_STORE_ADD_FLAG_USE_MERGE_HEURISTIC:		Use a heuristic when adding merge components
+ * @AS_STORE_ADD_FLAG_ONLY_NATIVE_LANGS:		Only load native languages
  *
  * The flags to use when adding applications to the store.
  **/
@@ -106,6 +107,7 @@ typedef enum {
 	AS_STORE_ADD_FLAG_PREFER_LOCAL		= 1 << 0,	/* Since: 0.2.2 */
 	AS_STORE_ADD_FLAG_USE_UNIQUE_ID		= 1 << 1,	/* Since: 0.6.1 */
 	AS_STORE_ADD_FLAG_USE_MERGE_HEURISTIC	= 1 << 2,	/* Since: 0.6.1 */
+	AS_STORE_ADD_FLAG_ONLY_NATIVE_LANGS	= 1 << 3,	/* Since: 0.6.5 */
 	/*< private >*/
 	AS_STORE_ADD_FLAG_LAST
 } AsStoreAddFlags;