Add appstream-build from the createrepo_as project

3 files changed, 453 insertions, 11 deletions

diff --git a/README.md b/README.md
index e5eb98b..94ad8ac 100644
--- a/README.md
+++ b/README.md
@@ -31,16 +31,6 @@ copy. To do the latter just do:
     make
     make install
 
-More Information
-----------------
-
-If you want to actually generate metadata rather than just consuming it, you
-probably want to be looking at: https://github.com/hughsie/createrepo_as or if
-you're completely lost, GNOME Software is a GUI tool that uses this library to
-implement a software center. See `src/plugins/gs-plugin-appstream.c` if you
-want some more examples on using this library where speed and latency really
-matter.
-
 Hacking
 -------
 
@@ -48,10 +38,137 @@ If you want a new feature, or have found a bug or a way to crash this library,
 please report as much information as you can to the issue tracker:
 https://github.com/hughsie/appstream-glib/issues -- patches very welcome.
 
-New functionality or crash fixes should include a test in `src/as-self-test.c`
+New functionality or crash fixes should include a test in `libappstream-builder/as-self-test.c`
 to ensure we don't regress in the future. New functionality should also be
 thread safe and also not leak *any* memory for success or failure cases.
 
+appstream-builder
+=================
+
+appstream-builder is a tool that allows us to create AppStream metadata from a
+directory of packages.
+It is typically used when generating distribution metadata, usually at the same
+time as modifyrepo or createrepo.
+
+What this tool does:
+
+ * Searches a directory of packages and reads just the RPM header of each.
+ * If a package contains an interesting file, just the relevant files are
+   decompressed from the package archive.
+ * A set of plugins are run on the extracted files, and if these match certain
+   criteria `AsbApplication` objects are created.
+ * Any screenshots referenced are downloaded to a local cache.
+   This is optional and can be disabled with `--nonet`.
+ * When all the packages are processed, some of the `AsbApplication` objects are
+   merged into single applications. This is how fonts are collected.
+ * The `AsbApplication` objects are serialized to XML and written to a
+   compressed archive.
+ * Any application icons or screenshots referenced are written to a .tar archive.
+
+Getting Started
+-----------
+
+To run appstream-builder you either need to install the package containing the
+binary and data files, or you can build a local copy. To do the latter just do:
+
+    dnf install automake autoconf libtool rpm-devel \
+                gtk3-devel sqlite-devel libsoup-devel
+    ./autogen.sh
+    make
+
+To actually run the extractor you can do:
+
+    ./appstream-builder --verbose   \
+                    --max-threads=8 \
+                    --log-dir=/tmp/logs \
+                    --packages-dir=/mnt/archive/Megarpms/21/Packages \
+                    --temp-dir=/mnt/ssd/AppStream/tmp \
+                    --output-dir=./repodata \
+                    --screenshot-url=http://megarpms.org/screenshots/ \
+                    --basename="megarpms-21"
+
+This will output a lot of progress text. Now, go and make a cup of tea and wait
+patiently if you have a lot of packages to process. After this is complete
+you should finally see:
+
+    Writing ./repodata/megarpms-21.xml.gz
+    Writing ./repodata/megarpms-21-icons.tar
+    Done!
+
+You now have two choices what to do with these files. You can either upload
+them with the rest of the metadata you ship (e.g. in the same directory as
+`repomd.xml` and `primary.sqlite.bz2`) which will work with Fedora 21.
+
+For Fedora 20, you have to actually install these files, so you can do something
+like this in the megarpms-release.spec file:
+
+    Source1:   http://www.megarpms.org/temp/megarpms-20.xml.gz
+    Source2:   http://www.megarpms.org/temp/megarpms-20-icons.tar.gz
+
+    %install
+    mkdir -p %{buildroot}%{_datadir}/app-info/xmls
+    cp %{SOURCE1} %{buildroot}%{_datadir}/app-info/xmls
+    mkdir -p %{buildroot}%{_datadir}/app-info/icons/megarpms-20
+    tar xvzf %{SOURCE2}
+    cd -
+
+or, if your distro ships a new enough libappstream-glib:
+
+    %install
+    DESTDIR=%{buildroot} appstream-util install %{SOURCE1} %{SOURCE2}
+
+This ensures that gnome-software can access both data files when starting up.
+
+What is an application
+-----------
+
+Applications are defined in the context of AppStream as such:
+
+ * Installs a desktop file and would be visible in a desktop
+ * Has an metadata extractor (e.g. libappstream-builder/plugins/asb-plugin-gstreamer.c)
+   and includes an AppData file
+
+Guidelines for applications
+-----------
+
+These guidelines explain how we filter applications from a package set.
+
+First, some key words:
+ * **SHOULD**: The application should do this if possible
+ * **MUST**: The application or addon must do this to be included
+ * **CANNOT**: the application or addon must not do this
+
+The current rules of inclusion are thus:
+
+ * Icons **MUST** be installed in `/usr/share/pixmaps/*`, `/usr/share/icons/*`,
+   `/usr/share/icons/hicolor/*/apps/*`, or `/usr/share/${app_name}/icons/*`
+ * Desktop files **MUST** be installed in `/usr/share/applications/`
+   or `/usr/share/applications/kde4/`
+ * Desktop files **MUST** have `Name`, `Comment` and `Icon` entries
+ * Valid applications with `NoDisplay=true` **MUST** have an AppData file.
+ * Applications with `Categories=Settings`, `Categories=ConsoleOnly` or
+   `Categories=DesktopSettings` **MUST** have an AppData file.
+ * Applications **MUST** have had an upstream release in the last 5 years or
+   have an AppData file.
+ * Application icon **MUST** be available in 48x48 or larger
+ * Applications must have at least one main or additional category listed
+   in the desktop file or supply an AppData file.
+   See http://standards.freedesktop.org/menu-spec/latest/apa.html and
+   http://standards.freedesktop.org/menu-spec/latest/apas02.html for the
+   full `Categories` list.
+ * Codecs **MUST** have an AppData file
+ * Input methods **MUST** have an AppData file
+ * If included, AppData files **MUST** be valid XML
+ * AppData files **MUST** be installed into `/usr/share/appdata`
+ * Application icons **CANNOT** use XPM or ICO format
+ * Applications **CANNOT** use obsolete toolkits such as GTK+-1.2 or QT3
+ * Applications that ship a desktop file **SHOULD** include an AppData file.
+ * Screenshots **SHOULD** be in 16:9 aspect ratio
+ * Application icons **SHOULD** have an alpha channel
+ * Applications **SHOULD** ship a 64x64 PNG format icon or SVG
+ * AppData files **SHOULD** include translations
+ * Desktop files **SHOULD** include translations
+
 License
 ----
 
diff --git a/client/Makefile.am b/client/Makefile.am
index d490f98..3196f6a 100644
--- a/client/Makefile.am
+++ b/client/Makefile.am
@@ -8,14 +8,19 @@ AM_CPPFLAGS =						\
 	-I$(top_builddir)				\
 	-I$(top_srcdir)/libappstream-glib		\
 	-I$(top_builddir)/libappstream-glib		\
+	-I$(top_srcdir)/libappstream-builder		\
+	-I$(top_builddir)/libappstream-builder		\
 	-DG_LOG_DOMAIN=\"As\"				\
 	-DVERSION="\"$(VERSION)\""			\
 	-DLOCALEDIR=\""$(localedir)"\"
 
 AS_GLIB_LIBS =						\
 	$(top_builddir)/libappstream-glib/libappstream-glib.la
+AS_BUILDER_LIBS =					\
+	$(top_builddir)/libappstream-builder/libappstream-builder.la
 
 bin_PROGRAMS =						\
+	appstream-builder				\
 	appstream-util
 
 appstream_util_SOURCES =				\
@@ -30,4 +35,17 @@ appstream_util_LDFLAGS =				\
 appstream_util_CFLAGS =					\
 	$(WARNINGFLAGS_C)
 
+appstream_builder_SOURCES =				\
+	as-builder.c
+appstream_builder_LDADD =				\
+	$(AS_GLIB_LIBS)					\
+	$(AS_BUILDER_LIBS)				\
+	$(GLIB_LIBS)					\
+	$(SOUP_LIBS)					\
+	$(LIBARCHIVE_LIBS)
+appstream_builder_LDFLAGS =				\
+	$(PIE_LDFLAGS)
+appstream_builder_CFLAGS =				\
+	$(WARNINGFLAGS_C)
+
 -include $(top_srcdir)/git.mk
diff --git a/client/as-builder.c b/client/as-builder.c
new file mode 100644
index 0000000..486018f
--- /dev/null
+++ b/client/as-builder.c
@@ -0,0 +1,307 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ *
+ * Copyright (C) 2014 Richard Hughes <richard@hughsie.com>
+ *
+ * Licensed under the GNU General Public License Version 2
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "config.h"
+
+#include <appstream-glib.h>
+#include <glib.h>
+#include <locale.h>
+
+#include "as-cleanup.h"
+#include "asb-context.h"
+#include "asb-utils.h"
+
+/**
+ * main:
+ **/
+int
+main (int argc, char **argv)
+{
+	AsbContext *ctx = NULL;
+	GOptionContext *option_context;
+	const gchar *filename;
+	gboolean add_cache_id = FALSE;
+	gboolean extra_checks = FALSE;
+	gboolean no_net = FALSE;
+	gboolean ret;
+	gboolean use_package_cache = FALSE;
+	gboolean verbose = FALSE;
+	gchar *tmp;
+	gdouble api_version = 0.0f;
+	gint max_threads = 4;
+	gint rc;
+	guint i;
+	_cleanup_dir_close_ GDir *dir = NULL;
+	_cleanup_error_free_ GError *error = NULL;
+	_cleanup_free_ gchar *basename = NULL;
+	_cleanup_free_ gchar *cache_dir = NULL;
+	_cleanup_free_ gchar *extra_appdata = NULL;
+	_cleanup_free_ gchar *extra_appstream = NULL;
+	_cleanup_free_ gchar *extra_screenshots = NULL;
+	_cleanup_free_ gchar *log_dir = NULL;
+	_cleanup_free_ gchar *old_metadata = NULL;
+	_cleanup_free_ gchar *output_dir = NULL;
+	_cleanup_free_ gchar *packages_dir = NULL;
+	_cleanup_free_ gchar *screenshot_uri = NULL;
+	_cleanup_free_ gchar *temp_dir = NULL;
+	_cleanup_ptrarray_unref_ GPtrArray *packages = NULL;
+	_cleanup_timer_destroy_ GTimer *timer = NULL;
+	const GOptionEntry options[] = {
+		{ "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose,
+			"Show extra debugging information", NULL },
+		{ "no-net", '\0', 0, G_OPTION_ARG_NONE, &no_net,
+			"Do not use the network to download screenshots", NULL },
+		{ "use-package-cache", '\0', 0, G_OPTION_ARG_NONE, &use_package_cache,
+			"Do not delete the decompressed package cache", NULL },
+		{ "extra-checks", '\0', 0, G_OPTION_ARG_NONE, &extra_checks,
+			"Perform extra checks on the source metadata", NULL },
+		{ "add-cache-id", '\0', 0, G_OPTION_ARG_NONE, &add_cache_id,
+			"Add a cache ID to each component", NULL },
+		{ "log-dir", '\0', 0, G_OPTION_ARG_STRING, &log_dir,
+			"Set the logging directory       [default: ./logs]", NULL },
+		{ "packages-dir", '\0', 0, G_OPTION_ARG_STRING, &packages_dir,
+			"Set the packages directory      [default: ./packages]", NULL },
+		{ "temp-dir", '\0', 0, G_OPTION_ARG_STRING, &temp_dir,
+			"Set the temporary directory     [default: ./tmp]", NULL },
+		{ "extra-appstream-dir", '\0', 0, G_OPTION_ARG_STRING, &extra_appstream,
+			"Use extra appstream data        [default: ./appstream-extra]", NULL },
+		{ "extra-appdata-dir", '\0', 0, G_OPTION_ARG_STRING, &extra_appdata,
+			"Use extra appdata data          [default: ./appdata-extra]", NULL },
+		{ "extra-screenshots-dir", '\0', 0, G_OPTION_ARG_STRING, &extra_screenshots,
+			"Use extra screenshots data      [default: ./screenshots-extra]", NULL },
+		{ "output-dir", '\0', 0, G_OPTION_ARG_STRING, &output_dir,
+			"Set the output directory        [default: .]", NULL },
+		{ "cache-dir", '\0', 0, G_OPTION_ARG_STRING, &output_dir,
+			"Set the cache directory         [default: ./cache]", NULL },
+		{ "basename", '\0', 0, G_OPTION_ARG_STRING, &basename,
+			"Set the origin name             [default: fedora-21]", NULL },
+		{ "max-threads", '\0', 0, G_OPTION_ARG_INT, &max_threads,
+			"Set the number of threads       [default: 4]", NULL },
+		{ "api-version", '\0', 0, G_OPTION_ARG_DOUBLE, &api_version,
+			"Set the AppStream version       [default: 0.4]", NULL },
+		{ "screenshot-uri", '\0', 0, G_OPTION_ARG_STRING, &screenshot_uri,
+			"Set the screenshot base URL     [default: none]", NULL },
+		{ "old-metadata", '\0', 0, G_OPTION_ARG_STRING, &old_metadata,
+			"Set the old metadata location   [default: none]", NULL },
+		{ NULL}
+	};
+
+	option_context = g_option_context_new (NULL);
+	g_option_context_add_main_entries (option_context, options, NULL);
+	ret = g_option_context_parse (option_context, &argc, &argv, &error);
+	if (!ret) {
+		g_print ("Failed to parse arguments: %s\n", error->message);
+		goto out;
+	}
+
+	if (verbose)
+		g_setenv ("G_MESSAGES_DEBUG", "all", TRUE);
+	if (extra_checks)
+		g_setenv ("ASB_PERFORM_EXTRA_CHECKS", "1", TRUE);
+
+#if !GLIB_CHECK_VERSION(2,40,0)
+	if (max_threads > 1) {
+		g_debug ("O_CLOEXEC not available, using 1 core");
+		max_threads = 1;
+	}
+#endif
+	/* set defaults */
+	if (api_version < 0.01)
+		api_version = 0.41;
+	if (packages_dir == NULL)
+		packages_dir = g_strdup ("./packages");
+	if (temp_dir == NULL)
+		temp_dir = g_strdup ("./tmp");
+	if (log_dir == NULL)
+		log_dir = g_strdup ("./logs");
+	if (output_dir == NULL)
+		output_dir = g_strdup (".");
+	if (cache_dir == NULL)
+		cache_dir = g_strdup ("./cache");
+	if (basename == NULL)
+		basename = g_strdup ("fedora-21");
+	if (screenshot_uri == NULL)
+		screenshot_uri = g_strdup ("http://alt.fedoraproject.org/pub/alt/screenshots/f21/");
+	if (extra_appstream == NULL)
+		extra_appstream = g_strdup ("./appstream-extra");
+	if (extra_appdata == NULL)
+		extra_appdata = g_strdup ("./appdata-extra");
+	if (extra_screenshots == NULL)
+		extra_screenshots = g_strdup ("./screenshots-extra");
+	setlocale (LC_ALL, "");
+
+	/* set up state */
+	if (use_package_cache) {
+		rc = g_mkdir_with_parents (temp_dir, 0700);
+		if (rc != 0) {
+			g_warning ("failed to create temp dir");
+			goto out;
+		}
+	} else {
+		ret = asb_utils_ensure_exists_and_empty (temp_dir, &error);
+		if (!ret) {
+			g_warning ("failed to create temp dir: %s", error->message);
+			goto out;
+		}
+	}
+	tmp = g_build_filename (temp_dir, "icons", NULL);
+	if (old_metadata != NULL) {
+		add_cache_id = TRUE;
+		ret = g_file_test (tmp, G_FILE_TEST_EXISTS);
+		if (!ret) {
+			g_warning ("%s has to exist to use old metadata", tmp);
+			goto out;
+		}
+	} else {
+		ret = asb_utils_ensure_exists_and_empty (tmp, &error);
+		if (!ret) {
+			g_warning ("failed to create icons dir: %s", error->message);
+			goto out;
+		}
+	}
+	g_free (tmp);
+	rc = g_mkdir_with_parents (log_dir, 0700);
+	if (rc != 0) {
+		g_warning ("failed to create log dir");
+		goto out;
+	}
+	rc = g_mkdir_with_parents (output_dir, 0700);
+	if (rc != 0) {
+		g_warning ("failed to create log dir");
+		goto out;
+	}
+	tmp = g_build_filename (output_dir, "screenshots", "112x63", NULL);
+	rc = g_mkdir_with_parents (tmp, 0700);
+	g_free (tmp);
+	if (rc != 0) {
+		g_warning ("failed to create screenshot cache dir");
+		goto out;
+	}
+	tmp = g_build_filename (output_dir, "screenshots", "624x351", NULL);
+	rc = g_mkdir_with_parents (tmp, 0700);
+	g_free (tmp);
+	if (rc != 0) {
+		g_warning ("failed to create screenshot cache dir");
+		goto out;
+	}
+	tmp = g_build_filename (output_dir, "screenshots", "752x423", NULL);
+	rc = g_mkdir_with_parents (tmp, 0700);
+	g_free (tmp);
+	if (rc != 0) {
+		g_warning ("failed to create screenshot cache dir");
+		goto out;
+	}
+	tmp = g_build_filename (output_dir, "screenshots", "source", NULL);
+	rc = g_mkdir_with_parents (tmp, 0700);
+	g_free (tmp);
+	if (rc != 0) {
+		g_warning ("failed to create screenshot cache dir");
+		goto out;
+	}
+	rc = g_mkdir_with_parents (cache_dir, 0700);
+	if (rc != 0) {
+		g_warning ("failed to create cache dir");
+		goto out;
+	}
+
+	ctx = asb_context_new ();
+	asb_context_set_no_net (ctx, no_net);
+	asb_context_set_api_version (ctx, api_version);
+	asb_context_set_add_cache_id (ctx, add_cache_id);
+	asb_context_set_extra_checks (ctx, extra_checks);
+	asb_context_set_use_package_cache (ctx, use_package_cache);
+	asb_context_set_old_metadata (ctx, old_metadata);
+	asb_context_set_extra_appstream (ctx, extra_appstream);
+	asb_context_set_extra_appdata (ctx, extra_appdata);
+	asb_context_set_extra_screenshots (ctx, extra_screenshots);
+	asb_context_set_screenshot_uri (ctx, screenshot_uri);
+	asb_context_set_log_dir (ctx, log_dir);
+	asb_context_set_temp_dir (ctx, temp_dir);
+	asb_context_set_output_dir (ctx, output_dir);
+	asb_context_set_cache_dir (ctx, cache_dir);
+	asb_context_set_basename (ctx, basename);
+	asb_context_set_max_threads (ctx, max_threads);
+	ret = asb_context_setup (ctx, &error);
+	if (!ret) {
+		g_warning ("failed to set up context: %s", error->message);
+		goto out;
+	}
+
+	/* scan each package */
+	packages = g_ptr_array_new_with_free_func (g_free);
+	if (argc == 1) {
+		dir = g_dir_open (packages_dir, 0, &error);
+		if (dir == NULL) {
+			g_warning ("failed to open packages: %s", error->message);
+			goto out;
+		}
+		while ((filename = g_dir_read_name (dir)) != NULL) {
+			tmp = g_build_filename (packages_dir,
+						filename, NULL);
+			g_ptr_array_add (packages, tmp);
+		}
+	} else {
+		for (i = 1; i < (guint) argc; i++)
+			g_ptr_array_add (packages, g_strdup (argv[i]));
+	}
+	g_print ("Scanning packages...\n");
+	timer = g_timer_new ();
+	for (i = 0; i < packages->len; i++) {
+		filename = g_ptr_array_index (packages, i);
+
+		/* anything in the cache */
+		if (asb_context_find_in_cache (ctx, filename)) {
+			g_debug ("Skipping %s as found in old md cache",
+				 filename);
+			continue;
+		}
+
+		/* add to list */
+		ret = asb_context_add_filename (ctx, filename, &error);
+		if (!ret) {
+			g_warning ("%s", error->message);
+			goto out;
+		}
+		if (g_timer_elapsed (timer, NULL) > 3.f) {
+			g_print ("Parsed %i/%i files...\n",
+				 i, packages->len);
+			g_timer_reset (timer);
+		}
+	}
+
+	/* disable anything not newest */
+	asb_context_disable_older_pkgs (ctx);
+
+	/* process all packages in the pool */
+	ret = asb_context_process (ctx, &error);
+	if (!ret) {
+		g_warning ("failed to process context: %s", error->message);
+		goto out;
+	}
+
+	/* success */
+	g_print ("Done!\n");
+out:
+	g_option_context_free (option_context);
+	if (ctx != NULL)
+		g_object_unref (ctx);
+	return 0;
+}