Add support for translated man pages, fixes #343275

* src/yelp-man-pager.c: (man_pager_parse):
* src/yelp-man-parser.c: (yelp_man_parser_parse_file),
(yelp_man_parser_parse_doc), (parser_parse_line),
(macro_ignore_handler), (macro_section_header_handler),
(parser_handle_linetag), (parser_read_until), (parser_append_text):
* src/yelp-man-parser.h:
* src/yelp-toc-pager.c: (add_man_page_to_toc),
(create_toc_from_index), (process_mandir_pending),
(process_cleanup):
* stylesheets/man2html.xsl:
Add support for translated man pages, fixes #343275

6 files changed, 340 insertions, 77 deletions

diff --git a/ChangeLog b/ChangeLog
index b1edd9df..fb9bd113 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2006-06-11  Brent Smith  <gnome@nextreality.net>
+
+	* src/yelp-man-pager.c: (man_pager_parse):
+	* src/yelp-man-parser.c: (yelp_man_parser_parse_file),
+	(yelp_man_parser_parse_doc), (parser_parse_line),
+	(macro_ignore_handler), (macro_section_header_handler),
+	(parser_handle_linetag), (parser_read_until), (parser_append_text):
+	* src/yelp-man-parser.h:
+	* src/yelp-toc-pager.c: (add_man_page_to_toc),
+	(create_toc_from_index), (process_mandir_pending),
+	(process_cleanup):
+	* stylesheets/man2html.xsl:
+	Add support for translated man pages, fixes #343275
+
 2006-06-11  Don Scorgie  <dscorgie@cvs.gnome.org>
 
 	* src/yelp-main.c: 
diff --git a/src/yelp-man-pager.c b/src/yelp-man-pager.c
index 59ed73ad..c1d7da5b 100644
--- a/src/yelp-man-pager.c
+++ b/src/yelp-man-pager.c
@@ -82,6 +82,65 @@ static YelpManSect *  man_section_process    (xmlNodePtr        node,
 
 static YelpPagerClass *parent_class;
 
+typedef struct _YelpLangEncodings YelpLangEncodings;
+
+struct _YelpLangEncodings {
+    gchar *language;
+    gchar *encoding;
+};
+
+/* http://www.w3.org/International/O-charset-lang.html */
+static const YelpLangEncodings langmap[] = {
+    { "C",     "ISO-8859-1" },
+    { "af",    "ISO-8859-1" },
+    { "ar",    "ISO-8859-6" },
+    { "bg",    "ISO-8859-5" },
+    { "be",    "ISO-8859-5" },
+    { "ca",    "ISO-8859-1" },
+    { "cs",    "ISO-8859-2" },
+    { "da",    "ISO-8859-1" },
+    { "de",    "ISO-8859-1" },
+    { "el",    "ISO-8859-7" },
+    { "en",    "ISO-8859-1" },
+    { "eo",    "ISO-8859-3" },
+    { "es",    "ISO-8859-1" },
+    { "et",    "ISO-8859-15" },
+    { "eu",    "ISO-8859-1" },
+    { "fi",    "ISO-8859-1" },
+    { "fo",    "ISO-8859-1" },
+    { "fr",    "ISO-8859-1" },
+    { "ga",    "ISO-8859-1" },
+    { "gd",    "ISO-8859-1" },
+    { "gl",    "ISO-8859-1" },
+    { "hu",    "ISO-8859-2" },
+    { "id",    "ISO-8859-1" }, /* is this right */
+    { "mt",    "ISO-8859-3" },
+    { "is",    "ISO-8859-1" },
+    { "it",    "ISO-8859-1" },
+    { "iw",    "ISO-8859-8" },
+    { "ja",    "EUC-JP" },
+    { "ko",    "EUC-KR" },
+    { "lt",    "ISO-8859-13" },
+    { "lv",    "ISO-8859-13" },
+    { "mk",    "ISO-8859-5" },
+    { "mt",    "ISO-8859-3" },
+    { "no",    "ISO-8859-1" },
+    { "pl",    "ISO-8859-2" },
+    { "pt_BR", "ISO-8859-1" },
+    { "ro",    "ISO-8859-2" },
+    { "ru",    "KOI8-R" },
+    { "sl",    "ISO-8859-2" },
+    { "sr",    "ISO-8859-2" }, /* Latin, not cyrillic */
+    { "sk",    "ISO-8859-2" },
+    { "sv",    "ISO-8859-1" },
+    { "tr",    "ISO-8859-9" },
+    { "uk",    "ISO-8859-5" },
+    { "zh_CN", "BIG5" },
+    { "zh_TW", "BIG5" },
+    { NULL,    NULL },
+};
+	
+
 GType
 yelp_man_pager_get_type (void)
 {
@@ -161,9 +220,12 @@ man_pager_parse (YelpPager *pager)
 {
     YelpDocInfo   *doc_info;
     gchar         *filename;
+    const gchar   *language;
+    const gchar   *encoding;
     YelpManParser *parser;
     xmlDocPtr      doc;
-    GError        *error;
+    GError        *error = NULL;
+    gint           i;
 
     g_return_val_if_fail (YELP_IS_MAN_PAGER (pager), FALSE);
 
@@ -172,8 +234,27 @@ man_pager_parse (YelpPager *pager)
 
     g_object_ref (pager);
 
+    /* We use the language to determine which encoding the manual
+     * page is in */ 
+    language = yelp_doc_info_get_language (doc_info);
+    g_print ("The language of the man page is %s\n", language);
+
+    /* default encoding if the language doesn't match below */
+    encoding = g_getenv("MAN_ENCODING");
+    if (encoding == NULL)
+	encoding = "ISO-8859-1";
+
+    if (language != NULL) {
+	for (i=0; langmap[i].language != NULL; i++) {
+	    if (g_str_equal (language, langmap[i].language)) {
+		encoding = langmap[i].encoding;
+		break;
+	    }
+	}
+    }
+    
     parser = yelp_man_parser_new ();
-    doc = yelp_man_parser_parse_file (parser, filename);
+    doc = yelp_man_parser_parse_file (parser, filename, encoding);
     yelp_man_parser_free (parser);
 
     if (doc == NULL) {
diff --git a/src/yelp-man-parser.c b/src/yelp-man-parser.c
index 1717f4dc..0b7e97cd 100644
--- a/src/yelp-man-parser.c
+++ b/src/yelp-man-parser.c
@@ -36,7 +36,7 @@
 
 #define d(x)
 
-#define PARSER_CUR (*(parser->cur) != '\0' \
+#define PARSER_CUR (g_utf8_get_char (parser->cur) != '\0' \
     && (parser->cur - parser->buffer < parser->length))
 
 static void        parser_parse_line        (YelpManParser *parser);
@@ -97,9 +97,15 @@ yelp_man_parser_new (void)
 
 xmlDocPtr
 yelp_man_parser_parse_file (YelpManParser   *parser,
-			    gchar           *file)
+			    gchar           *file,
+			    const gchar     *encoding)
 {
-    GError **errormsg = NULL;
+    GError *errormsg = NULL;
+    /*gchar *ptr = NULL;*/
+
+    g_return_val_if_fail (parser != NULL, NULL);
+    g_return_val_if_fail (file != NULL, NULL);
+    g_return_val_if_fail (encoding != NULL, NULL);
 	
     parser->channel = yelp_io_channel_new_file (file, NULL);
 
@@ -115,16 +121,51 @@ yelp_man_parser_parse_file (YelpManParser   *parser,
     while (g_io_channel_read_line (parser->channel,
 				   &(parser->buffer),
 				   &(parser->length),
-				   NULL, errormsg)
+				   NULL, &errormsg)
 	   == G_IO_STATUS_NORMAL) {
 
+	/* convert this line from the encoding indicated to UTF-8 */
+	if (!g_str_equal (encoding, "UTF-8")) {
+	    GError *converr = NULL;
+	    gchar *new_buffer = NULL;
+	    gsize bytes_written = 0;
+
+	    /* since our encoding is binary (NULL) in g_io_channel, then
+	     * our returned lined should end with \n.  Therefore we are making the
+	     * assumption that there are no partial characters at the end of this
+	     * string, and therefore can use calls like g_convert() which do not
+	     * preserve state - someone tell me if I'm wrong here */
+	    new_buffer = g_convert (parser->buffer, parser->length, "UTF-8", 
+	                            encoding, NULL, &bytes_written, &converr);
+	    if (converr != NULL) {
+		g_print ("Error occurred converting %s to UTF-8: %s\n", 
+		         encoding, converr->message);
+		g_error_free (converr);
+		break;
+	    } else if (parser->buffer == NULL) {
+		g_print ("parser->buffer == NULL\n");
+		break;
+	    }
+
+	    g_free (parser->buffer);
+	    parser->buffer = new_buffer;
+	    parser->length = bytes_written;
+	}
+	    
+	/* for debugging, make sure line is valid UTF-8 */
+	/*if (!g_utf8_validate (parser->buffer, (gssize)parser->length, &ptr)) {
+	    g_print ("str = %s\n", parser->buffer);
+	    g_print ("str ptr = %p\n", parser->buffer);
+	    g_print ("invalid char = %p (%c)\n", ptr, *ptr);
+	}*/
+	    
 	parser_parse_line (parser);
 
 	g_free (parser->buffer);
     }
 
     if (errormsg)
-	    g_print ("Error in g_io_channel_read_line()\n");
+	g_print ("Error in g_io_channel_read_line()\n");
 
     g_io_channel_shutdown (parser->channel, FALSE, NULL);
 
@@ -136,6 +177,7 @@ yelp_man_parser_parse_doc (YelpManParser *parser,
 			   YelpDocInfo   *doc_info)
 {
     gchar     *file;
+    gchar     *encoding = NULL;
     xmlDocPtr  doc = NULL;
 
     g_return_val_if_fail (parser != NULL, NULL);
@@ -147,7 +189,11 @@ yelp_man_parser_parse_doc (YelpManParser *parser,
     if (!file)
 	return NULL;
 
-    doc = yelp_man_parser_parse_file (parser, file);
+    encoding = (gchar *)g_getenv("MAN_ENCODING");
+    if (encoding == NULL)
+	encoding = "ISO-8859-1";
+
+    doc = yelp_man_parser_parse_file (parser, file, encoding);
 
     g_free (file);
 
@@ -173,10 +219,13 @@ parser_parse_line (YelpManParser *parser) {
     /* check to see if we are ignoring input */
     if (parser->ignore) {
 	gchar *ptr;
+	/* needs to be utf-8 compatible */
 	ptr = strstr (parser->buffer, parser->token);
     	if (ptr != NULL) {
-	    while (PARSER_CUR)
-		parser->anc = ++parser->cur;
+	    while (PARSER_CUR) {
+		parser->cur = g_utf8_next_char (parser->cur);
+		parser->anc = parser->cur;
+	    }
 	    g_free (parser->token);
 	    parser->ignore = FALSE;
 	} else {
@@ -209,7 +258,7 @@ parser_parse_line (YelpManParser *parser) {
 	parser_append_text (parser);
     
     if (PARSER_CUR) {
-	parser->cur++;
+	parser->cur = g_utf8_next_char (parser->cur);
 	parser_append_text (parser);
     }
 }
@@ -248,7 +297,8 @@ static void
 macro_ignore_handler (YelpManParser *parser, gchar *macro, GSList *args)
 { 
     while (PARSER_CUR) {
-	parser->anc = ++parser->cur;
+	parser->cur = g_utf8_next_char (parser->cur);
+	parser->anc = parser->cur;
     }
 }
 
@@ -364,6 +414,7 @@ macro_section_header_handler (YelpManParser *parser, gchar *macro, GSList *args)
 	str = args_concat_all (args);
 
     for (ptr = macro_uc; *ptr != '\0'; ptr++)
+	/* FIXME: utf-8 */
     	*ptr = g_ascii_toupper (*ptr);
     
     parser_stack_pop_node (parser, "IP");
@@ -734,7 +785,7 @@ struct MandocMacro {
     gint flags;
 };
 
-struct MandocMacro manual_macros[] = {
+static struct MandocMacro manual_macros[] = {
     { "Ad", MANDOC_PARSED | MANDOC_CALLABLE },
     { "An", MANDOC_PARSED | MANDOC_CALLABLE },
     { "Ar", MANDOC_PARSED | MANDOC_CALLABLE },
@@ -933,7 +984,7 @@ struct MacroHandler {
  * A great resource to figure out what each of these does is the groff
  * info page.  Also groff(7), man(7), and mdoc(7) are useful as well.
  */
-struct MacroHandler macro_handlers[] = {
+static struct MacroHandler macro_handlers[] = {
     { "\\\"", macro_ignore_handler },                /* groff: comment */ 
     { "ad", macro_ignore_handler },                  /* groff: set adjusting mode */ 
     { "Ad", macro_mandoc_utility_handler },          /* mandoc: Address */ 
@@ -1050,31 +1101,42 @@ parser_handle_linetag (YelpManParser *parser) {
 
     /* FIXME: figure out a better way to handle these cases */
     /* special case, if the line is simply ".\n" then return */
-    if (*(parser->cur+1) == '\n') {
-    	++parser->cur;
-	parser->anc = ++parser->cur;
+    if (g_utf8_get_char (g_utf8_next_char (parser->cur)) == '\n') {
+    	parser->cur = g_utf8_next_char (parser->cur);
+    	parser->cur = g_utf8_next_char (parser->cur);
+	parser->anc = parser->cur;
 	return;
     } 
     /* special case, if the line is simply "..\n" then return */
-    else if (*(parser->cur+1) == '.' && *(parser->cur+2) == '\n') {
-    	++parser->cur;
-    	++parser->cur;
-    	parser->anc = ++parser->cur;
+    else if (g_utf8_get_char (g_utf8_next_char(parser->cur)) == '.' && 
+	     g_utf8_get_char (g_utf8_next_char (g_utf8_next_char (parser->cur+2))) == '\n') {
+    	parser->cur = g_utf8_next_char (parser->cur);
+    	parser->cur = g_utf8_next_char (parser->cur);
+    	parser->cur = g_utf8_next_char (parser->cur);
+    	parser->anc = parser->cur;
     }
     
     /* skip any spaces after the control character . */
-    while (PARSER_CUR && *(parser->cur) == ' ')
-	    parser->cur++;
+    while (PARSER_CUR && g_utf8_get_char (parser->cur) == ' ')
+	    parser->cur = g_utf8_next_char (parser->cur);
     
     while (PARSER_CUR
-	   && *(parser->cur) != ' '
-	   && ( (*parser->cur != '\\') || ((*parser->cur == '\\') && (*(parser->cur+1) == '\"')) )
-	   && *(parser->cur) != '\n') {    
-	if ((*parser->cur == '\\') && (*(parser->cur+1) == '\"')) {
-	    parser->cur += 2;
+	   && g_utf8_get_char (parser->cur) != ' '
+	   && ( (g_utf8_get_char (parser->cur) != '\\') || 
+	        (
+		 (g_utf8_get_char(parser->cur) == '\\') && 
+		 (g_utf8_get_char(g_utf8_next_char (parser->cur)) == '\"')
+		) 
+	      )
+	   && g_utf8_get_char (parser->cur) != '\n') {    
+	if (
+	    (g_utf8_get_char (parser->cur) == '\\') && 
+	    (g_utf8_get_char (g_utf8_next_char (parser->cur)) == '\"')
+	   ) {
+	    parser->cur = g_utf8_next_char (g_utf8_next_char (parser->cur));
 	    break;
 	}
-	parser->cur++;
+	parser->cur = g_utf8_next_char (parser->cur);
     }
 
     /* copy the macro/request into str */
@@ -1086,36 +1148,39 @@ parser_handle_linetag (YelpManParser *parser) {
     
     /* FIXME: need to handle escaped characters */
     /* perform argument parsing and store argument in a singly linked list */
-    while (PARSER_CUR && *(parser->cur) != '\n') { 
+    while (PARSER_CUR && g_utf8_get_char (parser->cur) != '\n') { 
 	ptr = NULL;
 	arg = NULL;
 	    
 	/* skip any whitespace */
-	while (PARSER_CUR && *(parser->cur) == ' ')
-	    parser->anc = ++parser->cur;
+	while (PARSER_CUR && g_utf8_get_char (parser->cur) == ' ') {
+	    parser->cur = g_utf8_next_char (parser->cur);
+	    parser->anc = parser->cur;
+	}
 	
 get_argument:
 	/* search until we hit whitespace or an " */
 	while (PARSER_CUR && 
-               *(parser->cur) != '\n' &&
-	       *(parser->cur) != ' ' &&
-	       *(parser->cur) != '\"')
-		parser->cur++;
+               g_utf8_get_char (parser->cur) != '\n' &&
+	       g_utf8_get_char (parser->cur) != ' ' &&
+	       g_utf8_get_char (parser->cur) != '\"')
+		parser->cur = g_utf8_next_char (parser->cur);
 
 	/* this checks for escaped spaces */
 	if (PARSER_CUR && 
 	    ((parser->cur - parser->buffer) > 0) &&
-	    *(parser->cur) == ' ' &&
-	    *(parser->cur-1) == '\\') {
-		parser->cur++;
+	    g_utf8_get_char (parser->cur) == ' ' &&
+	    g_utf8_get_char (g_utf8_prev_char (parser->cur)) == '\\') {
+		parser->cur = g_utf8_next_char (parser->cur);
 		goto get_argument;
 	}
 	
-	if (*(parser->cur) == '\n' && (parser->cur == parser->anc)) {
+	if (g_utf8_get_char (parser->cur) == '\n' && 
+	    (parser->cur == parser->anc))
 	    break;
-	}
 	
-	if (*(parser->cur) == '\"' && *(parser->cur-1) == ' ') {
+	if (g_utf8_get_char (parser->cur) == '\"' && 
+	    g_utf8_get_char (g_utf8_prev_char (parser->cur)) == ' ') {
 	    /* quoted argument */
 	    ptr = strchr (parser->cur+1, '\"');
 	    if (ptr != NULL) {
@@ -1266,9 +1331,9 @@ parser_read_until (YelpManParser *parser,
     gchar c;
     
     while (PARSER_CUR
-	   && *(parser->cur) != '\n'
-	   && *(parser->cur) != delim) {
-	    parser->cur++;
+	   && g_utf8_get_char (parser->cur) != '\n'
+	   && g_utf8_get_char (parser->cur) != delim) {
+	    parser->cur = g_utf8_next_char (parser->cur);
     }
 
     if (parser->anc == parser->cur)
@@ -1566,7 +1631,7 @@ parser_append_text (YelpManParser *parser)
     c = *(parser->cur);
     *(parser->cur) = '\0';
 
-    if (*(parser->anc) != '\n')
+    if (g_utf8_get_char (parser->anc) != '\n')
 	parser_ensure_P (parser);
 
     node = xmlNewText (BAD_CAST parser->anc);
diff --git a/src/yelp-man-parser.h b/src/yelp-man-parser.h
index 437ea86f..26553e0a 100644
--- a/src/yelp-man-parser.h
+++ b/src/yelp-man-parser.h
@@ -32,7 +32,8 @@ typedef struct _YelpManParser YelpManParser;
 
 YelpManParser *     yelp_man_parser_new         (void);
 xmlDocPtr           yelp_man_parser_parse_file  (YelpManParser   *parser,
-						 gchar           *file);
+						 gchar           *file,
+						 const gchar     *encoding);
 xmlDocPtr           yelp_man_parser_parse_doc   (YelpManParser   *parser,
 						 YelpDocInfo     *doc);
 void                yelp_man_parser_free        (YelpManParser   *parser);
diff --git a/src/yelp-toc-pager.c b/src/yelp-toc-pager.c
index 6ecd71ce..60faf670 100644
--- a/src/yelp-toc-pager.c
+++ b/src/yelp-toc-pager.c
@@ -87,6 +87,7 @@ struct _YelpTocPagerPriv {
     GSList *mandir_langpath;     /* ptr to current entry in mandir_ptr */
     GHashTable *man_secthash;
     GHashTable *man_manhash;
+    GHashTable *man_dirlang;     /* key = man page directory, value = language */
 #endif
 
     xmlDocPtr     toc_doc;
@@ -978,6 +979,14 @@ add_man_page_to_toc (YelpTocPager *pager, gchar *dirname, gchar *filename)
 	    info = yelp_doc_info_get (url_full, TRUE);
 
 	    if (info) {
+		gchar *lang = NULL;
+
+		if (priv->man_dirlang)
+		    lang = g_hash_table_lookup (priv->man_dirlang, dirname);
+
+		if (lang)
+		    yelp_doc_info_set_language (info, lang);
+
 		yelp_doc_info_add_uri (info, url_short, YELP_URI_TYPE_MAN);
 		tmp = xmlNewChild (tmp, NULL, BAD_CAST "doc", NULL);
 		xmlNewProp (tmp, BAD_CAST "href", BAD_CAST url_full);
@@ -1038,11 +1047,14 @@ create_manindex_file (gchar *index_file, xmlDocPtr xmldoc)
 static int
 create_toc_from_index (YelpTocPager *pager, gchar *index_file)
 {
+    const gchar * const * langs = g_get_language_names ();
     xmlXPathContextPtr xpath = NULL;
     xmlXPathObjectPtr objsect = NULL;
     xmlDocPtr manindex_xml = NULL;
+    xmlNodePtr root = NULL;
+    xmlChar *language = NULL;
     gint update_flag = 0;
-    gint i, j, k;
+    gint i, j;
     
     YelpTocPagerPriv *priv  = YELP_TOC_PAGER (pager)->priv;
 
@@ -1054,6 +1066,22 @@ create_toc_from_index (YelpTocPager *pager, gchar *index_file)
 	return 0;
     }
     
+    /* check the language that this index file was generated for */
+    root = xmlDocGetRootElement (manindex_xml);
+    language = xmlGetProp (root, BAD_CAST "lang");
+    
+    /* if the language is not the same as the current language (specified
+     * by the LANGUAGE environment variable) then return so that the index 
+     * is recreated */
+    if (language == NULL || !g_str_equal (BAD_CAST language, langs[0])) {
+	g_print (_("Current language and index file language do not match.\n"
+		   "The index file will be recreated.\n"));
+	xmlFreeDoc (manindex_xml);
+	return 0;
+    }
+
+    xmlFree (language);
+    
     xpath = xmlXPathNewContext (manindex_xml);
     objsect = xmlXPathEvalExpression (BAD_CAST "/manindex/mansect", xpath);
 
@@ -1068,28 +1096,29 @@ create_toc_from_index (YelpTocPager *pager, gchar *index_file)
 	objdirs = xmlXPathEvalExpression (BAD_CAST "dir", xpath);
 
 	for (j=0; j < objdirs->nodesetval->nodeNr; j++) {
-	    xmlXPathObjectPtr objdirname;
-	    xmlXPathObjectPtr objdirmtime;
-	    xmlXPathObjectPtr objmanpages;
 	    xmlNodePtr dirnode = objdirs->nodesetval->nodeTab[j];
 	    xmlNodePtr node = NULL;
 	    xmlChar *dirname = NULL;
 	    xmlChar *dirmtime = NULL;
+	    xmlChar *lang = NULL;
 	    time_t mtime;
 	    struct stat buf;
 	    
-	    xpath->node = dirnode;
-	    objdirmtime = xmlXPathEvalExpression (BAD_CAST "@mtime", xpath);
-	    
-	    node = objdirmtime->nodesetval->nodeTab[0];
-	    dirmtime = xmlNodeListGetString (manindex_xml, 
-	                                     node->xmlChildrenNode, 1);
-	    
-	    objdirname = xmlXPathEvalExpression (BAD_CAST "name[1]", xpath);
+            lang     = xmlGetProp (dirnode, BAD_CAST "lang");
+	    dirmtime = xmlGetProp (dirnode, BAD_CAST "mtime");
+
+	    if (lang == NULL)
+		lang = xmlStrdup (BAD_CAST "C");
+
+	    for (node = dirnode->children; node != NULL; node = node->next) {
+		if (node->type == XML_ELEMENT_NODE && 
+		    g_str_equal ((gchar *)node->name, "name")) {
+		    dirname = xmlNodeGetContent (node);
+		}
+	    }
 
-	    node = objdirname->nodesetval->nodeTab[0];
-	    dirname = xmlNodeListGetString (manindex_xml, 
-	                                    node->xmlChildrenNode, 1);
+	    g_hash_table_insert (priv->man_dirlang, g_strdup ((gchar *)dirname), 
+	                         g_strdup ((gchar *)lang));
 	    
 	    /* if we can't stat the dirname for some reason, then skip adding
 	     * this directory to the TOC, remove the dirnode, and set the flag
@@ -1143,25 +1172,22 @@ create_toc_from_index (YelpTocPager *pager, gchar *index_file)
 	    /* otherwise just read from the index file */
 	    } else {
 	    
-		objmanpages = xmlXPathEvalExpression (BAD_CAST "page", xpath);
-
-		for (k=0; k < objmanpages->nodesetval->nodeNr; k++) {
-		    xmlNodePtr node = objmanpages->nodesetval->nodeTab[k];
+		for (node = dirnode->children; node != NULL; node = node->next) {
 		    xmlChar *manpage = NULL;
 
-		    manpage = xmlNodeListGetString (manindex_xml,
-		                                    node->xmlChildrenNode, 1);
-
-		    add_man_page_to_toc (pager, (gchar *)dirname, (gchar *)manpage);
-		    priv->manpage_count++;
-		    xmlFree (manpage);
-	        }
-		xmlXPathFreeObject (objmanpages);
+		    if (node->type == XML_ELEMENT_NODE && 
+		        g_str_equal ((gchar *)node->name, "page")) {
+			manpage = xmlNodeGetContent (node);
+			add_man_page_to_toc (pager, (gchar *)dirname, (gchar *)manpage);
+			priv->manpage_count++;
+			xmlFree (manpage);
+		    }
+		}
+	    
 	    }
+	    xmlFree (lang);
 	    xmlFree (dirmtime);
 	    xmlFree (dirname);
-	    xmlXPathFreeObject (objdirmtime);
-	    xmlXPathFreeObject (objdirname);
 	}
 
 	/* cleanup */
@@ -1242,6 +1268,10 @@ process_mandir_pending (YelpTocPager *pager)
 	gchar **manpaths = NULL;
 	gchar **mandirs = NULL;
 
+	if (!priv->man_dirlang)
+	    priv->man_dirlang = g_hash_table_new_full (g_str_hash, g_str_equal,
+	                                               g_free,     g_free);
+
 	/* check for the existence of the xml cache file in ~/.gnome2/yelp.d/
 	 * if it exists, use it as the source for man pages instead of 
 	 * searching the hard disk for them - should make startup much faster */
@@ -1256,6 +1286,7 @@ process_mandir_pending (YelpTocPager *pager)
 	mandirs = yelp_get_man_paths ();
 	priv->manindex_xml = xmlNewDoc (BAD_CAST "1.0");
 	priv->root = xmlNewNode (NULL, BAD_CAST "manindex");
+	xmlNewProp (priv->root, BAD_CAST "lang", BAD_CAST langs[0]);
 	priv->ins  = priv->root;
 
 	xmlDocSetRootElement (priv->manindex_xml, priv->root);
@@ -1273,6 +1304,33 @@ process_mandir_pending (YelpTocPager *pager)
 	manpaths = g_strsplit (manpath, G_SEARCHPATH_SEPARATOR_S, -1);
 	g_free (manpath);
 
+	/* order is important here, since we may encounter collisions when adding 
+	 * a man page to the hash table "man_manhash".
+	 *
+	 * We want the resulting list to be sorted in order of priority:
+	 * 1) Highest priority is given to the base directory name returned 
+	 *    from $MANPATH or the manpath program.  So if that is 
+	 *    /usr/local/share:/usr/share then the man page 
+	 *    /usr/local/share/man/man1/python.1.gz would have precedence over
+	 *    /usr/share/man/man1/python.1.gz 
+	 * 2) Next highest priority is given to the language.
+	 * 3) Lowest priority is given to the section name, i.e. "man0p" "man1"
+	 *    since a man page from one section should never conflict with a man page
+	 *    from another section (because of the .0p or .1 section in the filename)
+	 *
+	 * Here is an example of how it should be sorted when 
+	 *   $MANPATH=/usr/local/share:/usr/share and LANGUAGE="es"
+	 *   
+	 * Adding /usr/local/share/man/es/man0p
+	 * Adding /usr/share/man/es/man0p
+	 * Adding /usr/local/share/man/man0p
+	 * Adding /usr/share/man/man0p
+	 * Adding /usr/local/share/man/es/man1
+	 * Adding /usr/share/man/es/man1
+	 * Adding /usr/local/share/man/man1
+	 * Adding /usr/share/man/man1 
+	 */
+	
 	for (i=0; mandirs[i] != NULL; i++) {
 	    GSList *tmplist = NULL;
 
@@ -1285,12 +1343,31 @@ process_mandir_pending (YelpTocPager *pager)
 			                            mandirs[i],
 			                            NULL);
 
+		    g_hash_table_insert (priv->man_dirlang, g_strdup (dirname),
+		                         g_strdup (langs[j]));
+
+		    /* prepend to list for speed, reverse it at the end */
 		    tmplist = g_slist_prepend (tmplist, dirname);
 		}
 	    }
+	    tmplist = g_slist_reverse (tmplist);
 
 	    priv->mandir_list = g_slist_prepend (priv->mandir_list, tmplist);
 	}
+	priv->mandir_list = g_slist_reverse (priv->mandir_list);
+
+	/* debugging: print out lists in order */
+	/*GSList *list1 = priv->mandir_list;
+	GSList *list2 = NULL;
+
+	while (list1 != NULL) {
+	    list2 = list1->data;
+	    while (list2 && list2->data) {
+	        g_print ("Dir=%s\n", (gchar *)list2->data);
+	        list2 = g_slist_next (list2);
+	    }
+	    list1 = g_slist_next (list1);
+	}*/
 
 	g_strfreev (manpaths);
  
@@ -1311,6 +1388,7 @@ process_mandir_pending (YelpTocPager *pager)
 	if ((dir = g_dir_open (dirname, 0, NULL))) {
 	    struct stat buf;
 	    gchar mtime_str[20];
+	    gchar *lang = NULL;
 
 	    if (g_stat (dirname, &buf) < 0)
 		g_warning ("Unable to stat dir: \"%s\"\n", dirname);
@@ -1327,7 +1405,17 @@ process_mandir_pending (YelpTocPager *pager)
 
 	    g_snprintf (mtime_str, 20, "%u", (guint) buf.st_mtime);
 
+	    lang = g_hash_table_lookup (priv->man_dirlang, dirname);
+
 	    priv->ins = xmlNewChild (priv->ins, NULL, BAD_CAST "dir", NULL);
+
+	    if (lang == NULL) {
+		g_print ("dir %s is null\n", dirname);
+		lang = "C";
+	    }
+	    
+	    xmlNewProp (priv->ins, BAD_CAST "lang", BAD_CAST lang);
+
 	    xmlNewProp (priv->ins, BAD_CAST "mtime", BAD_CAST mtime_str);
 	    xmlAddChild (priv->ins, xmlNewText (BAD_CAST "\n      "));
 	    xmlNewChild (priv->ins, NULL, BAD_CAST "name", BAD_CAST dirname);
@@ -1799,11 +1887,19 @@ process_cleanup (YelpTocPager *pager)
 {
     YelpTocPagerPriv *priv = pager->priv;
 
+#ifdef ENABLE_MAN
+    /* clean up the man directory language hash table */
+    if (priv->man_dirlang) {
+        g_hash_table_destroy (priv->man_dirlang);
+        priv->man_dirlang = NULL;
+    }
+
     /* clean up the man page section hash table */
     if (priv->man_secthash) {
 	g_hash_table_destroy (priv->man_secthash);
 	priv->man_secthash = NULL;
     }
+#endif
 
     /* cleanup the stylesheet used to process the toc */
     if (priv->stylesheet) {
@@ -1811,6 +1907,12 @@ process_cleanup (YelpTocPager *pager)
 	priv->stylesheet = NULL;
     }
 
+    /* cleanup the parser context */
+    if (priv->parser) {
+	xmlFreeParserCtxt (priv->parser);
+	priv->parser = NULL;
+    }
+
     /* we only ever want to run this function once, so always return false */
     return FALSE;
 }
diff --git a/stylesheets/man2html.xsl b/stylesheets/man2html.xsl
index e422eca8..b9397928 100644
--- a/stylesheets/man2html.xsl
+++ b/stylesheets/man2html.xsl
@@ -6,7 +6,7 @@
                 extension-element-prefixes="yelp"
                 version="1.0">
 
-<xsl:output method="html"/>
+<xsl:output method="html" encoding="UTF-8"/>
 
 <xsl:include href="yelp-common.xsl"/>