Add:maptool:Option to keep tile timestampstile-versions

New maptool option -F allows to specify a reFerence map file to which tiles
of the new map file would be compared. Tiles which have the same contents as
in the previous map version, would also have timestamps inherited from the
old map file.

6 files changed, 180 insertions, 22 deletions

diff --git a/navit/map/binfile/binfile.c b/navit/map/binfile/binfile.c
index 9c4686c90..3ecdf1d94 100644
--- a/navit/map/binfile/binfile.c
+++ b/navit/map/binfile/binfile.c
@@ -340,8 +340,8 @@ binfile_extract(struct map_priv *m, char *dir, char *filename, int partial)
 			file_mkdir(fulld, 1);
 		}
 		if (full[len-2] != '/') {
-			lfh=zipfile_read_lfh(m->fi, zipfile_cd_offset(cd));
-			start=binfile_read_content(m, m->fi, zipfile_cd_offset(cd), lfh);
+			lfh=zipfile_read_lfh(m->fi, zipfile_lfh_offset(cd));
+			start=binfile_read_content(m, m->fi, zipfile_lfh_offset(cd), lfh);
 			dbg(lvl_debug,"fopen '%s'\n", full);
 			f=fopen(full,"w");
 			fwrite(start, lfh->zipuncmp, 1, f);
@@ -726,18 +726,18 @@ zipfile_to_tile(struct map_priv *m, struct zip_cd *cd, struct tile *t)
 	char *zipfn;
 	struct file *fi;
 	dbg(lvl_debug,"enter %p %p %p\n", m, cd, t);
-	dbg(lvl_debug,"cd->zipofst=0x"LONGLONG_HEX_FMT "\n", zipfile_cd_offset(cd));
+	dbg(lvl_debug,"cd->zipofst=0x"LONGLONG_HEX_FMT "\n", zipfile_lfh_offset(cd));
 	t->start=NULL;
 	t->mode=1;
 	if (m->fis)
 		fi=m->fis[cd->zipdsk];
 	else
 		fi=m->fi;
-	lfh=zipfile_read_lfh(fi, zipfile_cd_offset(cd));
-	zipfn=(char *)(file_data_read(fi,zipfile_cd_offset(cd)+sizeof(struct zip_lfh), lfh->zipfnln));
+	lfh=zipfile_read_lfh(fi, zipfile_lfh_offset(cd));
+	zipfn=(char *)(file_data_read(fi,zipfile_lfh_offset(cd)+sizeof(struct zip_lfh), lfh->zipfnln));
 	strncpy(buffer, zipfn, lfh->zipfnln);
 	buffer[lfh->zipfnln]='\0';
-	t->start=(int *)binfile_read_content(m, fi, zipfile_cd_offset(cd), lfh);
+	t->start=(int *)binfile_read_content(m, fi, zipfile_lfh_offset(cd), lfh);
 	t->end=t->start+lfh->zipuncmp/4;
 	t->fi=fi;
 	file_data_free(fi, (unsigned char *)zipfn);
@@ -891,7 +891,7 @@ download_request(struct map_download *download)
 		url.u.str=g_strdup_printf("%smemberid=%d",download->m->url,download->zipfile);
 		download->dl_size=-1;
 	} else {
-		long long offset=zipfile_cd_offset(download->cd_copy);
+		long long offset=zipfile_lfh_offset(download->cd_copy);
 		int size=download->cd_copy->zipcsiz+sizeof(struct zip_lfh)+download->cd_copy->zipcfnl;
 		url.u.str=g_strdup(download->m->url);
 		http_header.u.str=g_strdup_printf("Range: bytes="LONGLONG_FMT"-"LONGLONG_FMT,offset,offset+size-1);
@@ -978,7 +978,7 @@ download_finish(struct map_download *download)
 	download->cd_copy->zipcsiz=lfh->zipsize;
 	download->cd_copy->zipcunc=lfh->zipuncmp;
 	download->cd_copy->zipccrc=lfh->zipcrc;
-	lfh_offset = zipfile_cd_offset(download->cd_copy)+sizeof(struct zip_lfh);
+	lfh_offset = zipfile_lfh_offset(download->cd_copy)+sizeof(struct zip_lfh);
 	lfh_filename=(char *)file_data_read(download->file,lfh_offset,lfh->zipfnln);
 	memcpy(download->cd_copy+1,lfh_filename,lfh->zipfnln);
 	file_data_remove(download->file,(void *)lfh_filename);
diff --git a/navit/maptool/maptool.c b/navit/maptool/maptool.c
index 815eb9fff..a586b80dc 100644
--- a/navit/maptool/maptool.c
+++ b/navit/maptool/maptool.c
@@ -46,6 +46,7 @@
 #include "plugin.h"
 #include "util.h"
 #include "maptool.h"
+#include "debug.h"
 
 #define SLIZE_SIZE_DEFAULT_GB 1
 long long slice_size=SLIZE_SIZE_DEFAULT_GB*1024ll*1024*1024;
@@ -285,9 +286,11 @@ usage(FILE *f)
 #ifdef HAVE_POSTGRESQL
 	fprintf(f,"-d (--db) <conn. string>          : get osm data out of a postgresql database with osm simple scheme and given connect string\n");
 #endif
+	fprintf(f,"-g (--debug) <level>              : set global debug level to a given value\n");
 	fprintf(f,"-e (--end) <phase>                : end at specified phase\n");
 	fprintf(f,"-E (--experimental)               : Enable experimental features (%s)\n",
 		experimental_feature_description ? experimental_feature_description : "-not available in this version-");
+	fprintf(f,"-F (--reference-map) <file>       : compare resulting map to its previous version to preserve timestamps of unmodified tiles\n");
 	fprintf(f,"-i (--input-file) <file>          : specify the input file name (OSM), overrules default stdin\n");
 	fprintf(f,"-k (--keep-tmpfiles)              : do not delete tmp files after processing. useful to reuse them\n");
 	fprintf(f,"-M (--o5m)                        : input file os o5m\n");
@@ -343,6 +346,7 @@ struct maptool_params {
 	int countries_loaded;
 	int tilesdir_loaded;
 	int max_index_size;
+	char *reference_map;
 };
 
 static int
@@ -362,11 +366,13 @@ parse_option(struct maptool_params *p, char **argv, int argc, int *option_index)
 #ifdef HAVE_POSTGRESQL
 		{"db", 1, 0, 'd'},
 #endif
+		{"debug", 1, 0, 'g'},
 		{"dedupe-ways", 0, 0, 'w'},
 		{"dump", 0, 0, 'D'},
 		{"dump-coordinates", 0, 0, 'c'},
 		{"end", 1, 0, 'e'},
 		{"experimental", 0, 0, 'E'},
+		{"reference-map", 1, 0, 'F'},
 		{"help", 0, 0, 'h'},
 		{"keep-tmpfiles", 0, 0, 'k'},
 		{"nodes-only", 0, 0, 'N'},
@@ -386,7 +392,7 @@ parse_option(struct maptool_params *p, char **argv, int argc, int *option_index)
 		{"index-size", 0, 0, 'x'},
 		{0, 0, 0, 0}
 	};
-	c = getopt_long (argc, argv, "5:6B:DEMNO:PS:Wa:bc"
+	c = getopt_long (argc, argv, "5:6B:DEF:g:MNO:PS:Wa:bc"
 #ifdef HAVE_POSTGRESQL
 				      "d:"
 #endif
@@ -409,6 +415,9 @@ parse_option(struct maptool_params *p, char **argv, int argc, int *option_index)
 	case 'E':
 		experimental=1;
 		break;
+	case 'g':
+		debug_set_global_level(atoi(optarg),1);
+		break;
 	case 'M':
 		p->o5m=1;
 		break;	
@@ -451,6 +460,9 @@ parse_option(struct maptool_params *p, char **argv, int argc, int *option_index)
 	case 'e':
 		p->end=atoi(optarg);
 		break;
+	case 'F':
+		p->reference_map=g_strdup(optarg);
+		break;
 	case 'h':
 		return 2;
 	case 'm':
@@ -789,7 +801,7 @@ maptool_generate_tiles(struct maptool_params *p, char *suffix, char **filenames,
 }
 
 static void
-maptool_assemble_map(struct maptool_params *p, char *suffix, char **filenames, char **referencenames, int filename_count, int first, int last, char *suffix0)
+maptool_assemble_map(struct maptool_params *p, char *suffix, char **filenames, char **referencenames, int filename_count, int first, int last, char *suffix0, struct zip_hashed_cd *zhc)
 {
 	FILE *files[10];
 	FILE *references[10];
@@ -810,11 +822,14 @@ maptool_assemble_map(struct maptool_params *p, char *suffix, char **filenames, c
 			fprintf(stderr,"Fatal: Could not write output file.\n");
 			exit(1);
 		}
+		zip_set_reference_map(zip_info, zhc);
 		if (p->url) {
 			map_information_attrs[1].type=attr_url;
 			map_information_attrs[1].u.str=p->url;
 		}
 		index_init(zip_info, 1);
+		g_free(zipindex);
+		g_free(zipdir);
 	}
 	if (!strcmp(suffix,ch_suffix)) {  /* Makes compiler happy due to bug 35903 in gcc */
 		ch_assemble_map(suffix0,suffix,zip_info);
@@ -861,6 +876,7 @@ maptool_assemble_map(struct maptool_params *p, char *suffix, char **filenames, c
 		zip_write_index(zip_info);
 		zip_write_directory(zip_info);
 		zip_close(zip_info);
+		zip_destroy(zip_info);
 		if (p->md5file && zip_get_md5(zip_info, md5_data)) {
 			FILE *md5=fopen(p->md5file,"w");
 			int i;
@@ -921,7 +937,9 @@ int main(int argc, char **argv)
 	int i;
 	int suffix_start=0;
 	int option_index=0;
+	struct zip_hashed_cd *zhc=NULL;
 	main_init(argv[0]);
+	debug_init(argv[0]);
 #ifndef HAVE_GLIB
 	_g_slice_thread_init_nomessage();
 #endif
@@ -1100,12 +1118,15 @@ int main(int argc, char **argv)
 			p.tilesdir_loaded=1;
 		}
 		if (start_phase(&p,"assembling map")) {
+			if(!zhc && p.reference_map)
+				zhc=zip_hashed_cd_new(p.reference_map);
 			maptool_load_countries(&p);
 			maptool_load_tilesdir(&p, suffix);
-			maptool_assemble_map(&p, suffix, filenames, referencenames, filename_count, i == suffix_start, i == suffix_count-1, suffixes[0]);
+			maptool_assemble_map(&p, suffix, filenames, referencenames, filename_count, i == suffix_start, i == suffix_count-1, suffixes[0], zhc);
 		}
 		phase-=2;
 	}
+	zip_hashed_cd_free(zhc);
 	phase+=2;
 	start_phase(&p,"done");
 	return 0;
diff --git a/navit/maptool/maptool.h b/navit/maptool/maptool.h
index eca708591..f4fdf2eb7 100644
--- a/navit/maptool/maptool.h
+++ b/navit/maptool/maptool.h
@@ -417,7 +417,12 @@ FILE *zip_get_index(struct zip_info *info);
 int zip_get_zipnum(struct zip_info *info);
 void zip_set_zipnum(struct zip_info *info, int num);
 void zip_close(struct zip_info *info);
+int zip_load_reference_map(struct zip_info *info, char *reference_map_name);
 void zip_destroy(struct zip_info *info);
+struct zip_hashed_cd;
+void zip_set_reference_map(struct zip_info *zip_info, struct zip_hashed_cd *zhc);
+struct zip_hashed_cd *zip_hashed_cd_new(char *map_filename);
+void zip_hashed_cd_free(struct zip_hashed_cd *zhc);
 
 /* Break compilation on 32 bit architectures, as we're going to cast osmid's to gpointer to use them as keys to GHashTable's */
 struct maptool_force_64 {char s[sizeof(gpointer)<sizeof(osmid)?-1:1];};
diff --git a/navit/maptool/zip.c b/navit/maptool/zip.c
index f352e6353..9d034df71 100644
--- a/navit/maptool/zip.c
+++ b/navit/maptool/zip.c
@@ -17,6 +17,7 @@
  * Boston, MA  02110-1301, USA.
  */
 
+#include <glib.h>
 #include <zlib.h>
 #include <string.h>
 #include <stdlib.h>
@@ -24,6 +25,7 @@
 #include "maptool.h"
 #include "config.h"
 #include "zipfile.h"
+#include "attr.h"
 
 #ifdef HAVE_LIBCRYPTO
 #include <openssl/sha.h>
@@ -51,7 +53,119 @@ struct zip_info {
 	MD5_CTX md5_ctx;
 #endif
 	int md5;
+	struct zip_hashed_cd *reference_map;
 };
+/*
+ * Helper structure to zip file handle with hashtable of local file header pointers
+*/
+struct zip_hashed_cd {
+	struct file *fi;
+	/* hash table to access zip_lfh offsets by file name */
+	GHashTable *lfh_offsets;
+};
+
+void
+zip_set_reference_map(struct zip_info *zip_info, struct zip_hashed_cd *zhc)
+{
+	zip_info->reference_map=zhc;
+}
+
+/**
+ * @brief fill zip_hashed_cd structure with references to tile file local file headers.
+ * @param map_filename name of file holding reference map
+ * @return Pointer to allocated and filled zip_hashed_cd structure. Free with zip_hashed_cd_free().
+ */
+struct zip_hashed_cd *
+zip_hashed_cd_new(char *map_filename)
+{
+	struct zip_eoc *eoc;
+	struct zip64_eoc *eoc64;
+	struct zip_cd *cd;
+	struct attr a={attr_cache, {0}};
+	struct attr *opts[]={&a, NULL};
+	struct zip_hashed_cd *zhc=g_new(struct zip_hashed_cd,1);
+	unsigned long long offset=0;
+	zhc->fi=file_create(map_filename,opts);
+	if(!zhc->fi) {
+		g_free(zhc);
+		return NULL;
+	}
+	eoc=zipfile_read_eoc(zhc->fi);
+	if(!eoc) {
+		file_destroy(zhc->fi);
+		g_free(zhc);
+		return NULL;
+	}
+
+	eoc64=zipfile_read_eoc64(zhc->fi);
+
+	zhc->lfh_offsets=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+
+	while((cd=zipfile_read_cd(zhc->fi, eoc, eoc64, offset,-1))!=NULL) {
+		char *filename=g_strndup(cd->zipcfn, cd->zipcfnl);
+		if(!g_hash_table_insert(zhc->lfh_offsets, filename, (void*)zipfile_lfh_offset(cd)))
+			dbg(lvl_error,"Duplicate tile name: %s\n", filename);
+		offset+=sizeof(*cd)+zipfile_cd_name_and_extra_len(cd);
+		file_data_free(zhc->fi, (unsigned char *)cd);
+	}
+
+	file_data_free(zhc->fi, (unsigned char *)eoc64);
+	file_data_free(zhc->fi, (unsigned char *)eoc);
+
+	return zhc;
+}
+
+void
+zip_hashed_cd_free(struct zip_hashed_cd *zhc)
+{
+	if(zhc) {
+		file_destroy(zhc->fi);
+		g_hash_table_destroy(zhc->lfh_offsets);
+		g_free(zhc);
+	}
+}
+
+/**
+ * @brief Adjust timestamp of a given file if it was already known in the reference version of mapfile. 
+ * @param filename name of tile file being examined
+ * @param new_data uncompressed contents of the tile file being written
+ * @param new_len uncompressed size of the tile file being written
+ * @param zhc pointer to structure used to access reference map
+ * @param @out date pointer to variable containing date to adjust
+ * @param @out time pointer to variable containing time to adjust
+ * @return nothing.
+ */
+static void
+adjust_timestamp(char *filename, void *new_data, int new_len, struct zip_hashed_cd *zhc, short *date, short *time)
+{
+	long long offset;
+	struct zip_lfh *lfh;
+
+	if( !zhc || !zhc->fi || !zhc->lfh_offsets )
+		return;
+
+	if( !g_hash_table_lookup_extended(zhc->lfh_offsets, filename, NULL, (void **)&offset)) {
+		dbg(lvl_info,"Tile not found in reference map: %s\n",filename);
+		return;
+	}
+
+	if( (lfh=zipfile_read_lfh(zhc->fi,offset)) == NULL ) {
+		dbg(lvl_error,"Error reading tile header %s from reference map at offset "LONGLONG_FMT"\n",filename, offset);
+		return;
+	}
+
+	if( lfh->zipuncmp==new_len ) {
+		void *old_data=zipfile_read_content(zhc->fi, offset, lfh, NULL);
+		if( old_data && memcmp(old_data, new_data, new_len)==0 ) {
+			dbg(lvl_debug,"Adjusting timestamp for tile %s\n",filename);
+			*date=lfh->zipdate;
+			*time=lfh->ziptime;
+		}
+		file_data_free(zhc->fi, (unsigned char *)old_data);
+	}
+
+	file_data_free(zhc->fi, (unsigned char *)lfh);
+}
 
 static int
 zip_write(struct zip_info *info, void *data, int len)
@@ -97,6 +211,17 @@ compress2_int(Byte *dest, uLongf *destLen, const Bytef *source, uLong sourceLen,
 }
 #endif
 
+/**
+ * @brief Write a single member file to the zip file, compressing and encrypting it on the way.
+ *  Supports zip64 in the sense of handling zip files above 4Gb, but does not allow individual files above zip32 limits.
+ *  Does not support writing multivolume zip files.
+ * @param zip_info pointer to structure holding global zip file informations
+ * @param name name of file being written (not necessarily nul-terminated)
+ * @param filelen length of file name to pad to
+ * @param data uncopmpressed contents of file being written
+ * @param data_size uncopmpressed length of file being written
+ * @return nothing.
+ */
 void
 write_zipmember(struct zip_info *zip_info, char *name, int filelen, char *data, int data_size)
 {
@@ -121,8 +246,8 @@ write_zipmember(struct zip_info *zip_info, char *name, int filelen, char *data,
 		0x00,
 		0x0000,
 		0x0,
-		zip_info->time,
-		zip_info->date,
+		0,
+		0,
 		0x0,
 		0x0,
 		0x0,
@@ -155,6 +280,18 @@ write_zipmember(struct zip_info *zip_info, char *name, int filelen, char *data,
 	uLongf destlen=data_size+data_size/500+12;
 	char *compbuffer;
 
+	filename=g_alloca(filelen+1);
+	g_strlcpy(filename, name, filelen+1);
+	len=strlen(filename);
+	while (len < filelen) {
+		filename[len++]='_';
+	}
+	filename[filelen]='\0';
+
+	adjust_timestamp(filename, data, data_size, zip_info->reference_map, &lfh.zipdate, &lfh.ziptime);
+	cd.zipdat=lfh.zipdate;
+	cd.ziptim=lfh.ziptime;
+
 	compbuffer = malloc(destlen);
 	if (!compbuffer) {
 	  fprintf(stderr, "No more memory.\n");
@@ -215,13 +352,6 @@ write_zipmember(struct zip_info *zip_info, char *name, int filelen, char *data,
 		cd.zipcflg|=1;
 	}
 #endif
-	filename=g_alloca(filelen+1);
-	strcpy(filename, name);
-	len=strlen(filename);
-	while (len < filelen) {
-		filename[len++]='_';
-	}
-	filename[filelen]='\0';
 	zip_write(zip_info, &lfh, sizeof(lfh));
 	zip_write(zip_info, filename, filelen);
 	zip_info->offset+=sizeof(lfh)+filelen;
diff --git a/navit/zipfile.c b/navit/zipfile.c
index bda50b664..e4eabee1a 100644
--- a/navit/zipfile.c
+++ b/navit/zipfile.c
@@ -156,6 +156,8 @@ zipfile_read_cd(struct file *fi, struct zip_eoc *eoc, struct zip64_eoc *eoc64, i
 	long long cdoffset=eoc64?eoc64->zip64eofst:eoc->zipeofst;
 	if (len == -1) {
 		cd=(struct zip_cd *)file_data_read(fi,cdoffset+offset, sizeof(*cd));
+		if(!cd)
+			return NULL;
 		cd_to_cpu(cd);
 		len=zipfile_cd_name_and_extra_len(cd);
 		file_data_free(fi,(unsigned char *)cd);
@@ -201,7 +203,7 @@ zipfile_cd_ext(struct zip_cd *cd)
  * @return Offset of local file header in zip file.
  */
 long long
-zipfile_cd_offset(struct zip_cd *cd)
+zipfile_lfh_offset(struct zip_cd *cd)
 {
 	struct zip_cd_ext *ext=zipfile_cd_ext(cd);
 	if (ext)
diff --git a/navit/zipfile.h b/navit/zipfile.h
index 07ec5da57..4574db88a 100644
--- a/navit/zipfile.h
+++ b/navit/zipfile.h
@@ -181,7 +181,7 @@ struct zip64_eoc *zipfile_read_eoc64(struct file *fi);
 int zipfile_cd_name_and_extra_len(struct zip_cd *cd);
 struct zip_cd *zipfile_read_cd(struct file *fi, struct zip_eoc *eoc, struct zip64_eoc *eoc64, int offset, int len);
 struct zip_cd_ext *zipfile_cd_ext(struct zip_cd *cd);
-long long zipfile_cd_offset(struct zip_cd *cd);
+long long zipfile_lfh_offset(struct zip_cd *cd);
 struct zip_lfh *zipfile_read_lfh(struct file *fi, long long offset);
 unsigned char *zipfile_read_content(struct file *fi, long long offset, struct zip_lfh *lfh, char *passwd);
 void cd_to_cpu(struct zip_cd *zcd);