diff options
-rw-r--r-- | navit/map/binfile/binfile.c | 16 | ||||
-rw-r--r-- | navit/maptool/maptool.c | 27 | ||||
-rw-r--r-- | navit/maptool/maptool.h | 5 | ||||
-rw-r--r-- | navit/maptool/zip.c | 148 | ||||
-rw-r--r-- | navit/zipfile.c | 4 | ||||
-rw-r--r-- | navit/zipfile.h | 2 |
6 files changed, 180 insertions, 22 deletions
diff --git a/navit/map/binfile/binfile.c b/navit/map/binfile/binfile.c index 9c4686c90..3ecdf1d94 100644 --- a/navit/map/binfile/binfile.c +++ b/navit/map/binfile/binfile.c @@ -340,8 +340,8 @@ binfile_extract(struct map_priv *m, char *dir, char *filename, int partial) file_mkdir(fulld, 1); } if (full[len-2] != '/') { - lfh=zipfile_read_lfh(m->fi, zipfile_cd_offset(cd)); - start=binfile_read_content(m, m->fi, zipfile_cd_offset(cd), lfh); + lfh=zipfile_read_lfh(m->fi, zipfile_lfh_offset(cd)); + start=binfile_read_content(m, m->fi, zipfile_lfh_offset(cd), lfh); dbg(lvl_debug,"fopen '%s'\n", full); f=fopen(full,"w"); fwrite(start, lfh->zipuncmp, 1, f); @@ -726,18 +726,18 @@ zipfile_to_tile(struct map_priv *m, struct zip_cd *cd, struct tile *t) char *zipfn; struct file *fi; dbg(lvl_debug,"enter %p %p %p\n", m, cd, t); - dbg(lvl_debug,"cd->zipofst=0x"LONGLONG_HEX_FMT "\n", zipfile_cd_offset(cd)); + dbg(lvl_debug,"cd->zipofst=0x"LONGLONG_HEX_FMT "\n", zipfile_lfh_offset(cd)); t->start=NULL; t->mode=1; if (m->fis) fi=m->fis[cd->zipdsk]; else fi=m->fi; - lfh=zipfile_read_lfh(fi, zipfile_cd_offset(cd)); - zipfn=(char *)(file_data_read(fi,zipfile_cd_offset(cd)+sizeof(struct zip_lfh), lfh->zipfnln)); + lfh=zipfile_read_lfh(fi, zipfile_lfh_offset(cd)); + zipfn=(char *)(file_data_read(fi,zipfile_lfh_offset(cd)+sizeof(struct zip_lfh), lfh->zipfnln)); strncpy(buffer, zipfn, lfh->zipfnln); buffer[lfh->zipfnln]='\0'; - t->start=(int *)binfile_read_content(m, fi, zipfile_cd_offset(cd), lfh); + t->start=(int *)binfile_read_content(m, fi, zipfile_lfh_offset(cd), lfh); t->end=t->start+lfh->zipuncmp/4; t->fi=fi; file_data_free(fi, (unsigned char *)zipfn); @@ -891,7 +891,7 @@ download_request(struct map_download *download) url.u.str=g_strdup_printf("%smemberid=%d",download->m->url,download->zipfile); download->dl_size=-1; } else { - long long offset=zipfile_cd_offset(download->cd_copy); + long long offset=zipfile_lfh_offset(download->cd_copy); int size=download->cd_copy->zipcsiz+sizeof(struct zip_lfh)+download->cd_copy->zipcfnl; url.u.str=g_strdup(download->m->url); http_header.u.str=g_strdup_printf("Range: bytes="LONGLONG_FMT"-"LONGLONG_FMT,offset,offset+size-1); @@ -978,7 +978,7 @@ download_finish(struct map_download *download) download->cd_copy->zipcsiz=lfh->zipsize; download->cd_copy->zipcunc=lfh->zipuncmp; download->cd_copy->zipccrc=lfh->zipcrc; - lfh_offset = zipfile_cd_offset(download->cd_copy)+sizeof(struct zip_lfh); + lfh_offset = zipfile_lfh_offset(download->cd_copy)+sizeof(struct zip_lfh); lfh_filename=(char *)file_data_read(download->file,lfh_offset,lfh->zipfnln); memcpy(download->cd_copy+1,lfh_filename,lfh->zipfnln); file_data_remove(download->file,(void *)lfh_filename); diff --git a/navit/maptool/maptool.c b/navit/maptool/maptool.c index 815eb9fff..a586b80dc 100644 --- a/navit/maptool/maptool.c +++ b/navit/maptool/maptool.c @@ -46,6 +46,7 @@ #include "plugin.h" #include "util.h" #include "maptool.h" +#include "debug.h" #define SLIZE_SIZE_DEFAULT_GB 1 long long slice_size=SLIZE_SIZE_DEFAULT_GB*1024ll*1024*1024; @@ -285,9 +286,11 @@ usage(FILE *f) #ifdef HAVE_POSTGRESQL fprintf(f,"-d (--db) <conn. string> : get osm data out of a postgresql database with osm simple scheme and given connect string\n"); #endif + fprintf(f,"-g (--debug) <level> : set global debug level to a given value\n"); fprintf(f,"-e (--end) <phase> : end at specified phase\n"); fprintf(f,"-E (--experimental) : Enable experimental features (%s)\n", experimental_feature_description ? experimental_feature_description : "-not available in this version-"); + fprintf(f,"-F (--reference-map) <file> : compare resulting map to its previous version to preserve timestamps of unmodified tiles\n"); fprintf(f,"-i (--input-file) <file> : specify the input file name (OSM), overrules default stdin\n"); fprintf(f,"-k (--keep-tmpfiles) : do not delete tmp files after processing. useful to reuse them\n"); fprintf(f,"-M (--o5m) : input file os o5m\n"); @@ -343,6 +346,7 @@ struct maptool_params { int countries_loaded; int tilesdir_loaded; int max_index_size; + char *reference_map; }; static int @@ -362,11 +366,13 @@ parse_option(struct maptool_params *p, char **argv, int argc, int *option_index) #ifdef HAVE_POSTGRESQL {"db", 1, 0, 'd'}, #endif + {"debug", 1, 0, 'g'}, {"dedupe-ways", 0, 0, 'w'}, {"dump", 0, 0, 'D'}, {"dump-coordinates", 0, 0, 'c'}, {"end", 1, 0, 'e'}, {"experimental", 0, 0, 'E'}, + {"reference-map", 1, 0, 'F'}, {"help", 0, 0, 'h'}, {"keep-tmpfiles", 0, 0, 'k'}, {"nodes-only", 0, 0, 'N'}, @@ -386,7 +392,7 @@ parse_option(struct maptool_params *p, char **argv, int argc, int *option_index) {"index-size", 0, 0, 'x'}, {0, 0, 0, 0} }; - c = getopt_long (argc, argv, "5:6B:DEMNO:PS:Wa:bc" + c = getopt_long (argc, argv, "5:6B:DEF:g:MNO:PS:Wa:bc" #ifdef HAVE_POSTGRESQL "d:" #endif @@ -409,6 +415,9 @@ parse_option(struct maptool_params *p, char **argv, int argc, int *option_index) case 'E': experimental=1; break; + case 'g': + debug_set_global_level(atoi(optarg),1); + break; case 'M': p->o5m=1; break; @@ -451,6 +460,9 @@ parse_option(struct maptool_params *p, char **argv, int argc, int *option_index) case 'e': p->end=atoi(optarg); break; + case 'F': + p->reference_map=g_strdup(optarg); + break; case 'h': return 2; case 'm': @@ -789,7 +801,7 @@ maptool_generate_tiles(struct maptool_params *p, char *suffix, char **filenames, } static void -maptool_assemble_map(struct maptool_params *p, char *suffix, char **filenames, char **referencenames, int filename_count, int first, int last, char *suffix0) +maptool_assemble_map(struct maptool_params *p, char *suffix, char **filenames, char **referencenames, int filename_count, int first, int last, char *suffix0, struct zip_hashed_cd *zhc) { FILE *files[10]; FILE *references[10]; @@ -810,11 +822,14 @@ maptool_assemble_map(struct maptool_params *p, char *suffix, char **filenames, c fprintf(stderr,"Fatal: Could not write output file.\n"); exit(1); } + zip_set_reference_map(zip_info, zhc); if (p->url) { map_information_attrs[1].type=attr_url; map_information_attrs[1].u.str=p->url; } index_init(zip_info, 1); + g_free(zipindex); + g_free(zipdir); } if (!strcmp(suffix,ch_suffix)) { /* Makes compiler happy due to bug 35903 in gcc */ ch_assemble_map(suffix0,suffix,zip_info); @@ -861,6 +876,7 @@ maptool_assemble_map(struct maptool_params *p, char *suffix, char **filenames, c zip_write_index(zip_info); zip_write_directory(zip_info); zip_close(zip_info); + zip_destroy(zip_info); if (p->md5file && zip_get_md5(zip_info, md5_data)) { FILE *md5=fopen(p->md5file,"w"); int i; @@ -921,7 +937,9 @@ int main(int argc, char **argv) int i; int suffix_start=0; int option_index=0; + struct zip_hashed_cd *zhc=NULL; main_init(argv[0]); + debug_init(argv[0]); #ifndef HAVE_GLIB _g_slice_thread_init_nomessage(); #endif @@ -1100,12 +1118,15 @@ int main(int argc, char **argv) p.tilesdir_loaded=1; } if (start_phase(&p,"assembling map")) { + if(!zhc && p.reference_map) + zhc=zip_hashed_cd_new(p.reference_map); maptool_load_countries(&p); maptool_load_tilesdir(&p, suffix); - maptool_assemble_map(&p, suffix, filenames, referencenames, filename_count, i == suffix_start, i == suffix_count-1, suffixes[0]); + maptool_assemble_map(&p, suffix, filenames, referencenames, filename_count, i == suffix_start, i == suffix_count-1, suffixes[0], zhc); } phase-=2; } + zip_hashed_cd_free(zhc); phase+=2; start_phase(&p,"done"); return 0; diff --git a/navit/maptool/maptool.h b/navit/maptool/maptool.h index eca708591..f4fdf2eb7 100644 --- a/navit/maptool/maptool.h +++ b/navit/maptool/maptool.h @@ -417,7 +417,12 @@ FILE *zip_get_index(struct zip_info *info); int zip_get_zipnum(struct zip_info *info); void zip_set_zipnum(struct zip_info *info, int num); void zip_close(struct zip_info *info); +int zip_load_reference_map(struct zip_info *info, char *reference_map_name); void zip_destroy(struct zip_info *info); +struct zip_hashed_cd; +void zip_set_reference_map(struct zip_info *zip_info, struct zip_hashed_cd *zhc); +struct zip_hashed_cd *zip_hashed_cd_new(char *map_filename); +void zip_hashed_cd_free(struct zip_hashed_cd *zhc); /* Break compilation on 32 bit architectures, as we're going to cast osmid's to gpointer to use them as keys to GHashTable's */ struct maptool_force_64 {char s[sizeof(gpointer)<sizeof(osmid)?-1:1];}; diff --git a/navit/maptool/zip.c b/navit/maptool/zip.c index f352e6353..9d034df71 100644 --- a/navit/maptool/zip.c +++ b/navit/maptool/zip.c @@ -17,6 +17,7 @@ * Boston, MA 02110-1301, USA. */ +#include <glib.h> #include <zlib.h> #include <string.h> #include <stdlib.h> @@ -24,6 +25,7 @@ #include "maptool.h" #include "config.h" #include "zipfile.h" +#include "attr.h" #ifdef HAVE_LIBCRYPTO #include <openssl/sha.h> @@ -51,7 +53,119 @@ struct zip_info { MD5_CTX md5_ctx; #endif int md5; + struct zip_hashed_cd *reference_map; }; +/* + * Helper structure to zip file handle with hashtable of local file header pointers +*/ +struct zip_hashed_cd { + struct file *fi; + /* hash table to access zip_lfh offsets by file name */ + GHashTable *lfh_offsets; +}; + +void +zip_set_reference_map(struct zip_info *zip_info, struct zip_hashed_cd *zhc) +{ + zip_info->reference_map=zhc; +} + +/** + * @brief fill zip_hashed_cd structure with references to tile file local file headers. + * @param map_filename name of file holding reference map + * @return Pointer to allocated and filled zip_hashed_cd structure. Free with zip_hashed_cd_free(). + */ +struct zip_hashed_cd * +zip_hashed_cd_new(char *map_filename) +{ + struct zip_eoc *eoc; + struct zip64_eoc *eoc64; + struct zip_cd *cd; + struct attr a={attr_cache, {0}}; + struct attr *opts[]={&a, NULL}; + struct zip_hashed_cd *zhc=g_new(struct zip_hashed_cd,1); + unsigned long long offset=0; + zhc->fi=file_create(map_filename,opts); + if(!zhc->fi) { + g_free(zhc); + return NULL; + } + eoc=zipfile_read_eoc(zhc->fi); + if(!eoc) { + file_destroy(zhc->fi); + g_free(zhc); + return NULL; + } + + eoc64=zipfile_read_eoc64(zhc->fi); + + zhc->lfh_offsets=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + + while((cd=zipfile_read_cd(zhc->fi, eoc, eoc64, offset,-1))!=NULL) { + char *filename=g_strndup(cd->zipcfn, cd->zipcfnl); + if(!g_hash_table_insert(zhc->lfh_offsets, filename, (void*)zipfile_lfh_offset(cd))) + dbg(lvl_error,"Duplicate tile name: %s\n", filename); + offset+=sizeof(*cd)+zipfile_cd_name_and_extra_len(cd); + file_data_free(zhc->fi, (unsigned char *)cd); + } + + file_data_free(zhc->fi, (unsigned char *)eoc64); + file_data_free(zhc->fi, (unsigned char *)eoc); + + return zhc; +} + +void +zip_hashed_cd_free(struct zip_hashed_cd *zhc) +{ + if(zhc) { + file_destroy(zhc->fi); + g_hash_table_destroy(zhc->lfh_offsets); + g_free(zhc); + } +} + +/** + * @brief Adjust timestamp of a given file if it was already known in the reference version of mapfile. + * @param filename name of tile file being examined + * @param new_data uncompressed contents of the tile file being written + * @param new_len uncompressed size of the tile file being written + * @param zhc pointer to structure used to access reference map + * @param @out date pointer to variable containing date to adjust + * @param @out time pointer to variable containing time to adjust + * @return nothing. + */ +static void +adjust_timestamp(char *filename, void *new_data, int new_len, struct zip_hashed_cd *zhc, short *date, short *time) +{ + long long offset; + struct zip_lfh *lfh; + + if( !zhc || !zhc->fi || !zhc->lfh_offsets ) + return; + + if( !g_hash_table_lookup_extended(zhc->lfh_offsets, filename, NULL, (void **)&offset)) { + dbg(lvl_info,"Tile not found in reference map: %s\n",filename); + return; + } + + if( (lfh=zipfile_read_lfh(zhc->fi,offset)) == NULL ) { + dbg(lvl_error,"Error reading tile header %s from reference map at offset "LONGLONG_FMT"\n",filename, offset); + return; + } + + if( lfh->zipuncmp==new_len ) { + void *old_data=zipfile_read_content(zhc->fi, offset, lfh, NULL); + if( old_data && memcmp(old_data, new_data, new_len)==0 ) { + dbg(lvl_debug,"Adjusting timestamp for tile %s\n",filename); + *date=lfh->zipdate; + *time=lfh->ziptime; + } + file_data_free(zhc->fi, (unsigned char *)old_data); + } + + file_data_free(zhc->fi, (unsigned char *)lfh); +} static int zip_write(struct zip_info *info, void *data, int len) @@ -97,6 +211,17 @@ compress2_int(Byte *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, } #endif +/** + * @brief Write a single member file to the zip file, compressing and encrypting it on the way. + * Supports zip64 in the sense of handling zip files above 4Gb, but does not allow individual files above zip32 limits. + * Does not support writing multivolume zip files. + * @param zip_info pointer to structure holding global zip file informations + * @param name name of file being written (not necessarily nul-terminated) + * @param filelen length of file name to pad to + * @param data uncopmpressed contents of file being written + * @param data_size uncopmpressed length of file being written + * @return nothing. + */ void write_zipmember(struct zip_info *zip_info, char *name, int filelen, char *data, int data_size) { @@ -121,8 +246,8 @@ write_zipmember(struct zip_info *zip_info, char *name, int filelen, char *data, 0x00, 0x0000, 0x0, - zip_info->time, - zip_info->date, + 0, + 0, 0x0, 0x0, 0x0, @@ -155,6 +280,18 @@ write_zipmember(struct zip_info *zip_info, char *name, int filelen, char *data, uLongf destlen=data_size+data_size/500+12; char *compbuffer; + filename=g_alloca(filelen+1); + g_strlcpy(filename, name, filelen+1); + len=strlen(filename); + while (len < filelen) { + filename[len++]='_'; + } + filename[filelen]='\0'; + + adjust_timestamp(filename, data, data_size, zip_info->reference_map, &lfh.zipdate, &lfh.ziptime); + cd.zipdat=lfh.zipdate; + cd.ziptim=lfh.ziptime; + compbuffer = malloc(destlen); if (!compbuffer) { fprintf(stderr, "No more memory.\n"); @@ -215,13 +352,6 @@ write_zipmember(struct zip_info *zip_info, char *name, int filelen, char *data, cd.zipcflg|=1; } #endif - filename=g_alloca(filelen+1); - strcpy(filename, name); - len=strlen(filename); - while (len < filelen) { - filename[len++]='_'; - } - filename[filelen]='\0'; zip_write(zip_info, &lfh, sizeof(lfh)); zip_write(zip_info, filename, filelen); zip_info->offset+=sizeof(lfh)+filelen; diff --git a/navit/zipfile.c b/navit/zipfile.c index bda50b664..e4eabee1a 100644 --- a/navit/zipfile.c +++ b/navit/zipfile.c @@ -156,6 +156,8 @@ zipfile_read_cd(struct file *fi, struct zip_eoc *eoc, struct zip64_eoc *eoc64, i long long cdoffset=eoc64?eoc64->zip64eofst:eoc->zipeofst; if (len == -1) { cd=(struct zip_cd *)file_data_read(fi,cdoffset+offset, sizeof(*cd)); + if(!cd) + return NULL; cd_to_cpu(cd); len=zipfile_cd_name_and_extra_len(cd); file_data_free(fi,(unsigned char *)cd); @@ -201,7 +203,7 @@ zipfile_cd_ext(struct zip_cd *cd) * @return Offset of local file header in zip file. */ long long -zipfile_cd_offset(struct zip_cd *cd) +zipfile_lfh_offset(struct zip_cd *cd) { struct zip_cd_ext *ext=zipfile_cd_ext(cd); if (ext) diff --git a/navit/zipfile.h b/navit/zipfile.h index 07ec5da57..4574db88a 100644 --- a/navit/zipfile.h +++ b/navit/zipfile.h @@ -181,7 +181,7 @@ struct zip64_eoc *zipfile_read_eoc64(struct file *fi); int zipfile_cd_name_and_extra_len(struct zip_cd *cd); struct zip_cd *zipfile_read_cd(struct file *fi, struct zip_eoc *eoc, struct zip64_eoc *eoc64, int offset, int len); struct zip_cd_ext *zipfile_cd_ext(struct zip_cd *cd); -long long zipfile_cd_offset(struct zip_cd *cd); +long long zipfile_lfh_offset(struct zip_cd *cd); struct zip_lfh *zipfile_read_lfh(struct file *fi, long long offset); unsigned char *zipfile_read_content(struct file *fi, long long offset, struct zip_lfh *lfh, char *passwd); void cd_to_cpu(struct zip_cd *zcd); |