diff options
-rw-r--r-- | .gitignore | 2 | ||||
-rw-r--r-- | Makefile.am | 5 | ||||
-rw-r--r-- | NEWS | 25 | ||||
-rw-r--r-- | THANKS | 2 | ||||
-rw-r--r-- | TODO.md | 2 | ||||
-rw-r--r-- | blake2-impl.h | 133 | ||||
-rw-r--r-- | blake2.h | 156 | ||||
-rw-r--r-- | blake2b-ref.c | 394 | ||||
-rw-r--r-- | buf.c | 4 | ||||
-rw-r--r-- | checksum.c | 11 | ||||
-rw-r--r-- | checksum.h | 3 | ||||
-rw-r--r-- | configure.ac | 2 | ||||
-rw-r--r-- | delta.c | 8 | ||||
-rw-r--r-- | emit.c | 2 | ||||
-rw-r--r-- | job.h | 11 | ||||
-rw-r--r-- | librsync.h | 49 | ||||
-rw-r--r-- | libversions.txt | 1 | ||||
-rwxr-xr-x | mkprototab.pl | 1 | ||||
-rw-r--r-- | mksum.c | 51 | ||||
-rw-r--r-- | patch.c | 1 | ||||
-rw-r--r-- | protocol.h | 45 | ||||
-rw-r--r-- | rdiff.c | 43 | ||||
-rw-r--r-- | rdiff.magic | 12 | ||||
-rw-r--r-- | readsums.c | 23 | ||||
-rw-r--r-- | scoop.c | 3 | ||||
-rw-r--r-- | search.c | 15 | ||||
-rw-r--r-- | search.h | 6 | ||||
-rw-r--r-- | stream.c | 3 | ||||
-rw-r--r-- | sumset.c | 2 | ||||
-rw-r--r-- | sumset.h | 1 | ||||
-rw-r--r-- | testsuite/Makefile.am | 4 | ||||
-rwxr-xr-x | testsuite/changes.test | 9 | ||||
-rwxr-xr-x | testsuite/delta.test | 2 | ||||
-rwxr-xr-x | testsuite/help.test (renamed from testsuite/mksum.test) | 21 | ||||
-rw-r--r-- | testsuite/mksum.input/COPYING.sig | bin | 168 -> 0 bytes | |||
-rwxr-xr-x | testsuite/mutate.test | 17 | ||||
-rw-r--r-- | testsuite/signature.input/blake2/01.sig | bin | 0 -> 480 bytes | |||
-rw-r--r-- | testsuite/signature.input/md4/01.sig (renamed from testsuite/signature.input/01.sig) | bin | 168 -> 168 bytes | |||
-rwxr-xr-x | testsuite/signature.test | 21 | ||||
-rwxr-xr-x | testsuite/sources.test | 13 | ||||
-rw-r--r-- | testsuite/testcommon.sh | 7 | ||||
-rwxr-xr-x | testsuite/triple.test | 12 | ||||
-rw-r--r-- | whole.c | 11 |
43 files changed, 963 insertions, 170 deletions
@@ -1,4 +1,3 @@ -G .*.swp .deps Makefile @@ -42,3 +41,4 @@ librsync.la librsync-[0-9]*tar.gz librsync-[0-9]*/ /m4/ +tags diff --git a/Makefile.am b/Makefile.am index 7684b86..2ef8711 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,11 +39,12 @@ librsync_la_SOURCES = prototab.c prototab.h \ base64.c buf.c buf.h checksum.c checksum.h command.c \ command.h delta.c emit.c emit.h fileutil.c fileutil.h \ hex.c job.c job.h mdfour.c mdfour.h mksum.c msg.c netint.c netint.h \ - patch.c protocol.h readsums.c librsync.h librsync-config.h \ + patch.c readsums.c librsync.h librsync-config.h \ rollsum.c rollsum.h \ scoop.c search.c search.h stats.c stream.c stream.h sumset.c \ sumset.h trace.c trace.h tube.c types.h util.c util.h \ - version.c whole.c whole.h snprintf.h + version.c whole.c whole.h snprintf.h \ + blake2b-ref.c blake2.h blake2-impl.h librsync_la_LIBADD = @LIBOBJS@ @@ -1,5 +1,30 @@ Changes in 1.0.0 (not yet released) + * SECURITY: CVE-2014-8242: librsync previously used a truncated MD4 + "strong" check sum to match blocks. However, MD4 is not cryptographically + strong. It's possible that an attacker who can control the contents of one + part of a file could use it to control other regions of the file, if it's + transferred using librsync/rdiff. For example this might occur in a + database, mailbox, or VM image containing some attacker-controlled data. + + To mitigate this issue, signatures will by default be computed with a + 256-bit BLAKE2 hash. Old versions of librsync will complain about a + bad magic number when given these signature files. + + Backward compatibility can be obtained using the new + `rdiff sig --hash=md4` + option or through specifying the "signature magic" in the API, but + this should not be used when either the old or new file contain + untrusted data. + + Deltas generated from those signatures will also use BLAKE2 during + generation, but produce output that can be read by old versions. + + See https://github.com/librsync/librsync/issues/5 + + Thanks to Michael Samuel <miknet.net> for reporting this and offering an + initial patch. + * Improved rdiff man page from Debian. * Improved librsync.spec file for building RPMs. @@ -30,6 +30,8 @@ possible we have missed some contributors. If you know of anyone who has been omitted and deserves to be included, please send the details so they can be added. +For later releases, contributors are listed in NEWS. + Contributors for 0.9.6 * Wayne Davison <wayned@users.sourceforge.net> @@ -298,6 +298,8 @@ * Tests should fail if they can't find their inputs, or have zero inputs: at present they tend to succeed by default. + * Test varying strong-sum inputs: default, short, long. + * Security audit * If this code was to read differences or sums from random machines diff --git a/blake2-impl.h b/blake2-impl.h new file mode 100644 index 0000000..83d2dcb --- /dev/null +++ b/blake2-impl.h @@ -0,0 +1,133 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Written in 2012 by Samuel Neves <sneves@dei.uc.pt> + + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. + + You should have received a copy of the CC0 Public Domain Dedication along with + this software. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>. +*/ +#pragma once +#ifndef __BLAKE2_IMPL_H__ +#define __BLAKE2_IMPL_H__ + +#include <stdint.h> + +static inline uint32_t load32( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + return *( uint32_t * )( src ); +#else + const uint8_t *p = ( uint8_t * )src; + uint32_t w = *p++; + w |= ( uint32_t )( *p++ ) << 8; + w |= ( uint32_t )( *p++ ) << 16; + w |= ( uint32_t )( *p++ ) << 24; + return w; +#endif +} + +static inline uint64_t load64( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + return *( uint64_t * )( src ); +#else + const uint8_t *p = ( uint8_t * )src; + uint64_t w = *p++; + w |= ( uint64_t )( *p++ ) << 8; + w |= ( uint64_t )( *p++ ) << 16; + w |= ( uint64_t )( *p++ ) << 24; + w |= ( uint64_t )( *p++ ) << 32; + w |= ( uint64_t )( *p++ ) << 40; + w |= ( uint64_t )( *p++ ) << 48; + w |= ( uint64_t )( *p++ ) << 56; + return w; +#endif +} + +static inline void store32( void *dst, uint32_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + *( uint32_t * )( dst ) = w; +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static inline void store64( void *dst, uint64_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + *( uint64_t * )( dst ) = w; +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static inline uint64_t load48( const void *src ) +{ + const uint8_t *p = ( const uint8_t * )src; + uint64_t w = *p++; + w |= ( uint64_t )( *p++ ) << 8; + w |= ( uint64_t )( *p++ ) << 16; + w |= ( uint64_t )( *p++ ) << 24; + w |= ( uint64_t )( *p++ ) << 32; + w |= ( uint64_t )( *p++ ) << 40; + return w; +} + +static inline void store48( void *dst, uint64_t w ) +{ + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +} + +static inline uint32_t rotl32( const uint32_t w, const unsigned c ) +{ + return ( w << c ) | ( w >> ( 32 - c ) ); +} + +static inline uint64_t rotl64( const uint64_t w, const unsigned c ) +{ + return ( w << c ) | ( w >> ( 64 - c ) ); +} + +static inline uint32_t rotr32( const uint32_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 32 - c ) ); +} + +static inline uint64_t rotr64( const uint64_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 64 - c ) ); +} + +/* prevents compiler optimizing out memset() */ +static inline void secure_zero_memory( void *v, size_t n ) +{ + volatile uint8_t *p = ( volatile uint8_t * )v; + + while( n-- ) *p++ = 0; +} + +#endif + diff --git a/blake2.h b/blake2.h new file mode 100644 index 0000000..f8aba83 --- /dev/null +++ b/blake2.h @@ -0,0 +1,156 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Written in 2012 by Samuel Neves <sneves@dei.uc.pt> + + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. + + You should have received a copy of the CC0 Public Domain Dedication along with + this software. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>. +*/ +#pragma once +#ifndef __BLAKE2_H__ +#define __BLAKE2_H__ + +#include <stddef.h> +#include <stdint.h> + +#if defined(_MSC_VER) +#define ALIGN(x) __declspec(align(x)) +#else +#define ALIGN(x) __attribute__((aligned(x))) +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + +#pragma pack(push, 1) + typedef struct __blake2s_param + { + uint8_t digest_length; // 1 + uint8_t key_length; // 2 + uint8_t fanout; // 3 + uint8_t depth; // 4 + uint32_t leaf_length; // 8 + uint8_t node_offset[6];// 14 + uint8_t node_depth; // 15 + uint8_t inner_length; // 16 + // uint8_t reserved[0]; + uint8_t salt[BLAKE2S_SALTBYTES]; // 24 + uint8_t personal[BLAKE2S_PERSONALBYTES]; // 32 + } blake2s_param; + + ALIGN( 64 ) typedef struct __blake2s_state + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[2 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + uint8_t last_node; + } blake2s_state ; + + typedef struct __blake2b_param + { + uint8_t digest_length; // 1 + uint8_t key_length; // 2 + uint8_t fanout; // 3 + uint8_t depth; // 4 + uint32_t leaf_length; // 8 + uint64_t node_offset; // 16 + uint8_t node_depth; // 17 + uint8_t inner_length; // 18 + uint8_t reserved[14]; // 32 + uint8_t salt[BLAKE2B_SALTBYTES]; // 48 + uint8_t personal[BLAKE2B_PERSONALBYTES]; // 64 + } blake2b_param; + + ALIGN( 64 ) typedef struct __blake2b_state + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[2 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + uint8_t last_node; + } blake2b_state; + + typedef struct __blake2sp_state + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + } blake2sp_state; + + typedef struct __blake2bp_state + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + } blake2bp_state; +#pragma pack(pop) + + // Streaming API + int blake2s_init( blake2s_state *S, const uint8_t outlen ); + int blake2s_init_key( blake2s_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ); + int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + int blake2s_update( blake2s_state *S, const uint8_t *in, uint64_t inlen ); + int blake2s_final( blake2s_state *S, uint8_t *out, uint8_t outlen ); + + int blake2b_init( blake2b_state *S, const uint8_t outlen ); + int blake2b_init_key( blake2b_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ); + int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + int blake2b_update( blake2b_state *S, const uint8_t *in, uint64_t inlen ); + int blake2b_final( blake2b_state *S, uint8_t *out, uint8_t outlen ); + + int blake2sp_init( blake2sp_state *S, const uint8_t outlen ); + int blake2sp_init_key( blake2sp_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ); + int blake2sp_update( blake2sp_state *S, const uint8_t *in, uint64_t inlen ); + int blake2sp_final( blake2sp_state *S, uint8_t *out, uint8_t outlen ); + + int blake2bp_init( blake2bp_state *S, const uint8_t outlen ); + int blake2bp_init_key( blake2bp_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ); + int blake2bp_update( blake2bp_state *S, const uint8_t *in, uint64_t inlen ); + int blake2bp_final( blake2bp_state *S, uint8_t *out, uint8_t outlen ); + + // Simple API + int blake2s( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ); + int blake2b( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ); + + int blake2sp( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ); + int blake2bp( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ); + + static inline int blake2( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ) + { + return blake2b( out, in, key, outlen, inlen, keylen ); + } + +#if defined(__cplusplus) +} +#endif + +#endif + diff --git a/blake2b-ref.c b/blake2b-ref.c new file mode 100644 index 0000000..bcf81da --- /dev/null +++ b/blake2b-ref.c @@ -0,0 +1,394 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Written in 2012 by Samuel Neves <sneves@dei.uc.pt> + + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. + + You should have received a copy of the CC0 Public Domain Dedication along with + this software. If not, see <http://creativecommons.org/publicdomain/zero/1.0/>. +*/ + +#include <stdint.h> +#include <string.h> +#include <stdio.h> + +#include "blake2.h" +#include "blake2-impl.h" + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +static inline int blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = ~0ULL; + return 0; +} + +static inline int blake2b_clear_lastnode( blake2b_state *S ) +{ + S->f[1] = 0ULL; + return 0; +} + +/* Some helper functions, not necessarily useful */ +static inline int blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = ~0ULL; + return 0; +} + +static inline int blake2b_clear_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_clear_lastnode( S ); + + S->f[0] = 0ULL; + return 0; +} + +static inline int blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); + return 0; +} + + + +// Parameter-related functions +static inline int blake2b_param_set_digest_length( blake2b_param *P, const uint8_t digest_length ) +{ + P->digest_length = digest_length; + return 0; +} + +static inline int blake2b_param_set_fanout( blake2b_param *P, const uint8_t fanout ) +{ + P->fanout = fanout; + return 0; +} + +static inline int blake2b_param_set_max_depth( blake2b_param *P, const uint8_t depth ) +{ + P->depth = depth; + return 0; +} + +static inline int blake2b_param_set_leaf_length( blake2b_param *P, const uint32_t leaf_length ) +{ + store32( &P->leaf_length, leaf_length ); + return 0; +} + +static inline int blake2b_param_set_node_offset( blake2b_param *P, const uint64_t node_offset ) +{ + store64( &P->node_offset, node_offset ); + return 0; +} + +static inline int blake2b_param_set_node_depth( blake2b_param *P, const uint8_t node_depth ) +{ + P->node_depth = node_depth; + return 0; +} + +static inline int blake2b_param_set_inner_length( blake2b_param *P, const uint8_t inner_length ) +{ + P->inner_length = inner_length; + return 0; +} + +static inline int blake2b_param_set_salt( blake2b_param *P, const uint8_t salt[BLAKE2B_SALTBYTES] ) +{ + memcpy( P->salt, salt, BLAKE2B_SALTBYTES ); + return 0; +} + +static inline int blake2b_param_set_personal( blake2b_param *P, const uint8_t personal[BLAKE2B_PERSONALBYTES] ) +{ + memcpy( P->personal, personal, BLAKE2B_PERSONALBYTES ); + return 0; +} + +static inline int blake2b_init0( blake2b_state *S ) +{ + int i; + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; + + return 0; +} + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + blake2b_init0( S ); + uint8_t *p = ( uint8_t * )( P ); + size_t i; + + /* IV XOR ParamBlock */ + for( i = 0; i < 8; ++i ) + S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + + return 0; +} + + + +int blake2b_init( blake2b_state *S, const uint8_t outlen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + P->digest_length = outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store64( &P->node_offset, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2b_init_param( S, P ); +} + + +int blake2b_init_key( blake2b_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; + + P->digest_length = outlen; + P->key_length = keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store64( &P->node_offset, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2b_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +static int blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + uint64_t m[16]; + uint64_t v[16]; + int i; + + for( i = 0; i < 16; ++i ) + m[i] = load64( block + i * sizeof( m[i] ) ); + + for( i = 0; i < 8; ++i ) + v[i] = S->h[i]; + + v[ 8] = blake2b_IV[0]; + v[ 9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = S->t[0] ^ blake2b_IV[4]; + v[13] = S->t[1] ^ blake2b_IV[5]; + v[14] = S->f[0] ^ blake2b_IV[6]; + v[15] = S->f[1] ^ blake2b_IV[7]; +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2*i+0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2*i+1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while(0) +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + ROUND( 10 ); + ROUND( 11 ); + + for( i = 0; i < 8; ++i ) + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + +#undef G +#undef ROUND + return 0; +} + +/* inlen now in bytes */ +int blake2b_update( blake2b_state *S, const uint8_t *in, uint64_t inlen ) +{ + while( inlen > 0 ) + { + size_t left = S->buflen; + size_t fill = 2 * BLAKE2B_BLOCKBYTES - left; + + if( inlen > fill ) + { + memcpy( S->buf + left, in, fill ); // Fill buffer + S->buflen += fill; + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); // Compress + memcpy( S->buf, S->buf + BLAKE2B_BLOCKBYTES, BLAKE2B_BLOCKBYTES ); // Shift buffer left + S->buflen -= BLAKE2B_BLOCKBYTES; + in += fill; + inlen -= fill; + } + else // inlen <= fill + { + memcpy( S->buf + left, in, inlen ); + S->buflen += inlen; // Be lazy, do not compress + in += inlen; + inlen -= inlen; + } + } + + return 0; +} + +/* Is this correct? */ +int blake2b_final( blake2b_state *S, uint8_t *out, uint8_t outlen ) +{ + uint8_t buffer[BLAKE2B_OUTBYTES]; + int i; + + if( S->buflen > BLAKE2B_BLOCKBYTES ) + { + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); + S->buflen -= BLAKE2B_BLOCKBYTES; + memcpy( S->buf, S->buf + BLAKE2B_BLOCKBYTES, S->buflen ); + } + + blake2b_increment_counter( S, S->buflen ); + blake2b_set_lastblock( S ); + memset( S->buf + S->buflen, 0, 2 * BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ + blake2b_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, outlen ); + return 0; +} + +/* inlen, at least, should be uint64_t. Others can be size_t. */ +int blake2b( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ) +{ + blake2b_state S[1]; + + /* Verify parameters */ + if ( NULL == in ) return -1; + + if ( NULL == out ) return -1; + + if( NULL == key ) keylen = 0; + + if( keylen > 0 ) + { + if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2b_init( S, outlen ) < 0 ) return -1; + } + + blake2b_update( S, ( uint8_t * )in, inlen ); + blake2b_final( S, out, outlen ); + return 0; +} + +#if defined(BLAKE2B_SELFTEST) +#include <string.h> +#include "blake2-kat.h" +int main( int argc, char **argv ) +{ + uint8_t key[BLAKE2B_KEYBYTES]; + uint8_t buf[KAT_LENGTH]; + + for( size_t i = 0; i < BLAKE2B_KEYBYTES; ++i ) + key[i] = ( uint8_t )i; + + for( size_t i = 0; i < KAT_LENGTH; ++i ) + buf[i] = ( uint8_t )i; + + for( size_t i = 0; i < KAT_LENGTH; ++i ) + { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b( hash, buf, key, BLAKE2B_OUTBYTES, i, BLAKE2B_KEYBYTES ); + + if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) + { + puts( "error" ); + return -1; + } + } + + puts( "ok" ); + return 0; +} +#endif + @@ -1,7 +1,6 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- the library for network deltas - * $Id$ * * Copyright (C) 2000, 2001 by Martin Pool <mbp@sourcefrog.net> * @@ -22,7 +21,6 @@ /* | Pick a window, Jimmy, you're leaving. - | -- Martin Schwenke, regularly */ @@ -213,7 +211,7 @@ rs_result rs_file_copy_cb(void *arg, rs_long_t pos, size_t *len, void **buf) got = fread(*buf, 1, *len, f); if (got == -1) { - rs_error(strerror(errno)); + rs_error("read error: %s", strerror(errno)); return RS_IO_ERROR; } else if (got == 0) { rs_error("unexpected eof on fd%d", fileno(f)); @@ -30,6 +30,7 @@ #include "librsync.h" #include "checksum.h" +#include "blake2.h" /* This can possibly be used to restart the checksum system in the @@ -75,7 +76,15 @@ unsigned int rs_calc_weak_sum(void const *p, int len) * Since we can't retry a web transaction I'm not sure if it's very * useful in rproxy. */ -void rs_calc_strong_sum(void const *buf, size_t len, rs_strong_sum_t *sum) +void rs_calc_md4_sum(void const *buf, size_t len, rs_strong_sum_t *sum) { rs_mdfour((unsigned char *) sum, buf, len); } + +void rs_calc_blake2_sum(void const *buf, size_t len, rs_strong_sum_t *sum) +{ + blake2b_state ctx; + blake2b_init(&ctx, RS_MAX_STRONG_SUM_LENGTH); + blake2b_update(&ctx, (const uint8_t *)buf, len); + blake2b_final(&ctx, (uint8_t *)sum, RS_MAX_STRONG_SUM_LENGTH); +} @@ -22,7 +22,8 @@ rs_weak_sum_t rs_calc_weak_sum(void const *buf1, int len); -void rs_calc_strong_sum(void const *buf, size_t buf_len, rs_strong_sum_t *); +void rs_calc_md4_sum(void const *buf, size_t buf_len, rs_strong_sum_t *); +void rs_calc_blake2_sum(void const *buf, size_t buf_len, rs_strong_sum_t *); /* We should make this something other than zero to improve the * checksum algorithm: tridge suggests a prime number. */ diff --git a/configure.ac b/configure.ac index 136d3ef..a0763c7 100644 --- a/configure.ac +++ b/configure.ac @@ -12,7 +12,7 @@ AM_INIT_AUTOMAKE([foreign]) # GNU library versioning: This is NOT the librsync release number. # See libversions.txt and the libtool manual for an explanation of the # library versioning -librsync_libversion=1:2:0 +librsync_libversion=2:0:0 # Disable shared libs by default. AC_DISABLE_SHARED @@ -1,7 +1,6 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- library for network deltas - * $Id$ * * Copyright (C) 2000, 2001 by Martin Pool <mbp@sourcefrog.net> * Copyright (C) 2003 by Donovan Baarda <abo@minkirri.apana.org.au> @@ -80,6 +79,8 @@ #include "types.h" #include "rollsum.h" +const int RS_MD4_SUM_LENGTH = 16; +const int RS_BLAKE2_SUM_LENGTH = 32; /** * 2002-06-26: Donovan Baarda @@ -164,7 +165,7 @@ static rs_result rs_delta_s_scan(rs_job_t *job) result=rs_appendmiss(job,1); if (rs_roll_paranoia) { RollsumInit(&test); - RollsumUpdate(&test,job->scoop_next+job->scoop_pos, + RollsumUpdate(&test, job->scoop_next+job->scoop_pos, job->block_len); if (RollsumDigest(&test) != RollsumDigest(&job->weak_sum)) { rs_fatal("mismatch between rolled sum %#x and check %#x", @@ -458,7 +459,7 @@ rs_job_t *rs_delta_begin(rs_signature_t *sig) } job->strong_sum_len = sig->strong_sum_len; - if (job->strong_sum_len < 0 || job->strong_sum_len > RS_MD4_LENGTH) { + if (job->strong_sum_len < 0 || job->strong_sum_len > RS_MAX_STRONG_SUM_LENGTH) { rs_log(RS_LOG_ERR, "unreasonable strong_sum_len %d in signature", job->strong_sum_len); return NULL; @@ -467,4 +468,3 @@ rs_job_t *rs_delta_begin(rs_signature_t *sig) return job; } - @@ -1,7 +1,6 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- dynamic caching and delta update in HTTP - * $Id$ * * Copyright (C) 2000, 2001, 2004 by Martin Pool <mbp@sourcefrog.net> * @@ -39,7 +38,6 @@ #include "librsync.h" #include "command.h" -#include "protocol.h" #include "trace.h" #include "emit.h" #include "prototab.h" @@ -1,9 +1,8 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- the library for network deltas - * $Id$ * - * Copyright (C) 2000, 2001 by Martin Pool <mbp@sourcefrog.net> + * Copyright (C) 2000, 2001, 2014 by Martin Pool <mbp@sourcefrog.net> * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -70,15 +69,15 @@ struct rs_job { * scoop_buf[0..scoop_alloc], and scoop_next[0..scoop_avail] contains * data yet to be processed. scoop_next[scoop_pos..scoop_avail] is the * data yet to be scanned. */ - char *scoop_buf; /* the allocation pointer */ - char *scoop_next; /* the data pointer */ + rs_byte_t *scoop_buf; /* the allocation pointer */ + rs_byte_t *scoop_next; /* the data pointer */ size_t scoop_alloc; /* the allocation size */ size_t scoop_avail; /* the data size */ size_t scoop_pos; /* the scan position */ /** If USED is >0, then buf contains that much write data to * be sent out. */ - char write_buf[20]; + rs_byte_t write_buf[36]; int write_len; /** If \p copy_len is >0, then that much data should be copied @@ -91,6 +90,8 @@ struct rs_job { /** Callback used to copy data from the basis into the output. */ rs_copy_cb *copy_cb; void *copy_arg; + + int magic; }; @@ -2,7 +2,7 @@ * * librsync -- library for network deltas * - * Copyright (C) 2000, 2001 by Martin Pool <mbp@sourcefrog.net> + * Copyright 2000, 2001, 2014 by Martin Pool <mbp@sourcefrog.net> * Copyright (C) 2003 by Donovan Baarda <abo@minkirri.apana.org.au> * * This program is free software; you can redistribute it and/or modify @@ -32,9 +32,7 @@ * * \brief Main public interface to librsync. * \author Martin Pool <mbp@sourcefrog.net> - * \version librsync-0.9.6 - * - * $Id$ + * \version librsync-1.0.0 * * See \ref intro for an introduction to use of this library. */ @@ -52,6 +50,8 @@ extern "C" { extern char const rs_librsync_version[]; extern char const rs_licence_string[]; +typedef unsigned char rs_byte_t; + /** * \brief Log severity levels. @@ -73,6 +73,25 @@ typedef enum { + /* + | "The IETF already has more than enough + | RFCs that codify the obvious, make + | stupidity illegal, support truth, + | justice, and the IETF way, and generally + | demonstrate the author is a brilliant and + | valuable Contributor to The Standards + | Process." + | -- Vernon Schryver + */ + + +typedef enum { + RS_DELTA_MAGIC = 0x72730236, /* r s \2 6 */ + RS_MD4_SIG_MAGIC = 0x72730136, /* r s \1 6 */ + RS_BLAKE2_SIG_MAGIC = 0x72730137 /* r s \1 7 */ +} rs_magic_number; + + /** * \typedef rs_trace_fn_t * \brief Callback to write out log messages. @@ -190,10 +209,17 @@ typedef struct rs_stats { */ typedef struct rs_mdfour rs_mdfour_t; -#define RS_MD4_LENGTH 16 +/* Commenting this out deliberately - in case library users + * depend on this size - this could potentially overflow + * buffers now + * #define RS_MD4_LENGTH 16 + */ +extern const int RS_MD4_SUM_LENGTH, RS_BLAKE2_SUM_LENGTH; + +#define RS_MAX_STRONG_SUM_LENGTH 32 typedef unsigned int rs_weak_sum_t; -typedef unsigned char rs_strong_sum_t[RS_MD4_LENGTH]; +typedef unsigned char rs_strong_sum_t[RS_MAX_STRONG_SUM_LENGTH]; void rs_mdfour(unsigned char *out, void const *in, size_t); void rs_mdfour_begin(/* @out@ */ rs_mdfour_t * md); @@ -278,10 +304,6 @@ struct rs_buffers_s { */ typedef struct rs_buffers_s rs_buffers_t; -/** Default length of strong signatures, in bytes. The MD4 checksum - * is truncated to this size. */ -#define RS_DEFAULT_STRONG_LEN 8 - /** Default block length, if not determined by any other factors. */ #define RS_DEFAULT_BLOCK_LEN 2048 @@ -319,7 +341,8 @@ rs_result rs_job_free(rs_job_t *); int rs_accum_value(rs_job_t *, char *sum, size_t sum_len); -rs_job_t *rs_sig_begin(size_t new_block_len, size_t strong_sum_len); +rs_job_t *rs_sig_begin(size_t new_block_len, size_t strong_sum_len, + rs_long_t sig_magic); rs_job_t *rs_delta_begin(rs_signature_t *); @@ -367,7 +390,9 @@ extern int rs_inbuflen, rs_outbuflen; void rs_mdfour_file(FILE *in_file, char *result); rs_result rs_sig_file(FILE *old_file, FILE *sig_file, - size_t block_len, size_t strong_len, rs_stats_t *); + size_t block_len, size_t strong_len, + rs_long_t sign_magic, + rs_stats_t *); rs_result rs_loadsig_file(FILE *, rs_signature_t **, rs_stats_t *); diff --git a/libversions.txt b/libversions.txt index a51e8a8..584b2e5 100644 --- a/libversions.txt +++ b/libversions.txt @@ -19,3 +19,4 @@ RELEASE CURRENT REVISION AGE COMMENTS 0.9.4 1 1 0 0.9.5 1 1 0 0.9.6 1 2 0 refined and debugged +1.0.0 2 0 0 add BLAKE2 as default diff --git a/mkprototab.pl b/mkprototab.pl index 15cadb1..feaab0d 100755 --- a/mkprototab.pl +++ b/mkprototab.pl @@ -75,7 +75,6 @@ print TABLE <<EOT; #include <stdio.h> #include "librsync.h" -#include <protocol.h> #include <command.h> #include <prototab.h> @@ -1,9 +1,8 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- library for network deltas - * $Id$ * - * Copyright (C) 1999, 2000, 2001 by Martin Pool <mbp@sourcefrog.net> + * Copyright 1999-2001, 2014 by Martin Pool <mbp@sourcefrog.net> * Copyright (C) 1999 by Andrew Tridgell <tridge@samba.org> * * This program is free software; you can redistribute it and/or modify @@ -47,7 +46,6 @@ #include "util.h" #include "sumset.h" #include "job.h" -#include "protocol.h" #include "netint.h" #include "trace.h" #include "checksum.h" @@ -64,11 +62,11 @@ static rs_result rs_sig_s_generate(rs_job_t *); */ static rs_result rs_sig_s_header(rs_job_t *job) { - rs_squirt_n4(job, RS_SIG_MAGIC); + rs_squirt_n4(job, job->magic); rs_squirt_n4(job, job->block_len); rs_squirt_n4(job, job->strong_sum_len); rs_trace("sent header (magic %#x, block len = %d, strong sum len = %d)", - RS_SIG_MAGIC, (int) job->block_len, (int) job->strong_sum_len); + job->magic, (int) job->block_len, (int) job->strong_sum_len); job->stats.block_len = job->block_len; job->statefn = rs_sig_s_generate; @@ -88,13 +86,20 @@ rs_sig_do_block(rs_job_t *job, const void *block, size_t len) weak_sum = rs_calc_weak_sum(block, len); - rs_calc_strong_sum(block, len, &strong_sum); + if (job->magic == RS_BLAKE2_SIG_MAGIC) { + rs_calc_blake2_sum(block, len, &strong_sum); + } else if(job->magic == RS_MD4_SIG_MAGIC) { + rs_calc_md4_sum(block, len, &strong_sum); + } else { + rs_error("BUG: invalid job magic %#lx", (unsigned long) job->magic); + return RS_INTERNAL_ERROR; + } rs_squirt_n4(job, weak_sum); rs_tube_write(job, strong_sum, job->strong_sum_len); if (rs_trace_enabled()) { - char strong_sum_hex[RS_MD4_LENGTH * 2 + 1]; + char strong_sum_hex[RS_MAX_STRONG_SUM_LENGTH * 2 + 1]; rs_hexify(strong_sum_hex, strong_sum, job->strong_sum_len); rs_trace("sent weak sum 0x%08x and strong sum %s", weak_sum, strong_sum_hex); @@ -141,15 +146,41 @@ rs_sig_s_generate(rs_job_t *job) * * \sa rs_sig_file() */ -rs_job_t * rs_sig_begin(size_t new_block_len, size_t strong_sum_len) +rs_job_t * rs_sig_begin(size_t new_block_len, size_t strong_sum_len, + rs_long_t sig_magic) { rs_job_t *job; + int native_length; job = rs_job_new("signature", rs_sig_s_header); job->block_len = new_block_len; - assert(strong_sum_len > 0 && strong_sum_len <= RS_MD4_LENGTH); - job->strong_sum_len = strong_sum_len; + if (!sig_magic) + sig_magic = RS_BLAKE2_SIG_MAGIC; + + switch (sig_magic) { + case RS_BLAKE2_SIG_MAGIC: + native_length = RS_BLAKE2_SUM_LENGTH; + job->magic = sig_magic; + break; + case RS_MD4_SIG_MAGIC: + job->magic = sig_magic; + native_length = RS_MD4_SUM_LENGTH; + break; + default: + rs_error("invalid sig_magic %#lx", (unsigned long) sig_magic); + return NULL; + } + + if (!strong_sum_len) + job->strong_sum_len = native_length; + else { + assert(strong_sum_len <= native_length); + job->strong_sum_len = strong_sum_len; + } return job; } + +/* vim: expandtab shiftwidth=4 + */ @@ -36,7 +36,6 @@ #include "librsync.h" #include "util.h" #include "trace.h" -#include "protocol.h" #include "netint.h" #include "command.h" #include "sumset.h" diff --git a/protocol.h b/protocol.h deleted file mode 100644 index fe5ef66..0000000 --- a/protocol.h +++ /dev/null @@ -1,45 +0,0 @@ -/*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- - * - * librsync -- library for network deltas - * $Id$ - * - * Copyright (C) 1999, 2000, 2001 by Martin Pool <mbp@sourcefrog.net> - * Copyright (C) 1999 by Andrew Tridgell - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ - - -/* - * TODO: Have a way to copy from the old signature into the new - * one. This will be useful for the case where the files are in fact - * identical, which will be significantly common. - */ - - - /* - | "The IETF already has more than enough - | RFCs that codify the obvious, make - | stupidity illegal, support truth, - | justice, and the IETF way, and generally - | demonstrate the author is a brilliant and - | valuable Contributor to The Standards - | Process." - | -- Vernon Schryver - */ - - - -#define RS_DELTA_MAGIC 0x72730236 /* r s \2 6 */ -#define RS_SIG_MAGIC 0x72730136 /* r s \1 6 */ @@ -1,7 +1,6 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- the library for network deltas - * $Id$ * * Copyright (C) 1999, 2000, 2001 by Martin Pool <mbp@sourcefrog.net> * @@ -20,11 +19,11 @@ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ - /* + /* | .. after a year and a day, mourning is - | dangerous to the survivor and troublesome - | to the dead. - | -- Harold Bloom + | dangerous to the survivor and troublesome + | to the dead. + | -- Harold Bloom */ /* @@ -50,6 +49,7 @@ #include <stdlib.h> #include <stdio.h> +#include <string.h> #include <fcntl.h> #include <popt.h> @@ -72,7 +72,7 @@ #define PROGRAM "rdiff" static size_t block_len = RS_DEFAULT_BLOCK_LEN; -static size_t strong_len = RS_DEFAULT_STRONG_LEN; +static size_t strong_len = 0; static int show_stats = 0; @@ -85,12 +85,14 @@ enum { }; extern int rs_roll_paranoia; +char *rs_hash_name; const struct poptOption opts[] = { { "verbose", 'v', POPT_ARG_NONE, 0, 'v' }, { "version", 'V', POPT_ARG_NONE, 0, 'V' }, { "input-size", 'I', POPT_ARG_INT, &rs_inbuflen }, { "output-size", 'O', POPT_ARG_INT, &rs_outbuflen }, + { "hash", 'H', POPT_ARG_STRING, &rs_hash_name }, { "help", '?', POPT_ARG_NONE, 0, 'h' }, { 0, 'h', POPT_ARG_NONE, 0, 'h' }, { "block-size", 'b', POPT_ARG_INT, &block_len }, @@ -143,6 +145,8 @@ static void help(void) { " -V, --version Show program version\n" " -?, --help Show this help message\n" " -s, --statistics Show performance statistics\n" + "Signature generation options:\n" + " -H, --hash=ALG Hash algorithm: blake2 (default), md4\n" "Delta-encoding options:\n" " -b, --block-size=BYTES Signature block size\n" " -S, --sum-size=BYTES Set signature strength\n" @@ -160,6 +164,8 @@ static void rdiff_show_version(void) { char const *bzlib = "", *zlib = "", *trace = ""; +#if 0 + /* Compression isn't implemented so don't mention it. */ #ifdef HAVE_LIBZ zlib = ", gzip"; #endif @@ -167,6 +173,7 @@ static void rdiff_show_version(void) #ifdef HAVE_LIBBZ2 bzlib = ", bzip2"; #endif +#endif #ifndef DO_RS_TRACE trace = ", trace disabled"; @@ -239,13 +246,30 @@ static rs_result rdiff_sig(poptContext opcon) FILE *basis_file, *sig_file; rs_stats_t stats; rs_result result; + rs_long_t sig_magic; basis_file = rs_file_open(poptGetArg(opcon), "rb"); sig_file = rs_file_open(poptGetArg(opcon), "wb"); rdiff_no_more_args(opcon); - - result = rs_sig_file(basis_file, sig_file, block_len, strong_len, &stats); + + if (!rs_hash_name || !strcmp(rs_hash_name, "blake2")) { + sig_magic = RS_BLAKE2_SIG_MAGIC; + } else if (!strcmp(rs_hash_name, "md4")) { + /* By default, for compatibility with rdiff 0.9.8 and before, mdfour + * sums are truncated to only 8 bytes, making them even weaker, but + * making the signature file shorter. + */ + if (!strong_len) + strong_len = 8; + sig_magic = RS_MD4_SIG_MAGIC; + } else { + rs_error("unknown hash algorithm %s", rs_hash_name); + return RS_PARAM_ERROR; + } + + result = rs_sig_file(basis_file, sig_file, block_len, strong_len, + sig_magic, &stats); rs_file_close(sig_file); rs_file_close(basis_file); @@ -373,3 +397,6 @@ int main(const int argc, const char *argv[]) poptFreeContext(opcon); return result; } + +/* vim: et sw=4 + */ diff --git a/rdiff.magic b/rdiff.magic index e2d4216..93df761 100644 --- a/rdiff.magic +++ b/rdiff.magic @@ -3,13 +3,15 @@ # # librsync -- the library for network deltas # -# Copyright (C) 2001 by Martin Pool. You may do whatever you want with +# Copyright 2001, 2014 by Martin Pool. You may do whatever you want with # this file. -# -# $Id$ 0 belong 0x72730236 rdiff network-delta data -0 belong 0x72730136 rdiff network-delta signature data ->4 belong x (block length=%d, +0 belong 0x72730136 rdiff network-delta signature data (MD4, +>4 belong x block length=%d, +>8 belong x signature strength=%d) + +0 belong 0x72730137 rdiff network-delta signature data (BLAKE2, +>4 belong x block length=%d, >8 belong x signature strength=%d) @@ -38,7 +38,6 @@ #include "job.h" #include "trace.h" #include "netint.h" -#include "protocol.h" #include "util.h" #include "stream.h" @@ -73,7 +72,7 @@ static rs_result rs_loadsig_add_sum(rs_job_t *job, rs_strong_sum_t *strong) memcpy(asignature->strong_sum, strong, sig->strong_sum_len); if (rs_trace_enabled()) { - char hexbuf[RS_MD4_LENGTH * 2 + 2]; + char hexbuf[RS_MAX_STRONG_SUM_LENGTH * 2 + 2]; rs_hexify(hexbuf, strong, sig->strong_sum_len); rs_trace("read in checksum: weak=%#x, strong=%s", asignature->weak_sum, @@ -133,7 +132,7 @@ static rs_result rs_loadsig_s_stronglen(rs_job_t *job) return result; job->strong_sum_len = l; - if (l < 0 || l > RS_MD4_LENGTH) { + if (l < 0 || l > RS_MAX_STRONG_SUM_LENGTH) { rs_error("strong sum length %d is implausible", l); return RS_CORRUPT; } @@ -178,11 +177,17 @@ static rs_result rs_loadsig_s_magic(rs_job_t *job) if ((result = rs_suck_n4(job, &l)) != RS_DONE) { return result; - } else if (l != RS_SIG_MAGIC) { - rs_error("wrong magic number %#10x for signature", l); - return RS_BAD_MAGIC; - } else { - rs_trace("got signature magic %#10x", l); + } + + switch(l) { + case RS_MD4_SIG_MAGIC: + case RS_BLAKE2_SIG_MAGIC: + job->magic = job->signature->magic = l; + rs_trace("got signature magic %#10x", l); + break; + default: + rs_error("wrong magic number %#10x for signature", l); + return RS_BAD_MAGIC; } job->statefn = rs_loadsig_s_blocklen; @@ -208,7 +213,7 @@ rs_job_t *rs_loadsig_begin(rs_signature_t **signature) job = rs_job_new("loadsig", rs_loadsig_s_magic); *signature = job->signature = rs_alloc_struct(rs_signature_t); job->signature->count = 0; - + return job; } @@ -1,7 +1,6 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- the library for network deltas - * $Id$ * * Copyright (C) 2000, 2001 by Martin Pool <mbp@sourcefrog.net> * @@ -85,7 +84,7 @@ void rs_scoop_input(rs_job_t *job, size_t len) if (job->scoop_alloc < len) { /* need to allocate a new buffer, too */ - char *newbuf; + rs_byte_t *newbuf; int newsize = 2 * len; newbuf = rs_alloc(newsize, "scoop buffer"); if (job->scoop_avail) @@ -1,9 +1,8 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- the library for network deltas - * $Id$ * - * Copyright (C) 1999, 2000, 2001 by Martin Pool <mbp@sourcefrog.net> + * Copyright (C) 1999, 2000, 2001, 2014 by Martin Pool <mbp@sourcefrog.net> * Copyright (C) 1999 by Andrew Tridgell * * This program is free software; you can redistribute it and/or modify @@ -115,7 +114,8 @@ rs_build_hash_table(rs_signature_t * sums) */ int rs_search_for_block(rs_weak_sum_t weak_sum, - char const *inbuf, size_t block_len, + const rs_byte_t *inbuf, + size_t block_len, rs_signature_t const *sig, rs_stats_t * stats, rs_long_t * match_where) { @@ -140,7 +140,14 @@ rs_search_for_block(rs_weak_sum_t weak_sum, rs_trace("found weak match for %08x in token %d", weak_sum, token); if (!got_strong) { - rs_calc_strong_sum(inbuf, block_len, &strong_sum); + if(sig->magic == RS_BLAKE2_SIG_MAGIC) { + rs_calc_blake2_sum(inbuf, block_len, &strong_sum); + } else if (sig->magic == RS_MD4_SIG_MAGIC) { + rs_calc_md4_sum(inbuf, block_len, &strong_sum); + } else { + rs_error("Unknown signature algorithm - this is a BUG"); + return 0; /* FIXME: Is this the best way to handle this? */ + } got_strong = 1; } @@ -1,9 +1,8 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- the library for network deltas - * $Id$ * - * Copyright (C) 1999, 2000 by Martin Pool <mbp@sourcefrog.net> + * Copyright (C) 1999, 2000, 2014 by Martin Pool <mbp@sourcefrog.net> * Copyright (C) 1999 by Andrew Tridgell * * This program is free software; you can redistribute it and/or modify @@ -23,7 +22,8 @@ int rs_search_for_block(rs_weak_sum_t weak_sum, - char const *inbuf, size_t block_len, + const rs_byte_t *inbuf, + size_t block_len, rs_signature_t const *sums, rs_stats_t * stats, rs_long_t * match_where); @@ -1,7 +1,6 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- dynamic caching and delta update in HTTP - * $Id$ * * Copyright (C) 2000, 2001 by Martin Pool <mbp@sourcefrog.net> * @@ -96,8 +95,6 @@ #include "util.h" #include "trace.h" -static const int RS_STREAM_DOGTAG = 2001125; - /** * \brief Copy up to \p max_len bytes from input of \b stream to its output. @@ -61,7 +61,7 @@ void rs_sumset_dump(rs_signature_t const *sums) { int i; - char strong_hex[RS_MD4_LENGTH * 3]; + char strong_hex[RS_MAX_STRONG_SUM_LENGTH * 3]; rs_log(RS_LOG_INFO, "sumset info: block_len=%d, file length=%lu, " @@ -53,6 +53,7 @@ struct rs_signature { rs_block_sig_t *block_sigs; /* points to info for each chunk */ int *tag_table; rs_target_t *targets; + int magic; }; diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index 96078ff..2f5759d 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -30,8 +30,9 @@ isprefix_driver_LDADD = ../isprefix.o # XXX: should link replaced functions # tests are run first. TESTS = \ + help.test \ signature.test mutate.test sources.test isprefix.test \ - delta.test mksum.test triple.test changes.test + delta.test triple.test changes.test noinst_SCRIPTS = testcommon.sh mutate.pl $(TESTS) @@ -42,7 +43,6 @@ test_data = dot \ delta.input/02.delta delta.input/02.expect \ delta.input/03.delta delta.input/03.expect \ mdfour.input/01.expect mdfour.input/01.data \ - mksum.input/COPYING.sig \ triple.input/copying.in triple.input/half.in \ triple.input/hello.in triple.input/zero.in \ signature.input/01.in signature.input/01.sig diff --git a/testsuite/changes.test b/testsuite/changes.test index c2d167c..c3d7cd6 100755 --- a/testsuite/changes.test +++ b/testsuite/changes.test @@ -3,6 +3,8 @@ # librsync -- the library for network deltas # # Copyright (C) 2001, 2014 by Martin Pool <mbp@sourcefrog.net> + +# changes.test: Test converting in both directions between each pair of files. # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public License @@ -27,7 +29,10 @@ do old=$inputdir/01.in for new in $inputdir/*.in do - triple_test $buf $old $new - triple_test $buf $new $old + for hashopt in '' -Hmd4 -Hblake2 + do + triple_test $buf $old $new $hashopt + triple_test $buf $new $old $hashopt + done done done diff --git a/testsuite/delta.test b/testsuite/delta.test index c2cc123..3202423 100755 --- a/testsuite/delta.test +++ b/testsuite/delta.test @@ -2,6 +2,8 @@ # librsync -- the library for network deltas +# delta.test: Check application of some canned deltas to /dev/null + # Copyright (C) 1999, 2000, 2001, 2014 by Martin Pool <mbp@sourcefrog.net> # Copyright (C) 1999 by Andrew Tridgell <tridge@samba.org> diff --git a/testsuite/mksum.test b/testsuite/help.test index 01ae02e..81415a7 100755 --- a/testsuite/mksum.test +++ b/testsuite/help.test @@ -2,8 +2,10 @@ # librsync -- the library for network deltas # -# Copyright (C) 2000, 2001, 2014 by Martin Pool <mbp@sourcefrog.net> -# +# help.test: Test that `rdiff --help` etc work reasonably. + +# Copyright 2014 by Martin Pool <mbp@sourcefrog.net> + # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public License # as published by the Free Software Foundation; either version 2.1 of @@ -19,16 +21,9 @@ # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. . $srcdir/testcommon.sh -inputdir=$srcdir/mksum.input -[ -d $inputdir ] || fail_test $? no inputdir $inputdir -sig=$tmpdir/sig.tmp -expect=$inputdir/COPYING.sig -input=$srcdir/../COPYING +run_test ../rdiff --help +run_test ../rdiff --version -for inbuf in $bufsizes -do - cmd="../rdiff -I$inbuf signature $input $sig" - run_test $cmd - check_compare $sig $expect "$cmd" -done +run_test ../rdiff --help | grep ' --statistics' +run_test ../rdiff --version | grep 'Copyright' diff --git a/testsuite/mksum.input/COPYING.sig b/testsuite/mksum.input/COPYING.sig Binary files differdeleted file mode 100644 index 898bc51..0000000 --- a/testsuite/mksum.input/COPYING.sig +++ /dev/null diff --git a/testsuite/mutate.test b/testsuite/mutate.test index 5efc96a..b650d0d 100755 --- a/testsuite/mutate.test +++ b/testsuite/mutate.test @@ -1,9 +1,11 @@ #! /bin/sh -e # librsync -- the library for network deltas -# # Copyright (C) 2001, 2014 by Martin Pool <mbp@sourcefrog.net> -# + +# mutate.test: Make some deterministic pseudorandom changes to a file and +# compute deltas across them. + # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public License # as published by the Free Software Foundation; either version 2.1 of @@ -42,11 +44,14 @@ while test $i -lt 100 do perl "$srcdir/mutate.pl" $i 5 <"$old" >"$new" 2>>"$tmpdir/mutate.log" - run_test ../rdiff $debug signature $old $sig - run_test ../rdiff $debug delta $sig $new $delta - run_test ../rdiff $debug patch $old $delta "$out" + for hashopt in '' -Hmd4 -Hblake2 + do + run_test ../rdiff $debug signature $old $sig + run_test ../rdiff $debug delta $sig $new $delta + run_test ../rdiff $debug patch $old $delta "$out" - check_compare "$new" "$out" "mutate $i $old $new" + check_compare "$new" "$out" "mutate $i $old $new" + done i=`expr $i + 1` done diff --git a/testsuite/signature.input/blake2/01.sig b/testsuite/signature.input/blake2/01.sig Binary files differnew file mode 100644 index 0000000..f7a488f --- /dev/null +++ b/testsuite/signature.input/blake2/01.sig diff --git a/testsuite/signature.input/01.sig b/testsuite/signature.input/md4/01.sig Binary files differindex 898bc51..898bc51 100644 --- a/testsuite/signature.input/01.sig +++ b/testsuite/signature.input/md4/01.sig diff --git a/testsuite/signature.test b/testsuite/signature.test index aead30a..4741fb1 100755 --- a/testsuite/signature.test +++ b/testsuite/signature.test @@ -2,8 +2,12 @@ # librsync -- the library for network deltas # -# Copyright (C) 2001 by Martin Pool <mbp@sourcefrog.net> -# +# signature.test: Test that `rdiff signature` produces the exactly expected +# output: this is supposed to check that it remains compatible with previous +# versions. + +# Copyright (C) 2001, 2014 by Martin Pool <mbp@sourcefrog.net> + # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public License # as published by the Free Software Foundation; either version 2.1 of @@ -22,12 +26,15 @@ new=$tmpdir/signature -for input in "$srcdir/signature.input"/*.in +for hashfunc in md4 blake2 do - for inbuf in $bufsizes + for input in "$srcdir/signature.input"/*.in do - expect=`echo $input | sed -e 's/.in$/.sig/'` - run_test ../rdiff -I$inbuf signature "$input" "$new" - cmp "$expect" "$new" || fail_test "signature -I$inbuf $input" + for inbuf in $bufsizes + do + expect=`echo $input | sed -e 's/.in$/.sig/' -e "s,input,input/$hashfunc,"` + run_test ../rdiff --hash=$hashfunc -I$inbuf signature "$input" "$new" + check_compare "$expect" "$new" + done done done diff --git a/testsuite/sources.test b/testsuite/sources.test index eeba691..88b3e04 100755 --- a/testsuite/sources.test +++ b/testsuite/sources.test @@ -1,9 +1,10 @@ #! /bin/sh -e # librsync -- the library for network deltas -# # Copyright (C) 2001, 2014 by Martin Pool <mbp@sourcefrog.net> -# + +# sources.test: Run three-way tests on all librsync source files. + # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public License # as published by the Free Software Foundation; either version 2.1 of @@ -29,8 +30,12 @@ do test ! -r $new && continue test ! -r $old && continue test -n "$stats" && echo $old $new - triple_test $buf $old $new - triple_test $buf $new $old + + for hashopt in '' -Hmd4 -Hblake2 + do + triple_test $buf $old $new $hashopt + triple_test $buf $new $old $hashopt + done done done done diff --git a/testsuite/testcommon.sh b/testsuite/testcommon.sh index 8bf9e66..84839c2 100644 --- a/testsuite/testcommon.sh +++ b/testsuite/testcommon.sh @@ -76,11 +76,12 @@ triple_test () { buf="$1" old="$2" new="$3" + hashopt="$4" - run_test ../rdiff $debug -I$buf -O$buf $stats signature --block-size=$block_len \ + run_test ../rdiff $debug $hashopt -I$buf -O$buf $stats signature --block-size=$block_len \ $old $tmpdir/sig - run_test ../rdiff $debug -I$buf -O$buf $stats delta $tmpdir/sig $new $tmpdir/delta - run_test ../rdiff $debug -I$buf -O$buf $stats patch $old $tmpdir/delta $tmpdir/new + run_test ../rdiff $debug $hashopt -I$buf -O$buf $stats delta $tmpdir/sig $new $tmpdir/delta + run_test ../rdiff $debug $hashopt -I$buf -O$buf $stats patch $old $tmpdir/delta $tmpdir/new check_compare $new $tmpdir/new "triple -I$buf -O$buf $old $new" } diff --git a/testsuite/triple.test b/testsuite/triple.test index 37f758f..3cec31b 100755 --- a/testsuite/triple.test +++ b/testsuite/triple.test @@ -27,12 +27,14 @@ do do for new in $inputdir/*.in do - run_test ../rdiff $debug -I$buf -O$buf signature $old $tmpdir/sig - run_test ../rdiff $debug -I$buf -O$buf delta $tmpdir/sig $new $tmpdir/delta - run_test ../rdiff $debug -I$buf -O$buf patch $old $tmpdir/delta $tmpdir/new - - check_compare $new $tmpdir/new "triple -I$buf -O$buf $old $new" + for hashopt in $all_hashopts + do + run_test ../rdiff $debug $hashopt -I$buf -O$buf signature $old $tmpdir/sig + run_test ../rdiff $debug $hashopt -I$buf -O$buf delta $tmpdir/sig $new $tmpdir/delta + run_test ../rdiff $debug $hashopt -I$buf -O$buf patch $old $tmpdir/delta $tmpdir/new + check_compare $new $tmpdir/new "triple -I$buf -O$buf $old $new" + done done done done @@ -1,9 +1,8 @@ /*= -*- c-basic-offset: 4; indent-tabs-mode: nil; -*- * * librsync -- the library for network deltas - * $Id$ * - * Copyright (C) 2000, 2001 by Martin Pool <mbp@sourcefrog.net> + * Copyright 2000, 2001, 2014 by Martin Pool <mbp@sourcefrog.net> * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -100,16 +99,20 @@ rs_whole_run(rs_job_t *job, FILE *in_file, FILE *out_file) * * \param strong_len truncated length of strong checksums, in bytes * + * \param sig_magic A signature magic number indicating + * what format to use. + * * \sa rs_sig_begin() */ rs_result rs_sig_file(FILE *old_file, FILE *sig_file, size_t new_block_len, - size_t strong_len, rs_stats_t *stats) + size_t strong_len, rs_long_t sig_magic, + rs_stats_t *stats) { rs_job_t *job; rs_result r; - job = rs_sig_begin(new_block_len, strong_len); + job = rs_sig_begin(new_block_len, strong_len, sig_magic); r = rs_whole_run(job, old_file, sig_file); if (stats) memcpy(stats, &job->stats, sizeof *stats); |