diff options
| author | Sean McBride <sean@rogue-research.com> | 2012-11-29 18:20:11 -0500 |
|---|---|---|
| committer | Brad King <brad.king@kitware.com> | 2012-11-30 08:06:58 -0500 |
| commit | cf0f28ad3c13c7dc964b120e336cc802b20b442b (patch) | |
| tree | 059d58465817469f7456d51e6b9493c865c1a28c /libarchive | |
| parent | 39df0279539b519ad83ae12a5dedd50e6ecdc5d1 (diff) | |
| download | libarchive-cf0f28ad3c13c7dc964b120e336cc802b20b442b.tar.gz | |
Fix undefined left shift with signed ints
Caught by Clang's -fsanitize=shift. A small unsigned int was promoted,
according to C's regular promotion rules, to a signed int, it was then
left shifted. This sometimes pushed a 1 into the sign bit, which is
undefined behaviour. Fix by using unsigned temporaries.
Diffstat (limited to 'libarchive')
| -rw-r--r-- | libarchive/archive_endian.h | 36 |
1 files changed, 32 insertions, 4 deletions
diff --git a/libarchive/archive_endian.h b/libarchive/archive_endian.h index 68123b0d..750e190b 100644 --- a/libarchive/archive_endian.h +++ b/libarchive/archive_endian.h @@ -58,7 +58,13 @@ archive_be16dec(const void *pp) { unsigned char const *p = (unsigned char const *)pp; - return ((p[0] << 8) | p[1]); + /* Store into unsigned temporaries before left shifting, to avoid + promotion to signed int and then left shifting into the sign bit, + which is undefined behaviour. */ + unsigned int p1 = p[1]; + unsigned int p0 = p[0]; + + return ((p0 << 8) | p1); } static inline uint32_t @@ -66,7 +72,15 @@ archive_be32dec(const void *pp) { unsigned char const *p = (unsigned char const *)pp; - return ((p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]); + /* Store into unsigned temporaries before left shifting, to avoid + promotion to signed int and then left shifting into the sign bit, + which is undefined behaviour. */ + unsigned int p3 = p[3]; + unsigned int p2 = p[2]; + unsigned int p1 = p[1]; + unsigned int p0 = p[0]; + + return ((p0 << 24) | (p1 << 16) | (p2 << 8) | p3); } static inline uint64_t @@ -82,7 +96,13 @@ archive_le16dec(const void *pp) { unsigned char const *p = (unsigned char const *)pp; - return ((p[1] << 8) | p[0]); + /* Store into unsigned temporaries before left shifting, to avoid + promotion to signed int and then left shifting into the sign bit, + which is undefined behaviour. */ + unsigned int p1 = p[1]; + unsigned int p0 = p[0]; + + return ((p1 << 8) | p0); } static inline uint32_t @@ -90,7 +110,15 @@ archive_le32dec(const void *pp) { unsigned char const *p = (unsigned char const *)pp; - return ((p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0]); + /* Store into unsigned temporaries before left shifting, to avoid + promotion to signed int and then left shifting into the sign bit, + which is undefined behaviour. */ + unsigned int p3 = p[3]; + unsigned int p2 = p[2]; + unsigned int p1 = p[1]; + unsigned int p0 = p[0]; + + return ((p3 << 24) | (p2 << 16) | (p1 << 8) | p0); } static inline uint64_t |