RAR5 reader: fixed big-endian problems

The reader has assumed it's running on little-endian. The commit changes
direct memory reads to archive_le* function calls, which should allow
the reader to run on big-endian machines as well.

Changes were needed in the reader itself and in the file holding
reader's test cases.

The commit also removes 1 warning encountered when compiling under GCC
8 on PowerPC architecture.

Fixes #1097

2 files changed, 39 insertions, 33 deletions

diff --git a/libarchive/archive_read_support_format_rar5.c b/libarchive/archive_read_support_format_rar5.c
index 7681c948..86478a49 100644
--- a/libarchive/archive_read_support_format_rar5.c
+++ b/libarchive/archive_read_support_format_rar5.c
@@ -24,6 +24,7 @@
 */
 
 #include "archive_platform.h"
+#include "archive_endian.h"
 
 #ifdef HAVE_ERRNO_H
 #include <errno.h>
@@ -225,18 +226,10 @@ struct bit_reader {
     int in_addr;        /* Current byte pointer. */
 };
 
-/* RARv5 block header structure. */
+/* RARv5 block header structure. Use bf_* functions to get values from
+ * block_flags_u8 field. I.e. bf_byte_count, etc. */
 struct compressed_block_header {
-    union {
-        struct {
-            uint8_t bit_size : 3;
-            uint8_t byte_count : 3;
-            uint8_t is_last_block : 1;
-            uint8_t is_table_present : 1;
-        } block_flags;
-        uint8_t block_flags_u8;
-    };
-
+    uint8_t block_flags_u8; /* Fields encoded in little-endian bitfield */
     uint8_t block_cksum;
 };
 
@@ -429,26 +422,40 @@ static void cdeque_free(struct cdeque* d) {
     d->cap_mask = 0;
 }
 
+static inline
+uint8_t bf_bit_size(const struct compressed_block_header* hdr) {
+    return hdr->block_flags_u8 & 7;
+}
+
+static inline
+uint8_t bf_byte_count(const struct compressed_block_header* hdr) {
+    return (hdr->block_flags_u8 >> 3) & 7;
+}
+
+static inline
+uint8_t bf_is_last_block(const struct compressed_block_header* hdr) {
+    return (hdr->block_flags_u8 >> 6) & 1;
+}
+
+static inline
+uint8_t bf_is_table_present(const struct compressed_block_header* hdr) {
+    return (hdr->block_flags_u8 >> 7) & 1;
+}
+
 static inline struct rar5* get_context(struct archive_read* a) {
     return (struct rar5*) a->format->data;
 }
 
-// TODO: make sure these functions return a little endian number
-
 /* Convenience functions used by filter implementations. */
 
 static uint32_t read_filter_data(struct rar5* rar, uint32_t offset) {
-    uint32_t* dptr = (uint32_t*) &rar->cstate.window_buf[offset];
-    // TODO: bswap if big endian
-    return *dptr;
+    return archive_le32dec(&rar->cstate.window_buf[offset]);
 }
 
 static void write_filter_data(struct rar5* rar, uint32_t offset,
         uint32_t value)
 {
-    uint32_t* dptr = (uint32_t*) &rar->cstate.filtered_buf[offset];
-    // TODO: bswap if big endian
-    *dptr = value;
+    archive_le32enc(&rar->cstate.filtered_buf[offset], value);
 }
 
 static void circular_memcpy(uint8_t* dst, uint8_t* window, const int mask,
@@ -995,8 +1002,7 @@ static int read_u32(struct archive_read* a, uint32_t* pvalue) {
     if(!read_ahead(a, 4, &p))
         return 0;
 
-    *pvalue = *(const uint32_t*)p;
-
+    *pvalue = archive_le32dec(p);
     return ARCHIVE_OK == consume(a, 4) ? 1 : 0;
 }
 
@@ -1005,8 +1011,7 @@ static int read_u64(struct archive_read* a, uint64_t* pvalue) {
     if(!read_ahead(a, 8, &p))
         return 0;
 
-    *pvalue = *(const uint64_t*)p;
-
+    *pvalue = archive_le64dec(p);
     return ARCHIVE_OK == consume(a, 8) ? 1 : 0;
 }
 
@@ -1936,7 +1941,7 @@ static int create_decode_tables(uint8_t* bit_length,
         dist = bit_field - table->decode_len[cur_len - 1];
         dist >>= (16 - cur_len);
 
-        pos = table->decode_pos[cur_len] + dist;
+        pos = table->decode_pos[cur_len & 15] + dist;
         if(cur_len < rar5_countof(table->decode_pos) && pos < size) {
             table->quick_num[code] = table->decode_num[pos];
         } else {
@@ -2159,17 +2164,17 @@ static int parse_block_header(struct archive_read* a, const uint8_t* p,
 {
     memcpy(hdr, p, sizeof(struct compressed_block_header));
 
-    if(hdr->block_flags.byte_count > 2) {
+    if(bf_byte_count(hdr) > 2) {
         archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
                 "Unsupported block header size (was %d, max is 2)",
-                hdr->block_flags.byte_count);
+                bf_byte_count(hdr));
         return ARCHIVE_FATAL;
     }
 
     /* This should probably use bit reader interface in order to be more
      * future-proof. */
     *block_size = 0;
-    switch(hdr->block_flags.byte_count) {
+    switch(bf_byte_count(hdr)) {
         /* 1-byte block size */
         case 0:
             *block_size = *(const uint8_t*) &p[2];
@@ -2177,12 +2182,12 @@ static int parse_block_header(struct archive_read* a, const uint8_t* p,
 
         /* 2-byte block size */
         case 1:
-            *block_size = *(const uint16_t*) &p[2];
+            *block_size = archive_le16dec(&p[2]);
             break;
 
         /* 3-byte block size */
         case 2:
-            *block_size = *(const uint32_t*) &p[2];
+            *block_size = archive_le32dec(&p[2]);
             *block_size &= 0x00FFFFFF;
             break;
 
@@ -2379,7 +2384,7 @@ static int do_uncompress_block(struct archive_read* a, const uint8_t* p) {
 
     const int cmask = rar->cstate.window_mask;
     const struct compressed_block_header* hdr = &rar->last_block_hdr;
-    const uint8_t bit_size = 1 + hdr->block_flags.bit_size;
+    const uint8_t bit_size = 1 + bf_bit_size(hdr);
 
     while(1) {
         if(rar->cstate.write_ptr - rar->cstate.last_write_ptr >
@@ -2777,7 +2782,7 @@ static int process_block(struct archive_read* a) {
 
         /* Skip block header. Next data is huffman tables, if present. */
         ssize_t to_skip = sizeof(struct compressed_block_header) +
-            rar->last_block_hdr.block_flags.byte_count + 1;
+            bf_byte_count(&rar->last_block_hdr) + 1;
 
         if(ARCHIVE_OK != consume(a, to_skip))
             return ARCHIVE_EOF;
@@ -2833,7 +2838,7 @@ static int process_block(struct archive_read* a) {
         rar->bits.in_addr = 0;
         rar->bits.bit_addr = 0;
 
-        if(rar->last_block_hdr.block_flags.is_table_present) {
+        if(bf_is_table_present(&rar->last_block_hdr)) {
             /* Load Huffman tables. */
             ret = parse_tables(a, rar, p);
             if(ret != ARCHIVE_OK) {
diff --git a/libarchive/test/test_read_format_rar5.c b/libarchive/test/test_read_format_rar5.c
index 0ccedc76..7f2f32df 100644
--- a/libarchive/test/test_read_format_rar5.c
+++ b/libarchive/test/test_read_format_rar5.c
@@ -28,6 +28,7 @@
  * help. */
 #define __LIBARCHIVE_BUILD
 #include <archive_crc32.h>
+#include <archive_endian.h>
 
 #define PROLOGUE(reffile) \
     struct archive_entry *ae; \
@@ -81,7 +82,7 @@ int verify_data(const uint8_t* data_ptr, int magic, int size) {
         /* *lptr is a value inside unpacked test file, val is the
          * value that should be in the unpacked test file. */
 
-        if(*lptr != val)
+        if(archive_le32dec(lptr) != (uint32_t) val)
             return 0;
     }