Report out of buf space error in amqp_table_encode

amqp_encode_8 and amqp_encode_bytes return 1 on success, 0 otherwise.
Correct the checking of the return value of these functions to match the
style used in amqp_table.c

Report AMQP_STATUS_INVALID_PARAMETER in the event an invalid field kind
is input (instead of abort()).

Report AMQP_STATUS_TABLE_TOO_BIG if the table is too large to be encoded
in the buffer provided.

2 files changed, 25 insertions, 14 deletions

diff --git a/librabbitmq/amqp.h b/librabbitmq/amqp.h
index f469dda..59513c1 100644
--- a/librabbitmq/amqp.h
+++ b/librabbitmq/amqp.h
@@ -341,6 +341,8 @@ typedef enum amqp_status_enum_
   AMQP_STATUS_CONNECTION_CLOSED =         -0x0007,
   AMQP_STATUS_BAD_URL =                   -0x0008,
   AMQP_STATUS_SOCKET_ERROR =              -0x0009,
+  AMQP_STATUS_INVALID_PARAMETER =         -0x000A,
+  AMQP_STATUS_TABLE_TOO_BIG =             -0x000B,
 
   AMQP_STATUS_SSL_ERROR =                 -0x0200
 } amqp_status_enum;
diff --git a/librabbitmq/amqp_table.c b/librabbitmq/amqp_table.c
index 8495377..6071ece 100644
--- a/librabbitmq/amqp_table.c
+++ b/librabbitmq/amqp_table.c
@@ -299,12 +299,13 @@ static int amqp_encode_array(amqp_bytes_t encoded,
     }
   }
 
-  if (amqp_encode_32(encoded, &start, *offset - start - 4)) {
-    res = AMQP_STATUS_OK;
-  } else {
-    res = AMQP_STATUS_BAD_AMQP_DATA;
+  if (!amqp_encode_32(encoded, &start, *offset - start - 4)) {
+    res = AMQP_STATUS_TABLE_TOO_BIG;
+    goto out;
   }
 
+  res = AMQP_STATUS_OK;
+
 out:
   return res;
 }
@@ -319,13 +320,13 @@ int amqp_encode_table(amqp_bytes_t encoded,
   *offset += 4; /* size of the table gets filled in later on */
 
   for (i = 0; i < input->num_entries; i++) {
-    res = amqp_encode_8(encoded, offset, input->entries[i].key.len);
-    if (res < 0) {
+    if (!amqp_encode_8(encoded, offset, input->entries[i].key.len)) {
+      res = AMQP_STATUS_TABLE_TOO_BIG;
       goto out;
     }
 
-    res = amqp_encode_bytes(encoded, offset, input->entries[i].key);
-    if (res < 0) {
+    if (!amqp_encode_bytes(encoded, offset, input->entries[i].key)) {
+      res = AMQP_STATUS_TABLE_TOO_BIG;
       goto out;
     }
 
@@ -335,12 +336,13 @@ int amqp_encode_table(amqp_bytes_t encoded,
     }
   }
 
-  if (amqp_encode_32(encoded, &start, *offset - start - 4)) {
-    res = AMQP_STATUS_OK;
-  } else {
-    res = AMQP_STATUS_BAD_AMQP_DATA;
+  if (!amqp_encode_32(encoded, &start, *offset - start - 4)) {
+    res = AMQP_STATUS_TABLE_TOO_BIG;
+    goto out;
   }
 
+  res = AMQP_STATUS_OK;
+
 out:
   return res;
 }
@@ -355,7 +357,11 @@ static int amqp_encode_field_value(amqp_bytes_t encoded,
     goto out;
   }
 
-#define FIELD_ENCODER(bits, val) if (!amqp_encode_##bits(encoded, offset, val)) goto out; break
+#define FIELD_ENCODER(bits, val) if (!amqp_encode_##bits(encoded, offset, val)) { \
+                                    res = AMQP_STATUS_TABLE_TOO_BIG;              \
+                                    goto out;                                     \
+                                  }                                               \
+                                  break
 
   switch (entry->kind) {
   case AMQP_FIELD_KIND_BOOLEAN:
@@ -392,6 +398,7 @@ static int amqp_encode_field_value(amqp_bytes_t encoded,
   case AMQP_FIELD_KIND_DECIMAL:
     if (!amqp_encode_8(encoded, offset, entry->value.decimal.decimals)
         || !amqp_encode_32(encoded, offset, entry->value.decimal.value)) {
+      res = AMQP_STATUS_TABLE_TOO_BIG;
       goto out;
     }
     break;
@@ -403,6 +410,7 @@ static int amqp_encode_field_value(amqp_bytes_t encoded,
   case AMQP_FIELD_KIND_BYTES:
     if (!amqp_encode_32(encoded, offset, entry->value.bytes.len)
         || !amqp_encode_bytes(encoded, offset, entry->value.bytes)) {
+      res = AMQP_STATUS_TABLE_TOO_BIG;
       goto out;
     }
     break;
@@ -422,7 +430,8 @@ static int amqp_encode_field_value(amqp_bytes_t encoded,
     break;
 
   default:
-    abort();
+    res = AMQP_STATUS_INVALID_PARAMETER;
+    goto out;
   }
 
   res = AMQP_STATUS_OK;