lavf/prompeg: prompeg_write() must report data all was written

Previously, prompeg_write() would only report to caller that bytes we
written when a FEC packet was actually created.  Not all RTP packets are
expected to generate a FEC packet however, so this behavior was causing
avio to retry writing the RTP packet, eventually forcing the FEC state
machine to send a FEC packet erroneously (and so breaking out of the
retry loop).

This was resulting in incorrect FEC data being generated, and far too
many FEC packets to be sent (~100% FEC overhead).

fix #7863

Signed-off-by: David Holroyd <david.holroyd@m2amedia.tv>

1 files changed, 2 insertions, 4 deletions

diff --git a/libavformat/prompeg.c b/libavformat/prompeg.c
index 7b2e5e8344..59faa824bb 100644
--- a/libavformat/prompeg.c
+++ b/libavformat/prompeg.c
@@ -387,7 +387,7 @@ static int prompeg_write(URLContext *h, const uint8_t *buf, int size) {
     PrompegFec *fec_tmp;
     uint8_t *bitstring = NULL;
     int col_idx, col_out_idx, row_idx;
-    int ret, written = 0;
+    int ret = 0;
 
     if (s->init && ((ret = prompeg_init(h, buf, size)) < 0))
         goto end;
@@ -403,7 +403,6 @@ static int prompeg_write(URLContext *h, const uint8_t *buf, int size) {
         if (!s->first || s->packet_idx > 0) {
             if ((ret = prompeg_write_fec(h, s->fec_row, PROMPEG_FEC_ROW)) < 0)
                 goto end;
-            written += ret;
         }
         memcpy(s->fec_row->bitstring, bitstring, s->bitstring_size);
         s->fec_row->sn = AV_RB16(buf + 2);
@@ -434,7 +433,6 @@ static int prompeg_write(URLContext *h, const uint8_t *buf, int size) {
         col_out_idx = s->packet_idx / s->d;
         if ((ret = prompeg_write_fec(h, s->fec_col[col_out_idx], PROMPEG_FEC_COL)) < 0)
             goto end;
-        written += ret;
     }
 
     if (++s->packet_idx >= s->packet_idx_max) {
@@ -443,7 +441,7 @@ static int prompeg_write(URLContext *h, const uint8_t *buf, int size) {
             s->first = 0;
     }
 
-    ret = written;
+    ret = size;
 
 end:
     av_free(bitstring);