OSS-fuzz 45886 - validate funtion parameters for transfer functions

This was causing a buffer overrun when evaluating a transfer function
because the code assumed it would be a 1-in one-out function, whereas
the supplied function is a 1-in 3-out function, causing it to overrun
the output buffer.

This commit checks that the function is a 1-in, 1-out function and
returns an error if it isn't.

Similar fix applied for colour transfers, and a C++ commented out line
removed as it seemed to be entirely spurious.

1 files changed, 10 insertions, 1 deletions

diff --git a/pdf/pdf_gstate.c b/pdf/pdf_gstate.c
index 7801ee681..c1444b4b5 100644
--- a/pdf/pdf_gstate.c
+++ b/pdf/pdf_gstate.c
@@ -875,6 +875,11 @@ static int pdfi_set_all_transfers(pdf_context *ctx, pdf_array *a, pdf_dict *page
                     pdfi_countdown(o);
                     goto exit;
                 }
+                if (pfn[i]->params.m != 1 || pfn[i]->params.n != 1) {
+                    pdfi_countdown(o);
+                    code = gs_note_error(gs_error_rangecheck);
+                    goto exit;
+                }
             } else {
                 pdfi_countdown(o);
                 code = gs_note_error(gs_error_typecheck);
@@ -938,7 +943,6 @@ static int pdfi_set_all_transfers(pdf_context *ctx, pdf_array *a, pdf_dict *page
         }
     }
  exit:
-//    (void)pdfi_seek(ctx, ctx->main_stream, saved_stream_offset, SEEK_SET);
     for (i = 0; i < 4; i++) {
         pdfi_free_function(ctx, pfn[i]);
     }
@@ -957,6 +961,11 @@ static int pdfi_set_gray_transfer(pdf_context *ctx, pdf_obj *tr_obj, pdf_dict *p
     if (code < 0)
         return code;
 
+    if (pfn->params.m != 1 || pfn->params.n != 1) {
+        (void)pdfi_free_function(ctx, pfn);
+        return_error(gs_error_rangecheck);
+    }
+
     gs_settransfer_remap(ctx->pgs, gs_mapped_transfer, false);
     for (i = 0; i < transfer_map_size; i++) {
         float v, f;