[fix] Avoid int overflow when allocating large buffers

This patch introduces three macros: _cairo_malloc_ab,
_cairo_malloc_abc, _cairo_malloc_ab_plus_c and replaces various calls
to malloc(a*b), malloc(a*b*c), and malloc(a*b+c) with them.  The macros
return NULL if int overflow would occur during the allocation.  See
CODING_STYLE for more information.

1 files changed, 104 insertions, 0 deletions

diff --git a/src/cairo-malloc-private.h b/src/cairo-malloc-private.h
new file mode 100644
index 000000000..f503b3951
--- /dev/null
+++ b/src/cairo-malloc-private.h
@@ -0,0 +1,104 @@
+/* -*- Mode: c; tab-width: 8; c-basic-offset: 4; indent-tabs-mode: t; -*- */
+/* Cairo - a vector graphics library with display and print output
+ *
+ * Copyright © 2007 Mozilla Corporation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it either under the terms of the GNU Lesser General Public
+ * License version 2.1 as published by the Free Software Foundation
+ * (the "LGPL") or, at your option, under the terms of the Mozilla
+ * Public License Version 1.1 (the "MPL"). If you do not alter this
+ * notice, a recipient may use your version of this file under either
+ * the MPL or the LGPL.
+ *
+ * You should have received a copy of the LGPL along with this library
+ * in the file COPYING-LGPL-2.1; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ * You should have received a copy of the MPL along with this library
+ * in the file COPYING-MPL-1.1
+ *
+ * The contents of this file are subject to the Mozilla Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY
+ * OF ANY KIND, either express or implied. See the LGPL or the MPL for
+ * the specific language governing rights and limitations.
+ *
+ * The Original Code is the cairo graphics library.
+ *
+ * The Initial Developer of the Original Code is Mozilla Corporation
+ *
+ * Contributor(s):
+ *	Vladimir Vukicevic <vladimir@pobox.com>
+ */
+
+#ifndef CAIRO_MALLOC_PRIVATE_H
+#define CAIRO_MALLOC_PRIVATE_H
+
+#include "cairo-wideint-private.h"
+
+/**
+ * _cairo_malloc_ab:
+ * @n: number of elements to allocate
+ * @size: size of each element
+ *
+ * Allocates @a*@size memory using malloc(), taking care to not
+ * overflow when doing the multiplication.  Behaves much like
+ * calloc(), except that the returned memory is not set to zero.
+ * The memory should be freed using free().
+ *
+ * @size should be a constant so that the compiler can optimize
+ * out a constant division.
+ *
+ * Return value: A pointer to the newly allocated memory, or %NULL in
+ * case of malloc() failure or overflow.
+ */
+
+#define _cairo_malloc_ab(a, size) \
+  ((unsigned) (a) >= INT32_MAX / (unsigned) (size) ? NULL : \
+   malloc((unsigned) (a) * (unsigned) (size)))
+
+/**
+ * _cairo_malloc_abc:
+ * @a: first factor of number of elements to allocate
+ * @b: second factor of number of elements to allocate
+ * @size: size of each element
+ *
+ * Allocates @a*@b*@size memory using malloc(), taking care to not
+ * overflow when doing the multiplication.  Behaves like
+ * _cairo_malloc_ab().  The memory should be freed using free().
+ *
+ * @size should be a constant so that the compiler can optimize
+ * out a constant division.
+ *
+ * Return value: A pointer to the newly allocated memory, or %NULL in
+ * case of malloc() failure or overflow.
+ */
+
+#define _cairo_malloc_abc(a, b, size) \
+  ((unsigned) (a) >= INT32_MAX / (unsigned) (b) ? NULL : \
+   (unsigned) ((a)*(b)) >= INT32_MAX / (unsigned) (size) ? NULL : \
+   malloc((unsigned) (a) * (unsigned) (b) * (unsigned) size))
+
+/**
+ * _cairo_malloc_ab_plus_c:
+ * @n: number of elements to allocate
+ * @size: size of each element
+ * @k: additional size to allocate
+ *
+ * Allocates @a*@ksize+@k memory using malloc(), taking care to not
+ * overflow when doing the arithmetic.  Behaves like
+ * _cairo_malloc_ab().  The memory should be freed using free().
+ *
+ * Return value: A pointer to the newly allocated memory, or %NULL in
+ * case of malloc() failure or overflow.
+ */
+
+#define _cairo_malloc_ab_plus_c(n, size, k) \
+  ((unsigned) (n) >= INT32_MAX / (unsigned) (size) ? NULL : \
+   (unsigned) (k) >= INT32_MAX - (unsigned) (n) * (unsigned) (size) ? NULL : \
+   malloc((unsigned) (n) * (unsigned) (size) + (unsigned) (k)))
+
+#endif /* CAIRO_MALLOC_PRIVATE_H */