Bug#11739 SendBufferMemory set to 294967039 causes core where max = 4294967039

- added proper error message on all failed array pool mallocs

6 files changed, 31 insertions, 14 deletions

diff --git a/ndb/src/kernel/error/ErrorReporter.cpp b/ndb/src/kernel/error/ErrorReporter.cpp
index 248807db13d..6c8bb1fe615 100644
--- a/ndb/src/kernel/error/ErrorReporter.cpp
+++ b/ndb/src/kernel/error/ErrorReporter.cpp
@@ -165,7 +165,7 @@ ErrorReporter::setErrorHandlerShutdownType(NdbShutdownType nst)
 void childReportError(int error);
 
 void
-ErrorReporter::handleAssert(const char* message, const char* file, int line)
+ErrorReporter::handleAssert(const char* message, const char* file, int line, int ec)
 {
   char refMessage[100];
 
@@ -179,10 +179,10 @@ ErrorReporter::handleAssert(const char* message, const char* file, int line)
   BaseString::snprintf(refMessage, 100, "%s line: %d (block: %s)",
 	   file, line, blockName);
 #endif
-  WriteMessage(NDBD_EXIT_PRGERR, message, refMessage,
+  WriteMessage(ec, message, refMessage,
 	       theEmulatedJamIndex, theEmulatedJam);
 
-  childReportError(NDBD_EXIT_PRGERR);
+  childReportError(ec);
 
   NdbShutdown(s_errorHandlerShutdownType);
 }
diff --git a/ndb/src/kernel/error/ErrorReporter.hpp b/ndb/src/kernel/error/ErrorReporter.hpp
index 3f5e74f16b1..0ec84190238 100644
--- a/ndb/src/kernel/error/ErrorReporter.hpp
+++ b/ndb/src/kernel/error/ErrorReporter.hpp
@@ -18,6 +18,7 @@
 #define ERRORREPORTER_H
 
 #include <ndb_global.h>
+#include <ndbd_exit_codes.h>
 
 #include "TimeModule.hpp"
 #include <Emulator.hpp>
@@ -28,7 +29,7 @@ public:
   static void setErrorHandlerShutdownType(NdbShutdownType nst = NST_ErrorHandler);
   static void handleAssert(const char* message, 
 			   const char* file, 
-			   int line);
+			   int line, int ec = NDBD_EXIT_PRGERR);
   
   static void handleError(int faultID, 
 			  const char* problemData,
diff --git a/ndb/src/kernel/vm/ArrayPool.hpp b/ndb/src/kernel/vm/ArrayPool.hpp
index 924ed51ee15..f50617d9d7f 100644
--- a/ndb/src/kernel/vm/ArrayPool.hpp
+++ b/ndb/src/kernel/vm/ArrayPool.hpp
@@ -44,7 +44,7 @@ public:
    *
    * Note, can currently only be called once
    */
-  bool setSize(Uint32 noOfElements);
+  bool setSize(Uint32 noOfElements, bool exit_on_error = true);
 
   inline Uint32 getNoOfFree() const {
     return noOfFree;
@@ -218,13 +218,19 @@ ArrayPool<T>::~ArrayPool(){
 template <class T>
 inline
 bool
-ArrayPool<T>::setSize(Uint32 noOfElements){
+ArrayPool<T>::setSize(Uint32 noOfElements, bool exit_on_error){
   if(size == 0){
     if(noOfElements == 0)
       return true;
     theArray = (T *)NdbMem_Allocate(noOfElements * sizeof(T));
     if(theArray == 0)
-      return false;
+    {
+      if (!exit_on_error)
+	return false;
+      ErrorReporter::handleAssert("ArrayPool<T>::setSize malloc failed",
+				  __FILE__, __LINE__, NDBD_EXIT_MEMALLOC);
+      return false; // not reached
+    }
     size = noOfElements;
     noOfFree = noOfElements;
 
@@ -247,7 +253,11 @@ ArrayPool<T>::setSize(Uint32 noOfElements){
     
     return true;
   }
-  return false;
+  if (!exit_on_error)
+    return false;
+
+  ErrorReporter::handleAssert("ArrayPool<T>::setSize called twice", __FILE__, __LINE__);
+  return false; // not reached
 }
   
 template <class T>
diff --git a/ndb/src/kernel/vm/CArray.hpp b/ndb/src/kernel/vm/CArray.hpp
index a6e84e2c041..e8159da9a58 100644
--- a/ndb/src/kernel/vm/CArray.hpp
+++ b/ndb/src/kernel/vm/CArray.hpp
@@ -31,7 +31,7 @@ public:
    *
    * Note, can currently only be called once
    */
-  bool setSize(Uint32 noOfElements);
+  bool setSize(Uint32 noOfElements, bool exit_on_error = true);
 
   /**
    * Get size
@@ -82,13 +82,19 @@ CArray<T>::~CArray(){
 template <class T>
 inline
 bool
-CArray<T>::setSize(Uint32 noOfElements){
+CArray<T>::setSize(Uint32 noOfElements, bool exit_on_error){
   if(size == noOfElements)
     return true;
   
   theArray = (T *)NdbMem_Allocate(noOfElements * sizeof(T));
   if(theArray == 0)
-    return false;
+  {
+    if (!exit_on_error)
+      return false;
+    ErrorReporter::handleAssert("CArray<T>::setSize malloc failed",
+				__FILE__, __LINE__, NDBD_EXIT_MEMALLOC);
+    return false; // not reached
+  }
   size = noOfElements;
   return true;
 }
diff --git a/ndb/src/kernel/vm/SafeCounter.cpp b/ndb/src/kernel/vm/SafeCounter.cpp
index b09ad08b026..542e43f9172 100644
--- a/ndb/src/kernel/vm/SafeCounter.cpp
+++ b/ndb/src/kernel/vm/SafeCounter.cpp
@@ -25,8 +25,8 @@ SafeCounterManager::SafeCounterManager(class SimulatedBlock & block)
 {}
   
 bool
-SafeCounterManager::setSize(Uint32 maxNoOfActiveMutexes) {
-  return m_counterPool.setSize(maxNoOfActiveMutexes);
+SafeCounterManager::setSize(Uint32 maxNoOfActiveMutexes, bool exit_on_error) {
+  return m_counterPool.setSize(maxNoOfActiveMutexes, exit_on_error);
 }
 
 Uint32
diff --git a/ndb/src/kernel/vm/SafeCounter.hpp b/ndb/src/kernel/vm/SafeCounter.hpp
index 1f3cc15c2d6..3ee5e076ab8 100644
--- a/ndb/src/kernel/vm/SafeCounter.hpp
+++ b/ndb/src/kernel/vm/SafeCounter.hpp
@@ -63,7 +63,7 @@ class SafeCounterManager {
 public:
   SafeCounterManager(class SimulatedBlock &);
   
-  bool setSize(Uint32 maxNoOfActiveMutexes);
+  bool setSize(Uint32 maxNoOfActiveMutexes, bool exit_on_error = true);
   Uint32 getSize() const ;
 
   void execNODE_FAILREP(Signal*);