diff options
author | unknown <tomas@poseidon.ndb.mysql.com> | 2008-02-11 17:38:34 +0100 |
---|---|---|
committer | unknown <tomas@poseidon.ndb.mysql.com> | 2008-02-11 17:38:34 +0100 |
commit | 1be0f4f0bc4fcb39076d622b09cf65086793db58 (patch) | |
tree | ff290c4670a43599af0283ca8563d1350db80ff3 /storage | |
parent | 9c99798b7550526d70095a421cf361d9901d9697 (diff) | |
parent | 8dc4b6dceb05373bf33234a318691f8eb2cb3524 (diff) | |
download | mariadb-git-1be0f4f0bc4fcb39076d622b09cf65086793db58.tar.gz |
Merge poseidon.ndb.mysql.com:/home/tomas/cge-5.1
into poseidon.ndb.mysql.com:/home/tomas/mysql-5.1-new-ndb-merge
sql/ha_ndbcluster.cc:
Auto merged
storage/ndb/src/ndbapi/DictCache.cpp:
Auto merged
Diffstat (limited to 'storage')
40 files changed, 3185 insertions, 1811 deletions
diff --git a/storage/ndb/include/ndbapi/Ndb.hpp b/storage/ndb/include/ndbapi/Ndb.hpp index dca19c2ead0..f31638db283 100644 --- a/storage/ndb/include/ndbapi/Ndb.hpp +++ b/storage/ndb/include/ndbapi/Ndb.hpp @@ -1515,37 +1515,40 @@ public: TupleIdRange() {} Uint64 m_first_tuple_id; Uint64 m_last_tuple_id; + Uint64 m_highest_seen; void reset() { m_first_tuple_id = ~(Uint64)0; m_last_tuple_id = ~(Uint64)0; + m_highest_seen = 0; }; }; int initAutoIncrement(); int getAutoIncrementValue(const char* aTableName, - Uint64 & tupleId, Uint32 cacheSize, + Uint64 & autoValue, Uint32 cacheSize, Uint64 step = 1, Uint64 start = 1); int getAutoIncrementValue(const NdbDictionary::Table * aTable, - Uint64 & tupleId, Uint32 cacheSize, + Uint64 & autoValue, Uint32 cacheSize, Uint64 step = 1, Uint64 start = 1); int getAutoIncrementValue(const NdbDictionary::Table * aTable, - TupleIdRange & range, Uint64 & tupleId, + TupleIdRange & range, Uint64 & autoValue, Uint32 cacheSize, Uint64 step = 1, Uint64 start = 1); int readAutoIncrementValue(const char* aTableName, - Uint64 & tupleId); + Uint64 & autoValue); int readAutoIncrementValue(const NdbDictionary::Table * aTable, - Uint64 & tupleId); + Uint64 & autoValue); int readAutoIncrementValue(const NdbDictionary::Table * aTable, - TupleIdRange & range, Uint64 & tupleId); + TupleIdRange & range, Uint64 & autoValue); int setAutoIncrementValue(const char* aTableName, - Uint64 tupleId, bool increase); + Uint64 autoValue, bool modify); int setAutoIncrementValue(const NdbDictionary::Table * aTable, - Uint64 tupleId, bool increase); + Uint64 autoValue, bool modify); int setAutoIncrementValue(const NdbDictionary::Table * aTable, - TupleIdRange & range, Uint64 tupleId, - bool increase); + TupleIdRange & range, Uint64 autoValue, + bool modify); + bool checkUpdateAutoIncrementValue(TupleIdRange & range, Uint64 autoValue); private: int getTupleIdFromNdb(const NdbTableImpl* table, TupleIdRange & range, Uint64 & tupleId, @@ -1553,7 +1556,9 @@ private: int readTupleIdFromNdb(const NdbTableImpl* table, TupleIdRange & range, Uint64 & tupleId); int setTupleIdInNdb(const NdbTableImpl* table, - TupleIdRange & range, Uint64 tupleId, bool increase); + TupleIdRange & range, Uint64 tupleId, bool modify); + int checkTupleIdInNdb(TupleIdRange & range, + Uint64 tupleId); int opTupleIdOnNdb(const NdbTableImpl* table, TupleIdRange & range, Uint64 & opValue, Uint32 op); public: diff --git a/storage/ndb/include/ndbapi/ndbapi_limits.h b/storage/ndb/include/ndbapi/ndbapi_limits.h index e283913d059..98551866edc 100644 --- a/storage/ndb/include/ndbapi/ndbapi_limits.h +++ b/storage/ndb/include/ndbapi/ndbapi_limits.h @@ -26,6 +26,7 @@ #define NDB_MAX_TUPLE_SIZE (NDB_MAX_TUPLE_SIZE_IN_WORDS*4) #define NDB_MAX_ACTIVE_EVENTS 100 -#define NDB_MAX_SCANFILTER_SIZE_IN_WORDS 50000 +/* TUP ZATTR_BUFFER_SIZE 16384 (minus 1) minus place for getValue()s */ +#define NDB_MAX_SCANFILTER_SIZE_IN_WORDS (16384 - 1 - 1024) #endif diff --git a/storage/ndb/src/common/debugger/SignalLoggerManager.cpp b/storage/ndb/src/common/debugger/SignalLoggerManager.cpp index 471bea64f64..48cacb6bc1a 100644 --- a/storage/ndb/src/common/debugger/SignalLoggerManager.cpp +++ b/storage/ndb/src/common/debugger/SignalLoggerManager.cpp @@ -129,7 +129,7 @@ SignalLoggerManager::log(LogMode logMode, const char * params) const int count = getParameter(blocks, "BLOCK=", params); int cnt = 0; - if((count == 1 && blocks[0] == "ALL") || + if((count == 1 && !strcmp(blocks[0], "ALL")) || count == 0){ for (int number = 0; number < NO_OF_BLOCKS; ++number){ diff --git a/storage/ndb/src/common/debugger/signaldata/ScanTab.cpp b/storage/ndb/src/common/debugger/signaldata/ScanTab.cpp index 9668c9d0605..4f6b69ebfba 100644 --- a/storage/ndb/src/common/debugger/signaldata/ScanTab.cpp +++ b/storage/ndb/src/common/debugger/signaldata/ScanTab.cpp @@ -70,7 +70,7 @@ printSCANTABCONF(FILE * output, const Uint32 * theData, Uint32 len, Uint16 recei sig->transId1, sig->transId2); fprintf(output, " requestInfo: Eod: %d OpCount: %d\n", - (requestInfo & ScanTabConf::EndOfData == ScanTabConf::EndOfData), + (requestInfo & ScanTabConf::EndOfData) == ScanTabConf::EndOfData, (requestInfo & (~ScanTabConf::EndOfData))); size_t op_count= requestInfo & (~ScanTabConf::EndOfData); if(op_count){ diff --git a/storage/ndb/src/common/util/NdbOut.cpp b/storage/ndb/src/common/util/NdbOut.cpp index 7ca7c91e266..61de2be7572 100644 --- a/storage/ndb/src/common/util/NdbOut.cpp +++ b/storage/ndb/src/common/util/NdbOut.cpp @@ -29,7 +29,7 @@ static const char * fms[] = { "%d", "0x%08x", // Int32 "%u", "0x%08x", // Uint32 "%lld", "0x%016llx", // Int64 - "%llu", "0x%016llx" // Uint64 + "%llu", "0x%016llx", // Uint64 "%llu", "0x%016llx" // UintPtr }; diff --git a/storage/ndb/src/kernel/blocks/ERROR_codes.txt b/storage/ndb/src/kernel/blocks/ERROR_codes.txt index 727025cb73c..150400b9deb 100644 --- a/storage/ndb/src/kernel/blocks/ERROR_codes.txt +++ b/storage/ndb/src/kernel/blocks/ERROR_codes.txt @@ -3,10 +3,10 @@ Next NDBCNTR 1002 Next NDBFS 2000 Next DBACC 3002 Next DBTUP 4029 -Next DBLQH 5047 +Next DBLQH 5050 Next DBDICT 6008 Next DBDIH 7195 -Next DBTC 8054 +Next DBTC 8058 Next CMVMI 9000 Next BACKUP 10038 Next DBUTIL 11002 @@ -263,6 +263,9 @@ Delay execution of ABORTCONF signal 2 seconds to generate time-out. 8053: Crash in timeOutFoundLab, state CS_WAIT_COMMIT_CONF +5048: Crash in execCOMMIT +5049: SET_ERROR_INSERT_VALUE(5048) + ERROR CODES FOR TESTING TIME-OUT HANDLING IN DBTC ------------------------------------------------- @@ -319,6 +322,8 @@ ABORT OF TCKEYREQ 8038 : Simulate API disconnect just after SCAN_TAB_REQ +8057 : Send only 1 COMMIT per timeslice + 8052 : Simulate failure of TransactionBufferMemory allocation for OI lookup 8051 : Simulate failure of allocation for saveINDXKEYINFO diff --git a/storage/ndb/src/kernel/blocks/backup/Backup.cpp b/storage/ndb/src/kernel/blocks/backup/Backup.cpp index ec7b3f42374..5cc486ff83b 100644 --- a/storage/ndb/src/kernel/blocks/backup/Backup.cpp +++ b/storage/ndb/src/kernel/blocks/backup/Backup.cpp @@ -326,16 +326,18 @@ Backup::execCONTINUEB(Signal* signal) ptr.p->files.getPtr(filePtr, ptr.p->ctlFilePtr); FsBuffer & buf = filePtr.p->operation.dataBuffer; - if(buf.getFreeSize() + buf.getMinRead() < buf.getUsableSize()) { + if(buf.getFreeSize() < buf.getMaxWrite()) { jam(); TablePtr tabPtr LINT_SET_PTR; c_tablePool.getPtr(tabPtr, Tdata2); - DEBUG_OUT("Backup - Buffer full - " << buf.getFreeSize() - << " + " << buf.getMinRead() - << " < " << buf.getUsableSize() - << " - tableId = " << tabPtr.p->tableId); - + DEBUG_OUT("Backup - Buffer full - " + << buf.getFreeSize() + << " < " << buf.getMaxWrite() + << " (sz: " << buf.getUsableSize() + << " getMinRead: " << buf.getMinRead() + << ") - tableId = " << tabPtr.p->tableId); + signal->theData[0] = BackupContinueB::BUFFER_FULL_META; signal->theData[1] = Tdata1; signal->theData[2] = Tdata2; diff --git a/storage/ndb/src/kernel/blocks/backup/Backup.hpp b/storage/ndb/src/kernel/blocks/backup/Backup.hpp index 3fd9b2967fd..ad6e3b8fadc 100644 --- a/storage/ndb/src/kernel/blocks/backup/Backup.hpp +++ b/storage/ndb/src/kernel/blocks/backup/Backup.hpp @@ -557,8 +557,8 @@ public: NDB_TICKS m_reset_disk_speed_time; static const int DISK_SPEED_CHECK_DELAY = 100; - STATIC_CONST(NO_OF_PAGES_META_FILE = - (MAX_WORDS_META_FILE + BACKUP_WORDS_PER_PAGE - 1) / + STATIC_CONST(NO_OF_PAGES_META_FILE = + (2*MAX_WORDS_META_FILE + BACKUP_WORDS_PER_PAGE - 1) / BACKUP_WORDS_PER_PAGE); /** diff --git a/storage/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp b/storage/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp index 63d22bd0a37..1805d6ef4f8 100644 --- a/storage/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp +++ b/storage/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp @@ -3210,7 +3210,6 @@ Dbacc::readTablePk(Uint32 localkey1, Uint32 eh, Ptr<Operationrec> opPtr) { dump_lock_queue(opPtr); ndbrequire(opPtr.p->nextParallelQue == RNIL); - ndbrequire(opPtr.p->nextSerialQue == RNIL); ndbrequire(opPtr.p->m_op_bits & Operationrec::OP_ELEMENT_DISAPPEARED); ndbrequire(opPtr.p->m_op_bits & Operationrec::OP_COMMIT_DELETE_CHECK); ndbrequire((opPtr.p->m_op_bits & Operationrec::OP_STATE_MASK) == Operationrec::OP_STATE_RUNNING); diff --git a/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp b/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp index 83d38595c1f..bc8adf6fd32 100644 --- a/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp +++ b/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp @@ -5959,6 +5959,12 @@ void Dblqh::execCOMMIT(Signal* signal) TcConnectionrec * const regTcPtr = tcConnectptr.p; TRACE_OP(regTcPtr, "COMMIT"); + + CRASH_INSERTION(5048); + if (ERROR_INSERTED(5049)) + { + SET_ERROR_INSERT_VALUE(5048); + } commitReqLab(signal, gci); return; diff --git a/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp b/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp index e584883e3b6..043df5d5038 100644 --- a/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp +++ b/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp @@ -4495,7 +4495,7 @@ void Dbtc::commit020Lab(Signal* signal) if (localTcConnectptr.i != RNIL) { Tcount = Tcount + 1; - if (Tcount < 16) { + if (Tcount < 16 && !ERROR_INSERTED(8057)) { ptrCheckGuard(localTcConnectptr, TtcConnectFilesize, localTcConnectRecord); jam(); @@ -4514,6 +4514,9 @@ void Dbtc::commit020Lab(Signal* signal) }//if } else { jam(); + if (ERROR_INSERTED(8057)) + CLEAR_ERROR_INSERT_VALUE; + regApiPtr->apiConnectstate = CS_COMMIT_SENT; return; }//if diff --git a/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp b/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp index 45d124b8d7d..decb47e9758 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp +++ b/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp @@ -251,6 +251,7 @@ inline const Uint32* ALIGN_WORD(const void* ptr) #define ZUNSUPPORTED_BRANCH 892 #define ZSTORED_SEIZE_ATTRINBUFREC_ERROR 873 // Part of Scan +#define ZSTORED_TOO_MUCH_ATTRINFO_ERROR 874 #define ZREAD_ONLY_CONSTRAINT_VIOLATION 893 #define ZVAR_SIZED_NOT_SUPPORTED 894 @@ -1330,6 +1331,11 @@ typedef Ptr<HostBuffer> HostBufferPtr; struct Tuple_header { union { + /** + * List of prepared operations for this tuple. + * Points to most recent/last operation, ie. to walk the list must follow + * regOperPtr->prevActiveOp links. + */ Uint32 m_operation_ptr_i; // OperationPtrI Uint32 m_base_record_ref; // For disk tuple, ref to MM tuple }; @@ -1558,7 +1564,7 @@ public: /* * TUX checks if tuple is visible to scan. */ - bool tuxQueryTh(Uint32 fragPtrI, Uint32 tupAddr, Uint32 tupVersion, Uint32 transId1, Uint32 transId2, Uint32 savePointId); + bool tuxQueryTh(Uint32 fragPtrI, Uint32 pageId, Uint32 pageIndex, Uint32 tupVersion, Uint32 transId1, Uint32 transId2, bool dirty, Uint32 savepointId); int load_diskpage(Signal*, Uint32 opRec, Uint32 fragPtrI, Uint32 local_key, Uint32 flags); @@ -2202,17 +2208,20 @@ private: void checkImmediateTriggersAfterInsert(KeyReqStruct *req_struct, Operationrec* regOperPtr, - Tablerec* tablePtr); + Tablerec* tablePtr, + bool disk); void checkImmediateTriggersAfterUpdate(KeyReqStruct *req_struct, Operationrec* regOperPtr, - Tablerec* tablePtr); + Tablerec* tablePtr, + bool disk); void checkImmediateTriggersAfterDelete(KeyReqStruct *req_struct, Operationrec* regOperPtr, - Tablerec* tablePtr); + Tablerec* tablePtr, + bool disk); #if 0 void checkDeferredTriggers(Signal* signal, @@ -2226,7 +2235,8 @@ private: void fireImmediateTriggers(KeyReqStruct *req_struct, DLList<TupTriggerData>& triggerList, - Operationrec* regOperPtr); + Operationrec* regOperPtr, + bool disk); void fireDeferredTriggers(KeyReqStruct *req_struct, DLList<TupTriggerData>& triggerList, @@ -2239,12 +2249,13 @@ private: void executeTriggers(KeyReqStruct *req_struct, DLList<TupTriggerData>& triggerList, - Operationrec* regOperPtr); + Operationrec* regOperPtr, + bool disk); void executeTrigger(KeyReqStruct *req_struct, TupTriggerData* trigPtr, Operationrec* regOperPtr, - bool disk = true); + bool disk); bool readTriggerInfo(TupTriggerData* trigPtr, Operationrec* regOperPtr, @@ -2421,6 +2432,7 @@ private: void setNullBits(Uint32*, Tablerec* regTabPtr); bool checkNullAttributes(KeyReqStruct * const, Tablerec* const); + bool find_savepoint(OperationrecPtr& loopOpPtr, Uint32 savepointId); bool setup_read(KeyReqStruct* req_struct, Operationrec* regOperPtr, Fragrecord* regFragPtr, @@ -2533,7 +2545,8 @@ private: Operationrec* regOperPtr, Uint32 lenAttrInfo); void storedSeizeAttrinbufrecErrorLab(Signal* signal, - Operationrec* regOperPtr); + Operationrec* regOperPtr, + Uint32 errorCode); bool storedProcedureAttrInfo(Signal* signal, Operationrec* regOperPtr, const Uint32* data, @@ -2874,7 +2887,7 @@ private: void verify_page_lists(Disk_alloc_info&) {} #endif - void fix_commit_order(OperationrecPtr); + void findFirstOp(OperationrecPtr&); void commit_operation(Signal*, Uint32, Tuple_header*, PagePtr, Operationrec*, Fragrecord*, Tablerec*); @@ -3036,4 +3049,21 @@ Dbtup::get_dd_ptr(PagePtr* pagePtr, NdbOut& operator<<(NdbOut&, const Dbtup::Tablerec&); +inline +bool Dbtup::find_savepoint(OperationrecPtr& loopOpPtr, Uint32 savepointId) +{ + while (true) { + if (savepointId > loopOpPtr.p->savepointId) { + jam(); + return true; + } + loopOpPtr.i = loopOpPtr.p->prevActiveOp; + if (loopOpPtr.i == RNIL) { + break; + } + c_operation_pool.getPtr(loopOpPtr); + } + return false; +} + #endif diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupAbort.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupAbort.cpp index 59adfbfde89..93a160a4df3 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupAbort.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupAbort.cpp @@ -385,3 +385,38 @@ void Dbtup::send_TUPKEYREF(Signal* signal, TupKeyRef::SignalLength, JBB); } +/** + * Unlink one operation from the m_operation_ptr_i list in the tuple. + */ +void Dbtup::removeActiveOpList(Operationrec* const regOperPtr, + Tuple_header *tuple_ptr) +{ + OperationrecPtr raoOperPtr; + + if(!regOperPtr->m_copy_tuple_location.isNull()) + { + jam(); + c_undo_buffer.free_copy_tuple(®OperPtr->m_copy_tuple_location); + } + + if (regOperPtr->op_struct.in_active_list) { + regOperPtr->op_struct.in_active_list= false; + if (regOperPtr->nextActiveOp != RNIL) { + jam(); + raoOperPtr.i= regOperPtr->nextActiveOp; + c_operation_pool.getPtr(raoOperPtr); + raoOperPtr.p->prevActiveOp= regOperPtr->prevActiveOp; + } else { + jam(); + tuple_ptr->m_operation_ptr_i = regOperPtr->prevActiveOp; + } + if (regOperPtr->prevActiveOp != RNIL) { + jam(); + raoOperPtr.i= regOperPtr->prevActiveOp; + c_operation_pool.getPtr(raoOperPtr); + raoOperPtr.p->nextActiveOp= regOperPtr->nextActiveOp; + } + regOperPtr->prevActiveOp= RNIL; + regOperPtr->nextActiveOp= RNIL; + } +} diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupCommit.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupCommit.cpp index 812f071e037..f56e772c8b9 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupCommit.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupCommit.cpp @@ -97,39 +97,6 @@ void Dbtup::execTUP_WRITELOG_REQ(Signal* signal) } while (true); } -void Dbtup::removeActiveOpList(Operationrec* const regOperPtr, - Tuple_header *tuple_ptr) -{ - OperationrecPtr raoOperPtr; - - /** - * Release copy tuple - */ - if(!regOperPtr->m_copy_tuple_location.isNull()) - c_undo_buffer.free_copy_tuple(®OperPtr->m_copy_tuple_location); - - if (regOperPtr->op_struct.in_active_list) { - regOperPtr->op_struct.in_active_list= false; - if (regOperPtr->nextActiveOp != RNIL) { - jam(); - raoOperPtr.i= regOperPtr->nextActiveOp; - c_operation_pool.getPtr(raoOperPtr); - raoOperPtr.p->prevActiveOp= regOperPtr->prevActiveOp; - } else { - jam(); - tuple_ptr->m_operation_ptr_i = regOperPtr->prevActiveOp; - } - if (regOperPtr->prevActiveOp != RNIL) { - jam(); - raoOperPtr.i= regOperPtr->prevActiveOp; - c_operation_pool.getPtr(raoOperPtr); - raoOperPtr.p->nextActiveOp= regOperPtr->nextActiveOp; - } - regOperPtr->prevActiveOp= RNIL; - regOperPtr->nextActiveOp= RNIL; - } -} - /* ---------------------------------------------------------------- */ /* INITIALIZATION OF ONE CONNECTION RECORD TO PREPARE FOR NEXT OP. */ /* ---------------------------------------------------------------- */ @@ -142,6 +109,7 @@ void Dbtup::initOpConnection(Operationrec* regOperPtr) regOperPtr->op_struct.m_disk_preallocated= 0; regOperPtr->op_struct.m_load_diskpage_on_commit= 0; regOperPtr->op_struct.m_wait_log_buffer= 0; + regOperPtr->op_struct.in_active_list = false; regOperPtr->m_undo_buffer_space= 0; } @@ -170,6 +138,7 @@ Dbtup::dealloc_tuple(Signal* signal, Uint32 extra_bits = Tuple_header::FREED; if (bits & Tuple_header::DISK_PART) { + jam(); Local_key disk; memcpy(&disk, ptr->get_disk_ref_ptr(regTabPtr), sizeof(disk)); PagePtr tmpptr; @@ -182,6 +151,7 @@ Dbtup::dealloc_tuple(Signal* signal, if (! (bits & (Tuple_header::LCP_SKIP | Tuple_header::ALLOC)) && lcpScan_ptr_i != RNIL) { + jam(); ScanOpPtr scanOp; c_scanOpPool.getPtr(scanOp, lcpScan_ptr_i); Local_key rowid = regOperPtr->m_tuple_location; @@ -189,6 +159,7 @@ Dbtup::dealloc_tuple(Signal* signal, rowid.m_page_no = page->frag_page_id; if (rowid > scanpos) { + jam(); extra_bits = Tuple_header::LCP_KEEP; // Note REMOVE FREE ptr->m_operation_ptr_i = lcp_keep_list; regFragPtr->m_lcp_keep_list = rowid.ref(); @@ -229,11 +200,13 @@ Dbtup::commit_operation(Signal* signal, Uint32 mm_vars= regTabPtr->m_attributes[MM].m_no_of_varsize; if(mm_vars == 0) { + jam(); memcpy(tuple_ptr, copy, 4*fixsize); disk_ptr= (Tuple_header*)(((Uint32*)copy)+fixsize); } else { + jam(); /** * Var_part_ref is only stored in *allocated* tuple * so memcpy from copy, will over write it... @@ -258,6 +231,7 @@ Dbtup::commit_operation(Signal* signal, if(copy_bits & Tuple_header::MM_SHRINK) { + jam(); vpagePtrP->shrink_entry(tmp.m_page_idx, (sz + 3) >> 2); update_free_page_list(regFragPtr, vpagePtr); } @@ -268,6 +242,7 @@ Dbtup::commit_operation(Signal* signal, if (regTabPtr->m_no_of_disk_attributes && (copy_bits & Tuple_header::DISK_INLINE)) { + jam(); Local_key key; memcpy(&key, copy->get_disk_ref_ptr(regTabPtr), sizeof(Local_key)); Uint32 logfile_group_id= regFragPtr->m_logfile_group_id; @@ -278,22 +253,26 @@ Dbtup::commit_operation(Signal* signal, Uint32 sz, *dst; if(copy_bits & Tuple_header::DISK_ALLOC) { + jam(); disk_page_alloc(signal, regTabPtr, regFragPtr, &key, diskPagePtr, gci); } if(regTabPtr->m_attributes[DD].m_no_of_varsize == 0) { + jam(); sz= regTabPtr->m_offsets[DD].m_fix_header_size; dst= ((Fix_page*)diskPagePtr.p)->get_ptr(key.m_page_idx, sz); } else { + jam(); dst= ((Var_page*)diskPagePtr.p)->get_ptr(key.m_page_idx); sz= ((Var_page*)diskPagePtr.p)->get_entry_len(key.m_page_idx); } if(! (copy_bits & Tuple_header::DISK_ALLOC)) { + jam(); disk_page_undo_update(diskPagePtr.p, &key, dst, sz, gci, logfile_group_id); } @@ -307,6 +286,7 @@ Dbtup::commit_operation(Signal* signal, if(lcpScan_ptr_i != RNIL && (bits & Tuple_header::ALLOC)) { + jam(); ScanOpPtr scanOp; c_scanOpPool.getPtr(scanOp, lcpScan_ptr_i); Local_key rowid = regOperPtr->m_tuple_location; @@ -314,6 +294,7 @@ Dbtup::commit_operation(Signal* signal, rowid.m_page_no = pagePtr.p->frag_page_id; if(rowid > scanpos) { + jam(); copy_bits |= Tuple_header::LCP_SKIP; } } @@ -372,7 +353,10 @@ Dbtup::disk_page_commit_callback(Signal* signal, execTUP_COMMITREQ(signal); if(signal->theData[0] == 0) + { + jam(); c_lqh->tupcommit_conf_callback(signal, regOperPtr.p->userpointer); + } } void @@ -407,35 +391,21 @@ Dbtup::disk_page_log_buffer_callback(Signal* signal, c_lqh->tupcommit_conf_callback(signal, regOperPtr.p->userpointer); } +/** + * Move to the first operation performed on this tuple + */ void -Dbtup::fix_commit_order(OperationrecPtr opPtr) +Dbtup::findFirstOp(OperationrecPtr & firstPtr) { - ndbassert(!opPtr.p->is_first_operation()); - OperationrecPtr firstPtr = opPtr; + jam(); + printf("Detect out-of-order commit(%u) -> ", firstPtr.i); + ndbassert(!firstPtr.p->is_first_operation()); while(firstPtr.p->prevActiveOp != RNIL) { firstPtr.i = firstPtr.p->prevActiveOp; c_operation_pool.getPtr(firstPtr); } - - ndbout_c("fix_commit_order (swapping %d and %d)", - opPtr.i, firstPtr.i); - - /** - * Swap data between first and curr - */ - Uint32 prev= opPtr.p->prevActiveOp; - Uint32 next= opPtr.p->nextActiveOp; - Uint32 seco= firstPtr.p->nextActiveOp; - - Operationrec tmp = *opPtr.p; - * opPtr.p = * firstPtr.p; - * firstPtr.p = tmp; - - c_operation_pool.getPtr(seco)->prevActiveOp = opPtr.i; - c_operation_pool.getPtr(prev)->nextActiveOp = firstPtr.i; - if(next != RNIL) - c_operation_pool.getPtr(next)->prevActiveOp = firstPtr.i; + ndbout_c("%u", firstPtr.i); } /* ----------------------------------------------------------------- */ @@ -448,22 +418,17 @@ void Dbtup::execTUP_COMMITREQ(Signal* signal) TablerecPtr regTabPtr; KeyReqStruct req_struct; TransState trans_state; - Uint32 no_of_fragrec, no_of_tablerec, hash_value, gci; + Uint32 no_of_fragrec, no_of_tablerec; TupCommitReq * const tupCommitReq= (TupCommitReq *)signal->getDataPtr(); regOperPtr.i= tupCommitReq->opPtr; + Uint32 hash_value= tupCommitReq->hashValue; + Uint32 gci = tupCommitReq->gci; + jamEntry(); c_operation_pool.getPtr(regOperPtr); - if(!regOperPtr.p->is_first_operation()) - { - /** - * Out of order commit XXX check effect on triggers - */ - fix_commit_order(regOperPtr); - } - ndbassert(regOperPtr.p->is_first_operation()); regFragPtr.i= regOperPtr.p->fragmentPtr; trans_state= get_trans_state(regOperPtr.p); @@ -486,8 +451,10 @@ void Dbtup::execTUP_COMMITREQ(Signal* signal) #ifdef VM_TRACE if (tupCommitReq->diskpage == RNIL) { - m_pgman.m_ptr.setNull(); - req_struct.m_disk_page_ptr.setNull(); + m_pgman.m_ptr.i = RNIL; + m_pgman.m_ptr.p = 0; + req_struct.m_disk_page_ptr.i = RNIL; + req_struct.m_disk_page_ptr.p = 0; } #endif @@ -496,19 +463,63 @@ void Dbtup::execTUP_COMMITREQ(Signal* signal) PagePtr page; Tuple_header* tuple_ptr= (Tuple_header*) get_ptr(&page, ®OperPtr.p->m_tuple_location, regTabPtr.p); + + /** + * NOTE: This has to be run before potential time-slice when + * waiting for disk, as otherwise the "other-ops" in a multi-op + * commit might run while we're waiting for disk + * + */ + if (!regTabPtr.p->tuxCustomTriggers.isEmpty()) + { + if(get_tuple_state(regOperPtr.p) == TUPLE_PREPARED) + { + jam(); + + OperationrecPtr loopPtr = regOperPtr; + if (unlikely(!regOperPtr.p->is_first_operation())) + { + findFirstOp(loopPtr); + } + + /** + * Execute all tux triggers at first commit + * since previous tuple is otherwise removed... + */ + jam(); + goto first; + while(loopPtr.i != RNIL) + { + c_operation_pool.getPtr(loopPtr); + first: + executeTuxCommitTriggers(signal, + loopPtr.p, + regFragPtr.p, + regTabPtr.p); + set_tuple_state(loopPtr.p, TUPLE_TO_BE_COMMITTED); + loopPtr.i = loopPtr.p->nextActiveOp; + } + } + } bool get_page = false; if(regOperPtr.p->op_struct.m_load_diskpage_on_commit) { + jam(); Page_cache_client::Request req; - ndbassert(regOperPtr.p->is_first_operation() && - regOperPtr.p->is_last_operation()); + + /** + * Only last op on tuple needs "real" commit, + * hence only this one should have m_load_diskpage_on_commit + */ + ndbassert(tuple_ptr->m_operation_ptr_i == regOperPtr.i); /** * Check for page */ if(!regOperPtr.p->m_copy_tuple_location.isNull()) { + jam(); Tuple_header* tmp= (Tuple_header*) c_undo_buffer.get_ptr(®OperPtr.p->m_copy_tuple_location); @@ -518,23 +529,26 @@ void Dbtup::execTUP_COMMITREQ(Signal* signal) if (unlikely(regOperPtr.p->op_struct.op_type == ZDELETE && tmp->m_header_bits & Tuple_header::DISK_ALLOC)) { - jam(); + jam(); /** * Insert+Delete */ - regOperPtr.p->op_struct.m_load_diskpage_on_commit = 0; - regOperPtr.p->op_struct.m_wait_log_buffer = 0; - disk_page_abort_prealloc(signal, regFragPtr.p, + regOperPtr.p->op_struct.m_load_diskpage_on_commit = 0; + regOperPtr.p->op_struct.m_wait_log_buffer = 0; + disk_page_abort_prealloc(signal, regFragPtr.p, &req.m_page, req.m_page.m_page_idx); - - c_lgman->free_log_space(regFragPtr.p->m_logfile_group_id, + + c_lgman->free_log_space(regFragPtr.p->m_logfile_group_id, regOperPtr.p->m_undo_buffer_space); - if (0) ndbout_c("insert+delete"); goto skip_disk; + if (0) ndbout_c("insert+delete"); + jamEntry(); + goto skip_disk; } } else { + jam(); // initial delete ndbassert(regOperPtr.p->op_struct.op_type == ZDELETE); memcpy(&req.m_page, @@ -558,11 +572,14 @@ void Dbtup::execTUP_COMMITREQ(Signal* signal) /** * Timeslice */ + jam(); signal->theData[0] = 1; return; case -1: ndbrequire("NOT YET IMPLEMENTED" == 0); break; + default: + jam(); } get_page = true; @@ -579,8 +596,12 @@ void Dbtup::execTUP_COMMITREQ(Signal* signal) if(regOperPtr.p->op_struct.m_wait_log_buffer) { - ndbassert(regOperPtr.p->is_first_operation() && - regOperPtr.p->is_last_operation()); + jam(); + /** + * Only last op on tuple needs "real" commit, + * hence only this one should have m_wait_log_buffer + */ + ndbassert(tuple_ptr->m_operation_ptr_i == regOperPtr.i); Callback cb; cb.m_callbackData= regOperPtr.i; @@ -590,51 +611,39 @@ void Dbtup::execTUP_COMMITREQ(Signal* signal) Logfile_client lgman(this, c_lgman, regFragPtr.p->m_logfile_group_id); int res= lgman.get_log_buffer(signal, sz, &cb); + jamEntry(); switch(res){ case 0: + jam(); signal->theData[0] = 1; return; case -1: ndbrequire("NOT YET IMPLEMENTED" == 0); break; + default: + jam(); } } - if(!tuple_ptr) - { - tuple_ptr = (Tuple_header*) - get_ptr(&page, ®OperPtr.p->m_tuple_location,regTabPtr.p); - } + assert(tuple_ptr); skip_disk: req_struct.m_tuple_ptr = tuple_ptr; - if(get_tuple_state(regOperPtr.p) == TUPLE_PREPARED) - { - /** - * Execute all tux triggers at first commit - * since previous tuple is otherwise removed... - * btw...is this a "good" solution?? - * - * why can't we instead remove "own version" (when approriate ofcourse) - */ - if (!regTabPtr.p->tuxCustomTriggers.isEmpty()) { - jam(); - OperationrecPtr loopPtr= regOperPtr; - while(loopPtr.i != RNIL) - { - c_operation_pool.getPtr(loopPtr); - executeTuxCommitTriggers(signal, - loopPtr.p, - regFragPtr.p, - regTabPtr.p); - set_tuple_state(loopPtr.p, TUPLE_TO_BE_COMMITTED); - loopPtr.i = loopPtr.p->nextActiveOp; - } - } - } - - if(regOperPtr.p->is_last_operation()) + Uint32 nextOp = regOperPtr.p->nextActiveOp; + Uint32 prevOp = regOperPtr.p->prevActiveOp; + /** + * The trigger code (which is shared between detached/imediate) + * check op-list to check were to read before values from + * detached triggers should always read from original tuple value + * from before transaction start, not from any intermediate update + * + * Setting the op-list has this effect + */ + regOperPtr.p->nextActiveOp = RNIL; + regOperPtr.p->prevActiveOp = RNIL; + if(tuple_ptr->m_operation_ptr_i == regOperPtr.i) { + jam(); /** * Perform "real" commit */ @@ -643,24 +652,38 @@ skip_disk: checkDetachedTriggers(&req_struct, regOperPtr.p, regTabPtr.p, disk != RNIL); + tuple_ptr->m_operation_ptr_i = RNIL; + if(regOperPtr.p->op_struct.op_type != ZDELETE) { + jam(); commit_operation(signal, gci, tuple_ptr, page, regOperPtr.p, regFragPtr.p, regTabPtr.p); - removeActiveOpList(regOperPtr.p, tuple_ptr); } else { - removeActiveOpList(regOperPtr.p, tuple_ptr); + jam(); if (get_page) ndbassert(tuple_ptr->m_header_bits & Tuple_header::DISK_PART); dealloc_tuple(signal, gci, page.p, tuple_ptr, regOperPtr.p, regFragPtr.p, regTabPtr.p); } } - else + + if (nextOp != RNIL) { - removeActiveOpList(regOperPtr.p, tuple_ptr); + c_operation_pool.getPtr(nextOp)->prevActiveOp = prevOp; + } + + if (prevOp != RNIL) + { + c_operation_pool.getPtr(prevOp)->nextActiveOp = nextOp; + } + + if(!regOperPtr.p->m_copy_tuple_location.isNull()) + { + jam(); + c_undo_buffer.free_copy_tuple(®OperPtr.p->m_copy_tuple_location); } initOpConnection(regOperPtr.p); diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp index 8420e7f2bde..3a3c1155657 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp @@ -282,6 +282,7 @@ Dbtup::update_extent_pos(Disk_alloc_info& alloc, void Dbtup::restart_setup_page(Disk_alloc_info& alloc, PagePtr pagePtr) { + jam(); /** * Link to extent, clear uncommitted_used_space */ @@ -302,6 +303,7 @@ Dbtup::restart_setup_page(Disk_alloc_info& alloc, PagePtr pagePtr) ddassert(real_free >= estimated); if (real_free != estimated) { + jam(); extentPtr.p->m_free_space += (real_free - estimated); update_extent_pos(alloc, extentPtr); } @@ -374,6 +376,7 @@ Dbtup::disk_page_prealloc(Signal* signal, key->m_file_no= tmp.p->m_file_no; if (DBG_DISK) ndbout << " found dirty page " << *key << endl; + jam(); return 0; // Page in memory } } @@ -395,6 +398,7 @@ Dbtup::disk_page_prealloc(Signal* signal, * key = req.p->m_key; if (DBG_DISK) ndbout << " found transit page " << *key << endl; + jam(); return 0; } } @@ -404,6 +408,7 @@ Dbtup::disk_page_prealloc(Signal* signal, */ if (!c_page_request_pool.seize(req)) { + jam(); err= 1; //XXX set error code ndbout_c("no free request"); @@ -468,6 +473,7 @@ Dbtup::disk_page_prealloc(Signal* signal, err = c_lgman->alloc_log_space(logfile_group_id, sizeof(Disk_undo::AllocExtent)>>2); + jamEntry(); if(unlikely(err)) { return -err; @@ -568,6 +574,7 @@ Dbtup::disk_page_prealloc(Signal* signal, Uint32 newPageBits= alloc.calc_page_free_bits(new_size); if (newPageBits != (Uint32)pageBits) { + jam(); ddassert(ext.p->m_free_page_count[pageBits] > 0); ext.p->m_free_page_count[pageBits]--; ext.p->m_free_page_count[newPageBits]++; @@ -595,6 +602,7 @@ Dbtup::disk_page_prealloc(Signal* signal, int flags= Page_cache_client::ALLOC_REQ; if (pageBits == 0) { + jam(); //XXX empty page -> fast to map flags |= Page_cache_client::EMPTY_PAGE; preq.m_callback.m_callbackFunction = @@ -606,11 +614,13 @@ Dbtup::disk_page_prealloc(Signal* signal, switch(res) { case 0: + jam(); break; case -1: ndbassert(false); break; default: + jam(); execute(signal, preq.m_callback, res); // run callback } @@ -622,6 +632,7 @@ Dbtup::disk_page_prealloc_dirty_page(Disk_alloc_info & alloc, PagePtr pagePtr, Uint32 old_idx, Uint32 sz) { + jam(); ddassert(pagePtr.p->list_index == old_idx); Uint32 free= pagePtr.p->free_space; @@ -637,6 +648,7 @@ Dbtup::disk_page_prealloc_dirty_page(Disk_alloc_info & alloc, if (old_idx != new_idx) { + jam(); LocalDLList<Page> old_list(*pool, alloc.m_dirty_pages[old_idx]); LocalDLList<Page> new_list(*pool, alloc.m_dirty_pages[new_idx]); old_list.remove(pagePtr); @@ -660,6 +672,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc, Ptr<Page_request> req, Uint32 old_idx, Uint32 sz) { + jam(); ddassert(req.p->m_list_index == old_idx); Uint32 free= req.p->m_estimated_free_space; @@ -674,6 +687,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc, if (old_idx != new_idx) { + jam(); Page_request_list::Head *lists = alloc.m_page_requests; Local_page_request_list old_list(c_page_request_pool, lists[old_idx]); Local_page_request_list new_list(c_page_request_pool, lists[new_idx]); @@ -698,6 +712,7 @@ void Dbtup::disk_page_prealloc_callback(Signal* signal, Uint32 page_request, Uint32 page_id) { + jamEntry(); //ndbout_c("disk_alloc_page_callback id: %d", page_id); Ptr<Page_request> req; @@ -727,6 +742,7 @@ Dbtup::disk_page_prealloc_initial_callback(Signal*signal, Uint32 page_request, Uint32 page_id) { + jamEntry(); //ndbout_c("disk_alloc_page_callback_initial id: %d", page_id); /** * 1) lookup page request @@ -818,6 +834,7 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal, if (old_idx != new_idx || free != real_free) { + jam(); Ptr<Extent_info> extentPtr; c_extent_pool.getPtr(extentPtr, ext); @@ -825,6 +842,7 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal, if (old_idx != new_idx) { + jam(); ddassert(extentPtr.p->m_free_page_count[old_idx]); extentPtr.p->m_free_page_count[old_idx]--; extentPtr.p->m_free_page_count[new_idx]++; @@ -843,9 +861,11 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal, void Dbtup::disk_page_set_dirty(PagePtr pagePtr) { + jam(); Uint32 idx = pagePtr.p->list_index; if ((idx & 0x8000) == 0) { + jam(); /** * Already in dirty list */ @@ -874,7 +894,6 @@ Dbtup::disk_page_set_dirty(PagePtr pagePtr) Uint32 used = pagePtr.p->uncommitted_used_space; if (unlikely(pagePtr.p->m_restart_seq != globalData.m_restart_seq)) { - jam(); restart_setup_page(alloc, pagePtr); idx = alloc.calc_page_free_bits(free); used = 0; @@ -918,6 +937,7 @@ Dbtup::disk_page_unmap_callback(Uint32 when, type != File_formats::PT_Tup_varsize_page) || f_undo_done == false)) { + jam(); return ; } @@ -1014,6 +1034,7 @@ Dbtup::disk_page_unmap_callback(Uint32 when, << endl; } tsman.update_page_free_bits(&key, alloc.calc_page_free_bits(real_free)); + jamEntry(); } } @@ -1022,6 +1043,7 @@ Dbtup::disk_page_alloc(Signal* signal, Tablerec* tabPtrP, Fragrecord* fragPtrP, Local_key* key, PagePtr pagePtr, Uint32 gci) { + jam(); Uint32 logfile_group_id= fragPtrP->m_logfile_group_id; Disk_alloc_info& alloc= fragPtrP->m_disk_alloc_info; @@ -1050,6 +1072,7 @@ Dbtup::disk_page_free(Signal *signal, Tablerec *tabPtrP, Fragrecord * fragPtrP, Local_key* key, PagePtr pagePtr, Uint32 gci) { + jam(); if (DBG_DISK) ndbout << " disk_page_free " << *key << endl; @@ -1100,6 +1123,7 @@ Dbtup::disk_page_free(Signal *signal, if (old_idx != new_idx) { + jam(); ddassert(extentPtr.p->m_free_page_count[old_idx]); extentPtr.p->m_free_page_count[old_idx]--; extentPtr.p->m_free_page_count[new_idx]++; @@ -1126,6 +1150,7 @@ void Dbtup::disk_page_abort_prealloc(Signal *signal, Fragrecord* fragPtrP, Local_key* key, Uint32 sz) { + jam(); Page_cache_client::Request req; req.m_callback.m_callbackData= sz; req.m_callback.m_callbackFunction = @@ -1139,9 +1164,13 @@ Dbtup::disk_page_abort_prealloc(Signal *signal, Fragrecord* fragPtrP, switch(res) { case 0: + jam(); + break; case -1: + ndbrequire(false); break; default: + jam(); Ptr<GlobalPage> gpage; m_global_page_pool.getPtr(gpage, (Uint32)res); PagePtr pagePtr; @@ -1157,7 +1186,7 @@ Dbtup::disk_page_abort_prealloc_callback(Signal* signal, Uint32 sz, Uint32 page_id) { //ndbout_c("disk_alloc_page_callback id: %d", page_id); - + jamEntry(); Ptr<GlobalPage> gpage; m_global_page_pool.getPtr(gpage, page_id); @@ -1200,12 +1229,14 @@ Dbtup::disk_page_abort_prealloc_callback_1(Signal* signal, c_extent_pool.getPtr(extentPtr, ext); if (old_idx != new_idx) { + jam(); ddassert(extentPtr.p->m_free_page_count[old_idx]); extentPtr.p->m_free_page_count[old_idx]--; extentPtr.p->m_free_page_count[new_idx]++; if (old_idx == page_idx) { + jam(); ArrayPool<Page> *pool= (ArrayPool<Page>*)&m_global_page_pool; LocalDLList<Page> old_list(*pool, alloc.m_dirty_pages[old_idx]); LocalDLList<Page> new_list(*pool, alloc.m_dirty_pages[new_idx]); @@ -1215,6 +1246,7 @@ Dbtup::disk_page_abort_prealloc_callback_1(Signal* signal, } else { + jam(); pagePtr.p->list_index = new_idx | 0x8000; } } @@ -1272,6 +1304,7 @@ Uint64 Dbtup::disk_page_undo_alloc(Page* page, const Local_key* key, Uint32 sz, Uint32 gci, Uint32 logfile_group_id) { + jam(); Logfile_client lgman(this, c_lgman, logfile_group_id); Disk_undo::Alloc alloc; @@ -1293,6 +1326,7 @@ Dbtup::disk_page_undo_update(Page* page, const Local_key* key, const Uint32* src, Uint32 sz, Uint32 gci, Uint32 logfile_group_id) { + jam(); Logfile_client lgman(this, c_lgman, logfile_group_id); Disk_undo::Update update; @@ -1323,6 +1357,7 @@ Dbtup::disk_page_undo_free(Page* page, const Local_key* key, const Uint32* src, Uint32 sz, Uint32 gci, Uint32 logfile_group_id) { + jam(); Logfile_client lgman(this, c_lgman, logfile_group_id); Disk_undo::Free free; diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupExecQuery.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupExecQuery.cpp index a642d704eb9..8c096681b58 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupExecQuery.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupExecQuery.cpp @@ -75,9 +75,17 @@ void Dbtup::copyAttrinfo(Operationrec * regOperPtr, jam(); ndbrequire(copyAttrBufPtr.i < RnoOfAttrBufrec); ptrAss(copyAttrBufPtr, attrbufrec); - RbufLen= copyAttrBufPtr.p->attrbuf[ZBUF_DATA_LEN]; - Rnext= copyAttrBufPtr.p->attrbuf[ZBUF_NEXT]; - Rfirst= cfirstfreeAttrbufrec; + RbufLen = copyAttrBufPtr.p->attrbuf[ZBUF_DATA_LEN]; + Rnext = copyAttrBufPtr.p->attrbuf[ZBUF_NEXT]; + Rfirst = cfirstfreeAttrbufrec; + /* + * ATTRINFO comes from 2 mutually exclusive places: + * 1) TUPKEYREQ (also interpreted part) + * 2) STORED_PROCREQ before scan start + * Assert here that both have a check for overflow. + * The "<" instead of "<=" is intentional. + */ + ndbrequire(RinBufIndex + RbufLen < ZATTR_BUFFER_SIZE); MEMCOPY_NO_WORDS(&inBuffer[RinBufIndex], ©AttrBufPtr.p->attrbuf[0], RbufLen); @@ -221,7 +229,7 @@ Dbtup::calculateChecksum(Tuple_header* tuple_ptr, // includes tupVersion //printf("%p - ", tuple_ptr); - for (i= 0; i < rec_size-2; i++) { + for (i= 0; i < rec_size-Tuple_header::HeaderSize; i++) { checksum ^= tuple_header[i]; //printf("%.8x ", tuple_header[i]); } @@ -358,21 +366,7 @@ Dbtup::setup_read(KeyReqStruct *req_struct, dirty= false; } - OperationrecPtr prevOpPtr = currOpPtr; - bool found= false; - while(true) - { - if (savepointId > currOpPtr.p->savepointId) { - found= true; - break; - } - if (currOpPtr.p->is_first_operation()){ - break; - } - prevOpPtr= currOpPtr; - currOpPtr.i = currOpPtr.p->prevActiveOp; - c_operation_pool.getPtr(currOpPtr); - } + bool found= find_savepoint(currOpPtr, savepointId); Uint32 currOp= currOpPtr.p->op_struct.op_type; @@ -763,7 +757,8 @@ void Dbtup::execTUPKEYREQ(Signal* signal) } checkImmediateTriggersAfterInsert(&req_struct, regOperPtr, - regTabPtr); + regTabPtr, + disk_page != RNIL); set_change_mask_state(regOperPtr, SET_ALL_MASK); sendTUPKEYCONF(signal, &req_struct, regOperPtr); return; @@ -796,7 +791,8 @@ void Dbtup::execTUPKEYREQ(Signal* signal) } checkImmediateTriggersAfterUpdate(&req_struct, regOperPtr, - regTabPtr); + regTabPtr, + disk_page != RNIL); // XXX use terrorCode for now since all methods are void if (terrorCode != 0) { @@ -827,7 +823,8 @@ void Dbtup::execTUPKEYREQ(Signal* signal) */ checkImmediateTriggersAfterDelete(&req_struct, regOperPtr, - regTabPtr); + regTabPtr, + disk_page != RNIL); set_change_mask_state(regOperPtr, DELETE_CHANGES); sendTUPKEYCONF(signal, &req_struct, regOperPtr); return; @@ -1506,32 +1503,22 @@ int Dbtup::handleDeleteReq(Signal* signal, else { regOperPtr->tupVersion= req_struct->m_tuple_ptr->get_tuple_version(); - if(regTabPtr->m_no_of_disk_attributes) + } + + if(disk && regOperPtr->m_undo_buffer_space == 0) + { + regOperPtr->op_struct.m_wait_log_buffer = 1; + regOperPtr->op_struct.m_load_diskpage_on_commit = 1; + Uint32 sz= regOperPtr->m_undo_buffer_space= + (sizeof(Dbtup::Disk_undo::Free) >> 2) + + regTabPtr->m_offsets[DD].m_fix_header_size - 1; + + terrorCode= c_lgman->alloc_log_space(regFragPtr->m_logfile_group_id, + sz); + if(unlikely(terrorCode)) { - Uint32 sz; - if(regTabPtr->m_attributes[DD].m_no_of_varsize) - { - /** - * Need to have page in memory to read size - * to alloc undo space - */ - abort(); - } - else - sz= (sizeof(Dbtup::Disk_undo::Free) >> 2) + - regTabPtr->m_offsets[DD].m_fix_header_size - 1; - - regOperPtr->m_undo_buffer_space= sz; - - int res; - if((res= c_lgman->alloc_log_space(regFragPtr->m_logfile_group_id, - sz))) - { - terrorCode= res; - regOperPtr->m_undo_buffer_space= 0; - goto error; - } - + regOperPtr->m_undo_buffer_space= 0; + goto error; } } if (req_struct->attrinfo_len == 0) @@ -1540,7 +1527,9 @@ int Dbtup::handleDeleteReq(Signal* signal, } if (regTabPtr->need_expand(disk)) + { prepare_read(req_struct, regTabPtr, disk); + } { Uint32 RlogSize; diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupIndex.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupIndex.cpp index 0427f1c7612..0a4477db0d0 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupIndex.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupIndex.cpp @@ -15,6 +15,7 @@ #define DBTUP_C #define DBTUP_INDEX_CPP +#include <Dblqh.hpp> #include "Dbtup.hpp" #include <RefConvert.hpp> #include <ndb_limits.h> @@ -319,13 +320,25 @@ Dbtup::accReadPk(Uint32 tableId, Uint32 fragId, Uint32 fragPageId, Uint32 pageIn return ret; } +/* + * TUX index contains all tuple versions. A scan in TUX has scanned + * one of them and asks if it can be returned as scan result. This + * depends on trans id, dirty read flag, and savepoint within trans. + * + * Previously this faked a ZREAD operation and used getPage(). + * In TUP getPage() is run after ACC locking, but TUX comes here + * before ACC access. Instead of modifying getPage() it is more + * clear to do the full check here. + */ bool Dbtup::tuxQueryTh(Uint32 fragPtrI, - Uint32 tupAddr, + Uint32 pageId, + Uint32 pageIndex, Uint32 tupVersion, Uint32 transId1, Uint32 transId2, - Uint32 savePointId) + bool dirty, + Uint32 savepointId) { jamEntry(); FragrecordPtr fragPtr; @@ -334,35 +347,83 @@ Dbtup::tuxQueryTh(Uint32 fragPtrI, TablerecPtr tablePtr; tablePtr.i= fragPtr.p->fragTableId; ptrCheckGuard(tablePtr, cnoOfTablerec, tablerec); - // get page - Uint32 fragPageId= tupAddr >> MAX_TUPLES_BITS; - Uint32 pageIndex= tupAddr & ((1 << MAX_TUPLES_BITS ) - 1); - // use temp op rec - Operationrec tempOp; + PagePtr pagePtr; + pagePtr.i = pageId; + c_page_pool.getPtr(pagePtr); + KeyReqStruct req_struct; - tempOp.m_tuple_location.m_page_no= getRealpid(fragPtr.p, fragPageId); - tempOp.m_tuple_location.m_page_idx= pageIndex; - tempOp.savepointId= savePointId; - tempOp.op_struct.op_type= ZREAD; - req_struct.frag_page_id= fragPageId; - req_struct.trans_id1= transId1; - req_struct.trans_id2= transId2; - req_struct.dirty_op= 1; - - setup_fixed_part(&req_struct, &tempOp, tablePtr.p); - if (setup_read(&req_struct, &tempOp, fragPtr.p, tablePtr.p, false)) { - /* - * We use the normal getPage which will return the tuple to be used - * for this transaction and savepoint id. If its tuple version - * equals the requested then we have a visible tuple otherwise not. - */ + + { + Operationrec tmpOp; + tmpOp.m_tuple_location.m_page_no = pageId; + tmpOp.m_tuple_location.m_page_idx = pageIndex; + setup_fixed_part(&req_struct, &tmpOp, tablePtr.p); + } + + Tuple_header* tuple_ptr = req_struct.m_tuple_ptr; + + OperationrecPtr currOpPtr; + currOpPtr.i = tuple_ptr->m_operation_ptr_i; + if (currOpPtr.i == RNIL) { + jam(); + // tuple has no operation, any scan can see it + return true; + } + c_operation_pool.getPtr(currOpPtr); + + const bool sameTrans = + c_lqh->is_same_trans(currOpPtr.p->userpointer, transId1, transId2); + + bool res = false; + OperationrecPtr loopOpPtr = currOpPtr; + + if (!sameTrans) { jam(); - if (req_struct.m_tuple_ptr->get_tuple_version() == tupVersion) { + if (!dirty) { jam(); - return true; + if (currOpPtr.p->nextActiveOp == RNIL) { + jam(); + // last op - TUX makes ACC lock request in same timeslice + res = true; + } + } + else { + // loop to first op (returns false) + find_savepoint(loopOpPtr, 0); + const Uint32 op_type = loopOpPtr.p->op_struct.op_type; + + if (op_type != ZINSERT) { + jam(); + // read committed version + const Uint32 origVersion = tuple_ptr->get_tuple_version(); + if (origVersion == tupVersion) { + jam(); + res = true; + } + } } } - return false; + else { + jam(); + // for own trans, ignore dirty flag + + if (find_savepoint(loopOpPtr, savepointId)) { + jam(); + const Uint32 op_type = loopOpPtr.p->op_struct.op_type; + + if (op_type != ZDELETE) { + jam(); + // check if this op has produced the scanned version + Uint32 loopVersion = loopOpPtr.p->tupVersion; + if (loopVersion == tupVersion) { + jam(); + res = true; + } + } + } + } + + return res; } // ordered index build diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp index 00c8f108234..4c5fb7b645f 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp @@ -54,7 +54,14 @@ Dbtup::execACC_SCANREQ(Signal* signal) // flags Uint32 bits = 0; - if (!AccScanReq::getLcpScanFlag(req->requestInfo)) + + if (AccScanReq::getLcpScanFlag(req->requestInfo)) + { + jam(); + bits |= ScanOp::SCAN_LCP; + c_scanOpPool.getPtr(scanPtr, c_lcp_scan_op); + } + else { // seize from pool and link to per-fragment list LocalDLList<ScanOp> list(c_scanOpPool, frag.m_scanList); @@ -62,37 +69,26 @@ Dbtup::execACC_SCANREQ(Signal* signal) jam(); break; } - - if (!AccScanReq::getNoDiskScanFlag(req->requestInfo) - && tablePtr.p->m_no_of_disk_attributes) - { - bits |= ScanOp::SCAN_DD; - } - - bool mm = (bits & ScanOp::SCAN_DD); - if (tablePtr.p->m_attributes[mm].m_no_of_varsize > 0) { - bits |= ScanOp::SCAN_VS; - - // disk pages have fixed page format - ndbrequire(! (bits & ScanOp::SCAN_DD)); - } - if (! AccScanReq::getReadCommittedFlag(req->requestInfo)) { - if (AccScanReq::getLockMode(req->requestInfo) == 0) - bits |= ScanOp::SCAN_LOCK_SH; - else - bits |= ScanOp::SCAN_LOCK_EX; - } - } else { - jam(); - // LCP scan and disk + } + + if (!AccScanReq::getNoDiskScanFlag(req->requestInfo) + && tablePtr.p->m_no_of_disk_attributes) + { + bits |= ScanOp::SCAN_DD; + } + + bool mm = (bits & ScanOp::SCAN_DD); + if (tablePtr.p->m_attributes[mm].m_no_of_varsize > 0) { + bits |= ScanOp::SCAN_VS; - ndbrequire(frag.m_lcp_scan_op == c_lcp_scan_op); - c_scanOpPool.getPtr(scanPtr, frag.m_lcp_scan_op); - ndbrequire(scanPtr.p->m_fragPtrI == fragPtr.i); - bits |= ScanOp::SCAN_LCP; - if (tablePtr.p->m_attributes[MM].m_no_of_varsize > 0) { - bits |= ScanOp::SCAN_VS; - } + // disk pages have fixed page format + ndbrequire(! (bits & ScanOp::SCAN_DD)); + } + if (! AccScanReq::getReadCommittedFlag(req->requestInfo)) { + if (AccScanReq::getLockMode(req->requestInfo) == 0) + bits |= ScanOp::SCAN_LOCK_SH; + else + bits |= ScanOp::SCAN_LOCK_EX; } if (AccScanReq::getNRScanFlag(req->requestInfo)) @@ -112,6 +108,13 @@ Dbtup::execACC_SCANREQ(Signal* signal) jam(); scanPtr.p->m_endPage = RNIL; } + + if (AccScanReq::getLcpScanFlag(req->requestInfo)) + { + jam(); + ndbrequire((bits & ScanOp::SCAN_DD) == 0); + ndbrequire((bits & ScanOp::SCAN_LOCK) == 0); + } // set up scan op new (scanPtr.p) ScanOp(); @@ -1159,16 +1162,17 @@ Dbtup::releaseScanOp(ScanOpPtr& scanPtr) fragPtr.i = scanPtr.p->m_fragPtrI; ptrCheckGuard(fragPtr, cnoOfFragrec, fragrecord); - if(! (scanPtr.p->m_bits & ScanOp::SCAN_LCP)) + if(scanPtr.p->m_bits & ScanOp::SCAN_LCP) { - LocalDLList<ScanOp> list(c_scanOpPool, fragPtr.p->m_scanList); - list.release(scanPtr); + jam(); + fragPtr.p->m_lcp_scan_op = RNIL; + scanPtr.p->m_fragPtrI = RNIL; } else { - ndbrequire(fragPtr.p->m_lcp_scan_op == scanPtr.i); - fragPtr.p->m_lcp_scan_op = RNIL; - scanPtr.p->m_fragPtrI = RNIL; + jam(); + LocalDLList<ScanOp> list(c_scanOpPool, fragPtr.p->m_scanList); + list.release(scanPtr); } } @@ -1181,21 +1185,24 @@ Dbtup::execLCP_FRAG_ORD(Signal* signal) tablePtr.i = req->tableId; ptrCheckGuard(tablePtr, cnoOfTablerec, tablerec); - jam(); - FragrecordPtr fragPtr; - Uint32 fragId = req->fragmentId; - fragPtr.i = RNIL; - getFragmentrec(fragPtr, fragId, tablePtr.p); - ndbrequire(fragPtr.i != RNIL); - Fragrecord& frag = *fragPtr.p; - - ndbrequire(frag.m_lcp_scan_op == RNIL && c_lcp_scan_op != RNIL); - frag.m_lcp_scan_op = c_lcp_scan_op; - ScanOpPtr scanPtr; - c_scanOpPool.getPtr(scanPtr, frag.m_lcp_scan_op); - ndbrequire(scanPtr.p->m_fragPtrI == RNIL); - scanPtr.p->m_fragPtrI = fragPtr.i; - - scanFirst(signal, scanPtr); - scanPtr.p->m_state = ScanOp::First; + if (tablePtr.p->m_no_of_disk_attributes) + { + jam(); + FragrecordPtr fragPtr; + Uint32 fragId = req->fragmentId; + fragPtr.i = RNIL; + getFragmentrec(fragPtr, fragId, tablePtr.p); + ndbrequire(fragPtr.i != RNIL); + Fragrecord& frag = *fragPtr.p; + + ndbrequire(frag.m_lcp_scan_op == RNIL && c_lcp_scan_op != RNIL); + frag.m_lcp_scan_op = c_lcp_scan_op; + ScanOpPtr scanPtr; + c_scanOpPool.getPtr(scanPtr, frag.m_lcp_scan_op); + ndbrequire(scanPtr.p->m_fragPtrI == RNIL); + scanPtr.p->m_fragPtrI = fragPtr.i; + + scanFirst(signal, scanPtr); + scanPtr.p->m_state = ScanOp::First; + } } diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupStoredProcDef.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupStoredProcDef.cpp index 12d5f8aba38..a1e350853ce 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupStoredProcDef.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupStoredProcDef.cpp @@ -106,6 +106,11 @@ void Dbtup::scanProcedure(Signal* signal, regOperPtr->attrinbufLen = lenAttrInfo; regOperPtr->currentAttrinbufLen = 0; regOperPtr->storedProcPtr = storedPtr.i; + if (lenAttrInfo >= ZATTR_BUFFER_SIZE) { // yes ">=" + jam(); + // send REF and change state to ignore the ATTRINFO to come + storedSeizeAttrinbufrecErrorLab(signal, regOperPtr, ZSTORED_TOO_MUCH_ATTRINFO_ERROR); + } }//Dbtup::scanProcedure() void Dbtup::copyProcedure(Signal* signal, @@ -146,7 +151,7 @@ bool Dbtup::storedProcedureAttrInfo(Signal* signal, Uint32 RnoFree = cnoFreeAttrbufrec; if (ERROR_INSERTED(4004) && !copyProcedure) { CLEAR_ERROR_INSERT_VALUE; - storedSeizeAttrinbufrecErrorLab(signal, regOperPtr); + storedSeizeAttrinbufrecErrorLab(signal, regOperPtr, ZSTORED_SEIZE_ATTRINBUFREC_ERROR); return false; }//if regOperPtr->currentAttrinbufLen += length; @@ -162,7 +167,7 @@ bool Dbtup::storedProcedureAttrInfo(Signal* signal, regAttrPtr.p->attrbuf[ZBUF_NEXT] = RNIL; } else { jam(); - storedSeizeAttrinbufrecErrorLab(signal, regOperPtr); + storedSeizeAttrinbufrecErrorLab(signal, regOperPtr, ZSTORED_SEIZE_ATTRINBUFREC_ERROR); return false; }//if if (regOperPtr->firstAttrinbufrec == RNIL) { @@ -190,7 +195,7 @@ bool Dbtup::storedProcedureAttrInfo(Signal* signal, }//if if (ERROR_INSERTED(4005) && !copyProcedure) { CLEAR_ERROR_INSERT_VALUE; - storedSeizeAttrinbufrecErrorLab(signal, regOperPtr); + storedSeizeAttrinbufrecErrorLab(signal, regOperPtr, ZSTORED_SEIZE_ATTRINBUFREC_ERROR); return false; }//if @@ -212,7 +217,8 @@ bool Dbtup::storedProcedureAttrInfo(Signal* signal, }//Dbtup::storedProcedureAttrInfo() void Dbtup::storedSeizeAttrinbufrecErrorLab(Signal* signal, - Operationrec* regOperPtr) + Operationrec* regOperPtr, + Uint32 errorCode) { StoredProcPtr storedPtr; c_storedProcPool.getPtr(storedPtr, regOperPtr->storedProcPtr); @@ -224,7 +230,7 @@ void Dbtup::storedSeizeAttrinbufrecErrorLab(Signal* signal, regOperPtr->m_any_value = 0; set_trans_state(regOperPtr, TRANS_ERROR_WAIT_STORED_PROCREQ); signal->theData[0] = regOperPtr->userpointer; - signal->theData[1] = ZSTORED_SEIZE_ATTRINBUFREC_ERROR; + signal->theData[1] = errorCode; signal->theData[2] = regOperPtr->storedProcPtr; sendSignal(DBLQH_REF, GSN_STORED_PROCREF, signal, 3, JBB); }//Dbtup::storedSeizeAttrinbufrecErrorLab() diff --git a/storage/ndb/src/kernel/blocks/dbtup/DbtupTrigger.cpp b/storage/ndb/src/kernel/blocks/dbtup/DbtupTrigger.cpp index 09d71a19add..0ae6d0f4ac6 100644 --- a/storage/ndb/src/kernel/blocks/dbtup/DbtupTrigger.cpp +++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupTrigger.cpp @@ -369,7 +369,8 @@ Dbtup::dropTrigger(Tablerec* table, const DropTrigReq* req, BlockNumber sender) void Dbtup::checkImmediateTriggersAfterInsert(KeyReqStruct *req_struct, Operationrec *regOperPtr, - Tablerec *regTablePtr) + Tablerec *regTablePtr, + bool disk) { if(refToBlock(req_struct->TC_ref) != DBTC) { return; @@ -380,14 +381,16 @@ Dbtup::checkImmediateTriggersAfterInsert(KeyReqStruct *req_struct, jam(); fireImmediateTriggers(req_struct, regTablePtr->afterInsertTriggers, - regOperPtr); + regOperPtr, + disk); } } void Dbtup::checkImmediateTriggersAfterUpdate(KeyReqStruct *req_struct, Operationrec* regOperPtr, - Tablerec* regTablePtr) + Tablerec* regTablePtr, + bool disk) { if(refToBlock(req_struct->TC_ref) != DBTC) { return; @@ -398,21 +401,24 @@ Dbtup::checkImmediateTriggersAfterUpdate(KeyReqStruct *req_struct, jam(); fireImmediateTriggers(req_struct, regTablePtr->afterUpdateTriggers, - regOperPtr); + regOperPtr, + disk); } if ((regOperPtr->op_struct.primary_replica) && (!(regTablePtr->constraintUpdateTriggers.isEmpty()))) { jam(); fireImmediateTriggers(req_struct, regTablePtr->constraintUpdateTriggers, - regOperPtr); + regOperPtr, + disk); } } void Dbtup::checkImmediateTriggersAfterDelete(KeyReqStruct *req_struct, Operationrec* regOperPtr, - Tablerec* regTablePtr) + Tablerec* regTablePtr, + bool disk) { if(refToBlock(req_struct->TC_ref) != DBTC) { return; @@ -423,7 +429,8 @@ Dbtup::checkImmediateTriggersAfterDelete(KeyReqStruct *req_struct, jam(); executeTriggers(req_struct, regTablePtr->afterDeleteTriggers, - regOperPtr); + regOperPtr, + disk); } } @@ -547,7 +554,8 @@ end: void Dbtup::fireImmediateTriggers(KeyReqStruct *req_struct, DLList<TupTriggerData>& triggerList, - Operationrec* const regOperPtr) + Operationrec* const regOperPtr, + bool disk) { TriggerPtr trigPtr; triggerList.first(trigPtr); @@ -558,7 +566,8 @@ Dbtup::fireImmediateTriggers(KeyReqStruct *req_struct, jam(); executeTrigger(req_struct, trigPtr.p, - regOperPtr); + regOperPtr, + disk); }//if triggerList.next(trigPtr); }//while @@ -621,7 +630,8 @@ Dbtup::fireDetachedTriggers(KeyReqStruct *req_struct, void Dbtup::executeTriggers(KeyReqStruct *req_struct, DLList<TupTriggerData>& triggerList, - Operationrec* regOperPtr) + Operationrec* regOperPtr, + bool disk) { TriggerPtr trigPtr; triggerList.first(trigPtr); @@ -629,7 +639,8 @@ void Dbtup::executeTriggers(KeyReqStruct *req_struct, jam(); executeTrigger(req_struct, trigPtr.p, - regOperPtr); + regOperPtr, + disk); triggerList.next(trigPtr); } diff --git a/storage/ndb/src/kernel/blocks/dbtux/DbtuxScan.cpp b/storage/ndb/src/kernel/blocks/dbtux/DbtuxScan.cpp index a0643848530..43a861128a4 100644 --- a/storage/ndb/src/kernel/blocks/dbtux/DbtuxScan.cpp +++ b/storage/ndb/src/kernel/blocks/dbtux/DbtuxScan.cpp @@ -986,7 +986,8 @@ Dbtux::scanVisible(ScanOpPtr scanPtr, TreeEnt ent) const ScanOp& scan = *scanPtr.p; const Frag& frag = *c_fragPool.getPtr(scan.m_fragPtrI); Uint32 tableFragPtrI = frag.m_tupTableFragPtrI; - Uint32 tupAddr = getTupAddr(frag, ent); + Uint32 pageId = ent.m_tupLoc.getPageId(); + Uint32 pageOffset = ent.m_tupLoc.getPageOffset(); Uint32 tupVersion = ent.m_tupVersion; // check for same tuple twice in row if (scan.m_scanEnt.m_tupLoc == ent.m_tupLoc) @@ -996,8 +997,9 @@ Dbtux::scanVisible(ScanOpPtr scanPtr, TreeEnt ent) } Uint32 transId1 = scan.m_transId1; Uint32 transId2 = scan.m_transId2; + bool dirty = scan.m_readCommitted; Uint32 savePointId = scan.m_savePointId; - bool ret = c_tup->tuxQueryTh(tableFragPtrI, tupAddr, tupVersion, transId1, transId2, savePointId); + bool ret = c_tup->tuxQueryTh(tableFragPtrI, pageId, pageOffset, tupVersion, transId1, transId2, dirty, savePointId); jamEntry(); return ret; } diff --git a/storage/ndb/src/kernel/blocks/lgman.cpp b/storage/ndb/src/kernel/blocks/lgman.cpp index fc28906f4db..53cb1e113e1 100644 --- a/storage/ndb/src/kernel/blocks/lgman.cpp +++ b/storage/ndb/src/kernel/blocks/lgman.cpp @@ -918,7 +918,7 @@ Lgman::alloc_logbuffer_memory(Ptr<Logfile_group> ptr, Uint32 bytes) { Uint32 ptrI; Uint32 cnt = pages > 64 ? 64 : pages; - m_ctx.m_mm.alloc(&ptrI, &cnt, 1); + m_ctx.m_mm.alloc_pages(RG_DISK_OPERATIONS, &ptrI, &cnt, 1); if (cnt) { Buffer_idx range; @@ -1037,7 +1037,7 @@ Lgman::free_logbuffer_memory(Ptr<Logfile_group> ptr) ndbrequire(map.next(it)); tmp[1] = *it.data; - m_ctx.m_mm.release(range.m_ptr_i, range.m_idx); + m_ctx.m_mm.release_pages(RG_DISK_OPERATIONS, range.m_ptr_i, range.m_idx); map.next(it); } map.release(); diff --git a/storage/ndb/src/kernel/blocks/suma/SumaInit.cpp b/storage/ndb/src/kernel/blocks/suma/SumaInit.cpp index bf5c07b5b97..0248833978c 100644 --- a/storage/ndb/src/kernel/blocks/suma/SumaInit.cpp +++ b/storage/ndb/src/kernel/blocks/suma/SumaInit.cpp @@ -27,6 +27,8 @@ Suma::Suma(Block_context& ctx) : Restart(*this), c_gcp_list(c_gcp_pool) { + BLOCK_CONSTRUCTOR(Suma); + // Add received signals addRecSignal(GSN_READ_CONFIG_REQ, &Suma::execREAD_CONFIG_REQ); addRecSignal(GSN_STTOR, &Suma::execSTTOR); diff --git a/storage/ndb/src/kernel/vm/DLHashTable.hpp b/storage/ndb/src/kernel/vm/DLHashTable.hpp index df56dcb8846..b9d5f7c597f 100644 --- a/storage/ndb/src/kernel/vm/DLHashTable.hpp +++ b/storage/ndb/src/kernel/vm/DLHashTable.hpp @@ -46,8 +46,8 @@ public: /** * Seize element from pool - return i * - * Note must be either added using <b>add</b> or released - * using <b>release</b> + * Note *must* be added using <b>add</b> (even before hash.release) + * or be released using pool */ bool seize(Ptr<T> &); @@ -360,7 +360,14 @@ DLHashTableImpl<P, T, U>::remove(Ptr<T> & ptr) else { const Uint32 hv = ptr.p->hashValue() & mask; - hashValues[hv] = next; + if (hashValues[hv] == ptr.i) + { + hashValues[hv] = next; + } + else + { + // Will add assert in 5.1 + } } if(next != RNIL) @@ -386,7 +393,14 @@ DLHashTableImpl<P, T, U>::release(Ptr<T> & ptr) else { const Uint32 hv = ptr.p->hashValue() & mask; - hashValues[hv] = next; + if (hashValues[hv] == ptr.i) + { + hashValues[hv] = next; + } + else + { + // Will add assert in 5.1 + } } if(next != RNIL) diff --git a/storage/ndb/src/kernel/vm/DLHashTable2.hpp b/storage/ndb/src/kernel/vm/DLHashTable2.hpp index 23ced757d8b..05340317adb 100644 --- a/storage/ndb/src/kernel/vm/DLHashTable2.hpp +++ b/storage/ndb/src/kernel/vm/DLHashTable2.hpp @@ -42,8 +42,8 @@ public: /** * Seize element from pool - return i * - * Note must be either added using <b>add</b> or released - * using <b>release</b> + * Note *must* be added using <b>add</b> (even before hash.release) + * or be released using pool */ bool seize(Ptr<T> &); @@ -374,7 +374,14 @@ DLHashTable2<T, U>::remove(Ptr<T> & ptr){ prevP->nextHash = next; } else { const Uint32 hv = ptr.p->hashValue() & mask; - hashValues[hv] = next; + if (hashValues[hv] == ptr.i) + { + hashValues[hv] = next; + } + else + { + // Will add assert in 5.1 + } } if(next != RNIL){ @@ -395,7 +402,14 @@ DLHashTable2<T, U>::release(Ptr<T> & ptr){ prevP->nextHash = next; } else { const Uint32 hv = ptr.p->hashValue() & mask; - hashValues[hv] = next; + if (hashValues[hv] == ptr.i) + { + hashValues[hv] = next; + } + else + { + // Will add assert in 5.1 + } } if(next != RNIL){ diff --git a/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp b/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp index 14545e23820..717c354a180 100644 --- a/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp +++ b/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp @@ -48,7 +48,7 @@ NdbdSuperPool::NdbdSuperPool(class Ndbd_mem_manager & mm, { m_memRoot = m_mm.get_memroot(); - m_shift = Ndbd_mem_manager::log2((1 << (BMW_2LOG + 2)) / pageSize) - 1; + m_shift = Ndbd_mem_manager::ndb_log2((1 << (BMW_2LOG + 2)) / pageSize) - 1; m_add = (1 << m_shift) - 1; } diff --git a/storage/ndb/src/kernel/vm/Pool.cpp b/storage/ndb/src/kernel/vm/Pool.cpp index 29c4aec4ee3..f252a601ac2 100644 --- a/storage/ndb/src/kernel/vm/Pool.cpp +++ b/storage/ndb/src/kernel/vm/Pool.cpp @@ -20,7 +20,8 @@ void* Pool_context::alloc_page(Uint32 type_id, Uint32 *i) { - return m_block->m_ctx.m_mm.alloc_page(type_id, i); + return m_block->m_ctx.m_mm.alloc_page(type_id, i, + Ndbd_mem_manager::NDB_ZONE_LO); } void diff --git a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp index 70637a362d0..e2100e66baa 100644 --- a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp +++ b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp @@ -25,19 +25,22 @@ extern EventLogger g_eventLogger; extern EventLogger g_eventLogger; #endif -#ifdef NDBD_MALLOC_METHOD -#if NDBD_MALLOC_METHOD == sbrk -static const char * f_method = "sbrk"; +static int f_method_idx = 0; +#ifdef NDBD_MALLOC_METHOD_SBRK +static const char * f_method = "SMsm"; #else -static const char * f_method = "malloc"; -#endif -#elif SIZEOF_CHARP == 8 -static const char * f_method = "sbrk"; -#else -static const char * f_method = "malloc"; +static const char * f_method = "MSms"; #endif #define MAX_CHUNKS 10 +#define ZONE_LO 0 +#define ZONE_HI 1 + +/** + * POOL_RECORD_BITS == 13 => 32 - 13 = 19 bits for page + */ +#define ZONE_LO_BOUND (1u << 19) + struct InitChunk { Uint32 m_cnt; @@ -54,28 +57,42 @@ do_malloc(Uint32 pages, InitChunk* chunk) pages += 1; void * ptr = 0; Uint32 sz = pages; - if (strcmp(f_method, "sbrk") == 0) + +retry: + char method = f_method[f_method_idx]; + switch(method){ + case 0: + return false; + case 'S': + case 's': { ptr = 0; while (ptr == 0) { ptr = sbrk(sizeof(Alloc_page) * sz); + if (ptr == (void*)-1) { + if (method == 'S') + { + f_method_idx++; + goto retry; + } + ptr = 0; sz = 1 + (9 * sz) / 10; if (pages >= 32 && sz < 32) { sz = pages; - f_method = "malloc"; - g_eventLogger.info("sbrk(%lld) failed, trying malloc", - (Uint64)(sizeof(Alloc_page) * sz)); - break; + f_method_idx++; + goto retry; } } } + break; } - if (strcmp(f_method, "malloc") == 0) + case 'M': + case 'm': { ptr = 0; while (ptr == 0) @@ -83,15 +100,26 @@ do_malloc(Uint32 pages, InitChunk* chunk) ptr = malloc(sizeof(Alloc_page) * sz); if (ptr == 0) { + if (method == 'M') + { + f_method_idx++; + goto retry; + } + sz = 1 + (9 * sz) / 10; if (pages >= 32 && sz < 32) { - return false; + f_method_idx++; + goto retry; } } } + break; } - + default: + return false; + } + chunk->m_cnt = sz; chunk->m_ptr = (Alloc_page*)ptr; const UintPtr align = sizeof(Alloc_page) - 1; @@ -125,7 +153,7 @@ do_malloc(Uint32 pages, InitChunk* chunk) } Uint32 -Ndbd_mem_manager::log2(Uint32 input) +Ndbd_mem_manager::ndb_log2(Uint32 input) { input = input | (input >> 8); input = input | (input >> 4); @@ -151,6 +179,12 @@ Ndbd_mem_manager::Ndbd_mem_manager() } } +/** + * m_min = reserved + * m_curr = current + * m_max = max alloc, 0 = no limit + */ + void Ndbd_mem_manager::set_resource_limit(const Resource_limit& rl) { @@ -176,6 +210,40 @@ Ndbd_mem_manager::get_resource_limit(Uint32 id, Resource_limit& rl) const return false; } +static +inline +void +check_resource_limits(Resource_limit* rl) +{ +#ifdef VM_TRACE + Uint32 curr = 0; + Uint32 res_alloc = 0; + Uint32 shared_alloc = 0; + Uint32 sumres = 0; + for (Uint32 i = 1; i<XX_RL_COUNT; i++) + { + curr += rl[i].m_curr; + sumres += rl[i].m_min; + assert(rl[i].m_max == 0 || rl[i].m_curr <= rl[i].m_max); + if (rl[i].m_curr > rl[i].m_min) + { + shared_alloc += rl[i].m_curr - rl[i].m_min; + res_alloc += rl[i].m_min; + } + else + { + res_alloc += rl[i].m_curr; + } + } + assert(curr == rl[0].m_curr); + assert(res_alloc + shared_alloc == curr); + assert(res_alloc <= sumres); + assert(sumres == res_alloc + rl[0].m_min); + assert(rl[0].m_curr <= rl[0].m_max); +#endif +} + + bool Ndbd_mem_manager::init(bool alloc_less_memory) { @@ -292,6 +360,8 @@ Ndbd_mem_manager::init(bool alloc_less_memory) grow(chunks[i].m_start, chunks[i].m_cnt); } + check_resource_limits(m_resource_limit); + return true; } @@ -321,35 +391,68 @@ Ndbd_mem_manager::grow(Uint32 start, Uint32 cnt) cnt--; // last page is always marked as empty } - if (!m_used_bitmap_pages.get(start_bmp)) - { - if (start != (start_bmp << BPP_2LOG)) - { - ndbout_c("ndbd_malloc_impl.cpp:%d:grow(%d, %d) %d!=%d" - " - Unable to use due to bitmap pages missaligned!!", - __LINE__, start, cnt, start, (start_bmp << BPP_2LOG)); - g_eventLogger.error("ndbd_malloc_impl.cpp:%d:grow(%d, %d)" - " - Unable to use due to bitmap pages missaligned!!", - __LINE__, start, cnt); - return; - } + for (Uint32 i = 0; i<m_used_bitmap_pages.size(); i++) + if (m_used_bitmap_pages[i] == start_bmp) + goto found; + if (start != (start_bmp << BPP_2LOG)) + { + + ndbout_c("ndbd_malloc_impl.cpp:%d:grow(%d, %d) %d!=%d not using %uMb" + " - Unable to use due to bitmap pages missaligned!!", + __LINE__, start, cnt, start, (start_bmp << BPP_2LOG), + (cnt >> (20 - 15))); + g_eventLogger.error("ndbd_malloc_impl.cpp:%d:grow(%d, %d) not using %uMb" + " - Unable to use due to bitmap pages missaligned!!", + __LINE__, start, cnt, + (cnt >> (20 - 15))); + + dump(); + return; + } + #ifdef UNIT_TEST - ndbout_c("creating bitmap page %d", start_bmp); + ndbout_c("creating bitmap page %d", start_bmp); #endif - + + { Alloc_page* bmp = m_base_page + start; memset(bmp, 0, sizeof(Alloc_page)); - m_used_bitmap_pages.set(start_bmp); cnt--; start++; } - + m_used_bitmap_pages.push_back(start_bmp); + +found: if (cnt) { m_resource_limit[0].m_curr += cnt; m_resource_limit[0].m_max += cnt; - release(start, cnt); + if (start >= ZONE_LO_BOUND) + { + Uint64 mbytes = ((Uint64(cnt) * 32) + 1023) / 1024; + ndbout_c("Adding %uMb to ZONE_HI (%u,%u)", (Uint32)mbytes, start, cnt); + release(start, cnt); + } + else if (start + cnt <= ZONE_LO_BOUND) + { + Uint64 mbytes = ((Uint64(cnt)*32) + 1023) / 1024; + ndbout_c("Adding %uMb to ZONE_LO (%u,%u)", (Uint32)mbytes, start, cnt); + release(start, cnt); + } + else + { + Uint32 cnt0 = ZONE_LO_BOUND - start; + Uint32 cnt1 = start + cnt - ZONE_LO_BOUND; + Uint64 mbytes0 = ((Uint64(cnt0)*32) + 1023) / 1024; + Uint64 mbytes1 = ((Uint64(cnt1)*32) + 1023) / 1024; + ndbout_c("Adding %uMb to ZONE_LO (split %u,%u)", (Uint32)mbytes0, + start, cnt0); + ndbout_c("Adding %uMb to ZONE_HI (split %u,%u)", (Uint32)mbytes1, + ZONE_LO_BOUND, cnt1); + release(start, cnt0); + release(ZONE_LO_BOUND, cnt1); + } } } @@ -362,64 +465,82 @@ Ndbd_mem_manager::release(Uint32 start, Uint32 cnt) set(start, start+cnt-1); - release_impl(start, cnt); + Uint32 zone = start < ZONE_LO_BOUND ? 0 : 1; + release_impl(zone, start, cnt); } void -Ndbd_mem_manager::release_impl(Uint32 start, Uint32 cnt) +Ndbd_mem_manager::release_impl(Uint32 zone, Uint32 start, Uint32 cnt) { assert(start); Uint32 test = check(start-1, start+cnt); - if (test & 1) + if (start != ZONE_LO_BOUND && test & 1) { Free_page_data *fd = get_free_page_data(m_base_page + start - 1, start - 1); Uint32 sz = fd->m_size; Uint32 left = start - sz; - remove_free_list(left, fd->m_list); + remove_free_list(zone, left, fd->m_list); cnt += sz; start = left; } Uint32 right = start + cnt; - if (test & 2) + if (right != ZONE_LO_BOUND && test & 2) { Free_page_data *fd = get_free_page_data(m_base_page+right, right); Uint32 sz = fd->m_size; - remove_free_list(right, fd->m_list); + remove_free_list(zone, right, fd->m_list); cnt += sz; } - insert_free_list(start, cnt); + insert_free_list(zone, start, cnt); } void -Ndbd_mem_manager::alloc(Uint32* ret, Uint32 *pages, Uint32 min) +Ndbd_mem_manager::alloc(AllocZone zone, + Uint32* ret, Uint32 *pages, Uint32 min) +{ + if (zone == NDB_ZONE_ANY) + { + Uint32 save = * pages; + alloc_impl(ZONE_HI, ret, pages, min); + if (*pages) + return; + * pages = save; + } + + alloc_impl(ZONE_LO, ret, pages, min); +} + +void +Ndbd_mem_manager::alloc_impl(Uint32 zone, + Uint32* ret, Uint32 *pages, Uint32 min) { Int32 i; Uint32 start; Uint32 cnt = * pages; - Uint32 list = log2(cnt - 1); + Uint32 list = ndb_log2(cnt - 1); assert(cnt); assert(list <= 16); for (i = list; i < 16; i++) { - if ((start = m_buddy_lists[i])) + if ((start = m_buddy_lists[zone][i])) { /* ---------------------------------------------------------------- */ /* PROPER AMOUNT OF PAGES WERE FOUND. NOW SPLIT THE FOUND */ /* AREA AND RETURN THE PART NOT NEEDED. */ /* ---------------------------------------------------------------- */ - Uint32 sz = remove_free_list(start, i); + Uint32 sz = remove_free_list(zone, start, i); Uint32 extra = sz - cnt; assert(sz >= cnt); if (extra) { - insert_free_list(start + cnt, extra); + insert_free_list(zone, start + cnt, extra); clear_and_set(start, start+cnt-1); } else @@ -427,8 +548,7 @@ Ndbd_mem_manager::alloc(Uint32* ret, Uint32 *pages, Uint32 min) clear(start, start+cnt-1); } * ret = start; - m_resource_limit[0].m_curr += cnt; - assert(m_resource_limit[0].m_curr <= m_resource_limit[0].m_max); + assert(m_resource_limit[0].m_curr + cnt <= m_resource_limit[0].m_max); return; } } @@ -438,17 +558,17 @@ Ndbd_mem_manager::alloc(Uint32* ret, Uint32 *pages, Uint32 min) * search in other lists... */ - Int32 min_list = log2(min - 1); + Int32 min_list = ndb_log2(min - 1); assert((Int32)list >= min_list); for (i = list - 1; i >= min_list; i--) { - if ((start = m_buddy_lists[i])) + if ((start = m_buddy_lists[zone][i])) { - Uint32 sz = remove_free_list(start, i); + Uint32 sz = remove_free_list(zone, start, i); Uint32 extra = sz - cnt; if (sz > cnt) { - insert_free_list(start + cnt, extra); + insert_free_list(zone, start + cnt, extra); sz -= extra; clear_and_set(start, start+sz-1); } @@ -459,8 +579,7 @@ Ndbd_mem_manager::alloc(Uint32* ret, Uint32 *pages, Uint32 min) * ret = start; * pages = sz; - m_resource_limit[0].m_curr += sz; - assert(m_resource_limit[0].m_curr <= m_resource_limit[0].m_max); + assert(m_resource_limit[0].m_curr + sz <= m_resource_limit[0].m_max); return; } } @@ -468,12 +587,12 @@ Ndbd_mem_manager::alloc(Uint32* ret, Uint32 *pages, Uint32 min) } void -Ndbd_mem_manager::insert_free_list(Uint32 start, Uint32 size) +Ndbd_mem_manager::insert_free_list(Uint32 zone, Uint32 start, Uint32 size) { - Uint32 list = log2(size) - 1; + Uint32 list = ndb_log2(size) - 1; Uint32 last = start + size - 1; - Uint32 head = m_buddy_lists[list]; + Uint32 head = m_buddy_lists[zone][list]; Free_page_data* fd_first = get_free_page_data(m_base_page+start, start); fd_first->m_list = list; @@ -495,11 +614,11 @@ Ndbd_mem_manager::insert_free_list(Uint32 start, Uint32 size) fd->m_prev = start; } - m_buddy_lists[list] = start; + m_buddy_lists[zone][list] = start; } Uint32 -Ndbd_mem_manager::remove_free_list(Uint32 start, Uint32 list) +Ndbd_mem_manager::remove_free_list(Uint32 zone, Uint32 start, Uint32 list) { Free_page_data* fd = get_free_page_data(m_base_page+start, start); Uint32 size = fd->m_size; @@ -509,7 +628,7 @@ Ndbd_mem_manager::remove_free_list(Uint32 start, Uint32 list) if (prev) { - assert(m_buddy_lists[list] != start); + assert(m_buddy_lists[zone][list] != start); fd = get_free_page_data(m_base_page+prev, prev); assert(fd->m_next == start); assert(fd->m_list == list); @@ -517,8 +636,8 @@ Ndbd_mem_manager::remove_free_list(Uint32 start, Uint32 list) } else { - assert(m_buddy_lists[list] == start); - m_buddy_lists[list] = next; + assert(m_buddy_lists[zone][list] == start); + m_buddy_lists[zone][list] = next; } if (next) @@ -535,42 +654,62 @@ Ndbd_mem_manager::remove_free_list(Uint32 start, Uint32 list) void Ndbd_mem_manager::dump() const { - for(Uint32 i = 0; i<16; i++) + for (Uint32 zone = 0; zone < 2; zone ++) { - printf(" list: %d - ", i); - Uint32 head = m_buddy_lists[i]; - while(head) + for (Uint32 i = 0; i<16; i++) { - Free_page_data* fd = get_free_page_data(m_base_page+head, head); - printf("[ i: %d prev %d next %d list %d size %d ] ", - head, fd->m_prev, fd->m_next, fd->m_list, fd->m_size); - head = fd->m_next; + printf(" list: %d - ", i); + Uint32 head = m_buddy_lists[zone][i]; + while(head) + { + Free_page_data* fd = get_free_page_data(m_base_page+head, head); + printf("[ i: %d prev %d next %d list %d size %d ] ", + head, fd->m_prev, fd->m_next, fd->m_list, fd->m_size); + head = fd->m_next; + } + printf("EOL\n"); + } + + for (Uint32 i = 0; i<XX_RL_COUNT; i++) + { + printf("ri: %d min: %d curr: %d max: %d\n", + i, + m_resource_limit[i].m_min, + m_resource_limit[i].m_curr, + m_resource_limit[i].m_max); } - printf("EOL\n"); } } void* -Ndbd_mem_manager::alloc_page(Uint32 type, Uint32* i) +Ndbd_mem_manager::alloc_page(Uint32 type, Uint32* i, AllocZone zone) { Uint32 idx = type & RG_MASK; assert(idx && idx < XX_RL_COUNT); Resource_limit tot = m_resource_limit[0]; Resource_limit rl = m_resource_limit[idx]; - Uint32 add = (rl.m_curr < rl.m_min) ? 0 : 1; // Over min ? + Uint32 cnt = 1; + Uint32 res0 = (rl.m_curr < rl.m_min) ? 1 : 0; Uint32 limit = (rl.m_max == 0 || rl.m_curr < rl.m_max) ? 0 : 1; // Over limit Uint32 free = (tot.m_min + tot.m_curr < tot.m_max) ? 1 : 0; // Has free - if (likely(add == 0 || (limit == 0 && free == 1))) + assert(tot.m_min >= res0); + + if (likely(res0 == 1 || (limit == 0 && free == 1))) { - Uint32 cnt = 1; - alloc(i, &cnt, 1); - assert(cnt); - m_resource_limit[0].m_curr = tot.m_curr + add; - m_resource_limit[idx].m_curr = rl.m_curr + 1; - return m_base_page + *i; + alloc(zone, i, &cnt, 1); + if (likely(cnt)) + { + m_resource_limit[0].m_curr = tot.m_curr + cnt; + m_resource_limit[0].m_min = tot.m_min - res0; + m_resource_limit[idx].m_curr = rl.m_curr + cnt; + + check_resource_limits(m_resource_limit); + return m_base_page + *i; + } } + return 0; } @@ -582,10 +721,102 @@ Ndbd_mem_manager::release_page(Uint32 type, Uint32 i) Resource_limit tot = m_resource_limit[0]; Resource_limit rl = m_resource_limit[idx]; - Uint32 sub = (rl.m_curr < rl.m_min) ? 0 : 1; // Over min ? + Uint32 sub = (rl.m_curr <= rl.m_min) ? 1 : 0; // Over min ? release(i, 1); - m_resource_limit[0].m_curr = tot.m_curr - sub; + m_resource_limit[0].m_curr = tot.m_curr - 1; + m_resource_limit[0].m_min = tot.m_min + sub; m_resource_limit[idx].m_curr = rl.m_curr - 1; + + check_resource_limits(m_resource_limit); +} + +void +Ndbd_mem_manager::alloc_pages(Uint32 type, Uint32* i, Uint32 *cnt, Uint32 min) +{ + Uint32 idx = type & RG_MASK; + assert(idx && idx < XX_RL_COUNT); + Resource_limit tot = m_resource_limit[0]; + Resource_limit rl = m_resource_limit[idx]; + + Uint32 req = *cnt; + + Uint32 max = rl.m_max - rl.m_curr; + Uint32 res0 = rl.m_min - rl.m_curr; + Uint32 free_shared = tot.m_max - (tot.m_min + tot.m_curr); + + Uint32 res1; + if (rl.m_curr + req <= rl.m_min) + { + // all is reserved... + res0 = req; + res1 = 0; + } + else + { + req = rl.m_max ? max : req; + res0 = (rl.m_curr > rl.m_min) ? 0 : res0; + res1 = req - res0; + + if (unlikely(res1 > free_shared)) + { + res1 = free_shared; + req = res0 + res1; + } + } + + // req = pages to alloc + // res0 = portion that is reserved + // res1 = part that is over reserver + assert (res0 + res1 == req); + assert (tot.m_min >= res0); + + if (likely(req)) + { + // Hi order allocations can always use any zone + alloc(NDB_ZONE_ANY, i, &req, 1); + * cnt = req; + if (unlikely(req < res0)) // Got min than what was reserved :-( + { + res0 = req; + } + assert(tot.m_min >= res0); + assert(tot.m_curr + req <= tot.m_max); + + m_resource_limit[0].m_curr = tot.m_curr + req; + m_resource_limit[0].m_min = tot.m_min - res0; + m_resource_limit[idx].m_curr = rl.m_curr + req; + check_resource_limits(m_resource_limit); + return ; + } + * cnt = req; + return; +} + +void +Ndbd_mem_manager::release_pages(Uint32 type, Uint32 i, Uint32 cnt) +{ + Uint32 idx = type & RG_MASK; + assert(idx && idx < XX_RL_COUNT); + Resource_limit tot = m_resource_limit[0]; + Resource_limit rl = m_resource_limit[idx]; + + release(i, cnt); + + Uint32 currnew = rl.m_curr - cnt; + if (rl.m_curr > rl.m_min) + { + if (currnew < rl.m_min) + { + m_resource_limit[0].m_min = tot.m_min + (rl.m_min - currnew); + } + } + else + { + m_resource_limit[0].m_min = tot.m_min + cnt; + } + m_resource_limit[0].m_curr = tot.m_curr - cnt; + m_resource_limit[idx].m_curr = currnew; + check_resource_limits(m_resource_limit); } #ifdef UNIT_TEST @@ -781,3 +1012,4 @@ main(int argc, char** argv) template class Vector<Chunk>; #endif +template class Vector<Uint32>; diff --git a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp index d87927d9c45..78e41f1cabd 100644 --- a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp +++ b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp @@ -20,6 +20,7 @@ #include <Bitmask.hpp> #include <assert.h> #include "Pool.hpp" +#include <Vector.hpp> /** * 13 -> 8192 words -> 32768 bytes @@ -59,46 +60,53 @@ public: bool init(bool allow_alloc_less_than_requested = true); void* get_memroot() const { return (void*)m_base_page;} - void alloc(Uint32* ret, Uint32 *pages, Uint32 min_requested); - void release(Uint32 start, Uint32 cnt); - void dump() const ; - void* alloc_page(Uint32 type, Uint32* i); + enum AllocZone + { + NDB_ZONE_LO = 0, // Only allocate with page_id < (1 << 13) + NDB_ZONE_ANY = 1 // Allocate with any page_id + }; + + void* alloc_page(Uint32 type, Uint32* i, enum AllocZone); void release_page(Uint32 type, Uint32 i); - void* alloc_pages(Uint32 type, Uint32* i, Uint32 *cnt, Uint32 min = 1); - void release_pages(Uint32 type, Uint32 i, void*p, Uint32 cnt); + void alloc_pages(Uint32 type, Uint32* i, Uint32 *cnt, Uint32 min = 1); + void release_pages(Uint32 type, Uint32 i, Uint32 cnt); /** * Compute 2log of size * @note size = 0 -> 0 * @note size > 65536 -> 16 */ - static Uint32 log2(Uint32 size); + static Uint32 ndb_log2(Uint32 size); private: void grow(Uint32 start, Uint32 cnt); -#define XX_RL_COUNT 3 +#define XX_RL_COUNT 4 /** * Return pointer to free page data on page */ static Free_page_data* get_free_page_data(Alloc_page*, Uint32 idx); - Bitmask<1> m_used_bitmap_pages; + Vector<Uint32> m_used_bitmap_pages; - Uint32 m_buddy_lists[16]; + Uint32 m_buddy_lists[2][16]; Resource_limit m_resource_limit[XX_RL_COUNT]; // RG_COUNT in record_types.hpp Alloc_page * m_base_page; - void release_impl(Uint32 start, Uint32 cnt); - void insert_free_list(Uint32 start, Uint32 cnt); - Uint32 remove_free_list(Uint32 start, Uint32 list); + void release_impl(Uint32 zone, Uint32 start, Uint32 cnt); + void insert_free_list(Uint32 zone, Uint32 start, Uint32 cnt); + Uint32 remove_free_list(Uint32 zone, Uint32 start, Uint32 list); void set(Uint32 first, Uint32 last); void clear(Uint32 first, Uint32 last); void clear_and_set(Uint32 first, Uint32 last); Uint32 check(Uint32 first, Uint32 last); + + void alloc(AllocZone, Uint32* ret, Uint32 *pages, Uint32 min_requested); + void alloc_impl(Uint32 zone, Uint32* ret, Uint32 *pages, Uint32 min); + void release(Uint32 start, Uint32 cnt); }; inline diff --git a/storage/ndb/src/kernel/vm/pc.hpp b/storage/ndb/src/kernel/vm/pc.hpp index 194d5534646..cf03f676ae1 100644 --- a/storage/ndb/src/kernel/vm/pc.hpp +++ b/storage/ndb/src/kernel/vm/pc.hpp @@ -49,7 +49,7 @@ theEmulatedJamBlockNumber = number(); \ Uint32 tEmulatedJamIndex = theEmulatedJamIndex; \ *(Uint32*)(theEmulatedJam + tEmulatedJamIndex) = \ - ((theEmulatedJamBlockNumber << 20) | line); \ + ((theEmulatedJamBlockNumber << 20) | (line)); \ theEmulatedJamIndex = (tEmulatedJamIndex + 4) & JAM_MASK; } #else @@ -72,7 +72,7 @@ theEmulatedJamBlockNumber = number(); \ Uint32 tEmulatedJamIndex = theEmulatedJamIndex; \ *(Uint32*)((UintPtr)theEmulatedJam + (Uint32)tEmulatedJamIndex) = \ - ((theEmulatedJamBlockNumber << 20) | line); \ + ((theEmulatedJamBlockNumber << 20) | (line)); \ theEmulatedJamIndex = (tEmulatedJamIndex + 4) & JAM_MASK; } #endif @@ -231,6 +231,6 @@ #define MEMCOPY_PAGE(to, from, page_size_in_bytes) \ memcpy((void*)(to), (void*)(from), (size_t)(page_size_in_bytes)); #define MEMCOPY_NO_WORDS(to, from, no_of_words) \ - memcpy((to), (void*)(from), (size_t)(no_of_words << 2)); + memcpy((to), (void*)(from), (size_t)((no_of_words) << 2)); #endif diff --git a/storage/ndb/src/ndbapi/Ndb.cpp b/storage/ndb/src/ndbapi/Ndb.cpp index 15647861eef..e6a1c2cfcfd 100644 --- a/storage/ndb/src/ndbapi/Ndb.cpp +++ b/storage/ndb/src/ndbapi/Ndb.cpp @@ -942,6 +942,7 @@ Parameters: aTableName (IN) : The table name. step (IN) : Specifies the step between the autoincrement values. start (IN) : Start value for first value +Returns: 0 if succesful, -1 if error encountered Remark: Returns a new autoincrement value to the application. The autoincrement values can be increased by steps (default 1) and a number of values can be prefetched @@ -1072,9 +1073,18 @@ Ndb::getTupleIdFromNdb(const NdbTableImpl* table, DBUG_RETURN(0); } +/**************************************************************************** +int readAutoIncrementValue( const char* aTableName, + Uint64 & autoValue); + +Parameters: aTableName (IN) : The table name. + autoValue (OUT) : The current autoincrement value +Returns: 0 if succesful, -1 if error encountered +Remark: Returns the current autoincrement value to the application. +****************************************************************************/ int Ndb::readAutoIncrementValue(const char* aTableName, - Uint64 & tupleId) + Uint64 & autoValue) { DBUG_ENTER("Ndb::readAutoIncrementValue"); ASSERT_NOT_MYSQLD; @@ -1088,15 +1098,15 @@ Ndb::readAutoIncrementValue(const char* aTableName, } const NdbTableImpl* table = info->m_table_impl; TupleIdRange & range = info->m_tuple_id_range; - if (readTupleIdFromNdb(table, range, tupleId) == -1) + if (readTupleIdFromNdb(table, range, autoValue) == -1) DBUG_RETURN(-1); - DBUG_PRINT("info", ("value %lu", (ulong)tupleId)); + DBUG_PRINT("info", ("value %lu", (ulong)autoValue)); DBUG_RETURN(0); } int Ndb::readAutoIncrementValue(const NdbDictionary::Table * aTable, - Uint64 & tupleId) + Uint64 & autoValue) { DBUG_ENTER("Ndb::readAutoIncrementValue"); ASSERT_NOT_MYSQLD; @@ -1111,23 +1121,23 @@ Ndb::readAutoIncrementValue(const NdbDictionary::Table * aTable, DBUG_RETURN(-1); } TupleIdRange & range = info->m_tuple_id_range; - if (readTupleIdFromNdb(table, range, tupleId) == -1) + if (readTupleIdFromNdb(table, range, autoValue) == -1) DBUG_RETURN(-1); - DBUG_PRINT("info", ("value %lu", (ulong)tupleId)); + DBUG_PRINT("info", ("value %lu", (ulong)autoValue)); DBUG_RETURN(0); } int Ndb::readAutoIncrementValue(const NdbDictionary::Table * aTable, - TupleIdRange & range, Uint64 & tupleId) + TupleIdRange & range, Uint64 & autoValue) { DBUG_ENTER("Ndb::readAutoIncrementValue"); assert(aTable != 0); const NdbTableImpl* table = & NdbTableImpl::getImpl(*aTable); - if (readTupleIdFromNdb(table, range, tupleId) == -1) + if (readTupleIdFromNdb(table, range, autoValue) == -1) DBUG_RETURN(-1); - DBUG_PRINT("info", ("value %lu", (ulong)tupleId)); + DBUG_PRINT("info", ("value %lu", (ulong)autoValue)); DBUG_RETURN(0); } @@ -1155,9 +1165,20 @@ Ndb::readTupleIdFromNdb(const NdbTableImpl* table, DBUG_RETURN(0); } +/**************************************************************************** +int setAutoIncrementValue( const char* aTableName, + Uint64 autoValue, + bool modify); + +Parameters: aTableName (IN) : The table name. + autoValue (IN) : The new autoincrement value + modify (IN) : Modify existing value (not initialization) +Returns: 0 if succesful, -1 if error encountered +Remark: Sets a new autoincrement value for the application. +****************************************************************************/ int Ndb::setAutoIncrementValue(const char* aTableName, - Uint64 tupleId, bool increase) + Uint64 autoValue, bool modify) { DBUG_ENTER("Ndb::setAutoIncrementValue"); ASSERT_NOT_MYSQLD; @@ -1171,14 +1192,14 @@ Ndb::setAutoIncrementValue(const char* aTableName, } const NdbTableImpl* table = info->m_table_impl; TupleIdRange & range = info->m_tuple_id_range; - if (setTupleIdInNdb(table, range, tupleId, increase) == -1) + if (setTupleIdInNdb(table, range, autoValue, modify) == -1) DBUG_RETURN(-1); DBUG_RETURN(0); } int Ndb::setAutoIncrementValue(const NdbDictionary::Table * aTable, - Uint64 tupleId, bool increase) + Uint64 autoValue, bool modify) { DBUG_ENTER("Ndb::setAutoIncrementValue"); ASSERT_NOT_MYSQLD; @@ -1193,52 +1214,55 @@ Ndb::setAutoIncrementValue(const NdbDictionary::Table * aTable, DBUG_RETURN(-1); } TupleIdRange & range = info->m_tuple_id_range; - if (setTupleIdInNdb(table, range, tupleId, increase) == -1) + if (setTupleIdInNdb(table, range, autoValue, modify) == -1) DBUG_RETURN(-1); DBUG_RETURN(0); } int Ndb::setAutoIncrementValue(const NdbDictionary::Table * aTable, - TupleIdRange & range, Uint64 tupleId, - bool increase) + TupleIdRange & range, Uint64 autoValue, + bool modify) { DBUG_ENTER("Ndb::setAutoIncrementValue"); assert(aTable != 0); const NdbTableImpl* table = & NdbTableImpl::getImpl(*aTable); - if (setTupleIdInNdb(table, range, tupleId, increase) == -1) + if (setTupleIdInNdb(table, range, autoValue, modify) == -1) DBUG_RETURN(-1); DBUG_RETURN(0); } int Ndb::setTupleIdInNdb(const NdbTableImpl* table, - TupleIdRange & range, Uint64 tupleId, bool increase) + TupleIdRange & range, Uint64 tupleId, bool modify) { DBUG_ENTER("Ndb::setTupleIdInNdb"); - if (increase) + if (modify) { - if (range.m_first_tuple_id != range.m_last_tuple_id) + if (checkTupleIdInNdb(range, tupleId)) { - assert(range.m_first_tuple_id < range.m_last_tuple_id); - if (tupleId <= range.m_first_tuple_id + 1) - DBUG_RETURN(0); - if (tupleId <= range.m_last_tuple_id) + if (range.m_first_tuple_id != range.m_last_tuple_id) { - range.m_first_tuple_id = tupleId - 1; - DBUG_PRINT("info", - ("Setting next auto increment cached value to %lu", - (ulong)tupleId)); - DBUG_RETURN(0); + assert(range.m_first_tuple_id < range.m_last_tuple_id); + if (tupleId <= range.m_first_tuple_id + 1) + DBUG_RETURN(0); + if (tupleId <= range.m_last_tuple_id) + { + range.m_first_tuple_id = tupleId - 1; + DBUG_PRINT("info", + ("Setting next auto increment cached value to %lu", + (ulong)tupleId)); + DBUG_RETURN(0); + } } + /* + * if tupleId <= NEXTID, do nothing. otherwise update NEXTID to + * tupleId and set cached range to first = last = tupleId - 1. + */ + if (opTupleIdOnNdb(table, range, tupleId, 2) == -1) + DBUG_RETURN(-1); } - /* - * if tupleId <= NEXTID, do nothing. otherwise update NEXTID to - * tupleId and set cached range to first = last = tupleId - 1. - */ - if (opTupleIdOnNdb(table, range, tupleId, 2) == -1) - DBUG_RETURN(-1); } else { @@ -1277,6 +1301,39 @@ int Ndb::initAutoIncrement() return 0; } +bool +Ndb::checkUpdateAutoIncrementValue(TupleIdRange & range, Uint64 autoValue) +{ + return(checkTupleIdInNdb(range, autoValue) != 0); +} + +int +Ndb::checkTupleIdInNdb(TupleIdRange & range, Uint64 tupleId) +{ + DBUG_ENTER("Ndb::checkTupleIdIndNdb"); + if ((range.m_first_tuple_id != ~(Uint64)0) && + (range.m_first_tuple_id > tupleId)) + { + /* + * If we have ever cached a value in this object and this cached + * value is larger than the value we're trying to set then we + * need not check with the real value in the SYSTAB_0 table. + */ + DBUG_RETURN(0); + } + if (range.m_highest_seen > tupleId) + { + /* + * Although we've never cached any higher value we have read + * a higher value and again it isn't necessary to change the + * auto increment value. + */ + DBUG_RETURN(0); + } + DBUG_RETURN(1); +} + + int Ndb::opTupleIdOnNdb(const NdbTableImpl* table, TupleIdRange & range, Uint64 & opValue, Uint32 op) @@ -1342,15 +1399,15 @@ Ndb::opTupleIdOnNdb(const NdbTableImpl* table, tOperation->write_attr("NEXTID", 1); tOperation->interpret_exit_ok(); tOperation->def_label(0); - tOperation->interpret_exit_nok(9999); - + tOperation->interpret_exit_ok(); + tRecAttrResult = tOperation->getValue("NEXTID"); if (tConnection->execute( NdbTransaction::Commit ) == -1) { - if (tConnection->theError.code != 9999) - goto error_handler; + goto error_handler; } else { + range.m_highest_seen = tRecAttrResult->u_64_value(); DBUG_PRINT("info", ("Setting next auto increment value (db) to %lu", (ulong) opValue)); @@ -1363,7 +1420,7 @@ Ndb::opTupleIdOnNdb(const NdbTableImpl* table, tRecAttrResult = tOperation->getValue("NEXTID"); if (tConnection->execute( NdbTransaction::Commit ) == -1 ) goto error_handler; - opValue = tRecAttrResult->u_64_value(); // out + range.m_highest_seen = opValue = tRecAttrResult->u_64_value(); // out break; default: goto error_handler; @@ -1819,11 +1876,7 @@ Ndb::printState(const char* fmt, ...) NdbMutex_Lock(ndb_print_state_mutex); bool dups = false; unsigned i; - ndbout << buf << " ndb=" << hex << this << dec; -#ifndef NDB_WIN32 - ndbout << " thread=" << (int)pthread_self(); -#endif - ndbout << endl; + ndbout << buf << " ndb=" << hex << (void*)this << endl; for (unsigned n = 0; n < MAX_NDB_NODES; n++) { NdbTransaction* con = theConnectionArray[n]; if (con != 0) { diff --git a/storage/ndb/src/ndbapi/NdbDictionaryImpl.cpp b/storage/ndb/src/ndbapi/NdbDictionaryImpl.cpp index 953cfffd73b..0d226a97621 100644 --- a/storage/ndb/src/ndbapi/NdbDictionaryImpl.cpp +++ b/storage/ndb/src/ndbapi/NdbDictionaryImpl.cpp @@ -43,6 +43,7 @@ #include <NdbEnv.h> #include <NdbMem.h> #include <util/version.h> +#include <NdbSleep.h> #define DEBUG_PRINT 0 #define INCOMPATIBLE_VERSION -2 @@ -1777,7 +1778,23 @@ NdbDictInterface::dictSignal(NdbApiSignal* sig, { DBUG_ENTER("NdbDictInterface::dictSignal"); DBUG_PRINT("enter", ("useMasterNodeId: %d", node_specification)); - for(Uint32 i = 0; i<RETRIES; i++){ + + int sleep = 50; + int mod = 5; + + for(Uint32 i = 0; i<RETRIES; i++) + { + if (i > 0) + NdbSleep_MilliSleep(sleep + 10 * (rand() % mod)); + if (i == RETRIES / 2) + { + mod = 10; + } + if (i == 3*RETRIES/4) + { + sleep = 100; + } + m_buffer.clear(); // Protected area diff --git a/storage/ndb/src/ndbapi/ndberror.c b/storage/ndb/src/ndbapi/ndberror.c index 2ebf09c42ce..363e25fc519 100644 --- a/storage/ndb/src/ndbapi/ndberror.c +++ b/storage/ndb/src/ndbapi/ndberror.c @@ -314,6 +314,7 @@ ErrorBundle ErrorCodes[] = { { 242, DMEC, AE, "Zero concurrency in scan"}, { 244, DMEC, AE, "Too high concurrency in scan"}, { 269, DMEC, AE, "No condition and attributes to read in scan"}, + { 874, DMEC, AE, "Too much attrinfo (e.g. scan filter) for scan in tuple manager" }, { 4600, DMEC, AE, "Transaction is already started"}, { 4601, DMEC, AE, "Transaction is not started"}, { 4602, DMEC, AE, "You must call getNdbOperation before executeScan" }, diff --git a/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp b/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp index 775e7ec6288..644405636ba 100644 --- a/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp +++ b/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp @@ -58,14 +58,13 @@ syncSlaveWithMaster() know when the slave has caught up */ - MYSQL_RES * result; - MYSQL_ROW row; + SqlResultSet result; unsigned int masterEpoch = 0; unsigned int slaveEpoch = 0; unsigned int slaveEpochOld = 0; int maxLoops = 100; int loopCnt = 0; - + //Create a DbUtil object for the master DbUtil master("mysql",""); @@ -75,16 +74,13 @@ syncSlaveWithMaster() return NDBT_FAILED; } - //Get max epoch from master - if(master.doQuery("SELECT MAX(epoch) FROM mysql.ndb_binlog_index")) + //Get max epoch from master + if(master.doQuery("SELECT MAX(epoch) FROM mysql.ndb_binlog_index", result)) { return NDBT_FAILED; } - result = mysql_use_result(master.getMysql()); - row = mysql_fetch_row(result); - masterEpoch = atoi(row[0]); - mysql_free_result(result); - + masterEpoch = result.columnAsInt("epoch"); + /* Now we will pull current epoch from slave. If not the same as master, we will continue to retrieve the epoch @@ -103,15 +99,12 @@ syncSlaveWithMaster() while(slaveEpoch != masterEpoch && loopCnt < maxLoops) { - if(slave.doQuery("SELECT epoch FROM mysql.ndb_apply_status")) + if(slave.doQuery("SELECT epoch FROM mysql.ndb_apply_status",result)) { return NDBT_FAILED; } - result = mysql_use_result(slave.getMysql()); - row = mysql_fetch_row(result); - slaveEpoch = atoi(row[0]); - mysql_free_result(result); - + slaveEpoch = result.columnAsInt("epoch"); + if(slaveEpoch != slaveEpochOld) { slaveEpochOld = slaveEpoch; @@ -135,17 +128,15 @@ syncSlaveWithMaster() } int -verifySlaveLoad(BaseString *table) +verifySlaveLoad(BaseString &table) { - BaseString sqlStm; + //BaseString sqlStm; BaseString db; - MYSQL_RES * result; - MYSQL_ROW row; unsigned int masterCount = 0; unsigned int slaveCount = 0; db.assign("TEST_DB"); - sqlStm.assfmt("SELECT COUNT(*) FROM %s", table); + //sqlStm.assfmt("SELECT COUNT(*) FROM %s", table); //First thing to do is sync slave if(syncSlaveWithMaster()) @@ -163,15 +154,11 @@ verifySlaveLoad(BaseString *table) return NDBT_FAILED; } - if(master.doQuery(sqlStm.c_str())) + if((masterCount = master.selectCountTable(table.c_str())) == 0 ) { return NDBT_FAILED; } - result = mysql_use_result(master.getMysql()); - row = mysql_fetch_row(result); - masterCount = atoi(row[0]); - mysql_free_result(result); - + //Create a DB Object for slave DbUtil slave(db.c_str(),".slave"); @@ -181,15 +168,11 @@ verifySlaveLoad(BaseString *table) return NDBT_FAILED; } - if(slave.doQuery(sqlStm.c_str())) + if((slaveCount = slave.selectCountTable(table.c_str())) == 0 ) { return NDBT_FAILED; } - result = mysql_use_result(slave.getMysql()); - row = mysql_fetch_row(result); - slaveCount = atoi(row[0]); - mysql_free_result(result); - + if(slaveCount != masterCount) { g_err << "Verify Load -> Slave Count != Master Count " @@ -245,10 +228,9 @@ dropTEST_DB(NDBT_Context* ctx, NDBT_Step* step) } int -verifySlave(BaseString& sqlStm, BaseString& db) +verifySlave(BaseString& sqlStm, BaseString& db, BaseString& column) { - MYSQL_RES* resource; - MYSQL_ROW row; + SqlResultSet result; float masterSum; float slaveSum; @@ -269,30 +251,24 @@ verifySlave(BaseString& sqlStm, BaseString& db) return NDBT_FAILED; } - if(master.doQuery(sqlStm.c_str()) != NDBT_OK) + if(master.doQuery(sqlStm.c_str(),result) != NDBT_OK) { return NDBT_FAILED; } - resource = mysql_use_result(master.getMysql()); - row = mysql_fetch_row(resource); - masterSum = atoi(row[0]); - mysql_free_result(resource); - + masterSum = result.columnAsInt(column.c_str()); + //Login to slave if (!slave.connect()) { return NDBT_FAILED; } - if(slave.doQuery(sqlStm.c_str()) != NDBT_OK) + if(slave.doQuery(sqlStm.c_str(),result) != NDBT_OK) { return NDBT_FAILED; } - resource = mysql_use_result(slave.getMysql()); - row = mysql_fetch_row(resource); - slaveSum = atoi(row[0]); - mysql_free_result(resource); - + slaveSum = result.columnAsInt(column.c_str()); + if(masterSum != slaveSum) { g_err << "VerifySlave -> masterSum != slaveSum..." << endl; @@ -364,7 +340,7 @@ createTable_rep1(NDBT_Context* ctx, NDBT_Step* step) return NDBT_FAILED; } - if(verifySlaveLoad(&table)!= NDBT_OK) + if(verifySlaveLoad(table)!= NDBT_OK) { g_err << "Create Table -> Failed on verify slave load!" << endl; @@ -429,11 +405,13 @@ verifySlave_rep1(NDBT_Context* ctx, NDBT_Step* step) { BaseString sql; BaseString db; + BaseString column; sql.assign("SELECT SUM(c3) FROM rep1"); db.assign("TEST_DB"); + column.assign("c3"); - if (verifySlave(sql,db) != NDBT_OK) + if (verifySlave(sql,db,column) != NDBT_OK) return NDBT_FAILED; return NDBT_OK; } @@ -452,7 +430,7 @@ verifySlave_rep1(NDBT_Context* ctx, NDBT_Step* step) dropTEST_DB() {Drops TEST_DB database on master} - verifySlave(BaseString& sql, BaseSting& db) + verifySlave(BaseString& sql, BaseSting& db, BaseSting& column) {The SQL statement must sum a column and will verify that the sum of the column is equal on master & slave} */ diff --git a/storage/ndb/test/ndbapi/testInterpreter.cpp b/storage/ndb/test/ndbapi/testInterpreter.cpp index 0dc032ba7aa..22b810107b4 100644 --- a/storage/ndb/test/ndbapi/testInterpreter.cpp +++ b/storage/ndb/test/ndbapi/testInterpreter.cpp @@ -77,6 +77,11 @@ int runTestIncValue32(NDBT_Context* ctx, NDBT_Step* step){ const NdbDictionary::Table * pTab = ctx->getTab(); Ndb* pNdb = GETNDB(step); + if (strcmp(pTab->getName(), "T1") != 0) { + g_err << "runTestBug19537: skip, table != T1" << endl; + return NDBT_OK; + } + NdbConnection* pTrans = pNdb->startTransaction(); if (pTrans == NULL){ @@ -258,6 +263,84 @@ int runTestBug19537(NDBT_Context* ctx, NDBT_Step* step){ } +int runTestBug34107(NDBT_Context* ctx, NDBT_Step* step){ + int result = NDBT_OK; + const NdbDictionary::Table * pTab = ctx->getTab(); + Ndb* pNdb = GETNDB(step); + + int i; + for (i = 0; i <= 1; i++) { + g_info << "bug34107:" << (i == 0 ? " small" : " too big") << endl; + + NdbConnection* pTrans = pNdb->startTransaction(); + if (pTrans == NULL){ + ERR(pNdb->getNdbError()); + return NDBT_FAILED; + } + + NdbScanOperation* pOp = pTrans->getNdbScanOperation(pTab->getName()); + if (pOp == NULL) { + ERR(pTrans->getNdbError()); + pNdb->closeTransaction(pTrans); + return NDBT_FAILED; + } + + if (pOp->readTuples() == -1) { + ERR(pOp->getNdbError()); + pNdb->closeTransaction(pTrans); + return NDBT_FAILED; + } + + int n = i == 0 ? 10000 : 30000; + int k; + + for (k = 0; k < n; k++) { + + // inserts 1 word ATTRINFO + + if (pOp->interpret_exit_ok() == -1) { + ERR(pOp->getNdbError()); + pNdb->closeTransaction(pTrans); + return NDBT_FAILED; + } + } + + if (pTrans->execute(NoCommit) == -1) { + ERR(pTrans->getNdbError()); + pNdb->closeTransaction(pTrans); + return NDBT_FAILED; + } + + int ret; + while ((ret = pOp->nextResult()) == 0) + ; + g_info << "ret=" << ret << " err=" << pOp->getNdbError().code << endl; + + if (i == 0 && ret != 1) { + ERR(pTrans->getNdbError()); + pNdb->closeTransaction(pTrans); + return NDBT_FAILED; + } + + if (i == 1 && ret != -1) { + g_err << "unexpected big filter success" << endl; + pNdb->closeTransaction(pTrans); + return NDBT_FAILED; + } + if (i == 1 && pOp->getNdbError().code != 874) { + g_err << "unexpected big filter error code, wanted 874" << endl; + ERR(pTrans->getNdbError()); + pNdb->closeTransaction(pTrans); + return NDBT_FAILED; + } + + pNdb->closeTransaction(pTrans); + } + + return NDBT_OK; +} + + NDBT_TESTSUITE(testInterpreter); TESTCASE("IncValue32", "Test incValue for 32 bit integer\n"){ @@ -277,6 +360,12 @@ TESTCASE("Bug19537", INITIALIZER(runTestBug19537); FINALIZER(runClearTable); } +TESTCASE("Bug34107", + "Test too big scan filter (error 874)\n"){ + INITIALIZER(runLoadTable); + INITIALIZER(runTestBug34107); + FINALIZER(runClearTable); +} #if 0 TESTCASE("MaxTransactions", "Start transactions until no more can be created\n"){ diff --git a/storage/ndb/test/ndbapi/testNodeRestart.cpp b/storage/ndb/test/ndbapi/testNodeRestart.cpp index 751134c43c5..17c81fd0a26 100644 --- a/storage/ndb/test/ndbapi/testNodeRestart.cpp +++ b/storage/ndb/test/ndbapi/testNodeRestart.cpp @@ -23,6 +23,7 @@ #include <signaldata/DumpStateOrd.hpp> #include <Bitmask.hpp> #include <RefConvert.hpp> +#include <NdbEnv.h> int runLoadTable(NDBT_Context* ctx, NDBT_Step* step){ @@ -121,15 +122,57 @@ int runPkReadUntilStopped(NDBT_Context* ctx, NDBT_Step* step){ int runPkUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int records = ctx->getNumRecords(); + int multiop = ctx->getProperty("MULTI_OP", 1); + Ndb* pNdb = GETNDB(step); int i = 0; - HugoTransactions hugoTrans(*ctx->getTab()); - while (ctx->isTestStopped() == false) { + + HugoOperations hugoOps(*ctx->getTab()); + while (ctx->isTestStopped() == false) + { g_info << i << ": "; - int rows = (rand()%records)+1; - int batch = (rand()%rows)+1; - if (hugoTrans.pkUpdateRecords(GETNDB(step), rows, batch) != 0){ - return NDBT_FAILED; + int batch = (rand()%records)+1; + int row = rand() % records; + + if (batch > 25) + batch = 25; + + if(row + batch > records) + batch = records - row; + + if(hugoOps.startTransaction(pNdb) != 0) + goto err; + + if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0) + goto err; + + for (int j = 1; j<multiop; j++) + { + if(hugoOps.execute_NoCommit(pNdb) != 0) + goto err; + + if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0) + goto err; + } + + if(hugoOps.execute_Commit(pNdb) != 0) + goto err; + + hugoOps.closeTransaction(pNdb); + + continue; + +err: + NdbConnection* pCon = hugoOps.getTransaction(); + if(pCon == 0) + continue; + NdbError error = pCon->getNdbError(); + hugoOps.closeTransaction(pNdb); + if (error.status == NdbError::TemporaryError){ + NdbSleep_MilliSleep(50); + continue; } + return NDBT_FAILED; + i++; } return result; @@ -230,7 +273,7 @@ int runRestarter(NDBT_Context* ctx, NDBT_Step* step){ return NDBT_OK; } - if(restarter.waitClusterStarted(60) != 0){ + if(restarter.waitClusterStarted() != 0){ g_err << "Cluster failed to start" << endl; return NDBT_FAILED; } @@ -241,13 +284,27 @@ int runRestarter(NDBT_Context* ctx, NDBT_Step* step){ int id = lastId % restarter.getNumDbNodes(); int nodeId = restarter.getDbNodeId(id); ndbout << "Restart node " << nodeId << endl; - if(restarter.restartOneDbNode(nodeId, false, false, true) != 0){ + if(restarter.restartOneDbNode(nodeId, false, true, true) != 0){ g_err << "Failed to restartNextDbNode" << endl; result = NDBT_FAILED; break; } - if(restarter.waitClusterStarted(60) != 0){ + if (restarter.waitNodesNoStart(&nodeId, 1)) + { + g_err << "Failed to waitNodesNoStart" << endl; + result = NDBT_FAILED; + break; + } + + if (restarter.startNodes(&nodeId, 1)) + { + g_err << "Failed to start node" << endl; + result = NDBT_FAILED; + break; + } + + if(restarter.waitClusterStarted() != 0){ g_err << "Cluster failed to start" << endl; result = NDBT_FAILED; break; @@ -1883,6 +1940,178 @@ runBug32160(NDBT_Context* ctx, NDBT_Step* step) return NDBT_OK; } +int +runBug34216(NDBT_Context* ctx, NDBT_Step* step) +{ + int result = NDBT_OK; + int loops = ctx->getNumLoops(); + NdbRestarter restarter; + int i = 0; + int lastId = 0; + HugoOperations hugoOps(*ctx->getTab()); + int records = ctx->getNumRecords(); + Ndb* pNdb = GETNDB(step); + + if (restarter.getNumDbNodes() < 2) + { + ctx->stopTest(); + return NDBT_OK; + } + + if(restarter.waitClusterStarted() != 0){ + g_err << "Cluster failed to start" << endl; + return NDBT_FAILED; + } + + char buf[100]; + const char * off = NdbEnv_GetEnv("NDB_ERR_OFFSET", buf, sizeof(buf)); + int offset = off ? atoi(off) : 0; + + while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()) + { + int id = lastId % restarter.getNumDbNodes(); + int nodeId = restarter.getDbNodeId(id); + int err = 5048 + ((i+offset) % 2); + + int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; + + if(hugoOps.startTransaction(pNdb) != 0) + goto err; + + nodeId = hugoOps.getTransaction()->getConnectedNodeId(); + ndbout << "Restart node " << nodeId << " " << err <<endl; + + if (restarter.dumpStateOneNode(nodeId, val2, 2)) + return NDBT_FAILED; + + if(restarter.insertErrorInNode(nodeId, err) != 0){ + g_err << "Failed to restartNextDbNode" << endl; + result = NDBT_FAILED; + break; + } + + if (restarter.insertErrorInNode(nodeId, 8057) != 0) + { + g_err << "Failed to insert error 8057" << endl; + result = NDBT_FAILED; + break; + } + + int rows = 10; + int batch = 1; + int row = (records - rows) ? rand() % (records - rows) : 0; + + if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0) + goto err; + + for (int l = 1; l<5; l++) + { + if (hugoOps.execute_NoCommit(pNdb) != 0) + goto err; + + if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0) + goto err; + } + + hugoOps.execute_Commit(pNdb); + hugoOps.closeTransaction(pNdb); + + if (restarter.waitNodesNoStart(&nodeId, 1)) + { + g_err << "Failed to waitNodesNoStart" << endl; + result = NDBT_FAILED; + break; + } + + if (restarter.startNodes(&nodeId, 1)) + { + g_err << "Failed to startNodes" << endl; + result = NDBT_FAILED; + break; + } + + if(restarter.waitClusterStarted() != 0){ + g_err << "Cluster failed to start" << endl; + result = NDBT_FAILED; + break; + } + + lastId++; + i++; + } + + ctx->stopTest(); + + return result; +err: + return NDBT_FAILED; +} + + +int +runNF_commit(NDBT_Context* ctx, NDBT_Step* step) +{ + int result = NDBT_OK; + int loops = ctx->getNumLoops(); + NdbRestarter restarter; + if (restarter.getNumDbNodes() < 2) + { + ctx->stopTest(); + return NDBT_OK; + } + + if(restarter.waitClusterStarted() != 0){ + g_err << "Cluster failed to start" << endl; + return NDBT_FAILED; + } + + int i = 0; + while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()) + { + int nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes()); + int err = 5048; + + ndbout << "Restart node " << nodeId << " " << err <<endl; + + int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; + if (restarter.dumpStateOneNode(nodeId, val2, 2)) + return NDBT_FAILED; + + if(restarter.insertErrorInNode(nodeId, err) != 0){ + g_err << "Failed to restartNextDbNode" << endl; + result = NDBT_FAILED; + break; + } + + if (restarter.waitNodesNoStart(&nodeId, 1)) + { + g_err << "Failed to waitNodesNoStart" << endl; + result = NDBT_FAILED; + break; + } + + if (restarter.startNodes(&nodeId, 1)) + { + g_err << "Failed to startNodes" << endl; + result = NDBT_FAILED; + break; + } + + if(restarter.waitClusterStarted() != 0){ + g_err << "Cluster failed to start" << endl; + result = NDBT_FAILED; + break; + } + + i++; + } + + ctx->stopTest(); + + return result; +} + + NDBT_TESTSUITE(testNodeRestart); TESTCASE("NoLoad", "Test that one node at a time can be stopped and then restarted "\ @@ -2255,6 +2484,21 @@ TESTCASE("Bug29364", ""){ TESTCASE("Bug32160", ""){ INITIALIZER(runBug32160); } +TESTCASE("Bug34216", ""){ + INITIALIZER(runCheckAllNodesStarted); + INITIALIZER(runLoadTable); + STEP(runBug34216); + FINALIZER(runClearTable); +} +TESTCASE("mixedmultiop", ""){ + TC_PROPERTY("MULTI_OP", 5); + INITIALIZER(runCheckAllNodesStarted); + INITIALIZER(runLoadTable); + STEP(runNF_commit); + STEP(runPkUpdateUntilStopped); + STEP(runPkUpdateUntilStopped); + FINALIZER(runClearTable); +} NDBT_TESTSUITE_END(testNodeRestart); int main(int argc, const char** argv){ diff --git a/storage/ndb/test/ndbapi/testOIBasic.cpp b/storage/ndb/test/ndbapi/testOIBasic.cpp index 8f77f0bb062..4e1c8400768 100644 --- a/storage/ndb/test/ndbapi/testOIBasic.cpp +++ b/storage/ndb/test/ndbapi/testOIBasic.cpp @@ -36,10 +36,11 @@ struct Opt { // common options - unsigned m_batch; + uint m_batch; const char* m_bound; const char* m_case; bool m_collsp; + bool m_cont; bool m_core; const char* m_csname; CHARSET_INFO* m_cs; @@ -47,26 +48,29 @@ struct Opt { bool m_dups; NdbDictionary::Object::FragmentType m_fragtype; const char* m_index; - unsigned m_loop; + uint m_loop; bool m_msglock; bool m_nologging; bool m_noverify; - unsigned m_pctnull; - unsigned m_rows; - unsigned m_samples; - unsigned m_scanbatch; - unsigned m_scanpar; - unsigned m_scanstop; + uint m_pctnull; + uint m_rows; + uint m_samples; + uint m_scanbatch; + uint m_scanpar; + uint m_scanstop; int m_seed; - unsigned m_subloop; + const char* m_skip; + uint m_sloop; + uint m_ssloop; const char* m_table; - unsigned m_threads; + uint m_threads; int m_v; // int for lint Opt() : m_batch(32), m_bound("01234"), m_case(0), m_collsp(false), + m_cont(false), m_core(false), m_csname("random"), m_cs(0), @@ -85,7 +89,9 @@ struct Opt { m_scanpar(0), m_scanstop(0), m_seed(-1), - m_subloop(4), + m_skip(0), + m_sloop(4), + m_ssloop(4), m_table(0), m_threads(4), m_v(1) { @@ -107,6 +113,7 @@ printhelp() << " -bound xyz use only these bound types 0-4 [" << d.m_bound << "]" << endl << " -case abc only given test cases (letters a-z)" << endl << " -collsp use strnncollsp instead of strnxfrm" << endl + << " -cont on error continue to next test case [" << d.m_cont << "]" << endl << " -core core dump on error [" << d.m_core << "]" << endl << " -csname S charset or collation [" << d.m_csname << "]" << endl << " -die nnn exit immediately on NDB error code nnn" << endl @@ -122,7 +129,9 @@ printhelp() << " -scanbatch N scan batch 0=default [" << d.m_scanbatch << "]" << endl << " -scanpar N scan parallel 0=default [" << d.m_scanpar << "]" << endl << " -seed N srandom seed 0=loop number -1=random [" << d.m_seed << "]" << endl - << " -subloop N subtest (and subsubtest) loop count [" << d.m_subloop << "]" << endl + << " -skip abc skip given test cases (letters a-z)" << endl + << " -sloop N level 2 (sub)loop count [" << d.m_sloop << "]" << endl + << " -ssloop N level 3 (sub)loop count [" << d.m_ssloop << "]" << endl << " -table xyz only given table numbers (digits 0-9)" << endl << " -threads N number of threads [" << d.m_threads << "]" << endl << " -vN verbosity [" << d.m_v << "]" << endl @@ -142,17 +151,17 @@ static const char* hexstr = "0123456789abcdef"; // random ints -static unsigned -urandom(unsigned n) +static uint +urandom(uint n) { if (n == 0) return 0; - unsigned i = random() % n; + uint i = random() % n; return i; } static int -irandom(unsigned n) +irandom(uint n) { if (n == 0) return 0; @@ -163,7 +172,7 @@ irandom(unsigned n) } static bool -randompct(unsigned pct) +randompct(uint pct) { if (pct == 0) return false; @@ -172,15 +181,15 @@ randompct(unsigned pct) return urandom(100) < pct; } -static unsigned -random_coprime(unsigned n) +static uint +random_coprime(uint n) { - unsigned prime[] = { 101, 211, 307, 401, 503, 601, 701, 809, 907 }; - unsigned count = sizeof(prime) / sizeof(prime[0]); + uint prime[] = { 101, 211, 307, 401, 503, 601, 701, 809, 907 }; + uint count = sizeof(prime) / sizeof(prime[0]); if (n == 0) return 0; while (1) { - unsigned i = urandom(count); + uint i = urandom(count); if (n % prime[i] != 0) return prime[i]; } @@ -189,16 +198,16 @@ random_coprime(unsigned n) // random re-sequence of 0...(n-1) struct Rsq { - Rsq(unsigned n); - unsigned next(); + Rsq(uint n); + uint next(); private: - unsigned m_n; - unsigned m_i; - unsigned m_start; - unsigned m_prime; + uint m_n; + uint m_i; + uint m_start; + uint m_prime; }; -Rsq::Rsq(unsigned n) +Rsq::Rsq(uint n) { m_n = n; m_i = 0; @@ -206,7 +215,7 @@ Rsq::Rsq(unsigned n) m_prime = random_coprime(n); } -unsigned +uint Rsq::next() { assert(m_n != 0); @@ -216,30 +225,16 @@ Rsq::next() // log and error macros static NdbMutex *ndbout_mutex = NULL; - -static unsigned getthrno(); - -static const char* -getthrstr() -{ - static char buf[20]; - unsigned n = getthrno(); - if (n == (unsigned)-1) - strcpy(buf, ""); - else { - unsigned m = - g_opt.m_threads < 10 ? 1 : - g_opt.m_threads < 100 ? 2 : 3; - sprintf(buf, "[%0*u] ", m, n); - } - return buf; -} +static const char* getthrprefix(); #define LLN(n, s) \ do { \ if ((n) > g_opt.m_v) break; \ if (g_opt.m_msglock) NdbMutex_Lock(ndbout_mutex); \ - ndbout << getthrstr() << s << endl; \ + ndbout << getthrprefix(); \ + if ((n) > 2) \ + ndbout << "line " << __LINE__ << ": "; \ + ndbout << s << endl; \ if (g_opt.m_msglock) NdbMutex_Unlock(ndbout_mutex); \ } while(0) @@ -250,6 +245,8 @@ getthrstr() #define LL4(s) LLN(4, s) #define LL5(s) LLN(5, s) +#define HEX(x) hex << (x) << dec + // following check a condition and return -1 on failure #undef CHK // simple check @@ -281,53 +278,58 @@ getthrstr() class Thr; class Con; class Tab; +class ITab; class Set; class Tmr; struct Par : public Opt { - unsigned m_no; + uint m_no; Con* m_con; Con& con() const { assert(m_con != 0); return *m_con; } const Tab* m_tab; const Tab& tab() const { assert(m_tab != 0); return *m_tab; } + const ITab* m_itab; + const ITab& itab() const { assert(m_itab != 0); return *m_itab; } Set* m_set; Set& set() const { assert(m_set != 0); return *m_set; } Tmr* m_tmr; Tmr& tmr() const { assert(m_tmr != 0); return *m_tmr; } char m_currcase[2]; - unsigned m_lno; - unsigned m_slno; - unsigned m_totrows; + uint m_lno; + uint m_slno; + uint m_totrows; // value calculation - unsigned m_range; - unsigned m_pctrange; - unsigned m_pctbrange; + uint m_range; + uint m_pctrange; + uint m_pctbrange; int m_bdir; bool m_noindexkeyupdate; // choice of key bool m_randomkey; // do verify after read bool m_verify; - // deadlock possible - bool m_deadlock; - // abort percentabge - unsigned m_abortpct; + // errors to catch (see Con) + bool m_catcherr; + // abort percentage + uint m_abortpct; NdbOperation::LockMode m_lockmode; // scan options bool m_tupscan; bool m_ordered; bool m_descending; - // timer location + // threads used by current test case + uint m_usedthreads; Par(const Opt& opt) : Opt(opt), m_no(0), m_con(0), m_tab(0), + m_itab(0), m_set(0), m_tmr(0), m_lno(0), m_slno(0), - m_totrows(m_threads * m_rows), + m_totrows(0), m_range(m_rows), m_pctrange(40), m_pctbrange(80), @@ -335,39 +337,40 @@ struct Par : public Opt { m_noindexkeyupdate(false), m_randomkey(false), m_verify(false), - m_deadlock(false), + m_catcherr(0), m_abortpct(0), m_lockmode(NdbOperation::LM_Read), m_tupscan(false), m_ordered(false), - m_descending(false) + m_descending(false), + m_usedthreads(0) { m_currcase[0] = 0; } }; static bool -usetable(Par par, unsigned i) +usetable(Par par, uint i) { return par.m_table == 0 || strchr(par.m_table, '0' + i) != 0; } static bool -useindex(Par par, unsigned i) +useindex(Par par, uint i) { return par.m_index == 0 || strchr(par.m_index, '0' + i) != 0; } -static unsigned -thrrow(Par par, unsigned j) +static uint +thrrow(Par par, uint j) { - return par.m_threads * j + par.m_no; + return par.m_usedthreads * j + par.m_no; } static bool -isthrrow(Par par, unsigned i) +isthrrow(Par par, uint i) { - return i % par.m_threads == par.m_no; + return i % par.m_usedthreads == par.m_no; } // timer @@ -375,13 +378,13 @@ isthrrow(Par par, unsigned i) struct Tmr { void clr(); void on(); - void off(unsigned cnt = 0); + void off(uint cnt = 0); const char* time(); const char* pct(const Tmr& t1); const char* over(const Tmr& t1); NDB_TICKS m_on; - unsigned m_ms; - unsigned m_cnt; + uint m_ms; + uint m_cnt; char m_time[100]; char m_text[100]; Tmr() { clr(); } @@ -401,7 +404,7 @@ Tmr::on() } void -Tmr::off(unsigned cnt) +Tmr::off(uint cnt) { NDB_TICKS off = NdbTick_CurrentMillisecond(); assert(m_on != 0 && off >= m_on); @@ -446,53 +449,18 @@ Tmr::over(const Tmr& t1) return m_text; } -// list of ints - -struct Lst { - Lst(); - unsigned m_arr[1000]; - unsigned m_cnt; - void push(unsigned i); - unsigned cnt() const; - void reset(); -}; - -Lst::Lst() : - m_cnt(0) -{ -} - -void -Lst::push(unsigned i) -{ - assert(m_cnt < sizeof(m_arr)/sizeof(m_arr[0])); - m_arr[m_cnt++] = i; -} - -unsigned -Lst::cnt() const -{ - return m_cnt; -} - -void -Lst::reset() -{ - m_cnt = 0; -} - // character sets -static const unsigned maxcsnumber = 512; -static const unsigned maxcharcount = 32; -static const unsigned maxcharsize = 4; -static const unsigned maxxmulsize = 8; +static const uint maxcsnumber = 512; +static const uint maxcharcount = 32; +static const uint maxcharsize = 4; +static const uint maxxmulsize = 8; // single mb char struct Chr { - unsigned char m_bytes[maxcharsize]; - unsigned char m_xbytes[maxxmulsize * maxcharsize]; - unsigned m_size; + uchar m_bytes[maxcharsize]; + uchar m_xbytes[maxxmulsize * maxcharsize]; + uint m_size; Chr(); }; @@ -506,7 +474,7 @@ Chr::Chr() // charset and random valid chars to use struct Chs { CHARSET_INFO* m_cs; - unsigned m_xmul; + uint m_xmul; Chr* m_chr; Chs(CHARSET_INFO* cs); ~Chs(); @@ -523,22 +491,22 @@ Chs::Chs(CHARSET_INFO* cs) : m_xmul = 1; assert(m_xmul <= maxxmulsize); m_chr = new Chr [maxcharcount]; - unsigned i = 0; - unsigned miss1 = 0; - unsigned miss2 = 0; - unsigned miss3 = 0; - unsigned miss4 = 0; + uint i = 0; + uint miss1 = 0; + uint miss2 = 0; + uint miss3 = 0; + uint miss4 = 0; while (i < maxcharcount) { - unsigned char* bytes = m_chr[i].m_bytes; - unsigned char* xbytes = m_chr[i].m_xbytes; - unsigned& size = m_chr[i].m_size; + uchar* bytes = m_chr[i].m_bytes; + uchar* xbytes = m_chr[i].m_xbytes; + uint& size = m_chr[i].m_size; bool ok; size = m_cs->mbminlen + urandom(m_cs->mbmaxlen - m_cs->mbminlen + 1); assert(m_cs->mbminlen <= size && size <= m_cs->mbmaxlen); // prefer longer chars if (size == m_cs->mbminlen && m_cs->mbminlen < m_cs->mbmaxlen && urandom(5) != 0) continue; - for (unsigned j = 0; j < size; j++) { + for (uint j = 0; j < size; j++) { bytes[j] = urandom(256); } int not_used; @@ -550,13 +518,13 @@ Chs::Chs(CHARSET_INFO* cs) : } // check no proper prefix wellformed ok = true; - for (unsigned j = 1; j < size; j++) { + for (uint j = 1; j < size; j++) { if ((*cs->cset->well_formed_len)(cs, sbytes, sbytes + j, 1, ¬_used) == j) { ok = false; break; } } - if (! ok) { + if (!ok) { miss2++; continue; } @@ -566,37 +534,37 @@ Chs::Chs(CHARSET_INFO* cs) : int xlen = (*cs->coll->strnxfrm)(cs, xbytes, m_xmul * size, bytes, size); // check we got something ok = false; - for (unsigned j = 0; j < xlen; j++) { + for (uint j = 0; j < xlen; j++) { if (xbytes[j] != 0) { ok = true; break; } } - if (! ok) { + if (!ok) { miss3++; continue; } // check for duplicate (before normalize) ok = true; - for (unsigned j = 0; j < i; j++) { + for (uint j = 0; j < i; j++) { const Chr& chr = m_chr[j]; if (chr.m_size == size && memcmp(chr.m_bytes, bytes, size) == 0) { ok = false; break; } } - if (! ok) { + if (!ok) { miss4++; continue; } i++; } bool disorder = true; - unsigned bubbles = 0; + uint bubbles = 0; while (disorder) { disorder = false; - for (unsigned i = 1; i < maxcharcount; i++) { - unsigned len = sizeof(m_chr[i].m_xbytes); + for (uint i = 1; i < maxcharcount; i++) { + uint len = sizeof(m_chr[i].m_xbytes); if (memcmp(m_chr[i-1].m_xbytes, m_chr[i].m_xbytes, len) > 0) { Chr chr = m_chr[i]; m_chr[i] = m_chr[i-1]; @@ -627,7 +595,7 @@ static Chs* cslist[maxcsnumber]; static void resetcslist() { - for (unsigned i = 0; i < maxcsnumber; i++) { + for (uint i = 0; i < maxcsnumber; i++) { delete cslist[i]; cslist[i] = 0; } @@ -641,7 +609,7 @@ getcs(Par par) cs = par.m_cs; } else { while (1) { - unsigned n = urandom(maxcsnumber); + uint n = urandom(maxcsnumber); cs = get_charset(n, MYF(0)); if (cs != 0) { // prefer complex charsets @@ -667,24 +635,24 @@ struct Col { Longvarchar = NdbDictionary::Column::Longvarchar }; const class Tab& m_tab; - unsigned m_num; + uint m_num; const char* m_name; bool m_pk; Type m_type; - unsigned m_length; - unsigned m_bytelength; // multiplied by char width - unsigned m_attrsize; // base type size - unsigned m_headsize; // length bytes - unsigned m_bytesize; // full value size + uint m_length; + uint m_bytelength; // multiplied by char width + uint m_attrsize; // base type size + uint m_headsize; // length bytes + uint m_bytesize; // full value size bool m_nullable; const Chs* m_chs; - Col(const class Tab& tab, unsigned num, const char* name, bool pk, Type type, unsigned length, bool nullable, const Chs* chs); + Col(const class Tab& tab, uint num, const char* name, bool pk, Type type, uint length, bool nullable, const Chs* chs); ~Col(); bool equal(const Col& col2) const; void wellformed(const void* addr) const; }; -Col::Col(const class Tab& tab, unsigned num, const char* name, bool pk, Type type, unsigned length, bool nullable, const Chs* chs) : +Col::Col(const class Tab& tab, uint num, const char* name, bool pk, Type type, uint length, bool nullable, const Chs* chs) : m_tab(tab), m_num(num), m_name(strcpy(new char [strlen(name) + 1], name)), @@ -735,7 +703,7 @@ Col::wellformed(const void* addr) const { CHARSET_INFO* cs = m_chs->m_cs; const char* src = (const char*)addr; - unsigned len = m_bytelength; + uint len = m_bytelength; int not_used; assert((*cs->cset->well_formed_len)(cs, src, src + len, 0xffff, ¬_used) == len); } @@ -743,9 +711,9 @@ Col::wellformed(const void* addr) const case Col::Varchar: { CHARSET_INFO* cs = m_chs->m_cs; - const unsigned char* src = (const unsigned char*)addr; + const uchar* src = (const uchar*)addr; const char* ssrc = (const char*)src; - unsigned len = src[0]; + uint len = src[0]; int not_used; assert(len <= m_bytelength); assert((*cs->cset->well_formed_len)(cs, ssrc + 1, ssrc + 1 + len, 0xffff, ¬_used) == len); @@ -754,9 +722,9 @@ Col::wellformed(const void* addr) const case Col::Longvarchar: { CHARSET_INFO* cs = m_chs->m_cs; - const unsigned char* src = (const unsigned char*)addr; + const uchar* src = (const uchar*)addr; const char* ssrc = (const char*)src; - unsigned len = src[0] + (src[1] << 8); + uint len = src[0] + (src[1] << 8); int not_used; assert(len <= m_bytelength); assert((*cs->cset->well_formed_len)(cs, ssrc + 2, ssrc + 2 + len, 0xffff, ¬_used) == len); @@ -774,7 +742,7 @@ operator<<(NdbOut& out, const Col& col) out << "col[" << col.m_num << "] " << col.m_name; switch (col.m_type) { case Col::Unsigned: - out << " unsigned"; + out << " uint"; break; case Col::Char: { @@ -808,13 +776,13 @@ operator<<(NdbOut& out, const Col& col) struct ICol { const class ITab& m_itab; - unsigned m_num; + uint m_num; const Col& m_col; - ICol(const class ITab& itab, unsigned num, const Col& col); + ICol(const class ITab& itab, uint num, const Col& col); ~ICol(); }; -ICol::ICol(const class ITab& itab, unsigned num, const Col& col) : +ICol::ICol(const class ITab& itab, uint num, const Col& col) : m_itab(itab), m_num(num), m_col(col) @@ -842,47 +810,47 @@ struct ITab { const class Tab& m_tab; const char* m_name; Type m_type; - unsigned m_icols; + uint m_icols; const ICol** m_icol; - unsigned m_colmask; - ITab(const class Tab& tab, const char* name, Type type, unsigned icols); + uint m_keymask; + ITab(const class Tab& tab, const char* name, Type type, uint icols); ~ITab(); - void icoladd(unsigned k, const ICol* icolptr); + void icoladd(uint k, const ICol* icolptr); }; -ITab::ITab(const class Tab& tab, const char* name, Type type, unsigned icols) : +ITab::ITab(const class Tab& tab, const char* name, Type type, uint icols) : m_tab(tab), m_name(strcpy(new char [strlen(name) + 1], name)), m_type(type), m_icols(icols), m_icol(new const ICol* [icols + 1]), - m_colmask(0) + m_keymask(0) { - for (unsigned k = 0; k <= m_icols; k++) + for (uint k = 0; k <= m_icols; k++) m_icol[k] = 0; } ITab::~ITab() { delete [] m_name; - for (unsigned i = 0; i < m_icols; i++) + for (uint i = 0; i < m_icols; i++) delete m_icol[i]; delete [] m_icol; } void -ITab::icoladd(unsigned k, const ICol* icolptr) +ITab::icoladd(uint k, const ICol* icolptr) { assert(k == icolptr->m_num && k < m_icols && m_icol[k] == 0); m_icol[k] = icolptr; - m_colmask |= (1 << icolptr->m_col.m_num); + m_keymask |= (1 << icolptr->m_col.m_num); } static NdbOut& operator<<(NdbOut& out, const ITab& itab) { out << "itab " << itab.m_name << " icols=" << itab.m_icols; - for (unsigned k = 0; k < itab.m_icols; k++) { + for (uint k = 0; k < itab.m_icols; k++) { const ICol& icol = *itab.m_icol[k]; out << endl << icol; } @@ -893,56 +861,60 @@ operator<<(NdbOut& out, const ITab& itab) struct Tab { const char* m_name; - unsigned m_cols; + uint m_cols; const Col** m_col; - unsigned m_itabs; + uint m_pkmask; + uint m_itabs; const ITab** m_itab; - unsigned m_orderedindexes; - unsigned m_hashindexes; + uint m_orderedindexes; + uint m_hashindexes; // pk must contain an Unsigned column - unsigned m_keycol; - void coladd(unsigned k, Col* colptr); - void itabadd(unsigned j, ITab* itab); - Tab(const char* name, unsigned cols, unsigned itabs, unsigned keycol); + uint m_keycol; + void coladd(uint k, Col* colptr); + void itabadd(uint j, ITab* itab); + Tab(const char* name, uint cols, uint itabs, uint keycol); ~Tab(); }; -Tab::Tab(const char* name, unsigned cols, unsigned itabs, unsigned keycol) : +Tab::Tab(const char* name, uint cols, uint itabs, uint keycol) : m_name(strcpy(new char [strlen(name) + 1], name)), m_cols(cols), m_col(new const Col* [cols + 1]), + m_pkmask(0), m_itabs(itabs), m_itab(new const ITab* [itabs + 1]), m_orderedindexes(0), m_hashindexes(0), m_keycol(keycol) { - for (unsigned k = 0; k <= cols; k++) + for (uint k = 0; k <= cols; k++) m_col[k] = 0; - for (unsigned j = 0; j <= itabs; j++) + for (uint j = 0; j <= itabs; j++) m_itab[j] = 0; } Tab::~Tab() { delete [] m_name; - for (unsigned i = 0; i < m_cols; i++) + for (uint i = 0; i < m_cols; i++) delete m_col[i]; delete [] m_col; - for (unsigned i = 0; i < m_itabs; i++) + for (uint i = 0; i < m_itabs; i++) delete m_itab[i]; delete [] m_itab; } void -Tab::coladd(unsigned k, Col* colptr) +Tab::coladd(uint k, Col* colptr) { assert(k == colptr->m_num && k < m_cols && m_col[k] == 0); m_col[k] = colptr; + if (colptr->m_pk) + m_pkmask |= (1 << k); } void -Tab::itabadd(unsigned j, ITab* itabptr) +Tab::itabadd(uint j, ITab* itabptr) { assert(j < m_itabs && m_itab[j] == 0 && itabptr != 0); m_itab[j] = itabptr; @@ -956,11 +928,11 @@ static NdbOut& operator<<(NdbOut& out, const Tab& tab) { out << "tab " << tab.m_name << " cols=" << tab.m_cols; - for (unsigned k = 0; k < tab.m_cols; k++) { + for (uint k = 0; k < tab.m_cols; k++) { const Col& col = *tab.m_col[k]; out << endl << col; } - for (unsigned i = 0; i < tab.m_itabs; i++) { + for (uint i = 0; i < tab.m_itabs; i++) { if (tab.m_itab[i] == 0) continue; const ITab& itab = *tab.m_itab[i]; @@ -972,20 +944,20 @@ operator<<(NdbOut& out, const Tab& tab) // make table structs static const Tab** tablist = 0; -static unsigned tabcount = 0; +static uint tabcount = 0; static void verifytables() { - for (unsigned j = 0; j < tabcount; j++) { + for (uint j = 0; j < tabcount; j++) { const Tab* t = tablist[j]; if (t == 0) continue; assert(t->m_cols != 0 && t->m_col != 0); - for (unsigned k = 0; k < t->m_cols; k++) { + for (uint k = 0; k < t->m_cols; k++) { const Col* c = t->m_col[k]; assert(c != 0 && c->m_num == k); - assert(! (c->m_pk && c->m_nullable)); + assert(!(c->m_pk && c->m_nullable)); } assert(t->m_col[t->m_cols] == 0); { @@ -994,16 +966,16 @@ verifytables() assert(c->m_pk && c->m_type == Col::Unsigned); } assert(t->m_itabs != 0 && t->m_itab != 0); - for (unsigned i = 0; i < t->m_itabs; i++) { + for (uint i = 0; i < t->m_itabs; i++) { const ITab* x = t->m_itab[i]; if (x == 0) continue; assert(x != 0 && x->m_icols != 0 && x->m_icol != 0); - for (unsigned k = 0; k < x->m_icols; k++) { + for (uint k = 0; k < x->m_icols; k++) { const ICol* c = x->m_icol[k]; assert(c != 0 && c->m_num == k && c->m_col.m_num < t->m_cols); if (x->m_type == ITab::UniqueHashIndex) { - assert(! c->m_col.m_nullable); + assert(!c->m_col.m_nullable); } } } @@ -1019,11 +991,11 @@ makebuiltintables(Par par) tabcount = 3; if (tablist == 0) { tablist = new const Tab* [tabcount]; - for (unsigned j = 0; j < tabcount; j++) { + for (uint j = 0; j < tabcount; j++) { tablist[j] = 0; } } else { - for (unsigned j = 0; j < tabcount; j++) { + for (uint j = 0; j < tabcount; j++) { delete tablist[j]; tablist[j] = 0; } @@ -1202,7 +1174,8 @@ static Ndb_cluster_connection* g_ncc = 0; struct Con { Ndb* m_ndb; NdbDictionary::Dictionary* m_dic; - NdbConnection* m_tx; + NdbTransaction* m_tx; + Uint64 m_txid; NdbOperation* m_op; NdbIndexOperation* m_indexop; NdbScanOperation* m_scanop; @@ -1210,10 +1183,16 @@ struct Con { NdbScanFilter* m_scanfilter; enum ScanMode { ScanNo = 0, Committed, Latest, Exclusive }; ScanMode m_scanmode; - enum ErrType { ErrNone = 0, ErrDeadlock, ErrNospace, ErrOther }; + enum ErrType { + ErrNone = 0, + ErrDeadlock = 1, + ErrNospace = 2, + ErrOther = 4 + }; ErrType m_errtype; + char m_errname[100]; Con() : - m_ndb(0), m_dic(0), m_tx(0), m_op(0), m_indexop(0), + m_ndb(0), m_dic(0), m_tx(0), m_txid(0), m_op(0), m_indexop(0), m_scanop(0), m_indexscanop(0), m_scanfilter(0), m_scanmode(ScanNo), m_errtype(ErrNone) {} ~Con() { @@ -1237,18 +1216,20 @@ struct Con { int setBound(int num, int type, const void* value); int beginFilter(int group); int endFilter(); - int setFilter(int num, int cond, const void* value, unsigned len); - int execute(ExecType t); - int execute(ExecType t, bool& deadlock, bool& nospace); + int setFilter(int num, int cond, const void* value, uint len); + int execute(ExecType et); + int execute(ExecType et, uint& err); + int readTuple(Par par); int readTuples(Par par); int readIndexTuples(Par par); int executeScan(); int nextScanResult(bool fetchAllowed); - int nextScanResult(bool fetchAllowed, bool& deadlock); + int nextScanResult(bool fetchAllowed, uint& err); int updateScanTuple(Con& con2); int deleteScanTuple(Con& con2); void closeScan(); void closeTransaction(); + const char* errname(uint err); void printerror(NdbOut& out); }; @@ -1259,7 +1240,7 @@ Con::connect() m_ndb = new Ndb(g_ncc, "TEST_DB"); CHKCON(m_ndb->init() == 0, *this); CHKCON(m_ndb->waitUntilReady(30) == 0, *this); - m_tx = 0, m_op = 0; + m_tx = 0, m_txid = 0, m_op = 0; return 0; } @@ -1274,7 +1255,7 @@ void Con::disconnect() { delete m_ndb; - m_ndb = 0, m_dic = 0, m_tx = 0, m_op = 0; + m_ndb = 0, m_dic = 0, m_tx = 0, m_txid = 0, m_op = 0; } int @@ -1284,6 +1265,7 @@ Con::startTransaction() if (m_tx != 0) closeTransaction(); CHKCON((m_tx = m_ndb->startTransaction()) != 0, *this); + m_txid = m_tx->getTransactionId(); return 0; } @@ -1307,7 +1289,7 @@ int Con::getNdbIndexOperation(const ITab& itab, const Tab& tab) { assert(m_tx != 0); - unsigned tries = 0; + uint tries = 0; while (1) { if (getNdbIndexOperation1(itab, tab) == 0) break; @@ -1337,7 +1319,7 @@ int Con::getNdbIndexScanOperation(const ITab& itab, const Tab& tab) { assert(m_tx != 0); - unsigned tries = 0; + uint tries = 0; while (1) { if (getNdbIndexScanOperation1(itab, tab) == 0) break; @@ -1405,7 +1387,7 @@ Con::endFilter() } int -Con::setFilter(int num, int cond, const void* value, unsigned len) +Con::setFilter(int num, int cond, const void* value, uint len) { assert(m_tx != 0 && m_scanfilter != 0); CHKCON(m_scanfilter->cmp((NdbScanFilter::BinaryCondition)cond, num, value, len) == 0, *this); @@ -1413,34 +1395,46 @@ Con::setFilter(int num, int cond, const void* value, unsigned len) } int -Con::execute(ExecType t) +Con::execute(ExecType et) { assert(m_tx != 0); - CHKCON(m_tx->execute(t) == 0, *this); + CHKCON(m_tx->execute(et) == 0, *this); return 0; } int -Con::execute(ExecType t, bool& deadlock, bool& nospace) +Con::execute(ExecType et, uint& err) { - int ret = execute(t); - if (ret != 0 && deadlock && m_errtype == ErrDeadlock) { - LL3("caught deadlock"); - ret = 0; - } else { - deadlock = false; - } - if (ret != 0 && nospace && m_errtype == ErrNospace) { - LL3("caught nospace"); - ret = 0; - } else { - nospace = false; + int ret = execute(et); + // err in: errors to catch, out: error caught + const uint errin = err; + err = 0; + if (ret == -1) { + if (m_errtype == ErrDeadlock && (errin & ErrDeadlock)) { + LL3("caught deadlock"); + err = ErrDeadlock; + ret = 0; + } + if (m_errtype == ErrNospace && (errin & ErrNospace)) { + LL3("caught nospace"); + err = ErrNospace; + ret = 0; + } } CHK(ret == 0); return 0; } int +Con::readTuple(Par par) +{ + assert(m_tx != 0 && m_op != 0); + NdbOperation::LockMode lm = par.m_lockmode; + CHKCON(m_op->readTuple(lm) == 0, *this); + return 0; +} + +int Con::readTuples(Par par) { assert(m_tx != 0 && m_scanop != 0); @@ -1477,23 +1471,25 @@ Con::nextScanResult(bool fetchAllowed) int ret; assert(m_scanop != 0); CHKCON((ret = m_scanop->nextResult(fetchAllowed)) != -1, *this); - assert(ret == 0 || ret == 1 || (! fetchAllowed && ret == 2)); + assert(ret == 0 || ret == 1 || (!fetchAllowed && ret == 2)); return ret; } int -Con::nextScanResult(bool fetchAllowed, bool& deadlock) +Con::nextScanResult(bool fetchAllowed, uint& err) { int ret = nextScanResult(fetchAllowed); + // err in: errors to catch, out: error caught + const uint errin = err; + err = 0; if (ret == -1) { - if (deadlock && m_errtype == ErrDeadlock) { + if (m_errtype == ErrDeadlock && (errin & ErrDeadlock)) { LL3("caught deadlock"); + err = ErrDeadlock; ret = 0; } - } else { - deadlock = false; } - CHK(ret == 0 || ret == 1 || (! fetchAllowed && ret == 2)); + CHK(ret == 0 || ret == 1 || (!fetchAllowed && ret == 2)); return ret; } @@ -1502,6 +1498,7 @@ Con::updateScanTuple(Con& con2) { assert(con2.m_tx != 0); CHKCON((con2.m_op = m_scanop->updateCurrentTuple(con2.m_tx)) != 0, *this); + con2.m_txid = m_txid; // in the kernel return 0; } @@ -1510,6 +1507,7 @@ Con::deleteScanTuple(Con& con2) { assert(con2.m_tx != 0); CHKCON(m_scanop->deleteCurrentTuple(con2.m_tx) == 0, *this); + con2.m_txid = m_txid; // in the kernel return 0; } @@ -1527,15 +1525,26 @@ Con::closeTransaction() { assert(m_ndb != 0 && m_tx != 0); m_ndb->closeTransaction(m_tx); - m_tx = 0, m_op = 0; + m_tx = 0, m_txid = 0, m_op = 0; m_scanop = 0, m_indexscanop = 0; } +const char* +Con::errname(uint err) +{ + sprintf(m_errname, "0x%x", err); + if (err & ErrDeadlock) + strcat(m_errname, ",deadlock"); + if (err & ErrNospace) + strcat(m_errname, ",nospace"); + return m_errname; +} + void Con::printerror(NdbOut& out) { m_errtype = ErrOther; - unsigned any = 0; + uint any = 0; int code; int die = 0; if (m_ndb) { @@ -1563,7 +1572,7 @@ Con::printerror(NdbOut& out) } } } - if (! any) { + if (!any) { LL0("failed but no NDB error code"); } if (die) { @@ -1589,7 +1598,7 @@ invalidateindex(Par par) { Con& con = par.con(); const Tab& tab = par.tab(); - for (unsigned i = 0; i < tab.m_itabs; i++) { + for (uint i = 0; i < tab.m_itabs; i++) { if (tab.m_itab[i] == 0) continue; const ITab& itab = *tab.m_itab[i]; @@ -1639,7 +1648,7 @@ createtable(Par par) if (par.m_nologging) { t.setLogging(false); } - for (unsigned k = 0; k < tab.m_cols; k++) { + for (uint k = 0; k < tab.m_cols; k++) { const Col& col = *tab.m_col[k]; NdbDictionary::Column c(col.m_name); c.setType((NdbDictionary::Column::Type)col.m_type); @@ -1677,7 +1686,7 @@ static int dropindex(Par par) { const Tab& tab = par.tab(); - for (unsigned i = 0; i < tab.m_itabs; i++) { + for (uint i = 0; i < tab.m_itabs; i++) { if (tab.m_itab[i] == 0) continue; const ITab& itab = *tab.m_itab[i]; @@ -1699,7 +1708,7 @@ createindex(Par par, const ITab& itab) if (par.m_nologging || itab.m_type == ITab::OrderedIndex) { x.setLogging(false); } - for (unsigned k = 0; k < itab.m_icols; k++) { + for (uint k = 0; k < itab.m_icols; k++) { const ICol& icol = *itab.m_icol[k]; const Col& col = icol.m_col; x.addColumnName(col.m_name); @@ -1714,7 +1723,7 @@ static int createindex(Par par) { const Tab& tab = par.tab(); - for (unsigned i = 0; i < tab.m_itabs; i++) { + for (uint i = 0; i < tab.m_itabs; i++) { if (tab.m_itab[i] == 0) continue; const ITab& itab = *tab.m_itab[i]; @@ -1731,27 +1740,29 @@ struct Val { const Col& m_col; union { Uint32 m_uint32; - unsigned char* m_char; - unsigned char* m_varchar; - unsigned char* m_longvarchar; + uchar* m_char; + uchar* m_varchar; + uchar* m_longvarchar; }; + bool m_null; + // construct Val(const Col& col); ~Val(); void copy(const Val& val2); void copy(const void* addr); const void* dataaddr() const; - bool m_null; - int equal(Par par) const; - int equal(Par par, const ICol& icol) const; - int setval(Par par) const; - void calc(Par par, unsigned i); - void calckey(Par par, unsigned i); - void calckeychars(Par par, unsigned i, unsigned& n, unsigned char* buf); + void calc(Par par, uint i); + void calckey(Par par, uint i); + void calckeychars(Par par, uint i, uint& n, uchar* buf); void calcnokey(Par par); - void calcnokeychars(Par par, unsigned& n, unsigned char* buf); - int verify(Par par, const Val& val2) const; + void calcnokeychars(Par par, uint& n, uchar* buf); + // operations + int setval(Par par) const; + int setval(Par par, const ICol& icol) const; + // compare int cmp(Par par, const Val& val2) const; - int cmpchars(Par par, const unsigned char* buf1, unsigned len1, const unsigned char* buf2, unsigned len2) const; + int cmpchars(Par par, const uchar* buf1, uint len1, const uchar* buf2, uint len2) const; + int verify(Par par, const Val& val2) const; private: Val& operator=(const Val& val2); }; @@ -1759,20 +1770,26 @@ private: static NdbOut& operator<<(NdbOut& out, const Val& val); +// construct + Val::Val(const Col& col) : m_col(col) { switch (col.m_type) { case Col::Unsigned: + m_uint32 = 0x7e7e7e7e; break; case Col::Char: - m_char = new unsigned char [col.m_bytelength]; + m_char = new uchar [col.m_bytelength]; + memset(m_char, 0x7e, col.m_bytelength); break; case Col::Varchar: - m_varchar = new unsigned char [1 + col.m_bytelength]; + m_varchar = new uchar [1 + col.m_bytelength]; + memset(m_char, 0x7e, 1 + col.m_bytelength); break; case Col::Longvarchar: - m_longvarchar = new unsigned char [2 + col.m_bytelength]; + m_longvarchar = new uchar [2 + col.m_bytelength]; + memset(m_char, 0x7e, 2 + col.m_bytelength); break; default: assert(false); @@ -1858,52 +1875,17 @@ Val::dataaddr() const return 0; } -int -Val::equal(Par par) const -{ - Con& con = par.con(); - const Col& col = m_col; - assert(col.m_pk && ! m_null); - const char* addr = (const char*)dataaddr(); - LL5("equal [" << col << "] " << *this); - CHK(con.equal(col.m_num, addr) == 0); - return 0; -} - -int -Val::equal(Par par, const ICol& icol) const -{ - Con& con = par.con(); - assert(! m_null); - const char* addr = (const char*)dataaddr(); - LL5("equal [" << icol << "] " << *this); - CHK(con.equal(icol.m_num, addr) == 0); - return 0; -} - -int -Val::setval(Par par) const -{ - Con& con = par.con(); - const Col& col = m_col; - assert(! col.m_pk); - const char* addr = ! m_null ? (const char*)dataaddr() : 0; - LL5("setval [" << col << "] " << *this); - CHK(con.setValue(col.m_num, addr) == 0); - return 0; -} - void -Val::calc(Par par, unsigned i) +Val::calc(Par par, uint i) { const Col& col = m_col; col.m_pk ? calckey(par, i) : calcnokey(par); - if (! m_null) + if (!m_null) col.wellformed(dataaddr()); } void -Val::calckey(Par par, unsigned i) +Val::calckey(Par par, uint i) { const Col& col = m_col; m_null = false; @@ -1915,7 +1897,7 @@ Val::calckey(Par par, unsigned i) { const Chs* chs = col.m_chs; CHARSET_INFO* cs = chs->m_cs; - unsigned n = 0; + uint n = 0; calckeychars(par, i, n, m_char); // extend by appropriate space (*cs->cset->fill)(cs, (char*)&m_char[n], col.m_bytelength - n, 0x20); @@ -1923,7 +1905,7 @@ Val::calckey(Par par, unsigned i) break; case Col::Varchar: { - unsigned n = 0; + uint n = 0; calckeychars(par, i, n, m_varchar + 1); // set length and pad with nulls m_varchar[0] = n; @@ -1932,7 +1914,7 @@ Val::calckey(Par par, unsigned i) break; case Col::Longvarchar: { - unsigned n = 0; + uint n = 0; calckeychars(par, i, n, m_longvarchar + 2); // set length and pad with nulls m_longvarchar[0] = (n & 0xff); @@ -1947,13 +1929,13 @@ Val::calckey(Par par, unsigned i) } void -Val::calckeychars(Par par, unsigned i, unsigned& n, unsigned char* buf) +Val::calckeychars(Par par, uint i, uint& n, uchar* buf) { const Col& col = m_col; const Chs* chs = col.m_chs; CHARSET_INFO* cs = chs->m_cs; n = 0; - unsigned len = 0; + uint len = 0; while (len < col.m_length) { if (i % (1 + n) == 0) { break; @@ -1980,7 +1962,7 @@ Val::calcnokey(Par par) if (r < 0 && par.m_bdir > 0 || r > 0 && par.m_bdir < 0) r = -r; } - unsigned v = par.m_range + r; + uint v = par.m_range + r; switch (col.m_type) { case Col::Unsigned: m_uint32 = v; @@ -1989,7 +1971,7 @@ Val::calcnokey(Par par) { const Chs* chs = col.m_chs; CHARSET_INFO* cs = chs->m_cs; - unsigned n = 0; + uint n = 0; calcnokeychars(par, n, m_char); // extend by appropriate space (*cs->cset->fill)(cs, (char*)&m_char[n], col.m_bytelength - n, 0x20); @@ -1997,7 +1979,7 @@ Val::calcnokey(Par par) break; case Col::Varchar: { - unsigned n = 0; + uint n = 0; calcnokeychars(par, n, m_varchar + 1); // set length and pad with nulls m_varchar[0] = n; @@ -2006,7 +1988,7 @@ Val::calcnokey(Par par) break; case Col::Longvarchar: { - unsigned n = 0; + uint n = 0; calcnokeychars(par, n, m_longvarchar + 2); // set length and pad with nulls m_longvarchar[0] = (n & 0xff); @@ -2021,24 +2003,24 @@ Val::calcnokey(Par par) } void -Val::calcnokeychars(Par par, unsigned& n, unsigned char* buf) +Val::calcnokeychars(Par par, uint& n, uchar* buf) { const Col& col = m_col; const Chs* chs = col.m_chs; CHARSET_INFO* cs = chs->m_cs; n = 0; - unsigned len = 0; + uint len = 0; while (len < col.m_length) { if (urandom(1 + col.m_bytelength) == 0) { break; } - unsigned half = maxcharcount / 2; + uint half = maxcharcount / 2; int r = irandom((par.m_pctrange * half) / 100); if (par.m_bdir != 0 && urandom(10) != 0) { if (r < 0 && par.m_bdir > 0 || r > 0 && par.m_bdir < 0) r = -r; } - unsigned i = half + r; + uint i = half + r; assert(i < maxcharcount); const Chr& chr = chs->m_chr[i]; assert(n + chr.m_size <= col.m_bytelength); @@ -2048,13 +2030,39 @@ Val::calcnokeychars(Par par, unsigned& n, unsigned char* buf) } } +// operations + int -Val::verify(Par par, const Val& val2) const +Val::setval(Par par) const { - CHK(cmp(par, val2) == 0); + Con& con = par.con(); + const Col& col = m_col; + if (col.m_pk) { + assert(!m_null); + const char* addr = (const char*)dataaddr(); + LL5("setval pk [" << col << "] " << *this); + CHK(con.equal(col.m_num, addr) == 0); + } else { + const char* addr = !m_null ? (const char*)dataaddr() : 0; + LL5("setval non-pk [" << col << "] " << *this); + CHK(con.setValue(col.m_num, addr) == 0); + } + return 0; +} + +int +Val::setval(Par par, const ICol& icol) const +{ + Con& con = par.con(); + assert(!m_null); + const char* addr = (const char*)dataaddr(); + LL5("setval key [" << icol << "] " << *this); + CHK(con.equal(icol.m_num, addr) == 0); return 0; } +// compare + int Val::cmp(Par par, const Val& val2) const { @@ -2062,9 +2070,9 @@ Val::cmp(Par par, const Val& val2) const const Col& col2 = val2.m_col; assert(col.equal(col2)); if (m_null || val2.m_null) { - if (! m_null) + if (!m_null) return +1; - if (! val2.m_null) + if (!val2.m_null) return -1; return 0; } @@ -2084,21 +2092,21 @@ Val::cmp(Par par, const Val& val2) const break; case Col::Char: { - unsigned len = col.m_bytelength; + uint len = col.m_bytelength; return cmpchars(par, m_char, len, val2.m_char, len); } break; case Col::Varchar: { - unsigned len1 = m_varchar[0]; - unsigned len2 = val2.m_varchar[0]; + uint len1 = m_varchar[0]; + uint len2 = val2.m_varchar[0]; return cmpchars(par, m_varchar + 1, len1, val2.m_varchar + 1, len2); } break; case Col::Longvarchar: { - unsigned len1 = m_longvarchar[0] + (m_longvarchar[1] << 8); - unsigned len2 = val2.m_longvarchar[0] + (val2.m_longvarchar[1] << 8); + uint len1 = m_longvarchar[0] + (m_longvarchar[1] << 8); + uint len2 = val2.m_longvarchar[0] + (val2.m_longvarchar[1] << 8); return cmpchars(par, m_longvarchar + 2, len1, val2.m_longvarchar + 2, len2); } break; @@ -2110,17 +2118,17 @@ Val::cmp(Par par, const Val& val2) const } int -Val::cmpchars(Par par, const unsigned char* buf1, unsigned len1, const unsigned char* buf2, unsigned len2) const +Val::cmpchars(Par par, const uchar* buf1, uint len1, const uchar* buf2, uint len2) const { const Col& col = m_col; const Chs* chs = col.m_chs; CHARSET_INFO* cs = chs->m_cs; int k; - if (! par.m_collsp) { - unsigned char x1[maxxmulsize * 8000]; - unsigned char x2[maxxmulsize * 8000]; + if (!par.m_collsp) { + uchar x1[maxxmulsize * 8000]; + uchar x2[maxxmulsize * 8000]; // make strxfrm pad both to same length - unsigned len = maxxmulsize * col.m_bytelength; + uint len = maxxmulsize * col.m_bytelength; int n1 = NdbSqlUtil::strnxfrm_bug7284(cs, x1, chs->m_xmul * len, buf1, len1); int n2 = NdbSqlUtil::strnxfrm_bug7284(cs, x2, chs->m_xmul * len, buf2, len2); assert(n1 != -1 && n1 == n2); @@ -2131,8 +2139,17 @@ Val::cmpchars(Par par, const unsigned char* buf1, unsigned len1, const unsigned return k < 0 ? -1 : k > 0 ? +1 : 0; } +int +Val::verify(Par par, const Val& val2) const +{ + CHK(cmp(par, val2) == 0); + return 0; +} + +// print + static void -printstring(NdbOut& out, const unsigned char* str, unsigned len, bool showlen) +printstring(NdbOut& out, const uchar* str, uint len, bool showlen) { char buf[4 * 8000]; char *p = buf; @@ -2141,12 +2158,12 @@ printstring(NdbOut& out, const unsigned char* str, unsigned len, bool showlen) sprintf(p, "%u:", len); p += strlen(p); } - for (unsigned i = 0; i < len; i++) { - unsigned char c = str[i]; + for (uint i = 0; i < len; i++) { + uchar c = str[i]; if (c == '\\') { *p++ = '\\'; *p++ = c; - } else if (0x20 <= c && c < 0x7e) { + } else if (0x20 <= c && c <= 0x7e) { *p++ = c; } else { *p++ = '\\'; @@ -2173,19 +2190,19 @@ operator<<(NdbOut& out, const Val& val) break; case Col::Char: { - unsigned len = col.m_bytelength; + uint len = col.m_bytelength; printstring(out, val.m_char, len, false); } break; case Col::Varchar: { - unsigned len = val.m_varchar[0]; + uint len = val.m_varchar[0]; printstring(out, val.m_varchar + 1, len, true); } break; case Col::Longvarchar: { - unsigned len = val.m_longvarchar[0] + (val.m_longvarchar[1] << 8); + uint len = val.m_longvarchar[0] + (val.m_longvarchar[1] << 8); printstring(out, val.m_longvarchar + 2, len, true); } break; @@ -2202,16 +2219,36 @@ operator<<(NdbOut& out, const Val& val) struct Row { const Tab& m_tab; Val** m_val; - bool m_exist; - enum Op { NoOp = 0, ReadOp = 1, InsOp = 2, UpdOp = 4, DelOp = 8, AnyOp = 15 }; - Op m_pending; - Row* m_dbrow; // copy of db row before update + enum St { + StUndef = 0, + StDefine = 1, + StPrepare = 2, + StCommit = 3 + }; + enum Op { + OpNone = 0, + OpIns = 2, + OpUpd = 4, + OpDel = 8, + OpRead = 16, + OpReadEx = 32, + OpReadCom = 64, + OpDML = 2 | 4 | 8, + OpREAD = 16 | 32 | 64 + }; + St m_st; + Op m_op; + Uint64 m_txid; + Row* m_bi; + // construct Row(const Tab& tab); ~Row(); - void copy(const Row& row2); - void calc(Par par, unsigned i, unsigned mask = 0); - const Row& dbrow() const; - int verify(Par par, const Row& row2, bool pkonly) const; + void copy(const Row& row2, bool copy_bi); + void copyval(const Row& row2, uint colmask = ~0); + void calc(Par par, uint i, uint colmask = ~0); + // operations + int setval(Par par, uint colmask = ~0); + int setval(Par par, const ITab& itab); int insrow(Par par); int updrow(Par par); int updrow(Par par, const ITab& itab); @@ -2220,120 +2257,132 @@ struct Row { int selrow(Par par); int selrow(Par par, const ITab& itab); int setrow(Par par); + // compare int cmp(Par par, const Row& row2) const; int cmp(Par par, const Row& row2, const ITab& itab) const; + int verify(Par par, const Row& row2, bool pkonly) const; private: Row& operator=(const Row& row2); }; +static NdbOut& +operator<<(NdbOut& out, const Row* rowp); + +static NdbOut& +operator<<(NdbOut& out, const Row& row); + +// construct + Row::Row(const Tab& tab) : m_tab(tab) { m_val = new Val* [tab.m_cols]; - for (unsigned k = 0; k < tab.m_cols; k++) { + for (uint k = 0; k < tab.m_cols; k++) { const Col& col = *tab.m_col[k]; m_val[k] = new Val(col); } - m_exist = false; - m_pending = NoOp; - m_dbrow = 0; + m_st = StUndef; + m_op = OpNone; + m_txid = 0; + m_bi = 0; } Row::~Row() { const Tab& tab = m_tab; - for (unsigned k = 0; k < tab.m_cols; k++) { + for (uint k = 0; k < tab.m_cols; k++) { delete m_val[k]; } delete [] m_val; - delete m_dbrow; + delete m_bi; } void -Row::copy(const Row& row2) +Row::copy(const Row& row2, bool copy_bi) +{ + const Tab& tab = m_tab; + copyval(row2); + m_st = row2.m_st; + m_op = row2.m_op; + m_txid = row2.m_txid; + assert(m_bi == 0); + if (copy_bi && row2.m_bi != 0) { + m_bi = new Row(tab); + m_bi->copy(*row2.m_bi, copy_bi); + } +} + +void +Row::copyval(const Row& row2, uint colmask) { const Tab& tab = m_tab; assert(&tab == &row2.m_tab); - for (unsigned k = 0; k < tab.m_cols; k++) { + for (uint k = 0; k < tab.m_cols; k++) { Val& val = *m_val[k]; const Val& val2 = *row2.m_val[k]; - val.copy(val2); - } - m_exist = row2.m_exist; - m_pending = row2.m_pending; - if (row2.m_dbrow == 0) { - m_dbrow = 0; - } else { - assert(row2.m_dbrow->m_dbrow == 0); - if (m_dbrow == 0) - m_dbrow = new Row(tab); - m_dbrow->copy(*row2.m_dbrow); + if ((1 << k) & colmask) + val.copy(val2); } } void -Row::calc(Par par, unsigned i, unsigned mask) +Row::calc(Par par, uint i, uint colmask) { const Tab& tab = m_tab; - for (unsigned k = 0; k < tab.m_cols; k++) { - if (! (mask & (1 << k))) { + for (uint k = 0; k < tab.m_cols; k++) { + if ((1 << k) & colmask) { Val& val = *m_val[k]; val.calc(par, i); } } } -const Row& -Row::dbrow() const -{ - if (m_dbrow == 0) - return *this; - assert(m_pending == Row::UpdOp || m_pending == Row::DelOp); - return *m_dbrow; -} +// operations int -Row::verify(Par par, const Row& row2, bool pkonly) const +Row::setval(Par par, uint colmask) { const Tab& tab = m_tab; - const Row& row1 = *this; - assert(&row1.m_tab == &row2.m_tab && row1.m_exist && row2.m_exist); - for (unsigned k = 0; k < tab.m_cols; k++) { - const Col& col = row1.m_val[k]->m_col; - if (! pkonly || col.m_pk) { - const Val& val1 = *row1.m_val[k]; - const Val& val2 = *row2.m_val[k]; - CHK(val1.verify(par, val2) == 0); + Rsq rsq(tab.m_cols); + for (uint k = 0; k < tab.m_cols; k++) { + uint k2 = rsq.next(); + if ((1 << k2) & colmask) { + const Val& val = *m_val[k2]; + CHK(val.setval(par) == 0); } } return 0; } int +Row::setval(Par par, const ITab& itab) +{ + Con& con = par.con(); + Rsq rsq(itab.m_icols); + for (uint k = 0; k < itab.m_icols; k++) { + uint k2 = rsq.next(); + const ICol& icol = *itab.m_icol[k2]; + const Col& col = icol.m_col; + uint m = col.m_num; + const Val& val = *m_val[m]; + CHK(val.setval(par, icol) == 0); + } + return 0; +} + +int Row::insrow(Par par) { Con& con = par.con(); const Tab& tab = m_tab; - assert(! m_exist); CHK(con.getNdbOperation(tab) == 0); CHKCON(con.m_op->insertTuple() == 0, con); - Rsq rsq1(tab.m_cols); - for (unsigned k = 0; k < tab.m_cols; k++) { - unsigned k2 = rsq1.next(); - const Val& val = *m_val[k2]; - const Col& col = val.m_col; - if (col.m_pk) - CHK(val.equal(par) == 0); - } - Rsq rsq2(tab.m_cols); - for (unsigned k = 0; k < tab.m_cols; k++) { - unsigned k2 = rsq2.next(); - const Val& val = *m_val[k2]; - const Col& col = val.m_col; - if (! col.m_pk) - CHK(val.setval(par) == 0); - } - m_pending = InsOp; + CHK(setval(par, tab.m_pkmask) == 0); + CHK(setval(par, ~tab.m_pkmask) == 0); + assert(m_st == StUndef); + m_st = StDefine; + m_op = OpIns; + m_txid = con.m_txid; return 0; } @@ -2342,26 +2391,14 @@ Row::updrow(Par par) { Con& con = par.con(); const Tab& tab = m_tab; - assert(m_exist); CHK(con.getNdbOperation(tab) == 0); CHKCON(con.m_op->updateTuple() == 0, con); - Rsq rsq1(tab.m_cols); - for (unsigned k = 0; k < tab.m_cols; k++) { - unsigned k2 = rsq1.next(); - const Val& val = *m_val[k2]; - const Col& col = val.m_col; - if (col.m_pk) - CHK(val.equal(par) == 0); - } - Rsq rsq2(tab.m_cols); - for (unsigned k = 0; k < tab.m_cols; k++) { - unsigned k2 = rsq2.next(); - const Val& val = *m_val[k2]; - const Col& col = val.m_col; - if (! col.m_pk) - CHK(val.setval(par) == 0); - } - m_pending = UpdOp; + CHK(setval(par, tab.m_pkmask) == 0); + CHK(setval(par, ~tab.m_pkmask) == 0); + assert(m_st == StUndef); + m_st = StDefine; + m_op = OpUpd; + m_txid = con.m_txid; return 0; } @@ -2371,27 +2408,14 @@ Row::updrow(Par par, const ITab& itab) Con& con = par.con(); const Tab& tab = m_tab; assert(itab.m_type == ITab::UniqueHashIndex && &itab.m_tab == &tab); - assert(m_exist); CHK(con.getNdbIndexOperation(itab, tab) == 0); CHKCON(con.m_op->updateTuple() == 0, con); - Rsq rsq1(itab.m_icols); - for (unsigned k = 0; k < itab.m_icols; k++) { - unsigned k2 = rsq1.next(); - const ICol& icol = *itab.m_icol[k2]; - const Col& col = icol.m_col; - unsigned m = col.m_num; - const Val& val = *m_val[m]; - CHK(val.equal(par, icol) == 0); - } - Rsq rsq2(tab.m_cols); - for (unsigned k = 0; k < tab.m_cols; k++) { - unsigned k2 = rsq2.next(); - const Val& val = *m_val[k2]; - const Col& col = val.m_col; - if (! col.m_pk) - CHK(val.setval(par) == 0); - } - m_pending = UpdOp; + CHK(setval(par, itab) == 0); + CHK(setval(par, ~tab.m_pkmask) == 0); + assert(m_st == StUndef); + m_st = StDefine; + m_op = OpUpd; + m_txid = con.m_txid; return 0; } @@ -2400,18 +2424,13 @@ Row::delrow(Par par) { Con& con = par.con(); const Tab& tab = m_tab; - assert(m_exist); CHK(con.getNdbOperation(m_tab) == 0); CHKCON(con.m_op->deleteTuple() == 0, con); - Rsq rsq1(tab.m_cols); - for (unsigned k = 0; k < tab.m_cols; k++) { - unsigned k2 = rsq1.next(); - const Val& val = *m_val[k2]; - const Col& col = val.m_col; - if (col.m_pk) - CHK(val.equal(par) == 0); - } - m_pending = DelOp; + CHK(setval(par, tab.m_pkmask) == 0); + assert(m_st == StUndef); + m_st = StDefine; + m_op = OpDel; + m_txid = con.m_txid; return 0; } @@ -2421,19 +2440,13 @@ Row::delrow(Par par, const ITab& itab) Con& con = par.con(); const Tab& tab = m_tab; assert(itab.m_type == ITab::UniqueHashIndex && &itab.m_tab == &tab); - assert(m_exist); CHK(con.getNdbIndexOperation(itab, tab) == 0); CHKCON(con.m_op->deleteTuple() == 0, con); - Rsq rsq1(itab.m_icols); - for (unsigned k = 0; k < itab.m_icols; k++) { - unsigned k2 = rsq1.next(); - const ICol& icol = *itab.m_icol[k2]; - const Col& col = icol.m_col; - unsigned m = col.m_num; - const Val& val = *m_val[m]; - CHK(val.equal(par, icol) == 0); - } - m_pending = DelOp; + CHK(setval(par, itab) == 0); + assert(m_st == StUndef); + m_st = StDefine; + m_op = OpDel; + m_txid = con.m_txid; return 0; } @@ -2443,15 +2456,9 @@ Row::selrow(Par par) Con& con = par.con(); const Tab& tab = m_tab; CHK(con.getNdbOperation(m_tab) == 0); - CHKCON(con.m_op->readTuple() == 0, con); - Rsq rsq1(tab.m_cols); - for (unsigned k = 0; k < tab.m_cols; k++) { - unsigned k2 = rsq1.next(); - const Val& val = *m_val[k2]; - const Col& col = val.m_col; - if (col.m_pk) - CHK(val.equal(par) == 0); - } + CHKCON(con.readTuple(par) == 0, con); + CHK(setval(par, tab.m_pkmask) == 0); + // TODO state return 0; } @@ -2462,16 +2469,9 @@ Row::selrow(Par par, const ITab& itab) const Tab& tab = m_tab; assert(itab.m_type == ITab::UniqueHashIndex && &itab.m_tab == &tab); CHK(con.getNdbIndexOperation(itab, tab) == 0); - CHKCON(con.m_op->readTuple() == 0, con); - Rsq rsq1(itab.m_icols); - for (unsigned k = 0; k < itab.m_icols; k++) { - unsigned k2 = rsq1.next(); - const ICol& icol = *itab.m_icol[k2]; - const Col& col = icol.m_col; - unsigned m = col.m_num; - const Val& val = *m_val[m]; - CHK(val.equal(par, icol) == 0); - } + CHKCON(con.readTuple(par) == 0, con); + CHK(setval(par, itab) == 0); + // TODO state return 0; } @@ -2480,25 +2480,23 @@ Row::setrow(Par par) { Con& con = par.con(); const Tab& tab = m_tab; - Rsq rsq1(tab.m_cols); - for (unsigned k = 0; k < tab.m_cols; k++) { - unsigned k2 = rsq1.next(); - const Val& val = *m_val[k2]; - const Col& col = val.m_col; - if (! col.m_pk) - CHK(val.setval(par) == 0); - } - m_pending = UpdOp; + CHK(setval(par, ~tab.m_pkmask) == 0); + assert(m_st == StUndef); + m_st = StDefine; + m_op = OpUpd; + m_txid = con.m_txid; return 0; } +// compare + int Row::cmp(Par par, const Row& row2) const { const Tab& tab = m_tab; assert(&tab == &row2.m_tab); int c = 0; - for (unsigned k = 0; k < tab.m_cols; k++) { + for (uint k = 0; k < tab.m_cols; k++) { const Val& val = *m_val[k]; const Val& val2 = *row2.m_val[k]; if ((c = val.cmp(par, val2)) != 0) @@ -2512,10 +2510,10 @@ Row::cmp(Par par, const Row& row2, const ITab& itab) const { const Tab& tab = m_tab; int c = 0; - for (unsigned i = 0; i < itab.m_icols; i++) { + for (uint i = 0; i < itab.m_icols; i++) { const ICol& icol = *itab.m_icol[i]; const Col& col = icol.m_col; - unsigned k = col.m_num; + uint k = col.m_num; assert(k < tab.m_cols); const Val& val = *m_val[k]; const Val& val2 = *row2.m_val[k]; @@ -2525,19 +2523,70 @@ Row::cmp(Par par, const Row& row2, const ITab& itab) const return c; } +int +Row::verify(Par par, const Row& row2, bool pkonly) const +{ + const Tab& tab = m_tab; + const Row& row1 = *this; + assert(&row1.m_tab == &row2.m_tab); + for (uint k = 0; k < tab.m_cols; k++) { + const Col& col = row1.m_val[k]->m_col; + if (!pkonly || col.m_pk) { + const Val& val1 = *row1.m_val[k]; + const Val& val2 = *row2.m_val[k]; + CHK(val1.verify(par, val2) == 0); + } + } + return 0; +} + +// print + +static NdbOut& +operator<<(NdbOut& out, const Row::St st) +{ + if (st == Row::StUndef) + out << "StUndef"; + else if (st == Row::StDefine) + out << "StDefine"; + else if (st == Row::StPrepare) + out << "StPrepare"; + else if (st == Row::StCommit) + out << "StCommit"; + else + out << "st=" << st; + return out; +} + static NdbOut& operator<<(NdbOut& out, const Row::Op op) { - if (op == Row::NoOp) - out << "NoOp"; - else if (op == Row::InsOp) - out << "InsOp"; - else if (op == Row::UpdOp) - out << "UpdOp"; - else if (op == Row::DelOp) - out << "DelOp"; + if (op == Row::OpNone) + out << "OpNone"; + else if (op == Row::OpIns) + out << "OpIns"; + else if (op == Row::OpUpd) + out << "OpUpd"; + else if (op == Row::OpDel) + out << "OpDel"; + else if (op == Row::OpRead) + out << "OpRead"; + else if (op == Row::OpReadEx) + out << "OpReadEx"; + else if (op == Row::OpReadCom) + out << "OpReadCom"; else - out << op; + out << "op=" << op; + return out; +} + +static NdbOut& +operator<<(NdbOut& out, const Row* rowp) +{ + if (rowp == 0) + out << "[null]"; + else + out << *rowp; return out; } @@ -2545,26 +2594,18 @@ static NdbOut& operator<<(NdbOut& out, const Row& row) { const Tab& tab = row.m_tab; - for (unsigned i = 0; i < tab.m_cols; i++) { + out << "["; + for (uint i = 0; i < tab.m_cols; i++) { if (i > 0) out << " "; out << *row.m_val[i]; } - out << " exist=" << row.m_exist; - if (row.m_pending) - out << " pending=" << row.m_pending; - if (row.m_dbrow != 0) - out << " [dbrow=" << *row.m_dbrow << "]"; - return out; -} - -static NdbOut& -operator<<(NdbOut& out, const Row* rowptr) -{ - if (rowptr == 0) - out << "null"; - else - out << *rowptr; + out << " " << row.m_st; + out << " " << row.m_op; + out << " " << HEX(row.m_txid); + if (row.m_bi != 0) + out << " " << row.m_bi; + out << "]"; return out; } @@ -2572,45 +2613,38 @@ operator<<(NdbOut& out, const Row* rowptr) struct Set { const Tab& m_tab; - unsigned m_rows; + uint m_rows; Row** m_row; - unsigned* m_rowkey; // maps row number (from 0) in scan to tuple key + uint* m_rowkey; // maps row number (from 0) in scan to tuple key Row* m_keyrow; NdbRecAttr** m_rec; - Set(const Tab& tab, unsigned rows); + // construct + Set(const Tab& tab, uint rows); ~Set(); void reset(); - unsigned count() const; - // old and new values - bool exist(unsigned i) const; - void dbsave(unsigned i); - void calc(Par par, unsigned i, unsigned mask = 0); - bool pending(unsigned i, unsigned mask) const; - void notpending(unsigned i, ExecType et = Commit); - void notpending(const Lst& lst, ExecType et = Commit); - void dbdiscard(unsigned i); - void dbdiscard(const Lst& lst); - const Row& dbrow(unsigned i) const; + bool compat(Par par, uint i, const Row::Op op) const; + void push(uint i); + void copyval(uint i, uint colmask = ~0); // from bi + void calc(Par par, uint i, uint colmask = ~0); + uint count() const; + const Row* getrow(uint i, bool dirty = false) const; + // transaction + void post(Par par, ExecType et); // operations - int insrow(Par par, unsigned i); - int updrow(Par par, unsigned i); - int updrow(Par par, const ITab& itab, unsigned i); - int delrow(Par par, unsigned i); - int delrow(Par par, const ITab& itab, unsigned i); + int insrow(Par par, uint i); + int updrow(Par par, uint i); + int updrow(Par par, const ITab& itab, uint i); + int delrow(Par par, uint i); + int delrow(Par par, const ITab& itab, uint i); int selrow(Par par, const Row& keyrow); int selrow(Par par, const ITab& itab, const Row& keyrow); - // set and get - void setkey(Par par, const Row& keyrow); - void setkey(Par par, const ITab& itab, const Row& keyrow); - int setrow(Par par, unsigned i); + int setrow(Par par, uint i); int getval(Par par); - int getkey(Par par, unsigned* i); - int putval(unsigned i, bool force, unsigned n = ~0); - // sort rows in-place according to ordered index + int getkey(Par par, uint* i); + int putval(uint i, bool force, uint n = ~0); + // compare void sort(Par par, const ITab& itab); - void sort(Par par, const ITab& itab, unsigned lo, unsigned hi); - // verify - int verify(Par par, const Set& set2, bool pkonly) const; + int verify(Par par, const Set& set2, bool pkonly, bool dirty = false) const; int verifyorder(Par par, const ITab& itab, bool descending) const; // protect structure NdbMutex* m_mutex; @@ -2621,26 +2655,27 @@ struct Set { NdbMutex_Unlock(m_mutex); } private: + void sort(Par par, const ITab& itab, uint lo, uint hi); Set& operator=(const Set& set2); }; -Set::Set(const Tab& tab, unsigned rows) : +// construct + +Set::Set(const Tab& tab, uint rows) : m_tab(tab) { m_rows = rows; m_row = new Row* [m_rows]; - for (unsigned i = 0; i < m_rows; i++) { - // allocate on need to save space + for (uint i = 0; i < m_rows; i++) { m_row[i] = 0; } - m_rowkey = new unsigned [m_rows]; - for (unsigned n = 0; n < m_rows; n++) { - // initialize to null + m_rowkey = new uint [m_rows]; + for (uint n = 0; n < m_rows; n++) { m_rowkey[n] = ~0; } m_keyrow = new Row(tab); m_rec = new NdbRecAttr* [tab.m_cols]; - for (unsigned k = 0; k < tab.m_cols; k++) { + for (uint k = 0; k < tab.m_cols; k++) { m_rec[k] = 0; } m_mutex = NdbMutex_Create(); @@ -2649,7 +2684,7 @@ Set::Set(const Tab& tab, unsigned rows) : Set::~Set() { - for (unsigned i = 0; i < m_rows; i++) { + for (uint i = 0; i < m_rows; i++) { delete m_row[i]; } delete [] m_row; @@ -2662,132 +2697,203 @@ Set::~Set() void Set::reset() { - for (unsigned i = 0; i < m_rows; i++) { - if (m_row[i] != 0) { - Row& row = *m_row[i]; - row.m_exist = false; - } - } -} - -unsigned -Set::count() const -{ - unsigned count = 0; - for (unsigned i = 0; i < m_rows; i++) { - if (m_row[i] != 0) { - Row& row = *m_row[i]; - if (row.m_exist) - count++; - } + for (uint i = 0; i < m_rows; i++) { + m_row[i] = 0; } - return count; } -// old and new values - +// this sucks bool -Set::exist(unsigned i) const +Set::compat(Par par, uint i, const Row::Op op) const { - assert(i < m_rows); - if (m_row[i] == 0) // not allocated => not exist - return false; - return m_row[i]->m_exist; + Con& con = par.con(); + int ret = -1; + int place = 0; + do { + const Row* rowp = getrow(i); + if (rowp == 0) { + ret = op == Row::OpIns; + place = 1; + break; + } + const Row& row = *rowp; + if (!(op & Row::OpREAD)) { + if (row.m_st == Row::StDefine || row.m_st == Row::StPrepare) { + assert(row.m_op & Row::OpDML); + assert(row.m_txid != 0); + if (con.m_txid != row.m_txid) { + ret = false; + place = 2; + break; + } + if (row.m_op != Row::OpDel) { + ret = op == Row::OpUpd || op == Row::OpDel; + place = 3; + break; + } + ret = op == Row::OpIns; + place = 4; + break; + } + if (row.m_st == Row::StCommit) { + assert(row.m_op == Row::OpNone); + assert(row.m_txid == 0); + ret = op == Row::OpUpd || op == Row::OpDel; + place = 5; + break; + } + } + if (op & Row::OpREAD) { + bool dirty = + con.m_txid != row.m_txid && + par.m_lockmode == NdbOperation::LM_CommittedRead; + const Row* rowp2 = getrow(i, dirty); + if (rowp2 == 0 || rowp2->m_op == Row::OpDel) { + ret = false; + place = 6; + break; + } + ret = true; + place = 7; + break; + } + } while (0); + LL4("compat ret=" << ret << " place=" << place); + assert(ret == false || ret == true); + return ret; } void -Set::dbsave(unsigned i) +Set::push(uint i) { const Tab& tab = m_tab; - assert(i < m_rows && m_row[i] != 0); + assert(i < m_rows); + Row* bi = m_row[i]; + m_row[i] = new Row(tab); Row& row = *m_row[i]; - LL5("dbsave " << i << ": " << row); - assert(row.m_exist && ! row.m_pending && row.m_dbrow == 0); - // could swap pointers but making copy is safer - Row* rowptr = new Row(tab); - rowptr->copy(row); - row.m_dbrow = rowptr; + row.m_bi = bi; + if (bi != 0) + row.copyval(*bi); } void -Set::calc(Par par, unsigned i, unsigned mask) +Set::copyval(uint i, uint colmask) { - const Tab& tab = m_tab; - if (m_row[i] == 0) - m_row[i] = new Row(tab); + assert(m_row[i] != 0); Row& row = *m_row[i]; - row.calc(par, i, mask); -} - -bool -Set::pending(unsigned i, unsigned mask) const -{ - assert(i < m_rows); - if (m_row[i] == 0) // not allocated => not pending - return Row::NoOp; - return m_row[i]->m_pending & mask; + assert(row.m_bi != 0); + row.copyval(*row.m_bi, colmask); } void -Set::notpending(unsigned i, ExecType et) +Set::calc(Par par, uint i, uint colmask) { assert(m_row[i] != 0); Row& row = *m_row[i]; - if (et == Commit) { - if (row.m_pending == Row::InsOp) - row.m_exist = true; - if (row.m_pending == Row::DelOp) - row.m_exist = false; - } else { - if (row.m_pending == Row::InsOp) - row.m_exist = false; - if (row.m_pending == Row::DelOp) - row.m_exist = true; - } - row.m_pending = Row::NoOp; + row.calc(par, i, colmask); } -void -Set::notpending(const Lst& lst, ExecType et) +uint +Set::count() const { - for (unsigned j = 0; j < lst.m_cnt; j++) { - unsigned i = lst.m_arr[j]; - notpending(i, et); + uint count = 0; + for (uint i = 0; i < m_rows; i++) { + if (m_row[i] != 0) + count++; } + return count; } -void -Set::dbdiscard(unsigned i) +const Row* +Set::getrow(uint i, bool dirty) const { - assert(m_row[i] != 0); - Row& row = *m_row[i]; - LL5("dbdiscard " << i << ": " << row); - assert(row.m_dbrow != 0); - delete row.m_dbrow; - row.m_dbrow = 0; + assert(i < m_rows); + const Row* rowp = m_row[i]; + if (dirty) { + while (rowp != 0) { + bool b1 = rowp->m_op == Row::OpNone; + bool b2 = rowp->m_st == Row::StCommit; + assert(b1 == b2); + if (b1) { + assert(rowp->m_bi == 0); + break; + } + rowp = rowp->m_bi; + } + } + return rowp; } -const Row& -Set::dbrow(unsigned i) const -{ - assert(m_row[i] != 0); - Row& row = *m_row[i]; - return row.dbrow(); -} +// transaction void -Set::dbdiscard(const Lst& lst) +Set::post(Par par, ExecType et) { - for (unsigned j = 0; j < lst.m_cnt; j++) { - unsigned i = lst.m_arr[j]; - dbdiscard(i); + LL4("post"); + Con& con = par.con(); + assert(con.m_txid != 0); + uint i; + for (i = 0; i < m_rows; i++) { + Row* rowp = m_row[i]; + if (rowp == 0) { + LL5("skip " << i << " " << rowp); + continue; + } + if (rowp->m_st == Row::StCommit) { + assert(rowp->m_op == Row::OpNone); + assert(rowp->m_txid == 0); + assert(rowp->m_bi == 0); + LL5("skip committed " << i << " " << rowp); + continue; + } + assert(rowp->m_st == Row::StDefine || rowp->m_st == Row::StPrepare); + assert(rowp->m_txid != 0); + if (con.m_txid != rowp->m_txid) { + LL5("skip txid " << i << " " << HEX(con.m_txid) << " " << rowp); + continue; + } + // TODO read ops + assert(rowp->m_op & Row::OpDML); + LL4("post BEFORE " << rowp); + if (et == NoCommit) { + if (rowp->m_st == Row::StDefine) { + rowp->m_st = Row::StPrepare; + Row* bi = rowp->m_bi; + while (bi != 0 && bi->m_st == Row::StDefine) { + bi->m_st = Row::StPrepare; + bi = bi->m_bi; + } + } + } else if (et == Commit) { + if (rowp->m_op != Row::OpDel) { + rowp->m_st = Row::StCommit; + rowp->m_op = Row::OpNone; + rowp->m_txid = 0; + delete rowp->m_bi; + rowp->m_bi = 0; + } else { + delete rowp; + rowp = 0; + } + } else if (et == Rollback) { + while (rowp != 0 && rowp->m_st != Row::StCommit) { + Row* tmp = rowp; + rowp = rowp->m_bi; + tmp->m_bi = 0; + delete tmp; + } + } else { + assert(false); + } + m_row[i] = rowp; + LL4("post AFTER " << rowp); } } // operations int -Set::insrow(Par par, unsigned i) +Set::insrow(Par par, uint i) { assert(m_row[i] != 0); Row& row = *m_row[i]; @@ -2796,7 +2902,7 @@ Set::insrow(Par par, unsigned i) } int -Set::updrow(Par par, unsigned i) +Set::updrow(Par par, uint i) { assert(m_row[i] != 0); Row& row = *m_row[i]; @@ -2805,7 +2911,7 @@ Set::updrow(Par par, unsigned i) } int -Set::updrow(Par par, const ITab& itab, unsigned i) +Set::updrow(Par par, const ITab& itab, uint i) { assert(m_row[i] != 0); Row& row = *m_row[i]; @@ -2814,7 +2920,7 @@ Set::updrow(Par par, const ITab& itab, unsigned i) } int -Set::delrow(Par par, unsigned i) +Set::delrow(Par par, uint i) { assert(m_row[i] != 0); Row& row = *m_row[i]; @@ -2823,7 +2929,7 @@ Set::delrow(Par par, unsigned i) } int -Set::delrow(Par par, const ITab& itab, unsigned i) +Set::delrow(Par par, const ITab& itab, uint i) { assert(m_row[i] != 0); Row& row = *m_row[i]; @@ -2836,8 +2942,8 @@ Set::selrow(Par par, const Row& keyrow) { Con& con = par.con(); const Tab& tab = par.tab(); - setkey(par, keyrow); - LL5("selrow " << tab.m_name << ": keyrow: " << keyrow); + LL5("selrow " << tab.m_name << " keyrow " << keyrow); + m_keyrow->copyval(keyrow, tab.m_pkmask); CHK(m_keyrow->selrow(par) == 0); CHK(getval(par) == 0); return 0; @@ -2847,45 +2953,15 @@ int Set::selrow(Par par, const ITab& itab, const Row& keyrow) { Con& con = par.con(); - setkey(par, itab, keyrow); - LL5("selrow " << itab.m_name << ": keyrow: " << keyrow); + LL5("selrow " << itab.m_name << " keyrow " << keyrow); + m_keyrow->copyval(keyrow, itab.m_keymask); CHK(m_keyrow->selrow(par, itab) == 0); CHK(getval(par) == 0); return 0; } -// set and get - -void -Set::setkey(Par par, const Row& keyrow) -{ - const Tab& tab = m_tab; - for (unsigned k = 0; k < tab.m_cols; k++) { - const Col& col = *tab.m_col[k]; - if (col.m_pk) { - Val& val1 = *m_keyrow->m_val[k]; - const Val& val2 = *keyrow.dbrow().m_val[k]; - val1.copy(val2); - } - } -} - -void -Set::setkey(Par par, const ITab& itab, const Row& keyrow) -{ - const Tab& tab = m_tab; - for (unsigned k = 0; k < itab.m_icols; k++) { - const ICol& icol = *itab.m_icol[k]; - const Col& col = icol.m_col; - unsigned m = col.m_num; - Val& val1 = *m_keyrow->m_val[m]; - const Val& val2 = *keyrow.dbrow().m_val[m]; - val1.copy(val2); - } -} - int -Set::setrow(Par par, unsigned i) +Set::setrow(Par par, uint i) { Con& con = par.con(); assert(m_row[i] != 0); @@ -2899,18 +2975,18 @@ Set::getval(Par par) Con& con = par.con(); const Tab& tab = m_tab; Rsq rsq1(tab.m_cols); - for (unsigned k = 0; k < tab.m_cols; k++) { - unsigned k2 = rsq1.next(); + for (uint k = 0; k < tab.m_cols; k++) { + uint k2 = rsq1.next(); CHK(con.getValue(k2, m_rec[k2]) == 0); } return 0; } int -Set::getkey(Par par, unsigned* i) +Set::getkey(Par par, uint* i) { const Tab& tab = m_tab; - unsigned k = tab.m_keycol; + uint k = tab.m_keycol; assert(m_rec[k] != 0); const char* aRef = m_rec[k]->aRef(); Uint32 key = *(const Uint32*)aRef; @@ -2921,14 +2997,18 @@ Set::getkey(Par par, unsigned* i) } int -Set::putval(unsigned i, bool force, unsigned n) +Set::putval(uint i, bool force, uint n) { const Tab& tab = m_tab; - if (m_row[i] == 0) - m_row[i] = new Row(tab); + LL4("putval key=" << i << " row=" << n << " old=" << m_row[i]); + if (m_row[i] != 0) { + assert(force); + delete m_row[i]; + m_row[i] = 0; + } + m_row[i] = new Row(tab); Row& row = *m_row[i]; - CHK(! row.m_exist || force); - for (unsigned k = 0; k < tab.m_cols; k++) { + for (uint k = 0; k < tab.m_cols; k++) { Val& val = *row.m_val[k]; NdbRecAttr* rec = m_rec[k]; assert(rec != 0); @@ -2940,13 +3020,13 @@ Set::putval(unsigned i, bool force, unsigned n) val.copy(aRef); val.m_null = false; } - if (! row.m_exist) - row.m_exist = true; if (n != ~0) m_rowkey[n] = i; return 0; } +// compare + void Set::sort(Par par, const ITab& itab) { @@ -2955,12 +3035,12 @@ Set::sort(Par par, const ITab& itab) } void -Set::sort(Par par, const ITab& itab, unsigned lo, unsigned hi) +Set::sort(Par par, const ITab& itab, uint lo, uint hi) { assert(lo < m_rows && hi < m_rows && lo <= hi); Row* const p = m_row[lo]; - unsigned i = lo; - unsigned j = hi; + uint i = lo; + uint j = hi; while (i < j) { while (i < j && m_row[j]->cmp(par, *p, itab) >= 0) j--; @@ -2982,22 +3062,48 @@ Set::sort(Par par, const ITab& itab, unsigned lo, unsigned hi) sort(par, itab, i + 1, hi); } +/* + * set1 (self) is from dml and can contain un-committed operations. + * set2 is from read and contains no operations. "dirty" applies + * to set1: false = use latest row, true = use committed row. + */ int -Set::verify(Par par, const Set& set2, bool pkonly) const -{ - assert(&m_tab == &set2.m_tab && m_rows == set2.m_rows); - LL4("verify set1 count=" << count() << " vs set2 count=" << set2.count()); - for (unsigned i = 0; i < m_rows; i++) { +Set::verify(Par par, const Set& set2, bool pkonly, bool dirty) const +{ + const Set& set1 = *this; + assert(&set1.m_tab == &set2.m_tab && set1.m_rows == set2.m_rows); + LL3("verify dirty:" << dirty); + for (uint i = 0; i < set1.m_rows; i++) { + // the row versions we actually compare + const Row* row1p = set1.getrow(i, dirty); + const Row* row2p = set2.getrow(i); bool ok = true; - if (exist(i) != set2.exist(i)) { - ok = false; - } else if (exist(i)) { - if (dbrow(i).verify(par, set2.dbrow(i), pkonly) != 0) + int place = 0; + if (row1p == 0) { + if (row2p != 0) { + ok = false; + place = 1; + } + } else { + Row::Op op1 = row1p->m_op; + if (op1 != Row::OpDel) { + if (row2p == 0) { + ok = false; + place = 2; + } else if (row1p->verify(par, *row2p, pkonly) == -1) { + ok = false; + place = 3; + } + } else if (row2p != 0) { ok = false; + place = 4; + } } - if (! ok) { - LL1("verify failed: key=" << i << " row1=" << m_row[i] << " row2=" << set2.m_row[i]); - CHK(0 == 1); + if (!ok) { + LL1("verify " << i << " failed at " << place); + LL1("row1 " << row1p); + LL1("row2 " << row2p); + CHK(false); } } return 0; @@ -3007,18 +3113,17 @@ int Set::verifyorder(Par par, const ITab& itab, bool descending) const { const Tab& tab = m_tab; - for (unsigned n = 0; n < m_rows; n++) { - unsigned i2 = m_rowkey[n]; + for (uint n = 0; n < m_rows; n++) { + uint i2 = m_rowkey[n]; if (i2 == ~0) break; if (n == 0) continue; - unsigned i1 = m_rowkey[n - 1]; - assert(i1 < m_rows && i2 < m_rows); + uint i1 = m_rowkey[n - 1]; + assert(m_row[i1] != 0 && m_row[i2] != 0); const Row& row1 = *m_row[i1]; const Row& row2 = *m_row[i2]; - assert(row1.m_exist && row2.m_exist); - if (! descending) + if (!descending) CHK(row1.cmp(par, row2, itab) <= 0); else CHK(row1.cmp(par, row2, itab) >= 0); @@ -3026,10 +3131,12 @@ Set::verifyorder(Par par, const ITab& itab, bool descending) const return 0; } +// print + static NdbOut& operator<<(NdbOut& out, const Set& set) { - for (unsigned i = 0; i < set.m_rows; i++) { + for (uint i = 0; i < set.m_rows; i++) { const Row& row = *set.m_row[i]; if (i > 0) out << endl; @@ -3058,8 +3165,8 @@ int BVal::setbnd(Par par) const { Con& con = par.con(); - assert(g_compare_null || ! m_null); - const char* addr = ! m_null ? (const char*)dataaddr() : 0; + assert(g_compare_null || !m_null); + const char* addr = !m_null ? (const char*)dataaddr() : 0; const ICol& icol = m_icol; CHK(con.setBound(icol.m_num, m_type, addr) == 0); return 0; @@ -3068,7 +3175,7 @@ BVal::setbnd(Par par) const int BVal::setflt(Par par) const { - static unsigned index_bound_to_filter_bound[5] = { + static uint index_bound_to_filter_bound[5] = { NdbScanFilter::COND_GE, NdbScanFilter::COND_GT, NdbScanFilter::COND_LE, @@ -3076,12 +3183,12 @@ BVal::setflt(Par par) const NdbScanFilter::COND_EQ }; Con& con = par.con(); - assert(g_compare_null || ! m_null); - const char* addr = ! m_null ? (const char*)dataaddr() : 0; + assert(g_compare_null || !m_null); + const char* addr = !m_null ? (const char*)dataaddr() : 0; const ICol& icol = m_icol; const Col& col = icol.m_col; - unsigned length = col.m_bytesize; - unsigned cond = index_bound_to_filter_bound[m_type]; + uint length = col.m_bytesize; + uint cond = index_bound_to_filter_bound[m_type]; CHK(con.setFilter(col.m_num, cond, addr, length) == 0); return 0; } @@ -3104,27 +3211,27 @@ operator<<(NdbOut& out, const BVal& bval) struct BSet { const Tab& m_tab; const ITab& m_itab; - unsigned m_alloc; - unsigned m_bvals; + uint m_alloc; + uint m_bvals; BVal** m_bval; - BSet(const Tab& tab, const ITab& itab, unsigned rows); + BSet(const Tab& tab, const ITab& itab); ~BSet(); void reset(); void calc(Par par); - void calcpk(Par par, unsigned i); + void calcpk(Par par, uint i); int setbnd(Par par) const; int setflt(Par par) const; void filter(Par par, const Set& set, Set& set2) const; }; -BSet::BSet(const Tab& tab, const ITab& itab, unsigned rows) : +BSet::BSet(const Tab& tab, const ITab& itab) : m_tab(tab), m_itab(itab), m_alloc(2 * itab.m_icols), m_bvals(0) { m_bval = new BVal* [m_alloc]; - for (unsigned i = 0; i < m_alloc; i++) { + for (uint i = 0; i < m_alloc; i++) { m_bval[i] = 0; } } @@ -3138,7 +3245,7 @@ void BSet::reset() { while (m_bvals > 0) { - unsigned i = --m_bvals; + uint i = --m_bvals; delete m_bval[i]; m_bval[i] = 0; } @@ -3150,10 +3257,10 @@ BSet::calc(Par par) const ITab& itab = m_itab; par.m_pctrange = par.m_pctbrange; reset(); - for (unsigned k = 0; k < itab.m_icols; k++) { + for (uint k = 0; k < itab.m_icols; k++) { const ICol& icol = *itab.m_icol[k]; const Col& col = icol.m_col; - for (unsigned i = 0; i <= 1; i++) { + for (uint i = 0; i <= 1; i++) { if (m_bvals == 0 && urandom(100) == 0) return; if (m_bvals != 0 && urandom(3) == 0) @@ -3162,7 +3269,7 @@ BSet::calc(Par par) BVal& bval = *new BVal(icol); m_bval[m_bvals++] = &bval; bval.m_null = false; - unsigned sel; + uint sel; do { // equality bound only on i==0 sel = urandom(5 - i); @@ -3175,7 +3282,7 @@ BSet::calc(Par par) bval.m_type = 4; if (k + 1 < itab.m_icols) bval.m_type = 4; - if (! g_compare_null) + if (!g_compare_null) par.m_pctnull = 0; if (bval.m_type == 0 || bval.m_type == 1) par.m_bdir = -1; @@ -3199,11 +3306,11 @@ BSet::calc(Par par) } void -BSet::calcpk(Par par, unsigned i) +BSet::calcpk(Par par, uint i) { const ITab& itab = m_itab; reset(); - for (unsigned k = 0; k < itab.m_icols; k++) { + for (uint k = 0; k < itab.m_icols; k++) { const ICol& icol = *itab.m_icol[k]; const Col& col = icol.m_col; assert(col.m_pk); @@ -3220,14 +3327,14 @@ BSet::setbnd(Par par) const { if (m_bvals != 0) { Rsq rsq1(m_bvals); - for (unsigned j = 0; j < m_bvals; j++) { - unsigned j2 = rsq1.next(); + for (uint j = 0; j < m_bvals; j++) { + uint j2 = rsq1.next(); const BVal& bval = *m_bval[j2]; CHK(bval.setbnd(par) == 0); } // duplicate if (urandom(5) == 0) { - unsigned j3 = urandom(m_bvals); + uint j3 = urandom(m_bvals); const BVal& bval = *m_bval[j3]; CHK(bval.setbnd(par) == 0); } @@ -3243,14 +3350,14 @@ BSet::setflt(Par par) const CHK(con.beginFilter(NdbScanFilter::AND) == 0); if (m_bvals != 0) { Rsq rsq1(m_bvals); - for (unsigned j = 0; j < m_bvals; j++) { - unsigned j2 = rsq1.next(); + for (uint j = 0; j < m_bvals; j++) { + uint j2 = rsq1.next(); const BVal& bval = *m_bval[j2]; CHK(bval.setflt(par) == 0); } // duplicate if (urandom(5) == 0) { - unsigned j3 = urandom(m_bvals); + uint j3 = urandom(m_bvals); const BVal& bval = *m_bval[j3]; CHK(bval.setflt(par) == 0); } @@ -3266,57 +3373,59 @@ BSet::filter(Par par, const Set& set, Set& set2) const const ITab& itab = m_itab; assert(&tab == &set2.m_tab && set.m_rows == set2.m_rows); assert(set2.count() == 0); - for (unsigned i = 0; i < set.m_rows; i++) { - if (! set.exist(i)) - continue; + for (uint i = 0; i < set.m_rows; i++) { set.lock(); - const Row& row = set.dbrow(i); - set.unlock(); - if (! g_store_null_key) { - bool ok1 = false; - for (unsigned k = 0; k < itab.m_icols; k++) { - const ICol& icol = *itab.m_icol[k]; + do { + if (set.m_row[i] == 0) { + break; + } + const Row& row = *set.m_row[i]; + if (!g_store_null_key) { + bool ok1 = false; + for (uint k = 0; k < itab.m_icols; k++) { + const ICol& icol = *itab.m_icol[k]; + const Col& col = icol.m_col; + const Val& val = *row.m_val[col.m_num]; + if (!val.m_null) { + ok1 = true; + break; + } + } + if (!ok1) + break; + } + bool ok2 = true; + for (uint j = 0; j < m_bvals; j++) { + const BVal& bval = *m_bval[j]; + const ICol& icol = bval.m_icol; const Col& col = icol.m_col; const Val& val = *row.m_val[col.m_num]; - if (! val.m_null) { - ok1 = true; - break; + int ret = bval.cmp(par, val); + LL5("cmp: ret=" << ret << " " << bval << " vs " << val); + if (bval.m_type == 0) + ok2 = (ret <= 0); + else if (bval.m_type == 1) + ok2 = (ret < 0); + else if (bval.m_type == 2) + ok2 = (ret >= 0); + else if (bval.m_type == 3) + ok2 = (ret > 0); + else if (bval.m_type == 4) + ok2 = (ret == 0); + else { + assert(false); } + if (!ok2) + break; } - if (! ok1) - continue; - } - bool ok2 = true; - for (unsigned j = 0; j < m_bvals; j++) { - const BVal& bval = *m_bval[j]; - const ICol& icol = bval.m_icol; - const Col& col = icol.m_col; - const Val& val = *row.m_val[col.m_num]; - int ret = bval.cmp(par, val); - LL5("cmp: ret=" << ret << " " << bval << " vs " << val); - if (bval.m_type == 0) - ok2 = (ret <= 0); - else if (bval.m_type == 1) - ok2 = (ret < 0); - else if (bval.m_type == 2) - ok2 = (ret >= 0); - else if (bval.m_type == 3) - ok2 = (ret > 0); - else if (bval.m_type == 4) - ok2 = (ret == 0); - else { - assert(false); - } - if (! ok2) + if (!ok2) break; - } - if (! ok2) - continue; - if (set2.m_row[i] == 0) + assert(set2.m_row[i] == 0); set2.m_row[i] = new Row(tab); - Row& row2 = *set2.m_row[i]; - assert(! row2.m_exist); - row2.copy(row); + Row& row2 = *set2.m_row[i]; + row2.copy(row, true); + } while (0); + set.unlock(); } } @@ -3324,7 +3433,7 @@ static NdbOut& operator<<(NdbOut& out, const BSet& bset) { out << "bounds=" << bset.m_bvals; - for (unsigned j = 0; j < bset.m_bvals; j++) { + for (uint j = 0; j < bset.m_bvals; j++) { const BVal& bval = *bset.m_bval[j]; out << " [bound " << j << ": " << bval << "]"; } @@ -3341,59 +3450,40 @@ pkinsert(Par par) Set& set = par.set(); LL3("pkinsert " << tab.m_name); CHK(con.startTransaction() == 0); - Lst lst; - for (unsigned j = 0; j < par.m_rows; j++) { - unsigned j2 = ! par.m_randomkey ? j : urandom(par.m_rows); - unsigned i = thrrow(par, j2); + uint batch = 0; + for (uint j = 0; j < par.m_rows; j++) { + uint j2 = !par.m_randomkey ? j : urandom(par.m_rows); + uint i = thrrow(par, j2); set.lock(); - if (set.exist(i) || set.pending(i, Row::AnyOp)) { + if (!set.compat(par, i, Row::OpIns)) { + LL3("pkinsert SKIP " << i << " " << set.getrow(i)); set.unlock(); - continue; - } - set.calc(par, i); - CHK(set.insrow(par, i) == 0); - set.unlock(); - LL4("pkinsert " << i << ": " << *set.m_row[i]); - lst.push(i); - if (lst.cnt() == par.m_batch) { - bool deadlock = par.m_deadlock; - bool nospace = true; - ExecType et = randompct(par.m_abortpct) ? Rollback : Commit; - CHK(con.execute(et, deadlock, nospace) == 0); - con.closeTransaction(); - if (deadlock) { - LL1("pkinsert: stop on deadlock [at 1]"); - return 0; - } - if (nospace) { - LL1("pkinsert: cnt=" << j << " stop on nospace"); - return 0; - } + } else { + set.push(i); + set.calc(par, i); + CHK(set.insrow(par, i) == 0); + set.unlock(); + LL4("pkinsert key=" << i << " " << set.getrow(i)); + batch++; + } + bool lastbatch = (batch != 0 && j + 1 == par.m_rows); + if (batch == par.m_batch || lastbatch) { + uint err = par.m_catcherr; + ExecType et = !randompct(par.m_abortpct) ? Commit : Rollback; + CHK(con.execute(et, err) == 0); set.lock(); - set.notpending(lst, et); + set.post(par, !err ? et : Rollback); set.unlock(); - lst.reset(); - CHK(con.startTransaction() == 0); - } - } - if (lst.cnt() != 0) { - bool deadlock = par.m_deadlock; - bool nospace = true; - ExecType et = randompct(par.m_abortpct) ? Rollback : Commit; - CHK(con.execute(et, deadlock, nospace) == 0); - con.closeTransaction(); - if (deadlock) { - LL1("pkinsert: stop on deadlock [at 2]"); - return 0; - } - if (nospace) { - LL1("pkinsert: end: stop on nospace"); - return 0; + if (err) { + LL1("pkinsert key=" << i << " stop on " << con.errname(err)); + break; + } + batch = 0; + if (!lastbatch) { + con.closeTransaction(); + CHK(con.startTransaction() == 0); + } } - set.lock(); - set.notpending(lst, et); - set.unlock(); - return 0; } con.closeTransaction(); return 0; @@ -3407,59 +3497,40 @@ pkupdate(Par par) Set& set = par.set(); LL3("pkupdate " << tab.m_name); CHK(con.startTransaction() == 0); - Lst lst; - bool deadlock = false; - bool nospace = false; - for (unsigned j = 0; j < par.m_rows; j++) { - unsigned j2 = ! par.m_randomkey ? j : urandom(par.m_rows); - unsigned i = thrrow(par, j2); + uint batch = 0; + for (uint j = 0; j < par.m_rows; j++) { + uint j2 = !par.m_randomkey ? j : urandom(par.m_rows); + uint i = thrrow(par, j2); set.lock(); - if (! set.exist(i) || set.pending(i, Row::AnyOp)) { - set.unlock(); - continue; - } - set.dbsave(i); - set.calc(par, i); - CHK(set.updrow(par, i) == 0); - set.unlock(); - LL4("pkupdate " << i << ": " << *set.m_row[i]); - lst.push(i); - if (lst.cnt() == par.m_batch) { - deadlock = par.m_deadlock; - nospace = true; - ExecType et = randompct(par.m_abortpct) ? Rollback : Commit; - CHK(con.execute(et, deadlock, nospace) == 0); - if (deadlock) { - LL1("pkupdate: stop on deadlock [at 1]"); - break; - } - if (nospace) { - LL1("pkupdate: cnt=" << j << " stop on nospace [at 1]"); - break; - } - con.closeTransaction(); - set.lock(); - set.notpending(lst, et); - set.dbdiscard(lst); + if (!set.compat(par, i, Row::OpUpd)) { + LL3("pkupdate SKIP " << i << " " << set.getrow(i)); set.unlock(); - lst.reset(); - CHK(con.startTransaction() == 0); - } - } - if (! deadlock && ! nospace && lst.cnt() != 0) { - deadlock = par.m_deadlock; - nospace = true; - ExecType et = randompct(par.m_abortpct) ? Rollback : Commit; - CHK(con.execute(et, deadlock, nospace) == 0); - if (deadlock) { - LL1("pkupdate: stop on deadlock [at 2]"); - } else if (nospace) { - LL1("pkupdate: end: stop on nospace [at 2]"); } else { + set.push(i); + set.copyval(i, tab.m_pkmask); + set.calc(par, i, ~tab.m_pkmask); + CHK(set.updrow(par, i) == 0); + set.unlock(); + LL4("pkupdate key=" << i << " " << set.getrow(i)); + batch++; + } + bool lastbatch = (batch != 0 && j + 1 == par.m_rows); + if (batch == par.m_batch || lastbatch) { + uint err = par.m_catcherr; + ExecType et = !randompct(par.m_abortpct) ? Commit : Rollback; + CHK(con.execute(et, err) == 0); set.lock(); - set.notpending(lst, et); - set.dbdiscard(lst); + set.post(par, !err ? et : Rollback); set.unlock(); + if (err) { + LL1("pkupdate key=" << i << ": stop on " << con.errname(err)); + break; + } + batch = 0; + if (!lastbatch) { + con.closeTransaction(); + CHK(con.startTransaction() == 0); + } } } con.closeTransaction(); @@ -3474,49 +3545,39 @@ pkdelete(Par par) Set& set = par.set(); LL3("pkdelete " << tab.m_name); CHK(con.startTransaction() == 0); - Lst lst; - bool deadlock = false; - bool nospace = false; - for (unsigned j = 0; j < par.m_rows; j++) { - unsigned j2 = ! par.m_randomkey ? j : urandom(par.m_rows); - unsigned i = thrrow(par, j2); + uint batch = 0; + for (uint j = 0; j < par.m_rows; j++) { + uint j2 = !par.m_randomkey ? j : urandom(par.m_rows); + uint i = thrrow(par, j2); set.lock(); - if (! set.exist(i) || set.pending(i, Row::AnyOp)) { + if (!set.compat(par, i, Row::OpDel)) { + LL3("pkdelete SKIP " << i << " " << set.getrow(i)); set.unlock(); - continue; - } - CHK(set.delrow(par, i) == 0); - set.unlock(); - LL4("pkdelete " << i << ": " << *set.m_row[i]); - lst.push(i); - if (lst.cnt() == par.m_batch) { - deadlock = par.m_deadlock; - nospace = true; - ExecType et = randompct(par.m_abortpct) ? Rollback : Commit; - CHK(con.execute(et, deadlock, nospace) == 0); - if (deadlock) { - LL1("pkdelete: stop on deadlock [at 1]"); - break; - } - con.closeTransaction(); - set.lock(); - set.notpending(lst, et); - set.unlock(); - lst.reset(); - CHK(con.startTransaction() == 0); - } - } - if (! deadlock && ! nospace && lst.cnt() != 0) { - deadlock = par.m_deadlock; - nospace = true; - ExecType et = randompct(par.m_abortpct) ? Rollback : Commit; - CHK(con.execute(et, deadlock, nospace) == 0); - if (deadlock) { - LL1("pkdelete: stop on deadlock [at 2]"); } else { + set.push(i); + set.copyval(i, tab.m_pkmask); + CHK(set.delrow(par, i) == 0); + set.unlock(); + LL4("pkdelete key=" << i << " " << set.getrow(i)); + batch++; + } + bool lastbatch = (batch != 0 && j + 1 == par.m_rows); + if (batch == par.m_batch || lastbatch) { + uint err = par.m_catcherr; + ExecType et = !randompct(par.m_abortpct) ? Commit : Rollback; + CHK(con.execute(et, err) == 0); set.lock(); - set.notpending(lst, et); + set.post(par, !err ? et : Rollback); set.unlock(); + if (err) { + LL1("pkdelete key=" << i << " stop on " << con.errname(err)); + break; + } + batch = 0; + if (!lastbatch) { + con.closeTransaction(); + CHK(con.startTransaction() == 0); + } } } con.closeTransaction(); @@ -3533,9 +3594,11 @@ pkread(Par par) // expected const Set& set1 = set; Set set2(tab, set.m_rows); - for (unsigned i = 0; i < set.m_rows; i++) { + for (uint i = 0; i < set.m_rows; i++) { set.lock(); - if (! set.exist(i)) { + // TODO lock mode + if (!set.compat(par, i, Row::OpREAD)) { + LL3("pkread SKIP " << i << " " << set.getrow(i)); set.unlock(); continue; } @@ -3543,10 +3606,10 @@ pkread(Par par) CHK(con.startTransaction() == 0); CHK(set2.selrow(par, *set1.m_row[i]) == 0); CHK(con.execute(Commit) == 0); - unsigned i2 = (unsigned)-1; + uint i2 = (uint)-1; CHK(set2.getkey(par, &i2) == 0 && i == i2); CHK(set2.putval(i, false) == 0); - LL4("row " << set2.count() << ": " << *set2.m_row[i]); + LL4("row " << set2.count() << " " << set2.getrow(i)); con.closeTransaction(); } if (par.m_verify) @@ -3555,7 +3618,7 @@ pkread(Par par) } static int -pkreadfast(Par par, unsigned count) +pkreadfast(Par par, uint count) { Con& con = par.con(); const Tab& tab = par.tab(); @@ -3563,9 +3626,9 @@ pkreadfast(Par par, unsigned count) LL3("pkfast " << tab.m_name); Row keyrow(tab); // not batched on purpose - for (unsigned j = 0; j < count; j++) { - unsigned i = urandom(set.m_rows); - assert(set.exist(i)); + for (uint j = 0; j < count; j++) { + uint i = urandom(set.m_rows); + assert(set.compat(par, i, Row::OpREAD)); CHK(con.startTransaction() == 0); // define key keyrow.calc(par, i); @@ -3585,53 +3648,46 @@ static int hashindexupdate(Par par, const ITab& itab) { Con& con = par.con(); + const Tab& tab = par.tab(); Set& set = par.set(); LL3("hashindexupdate " << itab.m_name); CHK(con.startTransaction() == 0); - Lst lst; - bool deadlock = false; - bool nospace = false; - for (unsigned j = 0; j < par.m_rows; j++) { - unsigned j2 = ! par.m_randomkey ? j : urandom(par.m_rows); - unsigned i = thrrow(par, j2); + uint batch = 0; + for (uint j = 0; j < par.m_rows; j++) { + uint j2 = !par.m_randomkey ? j : urandom(par.m_rows); + uint i = thrrow(par, j2); set.lock(); - if (! set.exist(i) || set.pending(i, Row::AnyOp)) { + if (!set.compat(par, i, Row::OpUpd)) { + LL3("hashindexupdate SKIP " << i << " " << set.getrow(i)); set.unlock(); - continue; - } - set.dbsave(i); - // index key columns are not re-calculated - set.calc(par, i, itab.m_colmask); - CHK(set.updrow(par, itab, i) == 0); - set.unlock(); - LL4("hashindexupdate " << i << ": " << *set.m_row[i]); - lst.push(i); - if (lst.cnt() == par.m_batch) { - deadlock = par.m_deadlock; - CHK(con.execute(Commit, deadlock, nospace) == 0); - if (deadlock) { - LL1("hashindexupdate: stop on deadlock [at 1]"); - break; - } - con.closeTransaction(); - set.lock(); - set.notpending(lst); - set.dbdiscard(lst); - set.unlock(); - lst.reset(); - CHK(con.startTransaction() == 0); - } - } - if (! deadlock && lst.cnt() != 0) { - deadlock = par.m_deadlock; - CHK(con.execute(Commit, deadlock, nospace) == 0); - if (deadlock) { - LL1("hashindexupdate: stop on deadlock [at 2]"); } else { + // table pk and index key are not updated + set.push(i); + uint keymask = tab.m_pkmask | itab.m_keymask; + set.copyval(i, keymask); + set.calc(par, i, ~keymask); + CHK(set.updrow(par, itab, i) == 0); + set.unlock(); + LL4("hashindexupdate " << i << " " << set.getrow(i)); + batch++; + } + bool lastbatch = (batch != 0 && j + 1 == par.m_rows); + if (batch == par.m_batch || lastbatch) { + uint err = par.m_catcherr; + ExecType et = !randompct(par.m_abortpct) ? Commit : Rollback; + CHK(con.execute(et, err) == 0); set.lock(); - set.notpending(lst); - set.dbdiscard(lst); + set.post(par, !err ? et : Rollback); set.unlock(); + if (err) { + LL1("hashindexupdate " << i << " stop on " << con.errname(err)); + break; + } + batch = 0; + if (!lastbatch) { + con.closeTransaction(); + CHK(con.startTransaction() == 0); + } } } con.closeTransaction(); @@ -3645,45 +3701,39 @@ hashindexdelete(Par par, const ITab& itab) Set& set = par.set(); LL3("hashindexdelete " << itab.m_name); CHK(con.startTransaction() == 0); - Lst lst; - bool deadlock = false; - bool nospace = false; - for (unsigned j = 0; j < par.m_rows; j++) { - unsigned j2 = ! par.m_randomkey ? j : urandom(par.m_rows); - unsigned i = thrrow(par, j2); + uint batch = 0; + for (uint j = 0; j < par.m_rows; j++) { + uint j2 = !par.m_randomkey ? j : urandom(par.m_rows); + uint i = thrrow(par, j2); set.lock(); - if (! set.exist(i) || set.pending(i, Row::AnyOp)) { - set.unlock(); - continue; - } - CHK(set.delrow(par, itab, i) == 0); - set.unlock(); - LL4("hashindexdelete " << i << ": " << *set.m_row[i]); - lst.push(i); - if (lst.cnt() == par.m_batch) { - deadlock = par.m_deadlock; - CHK(con.execute(Commit, deadlock, nospace) == 0); - if (deadlock) { - LL1("hashindexdelete: stop on deadlock [at 1]"); - break; - } - con.closeTransaction(); - set.lock(); - set.notpending(lst); + if (!set.compat(par, i, Row::OpDel)) { + LL3("hashindexdelete SKIP " << i << " " << set.getrow(i)); set.unlock(); - lst.reset(); - CHK(con.startTransaction() == 0); - } - } - if (! deadlock && lst.cnt() != 0) { - deadlock = par.m_deadlock; - CHK(con.execute(Commit, deadlock, nospace) == 0); - if (deadlock) { - LL1("hashindexdelete: stop on deadlock [at 2]"); } else { + set.push(i); + set.copyval(i, itab.m_keymask); + CHK(set.delrow(par, itab, i) == 0); + set.unlock(); + LL4("hashindexdelete " << i << " " << set.getrow(i)); + batch++; + } + bool lastbatch = (batch != 0 && j + 1 == par.m_rows); + if (batch == par.m_batch || lastbatch) { + uint err = par.m_catcherr; + ExecType et = !randompct(par.m_abortpct) ? Commit : Rollback; + CHK(con.execute(et, err) == 0); set.lock(); - set.notpending(lst); + set.post(par, !err ? et : Rollback); set.unlock(); + if (err) { + LL1("hashindexdelete " << i << " stop on " << con.errname(err)); + break; + } + batch = 0; + if (!lastbatch) { + con.closeTransaction(); + CHK(con.startTransaction() == 0); + } } } con.closeTransaction(); @@ -3700,9 +3750,11 @@ hashindexread(Par par, const ITab& itab) // expected const Set& set1 = set; Set set2(tab, set.m_rows); - for (unsigned i = 0; i < set.m_rows; i++) { + for (uint i = 0; i < set.m_rows; i++) { set.lock(); - if (! set.exist(i)) { + // TODO lock mode + if (!set.compat(par, i, Row::OpREAD)) { + LL3("hashindexread SKIP " << i << " " << set.getrow(i)); set.unlock(); continue; } @@ -3710,10 +3762,10 @@ hashindexread(Par par, const ITab& itab) CHK(con.startTransaction() == 0); CHK(set2.selrow(par, itab, *set1.m_row[i]) == 0); CHK(con.execute(Commit) == 0); - unsigned i2 = (unsigned)-1; + uint i2 = (uint)-1; CHK(set2.getkey(par, &i2) == 0 && i == i2); CHK(set2.putval(i, false) == 0); - LL4("row " << set2.count() << ": " << *set2.m_row[i]); + LL4("row " << set2.count() << " " << *set2.m_row[i]); con.closeTransaction(); } if (par.m_verify) @@ -3731,40 +3783,39 @@ scanreadtable(Par par) const Set& set = par.set(); // expected const Set& set1 = set; - LL3("scanread " << tab.m_name << " lockmode=" << par.m_lockmode << " tupscan=" << par.m_tupscan << " expect=" << set1.count() << " verify=" << par.m_verify); + LL3("scanreadtable " << tab.m_name << " lockmode=" << par.m_lockmode << " tupscan=" << par.m_tupscan << " expect=" << set1.count() << " verify=" << par.m_verify); Set set2(tab, set.m_rows); CHK(con.startTransaction() == 0); CHK(con.getNdbScanOperation(tab) == 0); CHK(con.readTuples(par) == 0); set2.getval(par); CHK(con.executeScan() == 0); - unsigned n = 0; - bool deadlock = false; + uint n = 0; while (1) { int ret; - deadlock = par.m_deadlock; - CHK((ret = con.nextScanResult(true, deadlock)) == 0 || ret == 1); + uint err = par.m_catcherr; + CHK((ret = con.nextScanResult(true, err)) == 0 || ret == 1); if (ret == 1) break; - if (deadlock) { - LL1("scanreadtable: stop on deadlock"); + if (err) { + LL1("scanreadtable stop on " << con.errname(err)); break; } - unsigned i = (unsigned)-1; + uint i = (uint)-1; CHK(set2.getkey(par, &i) == 0); CHK(set2.putval(i, false, n) == 0); - LL4("row " << n << ": " << *set2.m_row[i]); + LL4("row " << n << " " << *set2.m_row[i]); n++; } con.closeTransaction(); if (par.m_verify) CHK(set1.verify(par, set2, false) == 0); - LL3("scanread " << tab.m_name << " done rows=" << n); + LL3("scanreadtable " << tab.m_name << " done rows=" << n); return 0; } static int -scanreadtablefast(Par par, unsigned countcheck) +scanreadtablefast(Par par, uint countcheck) { Con& con = par.con(); const Tab& tab = par.tab(); @@ -3777,7 +3828,7 @@ scanreadtablefast(Par par, unsigned countcheck) NdbRecAttr* rec; CHK(con.getValue((Uint32)0, rec) == 0); CHK(con.executeScan() == 0); - unsigned count = 0; + uint count = 0; while (1) { int ret; CHK((ret = con.nextScanResult(true)) == 0 || ret == 1); @@ -3797,7 +3848,7 @@ calcscanbounds(Par par, const ITab& itab, BSet& bset, const Set& set, Set& set1) while (true) { bset.calc(par); bset.filter(par, set, set1); - unsigned n = set1.count(); + uint n = set1.count(); // prefer proper subset if (0 < n && n < set.m_rows) break; @@ -3819,7 +3870,7 @@ scanreadindex(Par par, const ITab& itab, BSet& bset, bool calc) } else { bset.filter(par, set, set1); } - LL3("scanread " << itab.m_name << " " << bset << " lockmode=" << par.m_lockmode << " expect=" << set1.count() << " ordered=" << par.m_ordered << " descending=" << par.m_descending << " verify=" << par.m_verify); + LL3("scanreadindex " << itab.m_name << " " << bset << " lockmode=" << par.m_lockmode << " expect=" << set1.count() << " ordered=" << par.m_ordered << " descending=" << par.m_descending << " verify=" << par.m_verify); Set set2(tab, set.m_rows); CHK(con.startTransaction() == 0); CHK(con.getNdbIndexScanOperation(itab, tab) == 0); @@ -3827,22 +3878,21 @@ scanreadindex(Par par, const ITab& itab, BSet& bset, bool calc) CHK(bset.setbnd(par) == 0); set2.getval(par); CHK(con.executeScan() == 0); - unsigned n = 0; - bool deadlock = false; + uint n = 0; while (1) { int ret; - deadlock = par.m_deadlock; - CHK((ret = con.nextScanResult(true, deadlock)) == 0 || ret == 1); + uint err = par.m_catcherr; + CHK((ret = con.nextScanResult(true, err)) == 0 || ret == 1); if (ret == 1) break; - if (deadlock) { - LL1("scanreadindex: stop on deadlock"); + if (err) { + LL1("scanreadindex stop on " << con.errname(err)); break; } - unsigned i = (unsigned)-1; + uint i = (uint)-1; CHK(set2.getkey(par, &i) == 0); CHK(set2.putval(i, par.m_dups, n) == 0); - LL4("key " << i << " row " << n << ": " << *set2.m_row[i]); + LL4("key " << i << " row " << n << " " << *set2.m_row[i]); n++; } con.closeTransaction(); @@ -3851,12 +3901,12 @@ scanreadindex(Par par, const ITab& itab, BSet& bset, bool calc) if (par.m_ordered) CHK(set2.verifyorder(par, itab, par.m_descending) == 0); } - LL3("scanread " << itab.m_name << " done rows=" << n); + LL3("scanreadindex " << itab.m_name << " done rows=" << n); return 0; } static int -scanreadindexfast(Par par, const ITab& itab, const BSet& bset, unsigned countcheck) +scanreadindexfast(Par par, const ITab& itab, const BSet& bset, uint countcheck) { Con& con = par.con(); const Tab& tab = par.tab(); @@ -3871,7 +3921,7 @@ scanreadindexfast(Par par, const ITab& itab, const BSet& bset, unsigned countche NdbRecAttr* rec; CHK(con.getValue((Uint32)0, rec) == 0); CHK(con.executeScan() == 0); - unsigned count = 0; + uint count = 0; while (1) { int ret; CHK((ret = con.nextScanResult(true)) == 0 || ret == 1); @@ -3904,22 +3954,21 @@ scanreadfilter(Par par, const ITab& itab, BSet& bset, bool calc) CHK(bset.setflt(par) == 0); set2.getval(par); CHK(con.executeScan() == 0); - unsigned n = 0; - bool deadlock = false; + uint n = 0; while (1) { int ret; - deadlock = par.m_deadlock; - CHK((ret = con.nextScanResult(true, deadlock)) == 0 || ret == 1); + uint err = par.m_catcherr; + CHK((ret = con.nextScanResult(true, err)) == 0 || ret == 1); if (ret == 1) break; - if (deadlock) { - LL1("scanfilter: stop on deadlock"); + if (err) { + LL1("scanfilter stop on " << con.errname(err)); break; } - unsigned i = (unsigned)-1; + uint i = (uint)-1; CHK(set2.getkey(par, &i) == 0); CHK(set2.putval(i, par.m_dups, n) == 0); - LL4("key " << i << " row " << n << ": " << *set2.m_row[i]); + LL4("key " << i << " row " << n << " " << *set2.m_row[i]); n++; } con.closeTransaction(); @@ -3934,9 +3983,9 @@ static int scanreadindex(Par par, const ITab& itab) { const Tab& tab = par.tab(); - for (unsigned i = 0; i < par.m_subloop; i++) { + for (uint i = 0; i < par.m_ssloop; i++) { if (itab.m_type == ITab::OrderedIndex) { - BSet bset(tab, itab, par.m_rows); + BSet bset(tab, itab); CHK(scanreadfilter(par, itab, bset, true) == 0); CHK(scanreadindex(par, itab, bset, true) == 0); } @@ -3948,7 +3997,7 @@ static int scanreadindex(Par par) { const Tab& tab = par.tab(); - for (unsigned i = 0; i < tab.m_itabs; i++) { + for (uint i = 0; i < tab.m_itabs; i++) { if (tab.m_itab[i] == 0) continue; const ITab& itab = *tab.m_itab[i]; @@ -3985,7 +4034,7 @@ timescanpkindex(Par par) { const Tab& tab = par.tab(); const ITab& itab = *tab.m_itab[0]; // 1st index is on PK - BSet bset(tab, itab, par.m_rows); + BSet bset(tab, itab); par.tmr().on(); CHK(scanreadindexfast(par, itab, bset, par.m_totrows) == 0); par.tmr().off(par.set().m_rows); @@ -3996,7 +4045,7 @@ static int timepkreadtable(Par par) { par.tmr().on(); - unsigned count = par.m_samples; + uint count = par.m_samples; if (count == 0) count = par.m_totrows; CHK(pkreadfast(par, count) == 0); @@ -4009,13 +4058,13 @@ timepkreadindex(Par par) { const Tab& tab = par.tab(); const ITab& itab = *tab.m_itab[0]; // 1st index is on PK - BSet bset(tab, itab, par.m_rows); - unsigned count = par.m_samples; + BSet bset(tab, itab); + uint count = par.m_samples; if (count == 0) count = par.m_totrows; par.tmr().on(); - for (unsigned j = 0; j < count; j++) { - unsigned i = urandom(par.m_totrows); + for (uint j = 0; j < count; j++) { + uint i = urandom(par.m_totrows); bset.calcpk(par, i); CHK(scanreadindexfast(par, itab, bset, 1) == 0); } @@ -4031,7 +4080,7 @@ scanupdatetable(Par par) Con& con = par.con(); const Tab& tab = par.tab(); Set& set = par.set(); - LL3("scan update " << tab.m_name); + LL3("scanupdatetable " << tab.m_name); Set set2(tab, set.m_rows); par.m_lockmode = NdbOperation::LM_Exclusive; CHK(con.startTransaction() == 0); @@ -4039,87 +4088,70 @@ scanupdatetable(Par par) CHK(con.readTuples(par) == 0); set2.getval(par); CHK(con.executeScan() == 0); - unsigned count = 0; + uint count = 0; // updating trans Con con2; con2.connect(con); CHK(con2.startTransaction() == 0); - Lst lst; - bool deadlock = false; - bool nospace = false; + uint batch = 0; while (1) { int ret; - deadlock = par.m_deadlock; - CHK((ret = con.nextScanResult(true, deadlock)) == 0 || ret == 1); - if (ret == 1) + uint32 err = par.m_catcherr; + CHK((ret = con.nextScanResult(true, err)) != -1); + if (ret != 0) break; - if (deadlock) { - LL1("scanupdatetable: stop on deadlock [at 1]"); + if (err) { + LL1("scanupdatetable [scan] stop on " << con.errname(err)); break; } if (par.m_scanstop != 0 && urandom(par.m_scanstop) == 0) { con.closeScan(); break; } - do { - unsigned i = (unsigned)-1; + while (1) { + uint i = (uint)-1; CHK(set2.getkey(par, &i) == 0); - const Row& row = *set.m_row[i]; set.lock(); - if (! set.exist(i) || set.pending(i, Row::AnyOp)) { - LL4("scan update " << tab.m_name << ": skip: " << row); + if (!set.compat(par, i, Row::OpUpd)) { + LL3("scanupdatetable SKIP " << i << " " << set.getrow(i)); } else { CHKTRY(set2.putval(i, false) == 0, set.unlock()); CHKTRY(con.updateScanTuple(con2) == 0, set.unlock()); Par par2 = par; par2.m_con = &con2; - set.dbsave(i); - set.calc(par, i); + set.push(i); + set.calc(par, i, ~tab.m_pkmask); CHKTRY(set.setrow(par2, i) == 0, set.unlock()); - LL4("scan update " << tab.m_name << ": " << row); - lst.push(i); + LL4("scanupdatetable " << i << " " << set.getrow(i)); + batch++; } set.unlock(); - if (lst.cnt() == par.m_batch) { - deadlock = par.m_deadlock; - CHK(con2.execute(Commit, deadlock, nospace) == 0); - if (deadlock) { - LL1("scanupdatetable: stop on deadlock [at 2]"); - goto out; - } - con2.closeTransaction(); + CHK((ret = con.nextScanResult(false)) != -1); + bool lastbatch = (batch != 0 && ret != 0); + if (batch == par.m_batch || lastbatch) { + uint err = par.m_catcherr; + ExecType et = Commit; + CHK(con2.execute(et, err) == 0); set.lock(); - set.notpending(lst); - set.dbdiscard(lst); + set.post(par, !err ? et : Rollback); set.unlock(); - count += lst.cnt(); - lst.reset(); - CHK(con2.startTransaction() == 0); - } - CHK((ret = con.nextScanResult(false)) == 0 || ret == 1 || ret == 2); - if (ret == 2 && lst.cnt() != 0) { - deadlock = par.m_deadlock; - CHK(con2.execute(Commit, deadlock, nospace) == 0); - if (deadlock) { - LL1("scanupdatetable: stop on deadlock [at 3]"); + if (err) { + LL1("scanupdatetable [update] stop on " << con2.errname(err)); goto out; } + LL4("scanupdatetable committed batch"); + count += batch; + batch = 0; con2.closeTransaction(); - set.lock(); - set.notpending(lst); - set.dbdiscard(lst); - set.unlock(); - count += lst.cnt(); - lst.reset(); CHK(con2.startTransaction() == 0); } - } while (ret == 0); - if (ret == 1) - break; + if (ret != 0) + break; + } } out: con2.closeTransaction(); - LL3("scan update " << tab.m_name << " rows updated=" << count); + LL3("scanupdatetable " << tab.m_name << " rows updated=" << count); con.closeTransaction(); return 0; } @@ -4137,7 +4169,7 @@ scanupdateindex(Par par, const ITab& itab, BSet& bset, bool calc) } else { bset.filter(par, set, set1); } - LL3("scan update " << itab.m_name << " " << bset << " expect=" << set1.count() << " ordered=" << par.m_ordered << " descending=" << par.m_descending << " verify=" << par.m_verify); + LL3("scanupdateindex " << itab.m_name << " " << bset << " expect=" << set1.count() << " ordered=" << par.m_ordered << " descending=" << par.m_descending << " verify=" << par.m_verify); Set set2(tab, set.m_rows); par.m_lockmode = NdbOperation::LM_Exclusive; CHK(con.startTransaction() == 0); @@ -4146,83 +4178,67 @@ scanupdateindex(Par par, const ITab& itab, BSet& bset, bool calc) CHK(bset.setbnd(par) == 0); set2.getval(par); CHK(con.executeScan() == 0); - unsigned count = 0; + uint count = 0; // updating trans Con con2; con2.connect(con); CHK(con2.startTransaction() == 0); - Lst lst; - bool deadlock = false; - bool nospace = false; + uint batch = 0; while (1) { int ret; - deadlock = par.m_deadlock; - CHK((ret = con.nextScanResult(true, deadlock)) == 0 || ret == 1); - if (ret == 1) + uint err = par.m_catcherr; + CHK((ret = con.nextScanResult(true, err)) != -1); + if (ret != 0) break; - if (deadlock) { - LL1("scanupdateindex: stop on deadlock [at 1]"); + if (err) { + LL1("scanupdateindex [scan] stop on " << con.errname(err)); break; } if (par.m_scanstop != 0 && urandom(par.m_scanstop) == 0) { con.closeScan(); break; } - do { - unsigned i = (unsigned)-1; + while (1) { + uint i = (uint)-1; CHK(set2.getkey(par, &i) == 0); - const Row& row = *set.m_row[i]; set.lock(); - if (! set.exist(i) || set.pending(i, Row::AnyOp)) { - LL4("scan update " << itab.m_name << ": skip: " << row); + if (!set.compat(par, i, Row::OpUpd)) { + LL4("scanupdateindex SKIP " << set.getrow(i)); } else { CHKTRY(set2.putval(i, par.m_dups) == 0, set.unlock()); CHKTRY(con.updateScanTuple(con2) == 0, set.unlock()); Par par2 = par; par2.m_con = &con2; - set.dbsave(i); - set.calc(par, i, ! par.m_noindexkeyupdate ? 0 : itab.m_colmask); + set.push(i); + uint colmask = !par.m_noindexkeyupdate ? ~0 : ~itab.m_keymask; + set.calc(par, i, colmask); CHKTRY(set.setrow(par2, i) == 0, set.unlock()); - LL4("scan update " << itab.m_name << ": " << row); - lst.push(i); + LL4("scanupdateindex " << i << " " << set.getrow(i)); + batch++; } set.unlock(); - if (lst.cnt() == par.m_batch) { - deadlock = par.m_deadlock; - CHK(con2.execute(Commit, deadlock, nospace) == 0); - if (deadlock) { - LL1("scanupdateindex: stop on deadlock [at 2]"); - goto out; - } - con2.closeTransaction(); - LL4("scanupdateindex: committed batch [at 1]"); + CHK((ret = con.nextScanResult(false)) != -1); + bool lastbatch = (batch != 0 && ret != 0); + if (batch == par.m_batch || lastbatch) { + uint err = par.m_catcherr; + ExecType et = Commit; + CHK(con2.execute(et, err) == 0); set.lock(); - set.notpending(lst); - set.dbdiscard(lst); + set.post(par, !err ? et : Rollback); set.unlock(); - count += lst.cnt(); - lst.reset(); - CHK(con2.startTransaction() == 0); - } - CHK((ret = con.nextScanResult(false)) == 0 || ret == 1 || ret == 2); - if (ret == 2 && lst.cnt() != 0) { - deadlock = par.m_deadlock; - CHK(con2.execute(Commit, deadlock, nospace) == 0); - if (deadlock) { - LL1("scanupdateindex: stop on deadlock [at 3]"); + if (err) { + LL1("scanupdateindex [update] stop on " << con2.errname(err)); goto out; } + LL4("scanupdateindex committed batch"); + count += batch; + batch = 0; con2.closeTransaction(); - LL4("scanupdateindex: committed batch [at 2]"); - set.lock(); - set.notpending(lst); - set.dbdiscard(lst); - set.unlock(); - count += lst.cnt(); - lst.reset(); CHK(con2.startTransaction() == 0); } - } while (ret == 0); + if (ret != 0) + break; + } } out: con2.closeTransaction(); @@ -4231,7 +4247,7 @@ out: if (par.m_ordered) CHK(set2.verifyorder(par, itab, par.m_descending) == 0); } - LL3("scan update " << itab.m_name << " rows updated=" << count); + LL3("scanupdateindex " << itab.m_name << " rows updated=" << count); con.closeTransaction(); return 0; } @@ -4240,9 +4256,9 @@ static int scanupdateindex(Par par, const ITab& itab) { const Tab& tab = par.tab(); - for (unsigned i = 0; i < par.m_subloop; i++) { + for (uint i = 0; i < par.m_ssloop; i++) { if (itab.m_type == ITab::OrderedIndex) { - BSet bset(tab, itab, par.m_rows); + BSet bset(tab, itab); CHK(scanupdateindex(par, itab, bset, true) == 0); } else { CHK(hashindexupdate(par, itab) == 0); @@ -4255,7 +4271,7 @@ static int scanupdateindex(Par par) { const Tab& tab = par.tab(); - for (unsigned i = 0; i < tab.m_itabs; i++) { + for (uint i = 0; i < tab.m_itabs; i++) { if (tab.m_itab[i] == 0) continue; const ITab& itab = *tab.m_itab[i]; @@ -4294,14 +4310,14 @@ readverifyfull(Par par) CHK(scanreadtable(par) == 0); } // each thread scans different indexes - for (unsigned i = 0; i < tab.m_itabs; i++) { - if (i % par.m_threads != par.m_no) + for (uint i = 0; i < tab.m_itabs; i++) { + if (i % par.m_usedthreads != par.m_no) continue; if (tab.m_itab[i] == 0) continue; const ITab& itab = *tab.m_itab[i]; if (itab.m_type == ITab::OrderedIndex) { - BSet bset(tab, itab, par.m_rows); + BSet bset(tab, itab); CHK(scanreadindex(par, itab, bset, false) == 0); } else { CHK(hashindexread(par, itab) == 0); @@ -4317,7 +4333,7 @@ readverifyindex(Par par) return 0; par.m_verify = true; par.m_lockmode = NdbOperation::LM_CommittedRead; - unsigned sel = urandom(10); + uint sel = urandom(10); if (sel < 9) { par.m_ordered = true; par.m_descending = (sel < 5); @@ -4331,8 +4347,8 @@ pkops(Par par) { const Tab& tab = par.tab(); par.m_randomkey = true; - for (unsigned i = 0; i < par.m_subloop; i++) { - unsigned j = 0; + for (uint i = 0; i < par.m_ssloop; i++) { + uint j = 0; while (j < tab.m_itabs) { if (tab.m_itab[j] != 0) { const ITab& itab = *tab.m_itab[j]; @@ -4341,7 +4357,7 @@ pkops(Par par) } j++; } - unsigned sel = urandom(10); + uint sel = urandom(10); if (par.m_slno % 2 == 0) { // favor insert if (sel < 8) { @@ -4389,8 +4405,8 @@ static int pkupdatescanread(Par par) { par.m_dups = true; - par.m_deadlock = true; - unsigned sel = urandom(10); + par.m_catcherr |= Con::ErrDeadlock; + uint sel = urandom(10); if (sel < 5) { CHK(pkupdate(par) == 0); } else if (sel < 6) { @@ -4411,9 +4427,9 @@ static int mixedoperations(Par par) { par.m_dups = true; - par.m_deadlock = true; + par.m_catcherr |= Con::ErrDeadlock; par.m_scanstop = par.m_totrows; // randomly close scans - unsigned sel = urandom(10); + uint sel = urandom(10); if (sel < 2) { CHK(pkdelete(par) == 0); } else if (sel < 4) { @@ -4434,8 +4450,8 @@ static int parallelorderedupdate(Par par) { const Tab& tab = par.tab(); - unsigned k = 0; - for (unsigned i = 0; i < tab.m_itabs; i++) { + uint k = 0; + for (uint i = 0; i < tab.m_itabs; i++) { if (tab.m_itab[i] == 0) continue; const ITab& itab = *tab.m_itab[i]; @@ -4447,10 +4463,11 @@ parallelorderedupdate(Par par) par.m_noindexkeyupdate = true; par.m_ordered = true; par.m_descending = (par.m_slno != 0); + par.m_dups = false; par.m_verify = true; - BSet bset(tab, itab, par.m_rows); // empty bounds + BSet bset(tab, itab); // empty bounds // prefer empty bounds - unsigned sel = urandom(10); + uint sel = urandom(10); CHK(scanupdateindex(par, itab, bset, sel < 2) == 0); } } @@ -4469,6 +4486,418 @@ pkupdateindexbuild(Par par) return 0; } +// savepoint tests (single thread for now) + +struct Spt { + enum Res { Committed, Latest, Deadlock }; + bool m_same; // same transaction + NdbOperation::LockMode m_lm; + Res m_res; +}; + +static Spt sptlist[] = { + { 1, NdbOperation::LM_Read, Spt::Latest }, + { 1, NdbOperation::LM_Exclusive, Spt::Latest }, + { 1, NdbOperation::LM_CommittedRead, Spt::Latest }, + { 0, NdbOperation::LM_Read, Spt::Deadlock }, + { 0, NdbOperation::LM_Exclusive, Spt::Deadlock }, + { 0, NdbOperation::LM_CommittedRead, Spt::Committed } +}; +static uint sptcount = sizeof(sptlist)/sizeof(sptlist[0]); + +static int +savepointreadpk(Par par, Spt spt) +{ + LL3("savepointreadpk"); + Con& con = par.con(); + const Tab& tab = par.tab(); + Set& set = par.set(); + const Set& set1 = set; + Set set2(tab, set.m_rows); + uint n = 0; + for (uint i = 0; i < set.m_rows; i++) { + set.lock(); + if (!set.compat(par, i, Row::OpREAD)) { + LL4("savepointreadpk SKIP " << i << " " << set.getrow(i)); + set.unlock(); + continue; + } + set.unlock(); + CHK(set2.selrow(par, *set1.m_row[i]) == 0); + uint err = par.m_catcherr | Con::ErrDeadlock; + ExecType et = NoCommit; + CHK(con.execute(et, err) == 0); + if (err) { + if (err & Con::ErrDeadlock) { + CHK(spt.m_res == Spt::Deadlock); + // all rows have same behaviour + CHK(n == 0); + } + LL1("savepointreadpk stop on " << con.errname(err)); + break; + } + uint i2 = (uint)-1; + CHK(set2.getkey(par, &i2) == 0 && i == i2); + CHK(set2.putval(i, false) == 0); + LL4("row " << set2.count() << " " << set2.getrow(i)); + n++; + } + bool dirty = (!spt.m_same && spt.m_lm == NdbOperation::LM_CommittedRead); + if (spt.m_res != Spt::Deadlock) + CHK(set1.verify(par, set2, false, dirty) == 0); + return 0; +} + +static int +savepointreadhashindex(Par par, Spt spt) +{ + if (spt.m_lm == NdbOperation::LM_CommittedRead && !spt.m_same) { + LL1("skip hash index dirty read"); + return 0; + } + LL3("savepointreadhashindex"); + Con& con = par.con(); + const Tab& tab = par.tab(); + const ITab& itab = par.itab(); + Set& set = par.set(); + const Set& set1 = set; + Set set2(tab, set.m_rows); + uint n = 0; + for (uint i = 0; i < set.m_rows; i++) { + set.lock(); + if (!set.compat(par, i, Row::OpREAD)) { + LL3("savepointreadhashindex SKIP " << i << " " << set.getrow(i)); + set.unlock(); + continue; + } + set.unlock(); + CHK(set2.selrow(par, itab, *set1.m_row[i]) == 0); + uint err = par.m_catcherr | Con::ErrDeadlock; + ExecType et = NoCommit; + CHK(con.execute(et, err) == 0); + if (err) { + if (err & Con::ErrDeadlock) { + CHK(spt.m_res == Spt::Deadlock); + // all rows have same behaviour + CHK(n == 0); + } + LL1("savepointreadhashindex stop on " << con.errname(err)); + break; + } + uint i2 = (uint)-1; + CHK(set2.getkey(par, &i2) == 0 && i == i2); + CHK(set2.putval(i, false) == 0); + LL4("row " << set2.count() << " " << *set2.m_row[i]); + n++; + } + bool dirty = (!spt.m_same && spt.m_lm == NdbOperation::LM_CommittedRead); + if (spt.m_res != Spt::Deadlock) + CHK(set1.verify(par, set2, false, dirty) == 0); + return 0; +} + +static int +savepointscantable(Par par, Spt spt) +{ + LL3("savepointscantable"); + Con& con = par.con(); + const Tab& tab = par.tab(); + const Set& set = par.set(); + const Set& set1 = set; // not modifying current set + Set set2(tab, set.m_rows); // scan result + CHK(con.getNdbScanOperation(tab) == 0); + CHK(con.readTuples(par) == 0); + set2.getval(par); // getValue all columns + CHK(con.executeScan() == 0); + bool deadlock = false; + uint n = 0; + while (1) { + int ret; + uint err = par.m_catcherr | Con::ErrDeadlock; + CHK((ret = con.nextScanResult(true, err)) == 0 || ret == 1); + if (ret == 1) + break; + if (err) { + if (err & Con::ErrDeadlock) { + CHK(spt.m_res == Spt::Deadlock); + // all rows have same behaviour + CHK(n == 0); + deadlock = true; + } + LL1("savepointscantable stop on " << con.errname(err)); + break; + } + CHK(spt.m_res != Spt::Deadlock); + uint i = (uint)-1; + CHK(set2.getkey(par, &i) == 0); + CHK(set2.putval(i, false, n) == 0); + LL4("row " << n << " key " << i << " " << set2.getrow(i)); + n++; + } + if (set1.m_rows > 0) { + if (!deadlock) + CHK(spt.m_res != Spt::Deadlock); + else + CHK(spt.m_res == Spt::Deadlock); + } + LL2("savepointscantable " << n << " rows"); + bool dirty = (!spt.m_same && spt.m_lm == NdbOperation::LM_CommittedRead); + if (spt.m_res != Spt::Deadlock) + CHK(set1.verify(par, set2, false, dirty) == 0); + return 0; +} + +static int +savepointscanindex(Par par, Spt spt) +{ + LL3("savepointscanindex"); + Con& con = par.con(); + const Tab& tab = par.tab(); + const ITab& itab = par.itab(); + const Set& set = par.set(); + const Set& set1 = set; + Set set2(tab, set.m_rows); + CHK(con.getNdbIndexScanOperation(itab, tab) == 0); + CHK(con.readIndexTuples(par) == 0); + set2.getval(par); + CHK(con.executeScan() == 0); + bool deadlock = false; + uint n = 0; + while (1) { + int ret; + uint err = par.m_catcherr | Con::ErrDeadlock; + CHK((ret = con.nextScanResult(true, err)) == 0 || ret == 1); + if (ret == 1) + break; + if (err) { + if (err & Con::ErrDeadlock) { + CHK(spt.m_res == Spt::Deadlock); + // all rows have same behaviour + CHK(n == 0); + deadlock = true; + } + LL1("savepointscanindex stop on " << con.errname(err)); + break; + } + CHK(spt.m_res != Spt::Deadlock); + uint i = (uint)-1; + CHK(set2.getkey(par, &i) == 0); + CHK(set2.putval(i, par.m_dups, n) == 0); + LL4("row " << n << " key " << i << " " << set2.getrow(i)); + n++; + } + if (set1.m_rows > 0) { + if (!deadlock) + CHK(spt.m_res != Spt::Deadlock); + else + CHK(spt.m_res == Spt::Deadlock); + } + LL2("savepointscanindex " << n << " rows"); + bool dirty = (!spt.m_same && spt.m_lm == NdbOperation::LM_CommittedRead); + if (spt.m_res != Spt::Deadlock) + CHK(set1.verify(par, set2, false, dirty) == 0); + return 0; +} + +typedef int (*SptFun)(Par, Spt); + +static int +savepointtest(Par par, Spt spt, SptFun fun) +{ + Con& con = par.con(); + Par par2 = par; + Con con2; + if (!spt.m_same) { + con2.connect(con); // copy ndb reference + par2.m_con = &con2; + CHK(con2.startTransaction() == 0); + } + par2.m_lockmode = spt.m_lm; + CHK((*fun)(par2, spt) == 0); + if (!spt.m_same) { + con2.closeTransaction(); + } + return 0; +} + +static int +savepointtest(Par par, const char* op) +{ + Con& con = par.con(); + const Tab& tab = par.tab(); + Set& set = par.set(); + LL2("savepointtest op=\"" << op << "\""); + CHK(con.startTransaction() == 0); + const char* p = op; + char c; + while ((c = *p++) != 0) { + uint j; + for (j = 0; j < par.m_rows; j++) { + uint i = thrrow(par, j); + if (c == 'c') { + ExecType et = Commit; + CHK(con.execute(et) == 0); + set.lock(); + set.post(par, et); + set.unlock(); + CHK(con.startTransaction() == 0); + } else { + set.lock(); + set.push(i); + if (c == 'i') { + set.calc(par, i); + CHK(set.insrow(par, i) == 0); + } else if (c == 'u') { + set.copyval(i, tab.m_pkmask); + set.calc(par, i, ~tab.m_pkmask); + CHK(set.updrow(par, i) == 0); + } else if (c == 'd') { + set.copyval(i, tab.m_pkmask); + CHK(set.delrow(par, i) == 0); + } else { + assert(false); + } + set.unlock(); + } + } + } + { + ExecType et = NoCommit; + CHK(con.execute(et) == 0); + set.lock(); + set.post(par, et); + set.unlock(); + } + for (uint k = 0; k < sptcount; k++) { + Spt spt = sptlist[k]; + LL2("spt lm=" << spt.m_lm << " same=" << spt.m_same); + CHK(savepointtest(par, spt, &savepointreadpk) == 0); + CHK(savepointtest(par, spt, &savepointscantable) == 0); + for (uint i = 0; i < tab.m_itabs; i++) { + if (tab.m_itab[i] == 0) + continue; + const ITab& itab = *tab.m_itab[i]; + par.m_itab = &itab; + if (itab.m_type == ITab::OrderedIndex) + CHK(savepointtest(par, spt, &savepointscanindex) == 0); + else + CHK(savepointtest(par, spt, &savepointreadhashindex) == 0); + par.m_itab = 0; + } + } + { + ExecType et = Rollback; + CHK(con.execute(et) == 0); + set.lock(); + set.post(par, et); + set.unlock(); + } + con.closeTransaction(); + return 0; +} + +static int +savepointtest(Par par) +{ + assert(par.m_usedthreads == 1); + const char* oplist[] = { + // each based on previous and "c" not last + "i", + "icu", + "uuuuu", + "d", + "dciuuuuud", + 0 + }; + int i; + for (i = 0; oplist[i] != 0; i++) { + CHK(savepointtest(par, oplist[i]) == 0); + } + return 0; +} + +static int +halloweentest(Par par, const ITab& itab) +{ + LL2("halloweentest " << itab.m_name); + Con& con = par.con(); + const Tab& tab = par.tab(); + Set& set = par.set(); + CHK(con.startTransaction() == 0); + // insert 1 row + uint i = 0; + set.push(i); + set.calc(par, i); + CHK(set.insrow(par, i) == 0); + CHK(con.execute(NoCommit) == 0); + // scan via index until Set m_rows reached + uint scancount = 0; + bool stop = false; + while (!stop) { + par.m_lockmode = // makes no difference + scancount % 2 == 0 ? NdbOperation::LM_CommittedRead : + NdbOperation::LM_Read; + Set set1(tab, set.m_rows); // expected scan result + Set set2(tab, set.m_rows); // actual scan result + BSet bset(tab, itab); + calcscanbounds(par, itab, bset, set, set1); + CHK(con.getNdbIndexScanOperation(itab, tab) == 0); + CHK(con.readIndexTuples(par) == 0); + CHK(bset.setbnd(par) == 0); + set2.getval(par); + CHK(con.executeScan() == 0); + const uint savepoint = i; + LL3("scancount=" << scancount << " savepoint=" << savepoint); + uint n = 0; + while (1) { + int ret; + CHK((ret = con.nextScanResult(true)) == 0 || ret == 1); + if (ret == 1) + break; + uint k = (uint)-1; + CHK(set2.getkey(par, &k) == 0); + CHK(set2.putval(k, false, n) == 0); + LL3("row=" << n << " key=" << k); + CHK(k <= savepoint); + if (++i == set.m_rows) { + stop = true; + break; + } + set.push(i); + set.calc(par, i); + CHK(set.insrow(par, i) == 0); + CHK(con.execute(NoCommit) == 0); + n++; + } + con.closeScan(); + LL3("scanrows=" << n); + if (!stop) { + CHK(set1.verify(par, set2, false) == 0); + } + scancount++; + } + CHK(con.execute(Commit) == 0); + set.post(par, Commit); + assert(set.count() == set.m_rows); + CHK(pkdelete(par) == 0); + return 0; +} + +static int +halloweentest(Par par) +{ + assert(par.m_usedthreads == 1); + const Tab& tab = par.tab(); + for (uint i = 0; i < tab.m_itabs; i++) { + if (tab.m_itab[i] == 0) + continue; + const ITab& itab = *tab.m_itab[i]; + if (itab.m_type == ITab::OrderedIndex) + CHK(halloweentest(par, itab) == 0); + } + return 0; +} + // threads typedef int (*TFunc)(Par par); @@ -4477,22 +4906,22 @@ enum TMode { ST = 1, MT = 2 }; extern "C" { static void* runthread(void* arg); } struct Thr { - enum State { Wait, Start, Stop, Stopped, Exit }; + enum State { Wait, Start, Stop, Exit }; State m_state; Par m_par; - Uint64 m_id; + pthread_t m_id; NdbThread* m_thread; NdbMutex* m_mutex; NdbCondition* m_cond; TFunc m_func; int m_ret; void* m_status; - Thr(Par par, unsigned n); + char m_tmp[20]; // used for debug msg prefix + Thr(Par par, uint n); ~Thr(); int run(); void start(); void stop(); - void stopped(); void exit(); // void lock() { @@ -4513,10 +4942,9 @@ struct Thr { } }; -Thr::Thr(Par par, unsigned n) : +Thr::Thr(Par par, uint n) : m_state(Wait), m_par(par), - m_id(0), m_thread(0), m_mutex(0), m_cond(0), @@ -4533,7 +4961,7 @@ Thr::Thr(Par par, unsigned n) : m_cond = NdbCondition_Create(); assert(m_mutex != 0 && m_cond != 0); // run - const unsigned stacksize = 256 * 1024; + const uint stacksize = 256 * 1024; const NDB_THREAD_PRIO prio = NDB_THREAD_PRIO_LOW; m_thread = NdbThread_Create(runthread, (void**)this, stacksize, name, prio); } @@ -4558,7 +4986,7 @@ static void* runthread(void* arg) { Thr& thr = *(Thr*)arg; - thr.m_id = (Uint64)pthread_self(); + thr.m_id = pthread_self(); if (thr.run() < 0) { LL1("exit on error"); } else { @@ -4589,11 +5017,16 @@ Thr::run() LL4("start"); assert(m_state == Start); m_ret = (*m_func)(m_par); - m_state = Stopped; + m_state = Stop; LL4("stop"); signal(); unlock(); - CHK(m_ret == 0); + if (m_ret == -1) { + if (m_par.m_cont) + LL1("continue running due to -cont"); + else + return -1; + } } con.disconnect(); return 0; @@ -4612,16 +5045,7 @@ void Thr::stop() { lock(); - m_state = Stop; - signal(); - unlock(); -} - -void -Thr::stopped() -{ - lock(); - while (m_state != Stopped) + while (m_state != Stop) wait(); m_state = Wait; unlock(); @@ -4640,27 +5064,44 @@ Thr::exit() static Thr** g_thrlist = 0; -static unsigned -getthrno() +static Thr* +getthr() { if (g_thrlist != 0) { - Uint64 id = (Uint64)pthread_self(); - for (unsigned n = 0; n < g_opt.m_threads; n++) { + pthread_t id = pthread_self(); + for (uint n = 0; n < g_opt.m_threads; n++) { if (g_thrlist[n] != 0) { - const Thr& thr = *g_thrlist[n]; - if (thr.m_id == id) - return thr.m_par.m_no; + Thr& thr = *g_thrlist[n]; + if (pthread_equal(thr.m_id, id)) + return &thr; } } } - return (unsigned)-1; + return 0; +} + +// for debug messages (par.m_no not available) +static const char* +getthrprefix() +{ + Thr* thrp = getthr(); + if (thrp != 0) { + Thr& thr = *thrp; + uint n = thr.m_par.m_no; + uint m = + g_opt.m_threads < 10 ? 1 : + g_opt.m_threads < 100 ? 2 : 3; + sprintf(thr.m_tmp, "[%0*u] ", m, n); + return thr.m_tmp; + } + return ""; } static int -runstep(Par par, const char* fname, TFunc func, unsigned mode) +runstep(Par par, const char* fname, TFunc func, uint mode) { LL2("step: " << fname); - const int threads = (mode & ST ? 1 : par.m_threads); + const int threads = (mode & ST ? 1 : par.m_usedthreads); int n; for (n = 0; n < threads; n++) { LL4("start " << n); @@ -4673,11 +5114,11 @@ runstep(Par par, const char* fname, TFunc func, unsigned mode) thr.m_func = func; thr.start(); } - unsigned errs = 0; + uint errs = 0; for (n = threads - 1; n >= 0; n--) { LL4("stop " << n); Thr& thr = *g_thrlist[n]; - thr.stopped(); + thr.stop(); if (thr.m_ret != 0) errs++; } @@ -4689,7 +5130,7 @@ runstep(Par par, const char* fname, TFunc func, unsigned mode) CHK(runstep(par, #func, func, mode) == 0) #define SUBLOOP(par) \ - "subloop: " << par.m_lno << "/" << par.m_currcase << "/" << \ + "sloop: " << par.m_lno << "/" << par.m_currcase << "/" << \ par.m_tab->m_name << "/" << par.m_slno static int @@ -4698,7 +5139,7 @@ tbuild(Par par) RUNSTEP(par, droptable, ST); RUNSTEP(par, createtable, ST); RUNSTEP(par, invalidatetable, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); if (par.m_slno % 3 == 0) { RUNSTEP(par, createindex, ST); @@ -4718,7 +5159,7 @@ tbuild(Par par) } RUNSTEP(par, readverifyfull, MT); // leave last one - if (par.m_slno + 1 < par.m_subloop) { + if (par.m_slno + 1 < par.m_sloop) { RUNSTEP(par, pkdelete, MT); RUNSTEP(par, readverifyfull, MT); RUNSTEP(par, dropindex, ST); @@ -4737,7 +5178,7 @@ tindexscan(Par par) RUNSTEP(par, invalidateindex, MT); RUNSTEP(par, pkinsert, MT); RUNSTEP(par, readverifyfull, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, readverifyindex, MT); } @@ -4753,7 +5194,7 @@ tpkops(Par par) RUNSTEP(par, invalidatetable, MT); RUNSTEP(par, createindex, ST); RUNSTEP(par, invalidateindex, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, pkops, MT); LL2("rows=" << par.set().count()); @@ -4772,7 +5213,7 @@ tpkopsread(Par par) RUNSTEP(par, createindex, ST); RUNSTEP(par, invalidateindex, MT); RUNSTEP(par, readverifyfull, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, pkupdatescanread, MT); RUNSTEP(par, readverifyfull, MT); @@ -4792,7 +5233,7 @@ tmixedops(Par par) RUNSTEP(par, createindex, ST); RUNSTEP(par, invalidateindex, MT); RUNSTEP(par, readverifyfull, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, mixedoperations, MT); RUNSTEP(par, readverifyfull, MT); @@ -4807,7 +5248,7 @@ tbusybuild(Par par) RUNSTEP(par, createtable, ST); RUNSTEP(par, invalidatetable, MT); RUNSTEP(par, pkinsert, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, pkupdateindexbuild, MT); RUNSTEP(par, invalidateindex, MT); @@ -4828,7 +5269,7 @@ trollback(Par par) RUNSTEP(par, createindex, ST); RUNSTEP(par, invalidateindex, MT); RUNSTEP(par, readverifyfull, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, mixedoperations, MT); RUNSTEP(par, readverifyfull, MT); @@ -4846,7 +5287,7 @@ tparupdate(Par par) RUNSTEP(par, createindex, ST); RUNSTEP(par, invalidateindex, MT); RUNSTEP(par, readverifyfull, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, parallelorderedupdate, MT); RUNSTEP(par, readverifyfull, MT); @@ -4855,13 +5296,44 @@ tparupdate(Par par) } static int +tsavepoint(Par par) +{ + RUNSTEP(par, droptable, ST); + RUNSTEP(par, createtable, ST); + RUNSTEP(par, invalidatetable, MT); + RUNSTEP(par, createindex, ST); + RUNSTEP(par, invalidateindex, MT); + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { + LL1(SUBLOOP(par)); + RUNSTEP(par, savepointtest, MT); + RUNSTEP(par, readverifyfull, MT); + } + return 0; +} + +static int +thalloween(Par par) +{ + RUNSTEP(par, droptable, ST); + RUNSTEP(par, createtable, ST); + RUNSTEP(par, invalidatetable, MT); + RUNSTEP(par, createindex, ST); + RUNSTEP(par, invalidateindex, MT); + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { + LL1(SUBLOOP(par)); + RUNSTEP(par, halloweentest, MT); + } + return 0; +} + +static int ttimebuild(Par par) { Tmr t1; RUNSTEP(par, droptable, ST); RUNSTEP(par, createtable, ST); RUNSTEP(par, invalidatetable, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, pkinsert, MT); t1.on(); @@ -4881,7 +5353,7 @@ ttimemaint(Par par) RUNSTEP(par, droptable, ST); RUNSTEP(par, createtable, ST); RUNSTEP(par, invalidatetable, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, pkinsert, MT); t1.on(); @@ -4911,7 +5383,7 @@ ttimescan(Par par) RUNSTEP(par, droptable, ST); RUNSTEP(par, createtable, ST); RUNSTEP(par, invalidatetable, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, pkinsert, MT); RUNSTEP(par, createindex, ST); @@ -4938,7 +5410,7 @@ ttimepkread(Par par) RUNSTEP(par, droptable, ST); RUNSTEP(par, createtable, ST); RUNSTEP(par, invalidatetable, MT); - for (par.m_slno = 0; par.m_slno < par.m_subloop; par.m_slno++) { + for (par.m_slno = 0; par.m_slno < par.m_sloop; par.m_slno++) { LL1(SUBLOOP(par)); RUNSTEP(par, pkinsert, MT); RUNSTEP(par, createindex, ST); @@ -4981,7 +5453,9 @@ tcaselist[] = { TCase("e", tmixedops, "pk operations and scan operations"), TCase("f", tbusybuild, "pk operations and index build"), TCase("g", trollback, "operations with random rollbacks"), - TCase("h", tparupdate, "parallel ordered update (bug20446)"), + TCase("h", tparupdate, "parallel ordered update bug#20446"), + TCase("i", tsavepoint, "savepoint test locking bug#31477"), + TCase("j", thalloween, "savepoint test halloween problem"), TCase("t", ttimebuild, "time index build"), TCase("u", ttimemaint, "time index maintenance"), TCase("v", ttimescan, "time full scan table vs index on pk"), @@ -4989,14 +5463,14 @@ tcaselist[] = { TCase("z", tdrop, "drop test tables") }; -static const unsigned +static const uint tcasecount = sizeof(tcaselist) / sizeof(tcaselist[0]); static void printcases() { ndbout << "test cases:" << endl; - for (unsigned i = 0; i < tcasecount; i++) { + for (uint i = 0; i < tcasecount; i++) { const TCase& tcase = tcaselist[i]; ndbout << " " << tcase.m_name << " - " << tcase.m_desc << endl; } @@ -5008,13 +5482,13 @@ printtables() Par par(g_opt); makebuiltintables(par); ndbout << "tables and indexes (x=ordered z=hash x0=on pk):" << endl; - for (unsigned j = 0; j < tabcount; j++) { + for (uint j = 0; j < tabcount; j++) { if (tablist[j] == 0) continue; const Tab& tab = *tablist[j]; const char* tname = tab.m_name; ndbout << " " << tname; - for (unsigned i = 0; i < tab.m_itabs; i++) { + for (uint i = 0; i < tab.m_itabs; i++) { if (tab.m_itab[i] == 0) continue; const ITab& itab = *tab.m_itab[i]; @@ -5023,7 +5497,7 @@ printtables() iname += strlen(tname); ndbout << " " << iname; ndbout << "("; - for (unsigned k = 0; k < itab.m_icols; k++) { + for (uint k = 0; k < itab.m_icols; k++) { if (k != 0) ndbout << ","; const ICol& icol = *itab.m_icol[k]; @@ -5036,14 +5510,48 @@ printtables() } } +static bool +setcasepar(Par& par) +{ + Opt d; + const char* c = par.m_currcase; + switch (c[0]) { + case 'i': + { + if (par.m_usedthreads > 1) { + par.m_usedthreads = 1; + LL1("case " << c << " reduce threads to " << par.m_usedthreads); + } + const uint rows = 100; + if (par.m_rows > rows) { + par.m_rows = rows; + LL1("case " << c << " reduce rows to " << rows); + } + } + break; + case 'j': + { + if (par.m_usedthreads > 1) { + par.m_usedthreads = 1; + LL1("case " << c << " reduce threads to " << par.m_usedthreads); + } + } + break; + default: + break; + } + return true; +} + static int runtest(Par par) { + int totret = 0; if (par.m_seed == -1) { // good enough for daily run - unsigned short seed = (unsigned short)getpid(); + ushort seed = (ushort)getpid(); LL0("random seed: " << seed); - srandom((unsigned)seed); + srandom((uint)seed); } else if (par.m_seed != 0) { LL0("random seed: " << par.m_seed); srandom(par.m_seed); @@ -5061,9 +5569,10 @@ runtest(Par par) Con con; CHK(con.connect() == 0); par.m_con = &con; + par.m_catcherr |= Con::ErrNospace; // threads g_thrlist = new Thr* [par.m_threads]; - unsigned n; + uint n; for (n = 0; n < par.m_threads; n++) { g_thrlist[n] = 0; } @@ -5078,23 +5587,38 @@ runtest(Par par) LL1("random seed: " << par.m_lno); srandom(par.m_lno); } - for (unsigned i = 0; i < tcasecount; i++) { + for (uint i = 0; i < tcasecount; i++) { const TCase& tcase = tcaselist[i]; - if (par.m_case != 0 && strchr(par.m_case, tcase.m_name[0]) == 0) + if (par.m_case != 0 && strchr(par.m_case, tcase.m_name[0]) == 0 || + par.m_skip != 0 && strchr(par.m_skip, tcase.m_name[0]) != 0) { continue; + } sprintf(par.m_currcase, "%c", tcase.m_name[0]); + par.m_usedthreads = par.m_threads; + if (!setcasepar(par)) { + LL1("case " << tcase.m_name << " cannot run with given options"); + continue; + } + par.m_totrows = par.m_usedthreads * par.m_rows; makebuiltintables(par); LL1("case: " << par.m_lno << "/" << tcase.m_name << " - " << tcase.m_desc); - for (unsigned j = 0; j < tabcount; j++) { + for (uint j = 0; j < tabcount; j++) { if (tablist[j] == 0) continue; const Tab& tab = *tablist[j]; par.m_tab = &tab; par.m_set = new Set(tab, par.m_totrows); LL1("table: " << par.m_lno << "/" << tcase.m_name << "/" << tab.m_name); - CHK(tcase.m_func(par) == 0); + int ret = tcase.m_func(par); delete par.m_set; par.m_set = 0; + if (ret == -1) { + if (!par.m_cont) + return -1; + totret = -1; + LL1("continue to next case due to -cont"); + break; + } } } } @@ -5110,7 +5634,7 @@ runtest(Par par) delete [] g_thrlist; g_thrlist = 0; con.disconnect(); - return 0; + return totret; } static const char* g_progname = "testOIBasic"; @@ -5119,7 +5643,7 @@ int main(int argc, char** argv) { ndb_init(); - unsigned i; + uint i; ndbout << g_progname; for (i = 1; i < argc; i++) ndbout << " " << argv[i]; @@ -5156,6 +5680,10 @@ main(int argc, char** argv) g_opt.m_collsp = true; continue; } + if (strcmp(arg, "-cont") == 0) { + g_opt.m_cont = true; + continue; + } if (strcmp(arg, "-core") == 0) { g_opt.m_core = true; continue; @@ -5252,9 +5780,21 @@ main(int argc, char** argv) continue; } } - if (strcmp(arg, "-subloop") == 0) { + if (strcmp(arg, "-skip") == 0) { + if (++argv, --argc > 0) { + g_opt.m_skip = strdup(argv[0]); + continue; + } + } + if (strcmp(arg, "-sloop") == 0) { + if (++argv, --argc > 0) { + g_opt.m_sloop = atoi(argv[0]); + continue; + } + } + if (strcmp(arg, "-ssloop") == 0) { if (++argv, --argc > 0) { - g_opt.m_subloop = atoi(argv[0]); + g_opt.m_ssloop = atoi(argv[0]); continue; } } diff --git a/storage/ndb/test/run-test/daily-basic-tests.txt b/storage/ndb/test/run-test/daily-basic-tests.txt index 1aee1555f00..70c395fab17 100644 --- a/storage/ndb/test/run-test/daily-basic-tests.txt +++ b/storage/ndb/test/run-test/daily-basic-tests.txt @@ -1,7 +1,3 @@ -max-time: 3600 -cmd: atrt-mysql-test-run -args: --force - max-time: 600 cmd: atrt-testBackup args: -n NFMaster T1 @@ -744,10 +740,10 @@ max-time: 1000 cmd: testNdbApi args: -n Bug28443 -#max-time: 500 -#cmd: testInterpreter -#args: T1 -# +max-time: 500 +cmd: testInterpreter +args: T1 + max-time: 150000 cmd: testOperations args: @@ -1050,3 +1046,11 @@ max-time: 300 cmd: test_event args: -n Bug33793 T1 +max-time: 1200 +cmd: testNodeRestart +args: -n Bug34216 -l 10 T1 I3 D2 + +max-time: 1200 +cmd: testNodeRestart +args: -n mixedmultiop -l 10 T1 I2 I3 D2 + diff --git a/storage/ndb/test/tools/hugoPkUpdate.cpp b/storage/ndb/test/tools/hugoPkUpdate.cpp index b920a4f396a..3fa87da3145 100644 --- a/storage/ndb/test/tools/hugoPkUpdate.cpp +++ b/storage/ndb/test/tools/hugoPkUpdate.cpp @@ -106,7 +106,7 @@ int main(int argc, const char** argv){ NDBT_ThreadSet ths(_threads); // create Ndb object for each thread - if (ths.connect(&con, "TEST_DB") == -1) { + if (ths.connect(&con, db ? db : "TEST_DB") == -1) { ndbout << "connect failed: err=" << ths.get_err() << endl; return NDBT_ProgramExit(NDBT_FAILED); } diff --git a/storage/ndb/tools/waiter.cpp b/storage/ndb/tools/waiter.cpp index a292ab9140a..fc2a4b368b1 100644 --- a/storage/ndb/tools/waiter.cpp +++ b/storage/ndb/tools/waiter.cpp @@ -21,13 +21,11 @@ #include <NdbMain.h> #include <NdbOut.hpp> #include <NdbSleep.h> -#include <kernel/ndb_limits.h> #include <NDBT.hpp> -int -waitClusterStatus(const char* _addr, ndb_mgm_node_status _status, - unsigned int _timeout); +static int +waitClusterStatus(const char* _addr, ndb_mgm_node_status _status); enum ndb_waiter_options { OPT_WAIT_STATUS_NOT_STARTED = NDB_STD_OPTIONS_LAST, @@ -55,12 +53,13 @@ static struct my_option my_long_options[] = "Wait for cluster to enter single user mode", (uchar**) &_single_user, (uchar**) &_single_user, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 }, - { "timeout", 't', "Timeout to wait", + { "timeout", 't', "Timeout to wait in seconds", (uchar**) &_timeout, (uchar**) &_timeout, 0, GET_INT, REQUIRED_ARG, 120, 0, 0, 0, 0, 0 }, { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; + static void usage() { ndb_std_print_version(); @@ -70,16 +69,18 @@ static void usage() my_print_variables(my_long_options); } + int main(int argc, char** argv){ NDB_INIT(argv[0]); load_defaults("my",load_default_groups,&argc,&argv); const char* _hostName = NULL; - int ho_error; + #ifndef DBUG_OFF opt_debug= "d:t:O,/tmp/ndb_waiter.trace"; #endif - if ((ho_error=handle_options(&argc, &argv, my_long_options, - ndb_std_get_one_option))) + + if (handle_options(&argc, &argv, my_long_options, + ndb_std_get_one_option)) return NDBT_ProgramExit(NDBT_WRONGARGS); _hostName = argv[0]; @@ -105,7 +106,7 @@ int main(int argc, char** argv){ wait_status= NDB_MGM_NODE_STATUS_STARTED; } - if (waitClusterStatus(_hostName, wait_status, _timeout*10) != 0) + if (waitClusterStatus(_hostName, wait_status) != 0) return NDBT_ProgramExit(NDBT_FAILED); return NDBT_ProgramExit(NDBT_OK); } @@ -118,8 +119,6 @@ int main(int argc, char** argv){ NdbMgmHandle handle= NULL; Vector<ndb_mgm_node_state> ndbNodes; -Vector<ndb_mgm_node_state> mgmNodes; -Vector<ndb_mgm_node_state> apiNodes; int getStatus(){ @@ -128,8 +127,6 @@ getStatus(){ struct ndb_mgm_node_state * node; ndbNodes.clear(); - mgmNodes.clear(); - apiNodes.clear(); while(retries < 10){ status = ndb_mgm_get_status(handle); @@ -153,18 +150,16 @@ getStatus(){ ndbNodes.push_back(*node); break; case NDB_MGM_NODE_TYPE_MGM: - mgmNodes.push_back(*node); + /* Don't care about MGM nodes */ break; case NDB_MGM_NODE_TYPE_API: - apiNodes.push_back(*node); + /* Don't care about API nodes */ break; default: if(node->node_status == NDB_MGM_NODE_STATUS_UNKNOWN || node->node_status == NDB_MGM_NODE_STATUS_NO_CONTACT){ retries++; ndbNodes.clear(); - mgmNodes.clear(); - apiNodes.clear(); free(status); status = NULL; count = 0; @@ -183,24 +178,22 @@ getStatus(){ free(status); return 0; } - - g_err << "getStatus failed" << endl; + return -1; } -int +static int waitClusterStatus(const char* _addr, - ndb_mgm_node_status _status, - unsigned int _timeout) + ndb_mgm_node_status _status) { int _startphase = -1; - int _nodes[MAX_NDB_NODES]; - int _num_nodes = 0; + /* Ignore SIGPIPE */ + signal(SIGPIPE, SIG_IGN); handle = ndb_mgm_create_handle(); if (handle == NULL){ - g_err << "handle == NULL" << endl; + g_err << "Could not create ndb_mgm handle" << endl; return -1; } g_info << "Connecting to mgmsrv at " << _addr << endl; @@ -216,19 +209,11 @@ waitClusterStatus(const char* _addr, return -1; } - if (getStatus() != 0) - return -1; - - // Collect all nodes into nodes - for (size_t i = 0; i < ndbNodes.size(); i++){ - _nodes[i] = ndbNodes[i].node_id; - _num_nodes++; - } - - unsigned int attempts = 0; - unsigned int resetAttempts = 0; - const unsigned int MAX_RESET_ATTEMPTS = 10; - bool allInState = false; + int attempts = 0; + int resetAttempts = 0; + const int MAX_RESET_ATTEMPTS = 10; + bool allInState = false; + int timeout_ms= _timeout * 10; /* In number of 100 milliseconds */ while (allInState == false){ if (_timeout > 0 && attempts > _timeout){ /** @@ -236,8 +221,8 @@ waitClusterStatus(const char* _addr, * the state we want */ bool waitMore = false; - /** - * Make special check if we are waiting for + /** + * Make special check if we are waiting for * cluster to become started */ if(_status == NDB_MGM_NODE_STATUS_STARTED){ @@ -252,7 +237,7 @@ waitClusterStatus(const char* _addr, waitMore = false; } - } + } if (!waitMore || resetAttempts > MAX_RESET_ATTEMPTS){ g_err << "waitNodeState(" @@ -260,7 +245,7 @@ waitClusterStatus(const char* _addr, <<", "<<_startphase<<")" << " timeout after " << attempts <<" attemps" << endl; return -1; - } + } g_err << "waitNodeState(" << ndb_mgm_get_node_status_string(_status) @@ -269,62 +254,34 @@ waitClusterStatus(const char* _addr, << resetAttempts << endl; attempts = 0; resetAttempts++; - } - allInState = true; if (getStatus() != 0){ - g_err << "getStatus != 0" << endl; return -1; } - // ndbout << "waitNodeState; _num_nodes = " << _num_nodes << endl; - // for (int i = 0; i < _num_nodes; i++) - // ndbout << " node["<<i<<"] =" <<_nodes[i] << endl; + /* Assume all nodes are in state(if there is any) */ + allInState = (ndbNodes.size() > 0); - for (int i = 0; i < _num_nodes; i++){ - ndb_mgm_node_state* ndbNode = NULL; - for (size_t n = 0; n < ndbNodes.size(); n++){ - if (ndbNodes[n].node_id == _nodes[i]) - ndbNode = &ndbNodes[n]; - } + /* Loop through all nodes and check their state */ + for (size_t n = 0; n < ndbNodes.size(); n++) { + ndb_mgm_node_state* ndbNode = &ndbNodes[n]; - if(ndbNode == NULL){ - allInState = false; - continue; - } + assert(ndbNode != NULL); - g_info << "State node " << ndbNode->node_id << " " + g_info << "Node " << ndbNode->node_id << ": " << ndb_mgm_get_node_status_string(ndbNode->node_status)<< endl; - assert(ndbNode != NULL); - - if(_status == NDB_MGM_NODE_STATUS_STARTING && - ((ndbNode->node_status == NDB_MGM_NODE_STATUS_STARTING && - ndbNode->start_phase >= _startphase) || - (ndbNode->node_status == NDB_MGM_NODE_STATUS_STARTED))) - continue; - - if (_status == NDB_MGM_NODE_STATUS_STARTING){ - g_info << "status = " - << ndb_mgm_get_node_status_string(ndbNode->node_status) - <<", start_phase="<<ndbNode->start_phase<<endl; - if (ndbNode->node_status != _status) { - if (ndbNode->node_status < _status) - allInState = false; - else - g_info << "node_status(" << (unsigned)ndbNode->node_status - << ") != _status("<< (unsigned)_status << ")" <<endl; - } else if (ndbNode->start_phase < _startphase) - allInState = false; - } else { - if (ndbNode->node_status != _status) + if (ndbNode->node_status != _status) allInState = false; - } } - g_info << "Waiting for cluster enter state " - << ndb_mgm_get_node_status_string(_status)<< endl; - NdbSleep_MilliSleep(100); + + if (!allInState) { + g_info << "Waiting for cluster enter state " + << ndb_mgm_get_node_status_string(_status)<< endl; + NdbSleep_MilliSleep(100); + } + attempts++; } return 0; |