summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKim van der Riet <kpvdr@apache.org>2013-10-21 21:26:10 +0000
committerKim van der Riet <kpvdr@apache.org>2013-10-21 21:26:10 +0000
commit6d5d782b504677fcc4392086cb628dbbb79c800a (patch)
treee50621a47beb29aee73c463f89092e3b10da722a
parent127d7a7cd4f6a36ae3e44aaa6a12a28692b12b90 (diff)
downloadqpid-python-6d5d782b504677fcc4392086cb628dbbb79c800a.tar.gz
QPID-4984: WIP - Compiles, but functionally incomplete. Transactions not yet functional.
git-svn-id: https://svn.apache.org/repos/asf/qpid/branches/linearstore@1534383 13f79535-47bb-0310-9956-ffa450edef68
-rw-r--r--qpid/cpp/src/linearstore.cmake6
-rw-r--r--qpid/cpp/src/qpid/broker/Broker.cpp2
-rw-r--r--qpid/cpp/src/qpid/broker/Queue.cpp2
-rw-r--r--qpid/cpp/src/qpid/legacystore/jrnl/pmgr.cpp1
-rw-r--r--qpid/cpp/src/qpid/legacystore/jrnl/rmgr.cpp2
-rw-r--r--qpid/cpp/src/qpid/linearstore/EmptyFilePoolManagerImpl.cpp59
-rw-r--r--qpid/cpp/src/qpid/linearstore/EmptyFilePoolManagerImpl.h40
-rw-r--r--qpid/cpp/src/qpid/linearstore/JournalImpl.cpp21
-rw-r--r--qpid/cpp/src/qpid/linearstore/JournalImpl.h10
-rw-r--r--qpid/cpp/src/qpid/linearstore/JournalLogImpl.cpp59
-rw-r--r--qpid/cpp/src/qpid/linearstore/JournalLogImpl.h (renamed from qpid/cpp/src/qpid/linearstore/QpidLog.h)18
-rw-r--r--qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp34
-rw-r--r--qpid/cpp/src/qpid/linearstore/MessageStoreImpl.h10
-rw-r--r--qpid/cpp/src/qpid/linearstore/StorePlugin.cpp2
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/AtomicCounter.h58
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.cpp315
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.h37
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.cpp150
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.h32
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp90
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.h31
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolTypes.h14
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.cpp276
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.h33
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.cpp27
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.h17
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.cpp306
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.h57
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.cpp626
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.h103
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.cpp30
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/enq_map.cpp10
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/enq_map.h18
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.cpp32
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/enums.h10
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/jcfg.h69
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/jcntl.cpp617
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/jcntl.h265
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/jerrno.cpp13
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/jerrno.h7
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/jrec.h4
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/pmgr.cpp14
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/pmgr.h8
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/rcvdat.h111
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/rmgr.cpp707
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/rmgr.h102
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.cpp30
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.c1
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/wmgr.cpp256
-rw-r--r--qpid/cpp/src/qpid/linearstore/jrnl/wmgr.h73
50 files changed, 2026 insertions, 2789 deletions
diff --git a/qpid/cpp/src/linearstore.cmake b/qpid/cpp/src/linearstore.cmake
index 2237ed8a0f..6365905c57 100644
--- a/qpid/cpp/src/linearstore.cmake
+++ b/qpid/cpp/src/linearstore.cmake
@@ -78,11 +78,11 @@ if (BUILD_LINEARSTORE)
set (linear_jrnl_SOURCES
qpid/linearstore/jrnl/data_tok.cpp
qpid/linearstore/jrnl/deq_rec.cpp
- qpid/linearstore/jrnl/enq_map.cpp
- qpid/linearstore/jrnl/enq_rec.cpp
qpid/linearstore/jrnl/EmptyFilePool.cpp
qpid/linearstore/jrnl/EmptyFilePoolManager.cpp
qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp
+ qpid/linearstore/jrnl/enq_map.cpp
+ qpid/linearstore/jrnl/enq_rec.cpp
qpid/linearstore/jrnl/jcntl.cpp
qpid/linearstore/jrnl/jdir.cpp
qpid/linearstore/jrnl/jerrno.cpp
@@ -105,12 +105,12 @@ if (BUILD_LINEARSTORE)
qpid/linearstore/BindingDbt.cpp
qpid/linearstore/BufferValue.cpp
qpid/linearstore/DataTokenImpl.cpp
- qpid/linearstore/EmptyFilePoolManagerImpl.cpp
qpid/linearstore/IdDbt.cpp
qpid/linearstore/IdSequence.cpp
qpid/linearstore/JournalImpl.cpp
qpid/linearstore/MessageStoreImpl.cpp
qpid/linearstore/PreparedTransaction.cpp
+ qpid/linearstore/JournalLogImpl.cpp
qpid/linearstore/TxnCtxt.cpp
)
diff --git a/qpid/cpp/src/qpid/broker/Broker.cpp b/qpid/cpp/src/qpid/broker/Broker.cpp
index 85900f8061..2770c9d66b 100644
--- a/qpid/cpp/src/qpid/broker/Broker.cpp
+++ b/qpid/cpp/src/qpid/broker/Broker.cpp
@@ -998,7 +998,7 @@ Manageable::status_t Broker::queryQueue( const std::string& name,
return Manageable::STATUS_UNKNOWN_OBJECT;
}
q->query( results );
- return Manageable::STATUS_OK;;
+ return Manageable::STATUS_OK;
}
Manageable::status_t Broker::getTimestampConfig(bool& receive,
diff --git a/qpid/cpp/src/qpid/broker/Queue.cpp b/qpid/cpp/src/qpid/broker/Queue.cpp
index 5a43df44a4..d1f1afd61a 100644
--- a/qpid/cpp/src/qpid/broker/Queue.cpp
+++ b/qpid/cpp/src/qpid/broker/Queue.cpp
@@ -835,7 +835,7 @@ bool Queue::checkAutoDelete(const Mutex::ScopedLock& lock) const
bool Queue::isUnused(const Mutex::ScopedLock&) const
{
- return !owner && !users.isUsed();;
+ return !owner && !users.isUsed();
}
bool Queue::isEmpty(const Mutex::ScopedLock&) const
diff --git a/qpid/cpp/src/qpid/legacystore/jrnl/pmgr.cpp b/qpid/cpp/src/qpid/legacystore/jrnl/pmgr.cpp
index 3dc61e2661..3fa7c9b562 100644
--- a/qpid/cpp/src/qpid/legacystore/jrnl/pmgr.cpp
+++ b/qpid/cpp/src/qpid/legacystore/jrnl/pmgr.cpp
@@ -49,6 +49,7 @@ namespace journal
pmgr::page_cb::page_cb(u_int16_t index):
_index(index),
_state(UNUSED),
+ _frid(0),
_wdblks(0),
_rdblks(0),
_pdtokl(0),
diff --git a/qpid/cpp/src/qpid/legacystore/jrnl/rmgr.cpp b/qpid/cpp/src/qpid/legacystore/jrnl/rmgr.cpp
index 204affd1d1..d54de02455 100644
--- a/qpid/cpp/src/qpid/legacystore/jrnl/rmgr.cpp
+++ b/qpid/cpp/src/qpid/legacystore/jrnl/rmgr.cpp
@@ -227,6 +227,7 @@ rmgr::read(void** const datapp, std::size_t& dsize, void** const xidpp, std::siz
return RHM_IORES_EMPTY;
}
}
+ return RHM_IORES_SUCCESS;
}
int32_t
@@ -529,6 +530,7 @@ rmgr::skip(data_tok* dtokp)
return RHM_IORES_SUCCESS;
}
}
+ return RHM_IORES_SUCCESS;
}
iores
diff --git a/qpid/cpp/src/qpid/linearstore/EmptyFilePoolManagerImpl.cpp b/qpid/cpp/src/qpid/linearstore/EmptyFilePoolManagerImpl.cpp
deleted file mode 100644
index 2904afb944..0000000000
--- a/qpid/cpp/src/qpid/linearstore/EmptyFilePoolManagerImpl.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-#include "EmptyFilePoolManagerImpl.h"
-
-#include "QpidLog.h"
-#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h"
-
-namespace qpid {
-namespace linearstore {
-
-EmptyFilePoolManagerImpl::EmptyFilePoolManagerImpl(const std::string& qlsStorePath) :
- qpid::qls_jrnl::EmptyFilePoolManager(qlsStorePath)
-{}
-
-EmptyFilePoolManagerImpl::~EmptyFilePoolManagerImpl() {}
-
-void EmptyFilePoolManagerImpl::findEfpPartitions() {
- qpid::qls_jrnl::EmptyFilePoolManager::findEfpPartitions();
- QLS_LOG(info, "EFP Manager initialization complete");
- std::vector<qpid::qls_jrnl::EmptyFilePoolPartition*> partitionList;
- std::vector<qpid::qls_jrnl::EmptyFilePool*> filePoolList;
- getEfpPartitions(partitionList);
- if (partitionList.size() == 0) {
- QLS_LOG(error, "NO EFP PARTITIONS FOUND! No queue creation is possible.")
- } else {
- QLS_LOG(info, "> EFP Partitions found: " << partitionList.size());
- for (std::vector<qpid::qls_jrnl::EmptyFilePoolPartition*>::const_iterator i=partitionList.begin(); i!= partitionList.end(); ++i) {
- filePoolList.clear();
- (*i)->getEmptyFilePools(filePoolList);
- QLS_LOG(info, " * Partition " << (*i)->partitionNumber() << " containing " << filePoolList.size() << " pool" <<
- (filePoolList.size()>1 ? "s" : "") << " at \'" << (*i)->partitionDirectory() << "\'");
- for (std::vector<qpid::qls_jrnl::EmptyFilePool*>::const_iterator j=filePoolList.begin(); j!=filePoolList.end(); ++j) {
- QLS_LOG(info, " - EFP \'" << (*j)->dataSize_kib() << "k\' containing " << (*j)->numEmptyFiles() <<
- " files of size " << (*j)->dataSize_kib() << " KiB totaling " << (*j)->cumFileSize_kib() << " KiB");
- }
- }
- }
-}
-
-}} /* namespace qpid::linearstore */
diff --git a/qpid/cpp/src/qpid/linearstore/EmptyFilePoolManagerImpl.h b/qpid/cpp/src/qpid/linearstore/EmptyFilePoolManagerImpl.h
deleted file mode 100644
index f9087b2fbc..0000000000
--- a/qpid/cpp/src/qpid/linearstore/EmptyFilePoolManagerImpl.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-#ifndef QPID_LINEARSTORE_EMPTYFILEPOOLMANAGERIMPL_H_
-#define QPID_LINEARSTORE_EMPTYFILEPOOLMANAGERIMPL_H_
-
-#include "qpid/linearstore/jrnl/EmptyFilePoolManager.h"
-
-namespace qpid {
-namespace linearstore {
-
-class EmptyFilePoolManagerImpl: public qpid::qls_jrnl::EmptyFilePoolManager
-{
-public:
- EmptyFilePoolManagerImpl(const std::string& qlsStorePath);
- virtual ~EmptyFilePoolManagerImpl();
- void findEfpPartitions();
-};
-
-}} /* namespace qpid::linearstore */
-
-#endif /* QPID_LINEARSTORE_EMPTYFILEPOOLMANAGERIMPL_H_ */
diff --git a/qpid/cpp/src/qpid/linearstore/JournalImpl.cpp b/qpid/cpp/src/qpid/linearstore/JournalImpl.cpp
index 42ca113dc2..412a4922ca 100644
--- a/qpid/cpp/src/qpid/linearstore/JournalImpl.cpp
+++ b/qpid/cpp/src/qpid/linearstore/JournalImpl.cpp
@@ -21,20 +21,20 @@
#include "qpid/linearstore/JournalImpl.h"
+#include "qpid/linearstore/JournalLogImpl.h"
#include "qpid/linearstore/jrnl/jerrno.h"
#include "qpid/linearstore/jrnl/jexception.h"
#include "qpid/linearstore/jrnl/EmptyFilePool.h"
+#include "qpid/linearstore/StoreException.h"
#include "qpid/log/Statement.h"
#include "qpid/management/ManagementAgent.h"
-//#include "qmf/org/apache/qpid/linearstore/ArgsJournalExpand.h"
+#include "qpid/sys/Monitor.h"
+#include "qpid/sys/Timer.h"
+
#include "qmf/org/apache/qpid/linearstore/EventCreated.h"
#include "qmf/org/apache/qpid/linearstore/EventEnqThresholdExceeded.h"
#include "qmf/org/apache/qpid/linearstore/EventFull.h"
#include "qmf/org/apache/qpid/linearstore/EventRecovered.h"
-#include "qpid/sys/Monitor.h"
-#include "qpid/sys/Timer.h"
-#include "qpid/linearstore/QpidLog.h"
-#include "qpid/linearstore/StoreException.h"
using namespace qpid::qls_jrnl;
using namespace qpid::linearstore;
@@ -54,13 +54,14 @@ void GetEventsFireEvent::fire() { qpid::sys::Mutex::ScopedLock sl(_gefe_lock); i
JournalImpl::JournalImpl(qpid::sys::Timer& timer_,
const std::string& journalId,
const std::string& journalDirectory,
-// const std::string& journalBaseFilename,
+ JournalLogImpl& journalLogRef,
const qpid::sys::Duration getEventsTimeout,
const qpid::sys::Duration flushTimeout,
qpid::management::ManagementAgent* a,
DeleteCallback onDelete):
- jcntl(journalId, journalDirectory/*, journalBaseFilename*/),
+ jcntl(journalId, journalDirectory, journalLogRef),
timer(timer_),
+ _journalLogRef(journalLogRef),
getEventsTimerSetFlag(false),
// lastReadRid(0),
writeActivityFlag(false),
@@ -163,7 +164,7 @@ JournalImpl::initialize(qpid::qls_jrnl::EmptyFilePool* efpp_,
_mgmtObject->set_writePages(wcache_num_pages);
}
if (_agent != 0)
- _agent->raiseEvent(qmf::org::apache::qpid::linearstore::EventCreated(_jid, _jfsize_sblks * JRNL_SBLK_SIZE, _lpmgr.num_jfiles()),
+ _agent->raiseEvent::(qmf::org::apache::qpid::linearstore::EventCreated(_jid, _jfsize_sblks * JRNL_SBLK_SIZE, _lpmgr.num_jfiles()),
qpid::management::ManagementAgent::SEV_NOTE);
*/
}
@@ -558,11 +559,11 @@ JournalImpl::wr_aio_cb(std::vector<data_tok*>& dtokl)
switch (dtokp->wstate())
{
case data_tok::ENQ:
- std::cout << "<<<>>> JournalImpl::wr_aio_cb() ENQ dtokp rid=" << dtokp->rid() << std::endl << std::flush; // DEBUG
+//std::cout << "<<<>>> JournalImpl::wr_aio_cb() ENQ dtokp rid=0x" << std::hex << dtokp->rid() << std::dec << std::endl << std::flush; // DEBUG
dtokp->getSourceMessage()->enqueueComplete();
break;
case data_tok::DEQ:
- std::cout << "<<<>>> JournalImpl::wr_aio_cb() DEQ dtokp rid=" << dtokp->rid() << std::endl << std::flush; // DEBUG
+//std::cout << "<<<>>> JournalImpl::wr_aio_cb() DEQ dtokp rid=0x" << std::hex << dtokp->rid() << std::dec << std::endl << std::flush; // DEBUG
/* Don't need to signal until we have a way to ack completion of dequeue in AMQP
dtokp->getSourceMessage()->dequeueComplete();
if ( dtokp->getSourceMessage()->isDequeueComplete() ) // clear id after last dequeue
diff --git a/qpid/cpp/src/qpid/linearstore/JournalImpl.h b/qpid/cpp/src/qpid/linearstore/JournalImpl.h
index 4352eda074..fd2433803c 100644
--- a/qpid/cpp/src/qpid/linearstore/JournalImpl.h
+++ b/qpid/cpp/src/qpid/linearstore/JournalImpl.h
@@ -48,6 +48,7 @@ class EmptyFilePool;
namespace linearstore{
class JournalImpl;
+class JournalLogImpl;
class InactivityFireEvent : public qpid::sys::TimerTask
{
@@ -78,8 +79,9 @@ class JournalImpl : public qpid::broker::ExternalQueueStore, public qpid::qls_jr
public:
typedef boost::function<void (JournalImpl&)> DeleteCallback;
- private:
+ protected:
qpid::sys::Timer& timer;
+ JournalLogImpl& _journalLogRef;
bool getEventsTimerSetFlag;
boost::intrusive_ptr<qpid::sys::TimerTask> getEventsFireEventsPtr;
qpid::sys::Mutex _getf_lock;
@@ -98,6 +100,7 @@ class JournalImpl : public qpid::broker::ExternalQueueStore, public qpid::qls_jr
JournalImpl(qpid::sys::Timer& timer,
const std::string& journalId,
const std::string& journalDirectory,
+ JournalLogImpl& journalLogRef,
const qpid::sys::Duration getEventsTimeout,
const qpid::sys::Duration flushTimeout,
qpid::management::ManagementAgent* agent,
@@ -187,7 +190,7 @@ class JournalImpl : public qpid::broker::ExternalQueueStore, public qpid::qls_jr
void resetDeleteCallback() { deleteCallback = DeleteCallback(); }
- private:
+ protected:
// void free_read_buffers();
void createStore();
@@ -215,10 +218,11 @@ class TplJournalImpl : public JournalImpl
TplJournalImpl(qpid::sys::Timer& timer,
const std::string& journalId,
const std::string& journalDirectory,
+ JournalLogImpl& journalLogRef,
const qpid::sys::Duration getEventsTimeout,
const qpid::sys::Duration flushTimeout,
qpid::management::ManagementAgent* agent) :
- JournalImpl(timer, journalId, journalDirectory, getEventsTimeout, flushTimeout, agent)
+ JournalImpl(timer, journalId, journalDirectory, journalLogRef, getEventsTimeout, flushTimeout, agent)
{}
virtual ~TplJournalImpl() {}
diff --git a/qpid/cpp/src/qpid/linearstore/JournalLogImpl.cpp b/qpid/cpp/src/qpid/linearstore/JournalLogImpl.cpp
new file mode 100644
index 0000000000..b7935b1509
--- /dev/null
+++ b/qpid/cpp/src/qpid/linearstore/JournalLogImpl.cpp
@@ -0,0 +1,59 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+#include "qpid/linearstore/JournalLogImpl.h"
+
+namespace qpid {
+namespace linearstore {
+
+JournalLogImpl::JournalLogImpl(const qpid::qls_jrnl::JournalLog::log_level_t logLevelThreshold) : qpid::qls_jrnl::JournalLog(logLevelThreshold) {}
+JournalLogImpl::~JournalLogImpl() {}
+
+void
+JournalLogImpl::log(const qpid::qls_jrnl::JournalLog::log_level_t level,
+ const std::string& log_stmt) const {
+ switch (level) {
+ case LOG_CRITICAL: QPID_LOG(critical, "Linear Store: " << log_stmt); break;
+ case LOG_ERROR: QPID_LOG(error, "Linear Store: " << log_stmt); break;
+ case LOG_WARN: QPID_LOG(warning, "Linear Store: " << log_stmt); break;
+ case LOG_NOTICE: QPID_LOG(notice, "Linear Store: " << log_stmt); break;
+ case LOG_INFO: QPID_LOG(info, "Linear Store: " << log_stmt); break;
+ case LOG_DEBUG: QPID_LOG(debug, "Linear Store: " << log_stmt); break;
+ default: QPID_LOG(trace, "Linear Store: " << log_stmt);
+ }
+}
+
+void
+JournalLogImpl::log(const qpid::qls_jrnl::JournalLog::log_level_t level,
+ const std::string& jid,
+ const std::string& log_stmt) const {
+ switch (level) {
+ case LOG_CRITICAL: QPID_LOG(critical, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break;
+ case LOG_ERROR: QPID_LOG(error, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break;
+ case LOG_WARN: QPID_LOG(warning, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break;
+ case LOG_NOTICE: QPID_LOG(notice, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break;
+ case LOG_INFO: QPID_LOG(info, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break;
+ case LOG_DEBUG: QPID_LOG(debug, "Linear Store: Journal \'" << jid << "\":" << log_stmt); break;
+ default: QPID_LOG(trace, "Linear Store: Journal \'" << jid << "\":" << log_stmt);
+ }
+}
+
+}} // namespace qpid::linearstore
diff --git a/qpid/cpp/src/qpid/linearstore/QpidLog.h b/qpid/cpp/src/qpid/linearstore/JournalLogImpl.h
index b03ea7ac9d..a43af23ac6 100644
--- a/qpid/cpp/src/qpid/linearstore/QpidLog.h
+++ b/qpid/cpp/src/qpid/linearstore/JournalLogImpl.h
@@ -22,9 +22,27 @@
#ifndef QPID_LEGACYSTORE_LOG_H
#define QPID_LEGACYSTORE_LOG_H
+#include "qpid/linearstore/jrnl/JournalLog.h"
#include "qpid/log/Statement.h"
#define QLS_LOG(level, msg) QPID_LOG(level, "Linear Store: " << msg)
#define QLS_LOG2(level, queue, msg) QPID_LOG(level, "Linear Store: Journal \'" << queue << "\":" << msg)
+namespace qpid {
+namespace linearstore {
+
+class JournalLogImpl : public qpid::qls_jrnl::JournalLog
+{
+public:
+ JournalLogImpl(const qpid::qls_jrnl::JournalLog::log_level_t logLevelThreshold);
+ virtual ~JournalLogImpl();
+ virtual void log(const qpid::qls_jrnl::JournalLog::log_level_t logLevel,
+ const std::string& logStatement) const;
+ virtual void log(const qpid::qls_jrnl::JournalLog::log_level_t logLevel,
+ const std::string& journalId,
+ const std::string& logStatement) const;
+};
+
+}} // namespace qpid::linearstore
+
#endif // QPID_LEGACYSTORE_LOG_H
diff --git a/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp
index a0835530f7..cbc395d61a 100644
--- a/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp
+++ b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.cpp
@@ -28,7 +28,6 @@
#include "qpid/linearstore/IdDbt.h"
#include "qpid/linearstore/jrnl/EmptyFilePoolManager.h"
#include "qpid/linearstore/jrnl/txn_map.h"
-#include "qpid/linearstore/QpidLog.h"
#include "qpid/framing/FieldValue.h"
#include "qmf/org/apache/qpid/linearstore/Package.h"
#include "qpid/linearstore/StoreException.h"
@@ -73,6 +72,7 @@ MessageStoreImpl::MessageStoreImpl(qpid::broker::Broker* broker_, const char* en
isInit(false),
envPath(envpath_),
broker(broker_),
+ jrnlLog(qpid::qls_jrnl::JournalLog::LOG_NOTICE),
mgmtObject(),
agent(0)
{}
@@ -83,7 +83,7 @@ uint32_t MessageStoreImpl::chkJrnlWrPageCacheSize(const uint32_t param_, const s
if (p == 0) {
// For zero value, use default
- p = JRNL_WMGR_DEF_PAGE_SIZE_KIB;
+ p = QLS_WMGR_DEF_PAGE_SIZE_KIB;
QLS_LOG(warning, "parameter " << paramName_ << " (" << param_ << ") must be a power of 2 between 1 and 128; changing this parameter to default value (" << p << ")");
} else if ( p > 128 || (p & (p-1)) ) {
// For any positive value that is not a power of 2, use closest value
@@ -100,8 +100,8 @@ uint32_t MessageStoreImpl::chkJrnlWrPageCacheSize(const uint32_t param_, const s
uint16_t MessageStoreImpl::getJrnlWrNumPages(const uint32_t wrPageSizeKib_)
{
- uint32_t wrPageSizeSblks = wrPageSizeKib_ / JRNL_SBLK_SIZE_KIB; // convert from KiB to number sblks
- uint32_t defTotWCacheSizeSblks = JRNL_WMGR_DEF_PAGE_SIZE_SBLKS * JRNL_WMGR_DEF_PAGES;
+ uint32_t wrPageSizeSblks = wrPageSizeKib_ / QLS_SBLK_SIZE_KIB; // convert from KiB to number sblks
+ uint32_t defTotWCacheSizeSblks = QLS_WMGR_DEF_PAGE_SIZE_SBLKS * QLS_WMGR_DEF_PAGES;
switch (wrPageSizeKib_)
{
case 1:
@@ -127,13 +127,13 @@ qpid::qls_jrnl::efpPartitionNumber_t MessageStoreImpl::chkEfpPartition(const qpi
qpid::qls_jrnl::efpDataSize_kib_t MessageStoreImpl::chkEfpFileSizeKiB(const qpid::qls_jrnl::efpDataSize_kib_t efpFileSizeKib_,
const std::string& paramName_) {
- uint8_t rem = efpFileSizeKib_ % uint64_t(JRNL_SBLK_SIZE_KIB);
+ uint8_t rem = efpFileSizeKib_ % uint64_t(QLS_SBLK_SIZE_KIB);
if (rem != 0) {
uint64_t newVal = efpFileSizeKib_ - rem;
- if (rem >= (JRNL_SBLK_SIZE_KIB / 2))
- newVal += JRNL_SBLK_SIZE_KIB;
+ if (rem >= (QLS_SBLK_SIZE_KIB / 2))
+ newVal += QLS_SBLK_SIZE_KIB;
QLS_LOG(warning, "Parameter " << paramName_ << " (" << efpFileSizeKib_ << ") must be a multiple of " <<
- JRNL_SBLK_SIZE_KIB << "; changing this parameter to the closest allowable value (" <<
+ QLS_SBLK_SIZE_KIB << "; changing this parameter to the closest allowable value (" <<
newVal << ")");
return newVal;
}
@@ -154,7 +154,7 @@ void MessageStoreImpl::initManagement ()
mgmtObject->set_location(storeDir);
mgmtObject->set_tplIsInitialized(false);
mgmtObject->set_tplDirectory(getTplBaseDir());
- mgmtObject->set_tplWritePageSize(tplWCachePgSizeSblks * JRNL_SBLK_SIZE_BYTES);
+ mgmtObject->set_tplWritePageSize(tplWCachePgSizeSblks * QLS_SBLK_SIZE_BYTES);
mgmtObject->set_tplWritePages(tplWCacheNumPages);
agent->addObject(mgmtObject, 0, true);
@@ -193,9 +193,9 @@ bool MessageStoreImpl::init(const std::string& storeDir_,
// Set geometry members (converting to correct units where req'd)
defaultEfpPartitionNumber = efpPartition_;
defaultEfpFileSize_kib = efpFileSize_kib_;
- wCachePgSizeSblks = wCachePageSizeKib_ / JRNL_SBLK_SIZE_KIB; // convert from KiB to number sblks
+ wCachePgSizeSblks = wCachePageSizeKib_ / QLS_SBLK_SIZE_KIB; // convert from KiB to number sblks
wCacheNumPages = getJrnlWrNumPages(wCachePageSizeKib_);
- tplWCachePgSizeSblks = tplWCachePageSizeKib_ / JRNL_SBLK_SIZE_KIB; // convert from KiB to number sblks
+ tplWCachePgSizeSblks = tplWCachePageSizeKib_ / QLS_SBLK_SIZE_KIB; // convert from KiB to number sblks
tplWCacheNumPages = getJrnlWrNumPages(tplWCachePageSizeKib_);
if (storeDir_.size()>0) storeDir = storeDir_;
@@ -267,7 +267,7 @@ void MessageStoreImpl::init()
// NOTE: during normal initialization, agent == 0 because the store is initialized before the management infrastructure.
// However during a truncated initialization in a cluster, agent != 0. We always pass 0 as the agent for the
// TplStore to keep things consistent in a cluster. See https://bugzilla.redhat.com/show_bug.cgi?id=681026
- tplStorePtr.reset(new TplJournalImpl(broker->getTimer(), "TplStore", getTplBaseDir(), defJournalGetEventsTimeout, defJournalFlushTimeout, 0));
+ tplStorePtr.reset(new TplJournalImpl(broker->getTimer(), "TplStore", getTplBaseDir(), jrnlLog, defJournalGetEventsTimeout, defJournalFlushTimeout, 0));
isInit = true;
} catch (const DbException& e) {
if (e.get_errno() == DB_VERSION_MISMATCH)
@@ -291,7 +291,7 @@ void MessageStoreImpl::init()
}
} while (!isInit);
- efpMgr.reset(new EmptyFilePoolManagerImpl(getStoreTopLevelDir()));
+ efpMgr.reset(new qpid::qls_jrnl::EmptyFilePoolManager(getStoreTopLevelDir(), jrnlLog));
efpMgr->findEfpPartitions();
}
@@ -403,7 +403,7 @@ void MessageStoreImpl::create(qpid::broker::PersistableQueue& queue_,
return;
}
- jQueue = new JournalImpl(broker->getTimer(), queue_.getName(), getJrnlDir(queue_.getName()),
+ jQueue = new JournalImpl(broker->getTimer(), queue_.getName(), getJrnlDir(queue_.getName()), jrnlLog,
defJournalGetEventsTimeout, defJournalFlushTimeout, agent,
boost::bind(&MessageStoreImpl::journalDeleted, this, _1));
{
@@ -711,8 +711,8 @@ void MessageStoreImpl::recoverQueues(TxnCtxt& txn,
QLS_LOG(error, "Cannot recover empty (null) queue name - ignoring and attempting to continue.");
break;
}
- jQueue = new JournalImpl(broker->getTimer(), queueName, getJrnlDir(queueName), defJournalGetEventsTimeout,
- defJournalFlushTimeout, agent,
+ jQueue = new JournalImpl(broker->getTimer(), queueName, getJrnlDir(queueName),jrnlLog,
+ defJournalGetEventsTimeout, defJournalFlushTimeout, agent,
boost::bind(&MessageStoreImpl::journalDeleted, this, _1));
{
qpid::sys::Mutex::ScopedLock sl(journalListLock);
@@ -1207,7 +1207,7 @@ void MessageStoreImpl::loadContent(const qpid::broker::PersistableQueue& /*queue
void MessageStoreImpl::flush(const qpid::broker::PersistableQueue& queue_)
{
- QLS_LOG(info, "*** MessageStoreImpl::flush() queue=\"" << queue_.getName() << "\"");
+// QLS_LOG(info, "*** MessageStoreImpl::flush() queue=\"" << queue_.getName() << "\"");
if (queue_.getExternalQueueStore() == 0) return;
checkInit();
std::string qn = queue_.getName();
diff --git a/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.h b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.h
index 0028b809e9..d1ce7a2b8a 100644
--- a/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.h
+++ b/qpid/cpp/src/qpid/linearstore/MessageStoreImpl.h
@@ -22,14 +22,15 @@
#ifndef QPID_LEGACYSTORE_MESSAGESTOREIMPL_H
#define QPID_LEGACYSTORE_MESSAGESTOREIMPL_H
+#include <iomanip>
#include <string>
#include "db-inc.h"
#include "qpid/linearstore/Cursor.h"
-#include "qpid/linearstore/EmptyFilePoolManagerImpl.h"
#include "qpid/linearstore/IdDbt.h"
#include "qpid/linearstore/IdSequence.h"
#include "qpid/linearstore/JournalImpl.h"
+#include "qpid/linearstore/JournalLogImpl.h"
#include "qpid/linearstore/jrnl/jcfg.h"
#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h"
#include "qpid/linearstore/PreparedTransaction.h"
@@ -101,10 +102,10 @@ class MessageStoreImpl : public qpid::broker::MessageStore, public qpid::managem
// Default store settings
static const bool defTruncateFlag = false;
- static const uint32_t defWCachePageSizeKib = JRNL_WMGR_DEF_PAGE_SIZE_KIB;
+ static const uint32_t defWCachePageSizeKib = QLS_WMGR_DEF_PAGE_SIZE_KIB;
static const uint32_t defTplWCachePageSizeKib = defWCachePageSizeKib / 8;
static const uint16_t defEfpPartition = 1;
- static const uint64_t defEfpFileSizeKib = 512 * JRNL_SBLK_SIZE_KIB;
+ static const uint64_t defEfpFileSizeKib = 512 * QLS_SBLK_SIZE_KIB;
static const std::string storeTopLevelDir;
static qpid::sys::Duration defJournalGetEventsTimeout;
@@ -143,7 +144,8 @@ class MessageStoreImpl : public qpid::broker::MessageStore, public qpid::managem
bool isInit;
const char* envPath;
qpid::broker::Broker* broker;
- boost::shared_ptr<EmptyFilePoolManagerImpl> efpMgr;
+ JournalLogImpl jrnlLog;
+ boost::shared_ptr<qpid::qls_jrnl::EmptyFilePoolManager> efpMgr;
qmf::org::apache::qpid::linearstore::Store::shared_ptr mgmtObject;
qpid::management::ManagementAgent* agent;
diff --git a/qpid/cpp/src/qpid/linearstore/StorePlugin.cpp b/qpid/cpp/src/qpid/linearstore/StorePlugin.cpp
index bf574d9740..9826240068 100644
--- a/qpid/cpp/src/qpid/linearstore/StorePlugin.cpp
+++ b/qpid/cpp/src/qpid/linearstore/StorePlugin.cpp
@@ -23,7 +23,7 @@
#include "qpid/Plugin.h"
#include "qpid/Options.h"
#include "qpid/DataDir.h"
-#include "qpid/linearstore/QpidLog.h"
+#include "qpid/linearstore/JournalLogImpl.h"
#include "qpid/linearstore/MessageStoreImpl.h"
using qpid::linearstore::MessageStoreImpl;
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/AtomicCounter.h b/qpid/cpp/src/qpid/linearstore/jrnl/AtomicCounter.h
index cdae41f944..2f9d406b7c 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/AtomicCounter.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/AtomicCounter.h
@@ -31,105 +31,93 @@ template <class T>
class AtomicCounter
{
private:
- T count;
+ T count_;
mutable smutex countMutex;
public:
- AtomicCounter(const T& initValue = T(0)) : count(initValue) {}
+ AtomicCounter(const T& initValue = T(0)) : count_(initValue) {}
virtual ~AtomicCounter() {}
T get() const {
slock l(countMutex);
- return count;
+ return count_;
}
T increment() {
slock l(countMutex);
- return ++count;
+ return ++count_;
}
T add(const T& a) {
slock l(countMutex);
- count += a;
- return count;
+ count_ += a;
+ return count_;
}
T addLimit(const T& a, const T& limit, const uint32_t jerr) {
slock l(countMutex);
- if (count + a > limit) throw jexception(jerr, "AtomicCounter", "addLimit");
- count += a;
- return count;
+ if (count_ + a > limit) throw jexception(jerr, "AtomicCounter", "addLimit");
+ count_ += a;
+ return count_;
}
T decrement() {
slock l(countMutex);
- return --count;
+ return --count_;
}
T decrementLimit(const T& limit = T(0), const uint32_t jerr = jerrno::JERR__UNDERFLOW) {
slock l(countMutex);
- if (count < limit + 1) {
+ if (count_ < limit + 1) {
throw jexception(jerr, "AtomicCounter", "decrementLimit");
}
- return --count;
+ return --count_;
}
T subtract(const T& s) {
slock l(countMutex);
- count -= s;
- return count;
+ count_ -= s;
+ return count_;
}
T subtractLimit(const T& s, const T& limit = T(0), const uint32_t jerr = jerrno::JERR__UNDERFLOW) {
slock l(countMutex);
- if (count < limit + s) throw jexception(jerr, "AtomicCounter", "subtractLimit");
- count -= s;
- return count;
+ if (count_ < limit + s) throw jexception(jerr, "AtomicCounter", "subtractLimit");
+ count_ -= s;
+ return count_;
}
bool operator==(const T& o) const {
slock l(countMutex);
- return count == o;
+ return count_ == o;
}
bool operator<(const T& o) const {
slock l(countMutex);
- return count < o;
+ return count_ < o;
}
bool operator<=(const T& o) const {
slock l(countMutex);
- return count <= o;
+ return count_ <= o;
}
friend T operator-(const T& a, const AtomicCounter& b) {
slock l(b.countMutex);
- return a - b.count;
+ return a - b.count_;
}
friend T operator-(const AtomicCounter& a, const T& b) {
slock l(a.countMutex);
- return a.count - b;
+ return a.count_ - b;
}
friend T operator-(const AtomicCounter&a, const AtomicCounter& b) {
slock l1(a.countMutex);
slock l2(b.countMutex);
- return a.count - b.count;
+ return a.count_ - b.count_;
}
-
-/*
- friend std::ostream& operator<<(std::ostream& out, const AtomicCounter& a) {
- T temp; // Use temp so lock is not held while streaming to out.
- {
- slock l(a.countMutex);
- temp = a.count;
- }
- out << temp;
- return out;
- }
-*/
};
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.cpp
index 8d9cf8ce43..c78028362a 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.cpp
@@ -23,229 +23,274 @@
#include <cctype>
#include <fstream>
+#include "qpid/linearstore/jrnl/EmptyFilePoolPartition.h"
#include "qpid/linearstore/jrnl/jcfg.h"
#include "qpid/linearstore/jrnl/jdir.h"
#include "qpid/linearstore/jrnl/JournalFile.h"
+#include "qpid/linearstore/jrnl/JournalLog.h"
#include "qpid/linearstore/jrnl/slock.h"
#include "qpid/linearstore/jrnl/utils/file_hdr.h"
#include <sys/stat.h>
+#include <unistd.h>
#include <uuid/uuid.h>
#include <vector>
-#include <iostream> // DEBUG
-
namespace qpid {
namespace qls_jrnl {
-EmptyFilePool::EmptyFilePool(const std::string& efpDirectory_,
- const EmptyFilePoolPartition* partitionPtr_) :
- efpDirectory(efpDirectory_),
- efpDataSize_kib(fileSizeKbFromDirName(efpDirectory_, partitionPtr_->partitionNumber())),
- partitionPtr(partitionPtr_)
+EmptyFilePool::EmptyFilePool(const std::string& efpDirectory,
+ const EmptyFilePoolPartition* partitionPtr,
+ JournalLog& journalLogRef) :
+ efpDirectory_(efpDirectory),
+ efpDataSize_kib_(fileSizeKbFromDirName(efpDirectory, partitionPtr->getPartitionNumber())),
+ partitionPtr_(partitionPtr),
+ journalLogRef_(journalLogRef)
{}
EmptyFilePool::~EmptyFilePool() {}
-void
-EmptyFilePool::initialize() {
- //std::cout << "Reading " << efpDirectory << std::endl; // DEBUG
+void EmptyFilePool::initialize() {
std::vector<std::string> dirList;
- jdir::read_dir(efpDirectory, dirList, false, true, false, false);
+ jdir::read_dir(efpDirectory_, dirList, false, true, false, false);
for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) {
size_t dotPos = i->rfind(".");
if (dotPos != std::string::npos) {
if (i->substr(dotPos).compare(".jrnl") == 0 && i->length() == 41) {
- std::string emptyFile(efpDirectory + "/" + (*i));
+ std::string emptyFile(efpDirectory_ + "/" + (*i));
if (validateEmptyFile(emptyFile)) {
pushEmptyFile(emptyFile);
}
}
}
}
- //std::cout << "Found " << emptyFileList.size() << " files" << std::endl; // DEBUG
}
-efpDataSize_kib_t
-EmptyFilePool::dataSize_kib() const {
- return efpDataSize_kib;
+efpDataSize_kib_t EmptyFilePool::dataSize_kib() const {
+ return efpDataSize_kib_;
}
-efpFileSize_kib_t
-EmptyFilePool::fileSize_kib() const {
- return efpDataSize_kib + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_KIB);
+efpFileSize_kib_t EmptyFilePool::fileSize_kib() const {
+ return efpDataSize_kib_ + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB);
}
-efpDataSize_sblks_t
-EmptyFilePool::dataSize_sblks() const {
- return efpDataSize_kib / JRNL_SBLK_SIZE_KIB;
+efpDataSize_sblks_t EmptyFilePool::dataSize_sblks() const {
+ return efpDataSize_kib_ / QLS_SBLK_SIZE_KIB;
}
-efpFileSize_sblks_t
-EmptyFilePool::fileSize_sblks() const {
- return (efpDataSize_kib / JRNL_SBLK_SIZE_KIB) + QLS_JRNL_FHDR_RES_SIZE_SBLKS;
+efpFileSize_sblks_t EmptyFilePool::fileSize_sblks() const {
+ return (efpDataSize_kib_ / QLS_SBLK_SIZE_KIB) + QLS_JRNL_FHDR_RES_SIZE_SBLKS;
}
-efpFileCount_t
-EmptyFilePool::numEmptyFiles() const {
- slock l(emptyFileListMutex);
- return efpFileCount_t(emptyFileList.size());
+efpFileCount_t EmptyFilePool::numEmptyFiles() const {
+ slock l(emptyFileListMutex_);
+ return efpFileCount_t(emptyFileList_.size());
}
-efpDataSize_kib_t
-EmptyFilePool::cumFileSize_kib() const {
- slock l(emptyFileListMutex);
- return efpDataSize_kib_t(emptyFileList.size()) * efpDataSize_kib;
+efpDataSize_kib_t EmptyFilePool::cumFileSize_kib() const {
+ slock l(emptyFileListMutex_);
+ return efpDataSize_kib_t(emptyFileList_.size()) * efpDataSize_kib_;
}
-efpPartitionNumber_t
-EmptyFilePool::getPartitionNumber() const {
- return partitionPtr->partitionNumber();
+efpPartitionNumber_t EmptyFilePool::getPartitionNumber() const {
+ return partitionPtr_->getPartitionNumber();
}
-const EmptyFilePoolPartition*
-EmptyFilePool::getPartition() const {
- return partitionPtr;
+const EmptyFilePoolPartition* EmptyFilePool::getPartition() const {
+ return partitionPtr_;
}
-const efpIdentity_t
-EmptyFilePool::getIdentity() const {
- return efpIdentity_t(partitionPtr->partitionNumber(), efpDataSize_kib);
+const efpIdentity_t EmptyFilePool::getIdentity() const {
+ return efpIdentity_t(partitionPtr_->getPartitionNumber(), efpDataSize_kib_);
}
-std::string
-EmptyFilePool::takeEmptyFile(const std::string& destDirectory) {
+std::string EmptyFilePool::takeEmptyFile(const std::string& destDirectory) {
std::string emptyFileName = popEmptyFile();
std::string newFileName = destDirectory + emptyFileName.substr(emptyFileName.rfind('/')); // NOTE: substr() includes leading '/'
- if (::rename(emptyFileName.c_str(), newFileName.c_str())) {
- pushEmptyFile(emptyFileName);
- std::ostringstream oss;
- oss << "file=\"" << emptyFileName << "\" dest=\"" << newFileName << "\"" << FORMAT_SYSERR(errno);
- throw jexception(jerrno::JERR_JDIR_FMOVE, oss.str(), "EmptyFilePool", "takeEmptyFile");
+ if (moveEmptyFile(emptyFileName.c_str(), newFileName.c_str())) {
+ // Try again with new UUID for file name
+ newFileName = destDirectory + "/" + getEfpFileName();
+ if (moveEmptyFile(emptyFileName.c_str(), newFileName.c_str())) {
+ pushEmptyFile(emptyFileName);
+ std::ostringstream oss;
+ oss << "file=\"" << emptyFileName << "\" dest=\"" << newFileName << "\"" << FORMAT_SYSERR(errno);
+ throw jexception(jerrno::JERR_JDIR_FMOVE, oss.str(), "EmptyFilePool", "takeEmptyFile");
+ }
}
return newFileName;
}
-bool
-EmptyFilePool::returnEmptyFile(const JournalFile* srcFile) {
- std::string emptyFileName(efpDirectory + srcFile->getFileName());
- // TODO: reset file here
- if (::rename(srcFile->getFqFileName().c_str(), emptyFileName.c_str())) {
- std::ostringstream oss;
- oss << "file=\"" << srcFile << "\" dest=\"" << emptyFileName << "\"" << FORMAT_SYSERR(errno);
- throw jexception(jerrno::JERR_JDIR_FMOVE, oss.str(), "EmptyFilePool", "returnEmptyFile");
+void EmptyFilePool::returnEmptyFile(const std::string& fqSrcFile) {
+ std::string emptyFileName(efpDirectory_ + fqSrcFile.substr(fqSrcFile.rfind('/'))); // NOTE: substr() includes leading '/'
+ if (moveEmptyFile(fqSrcFile.c_str(), emptyFileName.c_str())) {
+ // Try again with new UUID for file name
+ emptyFileName = efpDirectory_ + "/" + getEfpFileName();
+ if (moveEmptyFile(fqSrcFile.c_str(), emptyFileName.c_str())) {
+ // Failed twice in a row - delete file
+ ::unlink(fqSrcFile.c_str());
+ return;
+ }
}
+ resetEmptyFileHeader(emptyFileName);
pushEmptyFile(emptyFileName);
- return true;
}
-// protected
+// --- protected functions ---
-void
-EmptyFilePool::pushEmptyFile(const std::string fqFileName_) {
- slock l(emptyFileListMutex);
- emptyFileList.push_back(fqFileName_);
+void EmptyFilePool::createEmptyFile() {
+ ::file_hdr_t fh;
+ ::file_hdr_create(&fh, QLS_FILE_MAGIC, QLS_JRNL_VERSION, QLS_JRNL_FHDR_RES_SIZE_SBLKS, partitionPtr_->getPartitionNumber(), efpDataSize_kib_);
+ std::string efpfn = getEfpFileName();
+ std::ofstream ofs(efpfn.c_str(), std::ofstream::out | std::ofstream::binary);
+ if (ofs.good()) {
+ ofs.write((char*)&fh, sizeof(::file_hdr_t));
+ uint64_t rem = ((efpDataSize_kib_ + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB)) * 1024) - sizeof(::file_hdr_t);
+ while (rem--)
+ ofs.put('\0');
+ ofs.close();
+ pushEmptyFile(efpfn);
+//std::cout << "WARNING: EFP " << efpDirectory << " is empty - created new journal file " << efpfn.substr(efpfn.rfind('/') + 1) << " on the fly" << std::endl;
+ } else {
+//std::cerr << "ERROR: Unable to open file \"" << efpfn << "\"" << std::endl; // DEBUG
+ }
}
-std::string
-EmptyFilePool::popEmptyFile() {
+std::string EmptyFilePool::getEfpFileName() {
+ uuid_t uuid;
+ ::uuid_generate(uuid); // NOTE: uuid_generate() is not thread safe
+ char uuid_str[37]; // 36 char uuid + trailing \0
+ ::uuid_unparse(uuid, uuid_str);
+ std::ostringstream oss;
+ oss << efpDirectory_ << "/" << uuid_str << QLS_JRNL_FILE_EXTENSION;
+ return oss.str();
+}
+
+std::string EmptyFilePool::popEmptyFile() {
std::string emptyFileName;
bool isEmpty = false;
{
- slock l(emptyFileListMutex);
- isEmpty = emptyFileList.empty();
+ slock l(emptyFileListMutex_);
+ isEmpty = emptyFileList_.empty();
}
if (isEmpty) {
createEmptyFile();
}
{
- slock l(emptyFileListMutex);
- emptyFileName = emptyFileList.front();
- emptyFileList.pop_front();
+ slock l(emptyFileListMutex_);
+ emptyFileName = emptyFileList_.front();
+ emptyFileList_.pop_front();
}
return emptyFileName;
}
-void
-EmptyFilePool::createEmptyFile() {
- ::file_hdr_t fh;
- ::file_hdr_create(&fh, QLS_FILE_MAGIC, QLS_JRNL_VERSION, QLS_JRNL_FHDR_RES_SIZE_SBLKS, partitionPtr->partitionNumber(), efpDataSize_kib);
- std::string efpfn = getEfpFileName();
- std::ofstream ofs(efpfn.c_str(), std::ofstream::out | std::ofstream::binary);
- if (ofs.good()) {
- ofs.write((char*)&fh, sizeof(::file_hdr_t));
- uint64_t rem = ((efpDataSize_kib + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_KIB)) * 1024) - sizeof(::file_hdr_t);
- while (rem--)
- ofs.put('\0');
- ofs.close();
- pushEmptyFile(efpfn);
- std::cout << "WARNING: EFP " << efpDirectory << " is empty - created new journal file " <<
- efpfn.substr(efpfn.rfind('/') + 1) << " on the fly" << std::endl;
+void EmptyFilePool::pushEmptyFile(const std::string fqFileName) {
+ slock l(emptyFileListMutex_);
+ emptyFileList_.push_back(fqFileName);
+}
+
+void EmptyFilePool::resetEmptyFileHeader(const std::string& fqFileName) {
+ std::fstream fs(fqFileName.c_str(), std::fstream::in | std::fstream::out | std::fstream::binary);
+ if (fs.good()) {
+ const std::streamsize buffsize = QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_BYTES;
+ char buff[buffsize];
+ fs.read((char*)buff, buffsize);
+ std::streampos bytesRead = fs.tellg();
+ if (bytesRead == buffsize) {
+ ::file_hdr_reset((::file_hdr_t*)buff);
+ ::memset(buff + sizeof(::file_hdr_t), 0, MAX_FILE_HDR_LEN - sizeof(::file_hdr_t)); // set rest of buffer to 0
+ fs.seekp(0, std::fstream::beg);
+ fs.write(buff, buffsize);
+ std::streampos bytesWritten = fs.tellp();
+ if (bytesWritten != buffsize) {
+//std::cerr << "ERROR: Unable to write file header of file \"" << fqFileName_ << "\": tried to write " << buffsize << " bytes; wrote " << bytesWritten << " bytes." << std::endl;
+ }
+ } else {
+//std::cerr << "ERROR: Unable to read file header of file \"" << fqFileName_ << "\": tried to read " << sizeof(::file_hdr_t) << " bytes; read " << bytesRead << " bytes." << std::endl;
+ }
+ fs.close();
} else {
- std::cerr << "ERROR: Unable to open file \"" << efpfn << "\"" << std::endl; // DEBUG
+//std::cerr << "ERROR: Unable to open file \"" << fqFileName_ << "\" for reading" << std::endl; // DEBUG
}
}
-bool
-EmptyFilePool::validateEmptyFile(const std::string& emptyFileName_) const {
+bool EmptyFilePool::validateEmptyFile(const std::string& emptyFileName) const {
+ std::ostringstream oss;
struct stat s;
- if (::stat(emptyFileName_.c_str(), &s))
+ if (::stat(emptyFileName.c_str(), &s))
{
- std::ostringstream oss;
- oss << "stat: file=\"" << emptyFileName_ << "\"" << FORMAT_SYSERR(errno);
+ oss << "stat: file=\"" << emptyFileName << "\"" << FORMAT_SYSERR(errno);
throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "EmptyFilePool", "validateEmptyFile");
}
- efpDataSize_kib_t expectedSize = (JRNL_SBLK_SIZE_KIB + efpDataSize_kib) * 1024;
+
+ // Size matches pool
+ efpDataSize_kib_t expectedSize = (QLS_SBLK_SIZE_KIB + efpDataSize_kib_) * 1024;
if ((efpDataSize_kib_t)s.st_size != expectedSize) {
- //std::cout << "ERROR: File " << emptyFileName << ": Incorrect size: Expected=" << expectedSize << "; actual=" << s.st_size << std::endl; // DEBUG
+ oss << "ERROR: File " << emptyFileName << ": Incorrect size: Expected=" << expectedSize << "; actual=" << s.st_size;
+ journalLogRef_.log(JournalLog::LOG_ERROR, oss.str());
return false;
}
- std::ifstream ifs(emptyFileName_.c_str(), std::ifstream::in | std::ifstream::binary);
- if (!ifs) {
- //std::cout << "ERROR: File " << emptyFileName << ": Unable to open for reading" << std::endl;
+ // Open file and read header
+ std::fstream fs(emptyFileName.c_str(), std::fstream::in | std::fstream::out | std::fstream::binary);
+ if (!fs) {
+ oss << "ERROR: File " << emptyFileName << ": Unable to open for reading";
+ journalLogRef_.log(JournalLog::LOG_ERROR, oss.str());
+ return false;
+ }
+ const std::streamsize buffsize = QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_BYTES;
+ char buff[buffsize];
+ fs.read((char*)buff, buffsize);
+ std::streampos bytesRead = fs.tellg();
+ if (bytesRead != buffsize) {
+ oss << "ERROR: Unable to read file header of file \"" << emptyFileName << "\": tried to read " << buffsize << " bytes; read " << bytesRead << " bytes";
+ journalLogRef_.log(JournalLog::LOG_ERROR, oss.str());
+ fs.close();
return false;
}
- const uint8_t fhFileNameBuffLen = 50;
- char fhFileNameBuff[fhFileNameBuffLen];
- ::file_hdr_t fh;
- ifs.read((char*)&fh, sizeof(::file_hdr_t));
- uint16_t fhFileNameLen = fh._queue_name_len > fhFileNameBuffLen ? fhFileNameBuffLen : fh._queue_name_len;
- ifs.read(fhFileNameBuff, fhFileNameLen);
- std::string fhFileName(fhFileNameBuff, fhFileNameLen);
- ifs.close();
-
- if (fh._rhdr._magic != QLS_FILE_MAGIC ||
- fh._rhdr._version != QLS_JRNL_VERSION ||
- fh._efp_partition != partitionPtr->partitionNumber() ||
- fh._file_size_kib != efpDataSize_kib ||
- !::is_file_hdr_reset(&fh))
+ // Check file header
+ const bool jrnlMagicError = ((::file_hdr_t*)buff)->_rhdr._magic != QLS_FILE_MAGIC;
+ const bool jrnlVersionError = ((::file_hdr_t*)buff)->_rhdr._version != QLS_JRNL_VERSION;
+ const bool jrnlPartitionError = ((::file_hdr_t*)buff)->_efp_partition != partitionPtr_->getPartitionNumber();
+ const bool jrnlFileSizeError = ((::file_hdr_t*)buff)->_file_size_kib != efpDataSize_kib_;
+ if (jrnlMagicError || jrnlVersionError || jrnlPartitionError || jrnlFileSizeError)
{
- //std::cout << "ERROR: File " << emptyFileName << ": Invalid file header" << std::endl;
+ oss << "ERROR: File " << emptyFileName << ": Invalid file header - mismatched header fields: " <<
+ (jrnlMagicError ? "magic " : "") <<
+ (jrnlVersionError ? "version " : "") <<
+ (jrnlPartitionError ? "partition" : "") <<
+ (jrnlFileSizeError ? "file-size" : "");
+ journalLogRef_.log(JournalLog::LOG_ERROR, oss.str());
+ fs.close();
return false;
}
- return true;
-}
+ // Check file header is reset
+ if (!::is_file_hdr_reset((::file_hdr_t*)buff)) {
+ ::file_hdr_reset((::file_hdr_t*)buff);
+ ::memset(buff + sizeof(::file_hdr_t), 0, MAX_FILE_HDR_LEN - sizeof(::file_hdr_t)); // set rest of buffer to 0
+ fs.seekp(0, std::fstream::beg);
+ fs.write(buff, buffsize);
+ std::streampos bytesWritten = fs.tellp();
+ if (bytesWritten != buffsize) {
+ oss << "ERROR: Unable to write file header of file \"" << emptyFileName << "\": tried to write " << buffsize << " bytes; wrote " << bytesWritten << " bytes";
+ journalLogRef_.log(JournalLog::LOG_ERROR, oss.str());
+ fs.close();
+ return false;
+ }
+ oss << "WARNING: File " << emptyFileName << ": File header not reset";
+ journalLogRef_.log(JournalLog::LOG_WARN, oss.str());
+ }
-std::string
-EmptyFilePool::getEfpFileName() {
- uuid_t uuid;
- ::uuid_generate(uuid); // NOTE: NOT THREAD SAFE
- char uuid_str[37]; // 36 char uuid + trailing \0
- ::uuid_unparse(uuid, uuid_str);
- std::ostringstream oss;
- oss << efpDirectory << "/" << uuid_str << QLS_JRNL_FILE_EXTENSION;
- return oss.str();
+ // Close file
+ fs.close();
+ return true;
}
-// protected
// static
-efpDataSize_kib_t
-EmptyFilePool::fileSizeKbFromDirName(const std::string& dirName_,
- const efpPartitionNumber_t partitionNumber_) {
+efpDataSize_kib_t EmptyFilePool::fileSizeKbFromDirName(const std::string& dirName,
+ const efpPartitionNumber_t partitionNumber) {
// Check for dirName format 'NNNk', where NNN is a number, convert NNN into an integer. NNN cannot be 0.
- std::string n(dirName_.substr(dirName_.rfind('/')+1));
+ std::string n(dirName.substr(dirName.rfind('/')+1));
bool valid = true;
for (uint16_t charNum = 0; charNum < n.length(); ++charNum) {
if (charNum < n.length()-1) {
@@ -258,12 +303,24 @@ EmptyFilePool::fileSizeKbFromDirName(const std::string& dirName_,
}
}
efpDataSize_kib_t s = ::atol(n.c_str());
- if (!valid || s == 0 || s % JRNL_SBLK_SIZE_KIB != 0) {
+ if (!valid || s == 0 || s % QLS_SBLK_SIZE_KIB != 0) {
std::ostringstream oss;
- oss << "Partition: " << partitionNumber_ << "; EFP directory: \'" << n << "\'";
+ oss << "Partition: " << partitionNumber << "; EFP directory: \'" << n << "\'";
throw jexception(jerrno::JERR_EFP_BADEFPDIRNAME, oss.str(), "EmptyFilePool", "fileSizeKbFromDirName");
}
return s;
}
+// static
+int EmptyFilePool::moveEmptyFile(const std::string& from,
+ const std::string& to) {
+ if (::rename(from.c_str(), to.c_str())) {
+ if (errno == EEXIST) return errno; // File name exists
+ std::ostringstream oss;
+ oss << "file=\"" << from << "\" dest=\"" << to << "\"" << FORMAT_SYSERR(errno);
+ throw jexception(jerrno::JERR_JDIR_FMOVE, oss.str(), "EmptyFilePool", "returnEmptyFile");
+ }
+ return 0;
+}
+
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.h b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.h
index 4de12fdbf3..e4eeeaa042 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePool.h
@@ -30,15 +30,16 @@ namespace qls_jrnl {
}} // namespace qpid::qls_jrnl
#include <deque>
-#include "qpid/linearstore/jrnl/EmptyFilePoolPartition.h"
#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h"
#include "qpid/linearstore/jrnl/smutex.h"
#include <string>
namespace qpid {
namespace qls_jrnl {
+class EmptyFilePoolPartition;
class jdir;
class JournalFile;
+class JournalLog;
class EmptyFilePool
{
@@ -46,17 +47,19 @@ protected:
typedef std::deque<std::string> emptyFileList_t;
typedef emptyFileList_t::iterator emptyFileListItr_t;
- const std::string efpDirectory;
- const efpDataSize_kib_t efpDataSize_kib;
- const EmptyFilePoolPartition* partitionPtr;
+ const std::string efpDirectory_;
+ const efpDataSize_kib_t efpDataSize_kib_;
+ const EmptyFilePoolPartition* partitionPtr_;
+ JournalLog& journalLogRef_;
private:
- emptyFileList_t emptyFileList;
- smutex emptyFileListMutex;
+ emptyFileList_t emptyFileList_;
+ smutex emptyFileListMutex_;
public:
- EmptyFilePool(const std::string& efpDirectory_,
- const EmptyFilePoolPartition* partitionPtr_);
+ EmptyFilePool(const std::string& efpDirectory,
+ const EmptyFilePoolPartition* partitionPtr,
+ JournalLog& journalLogRef);
virtual ~EmptyFilePool();
void initialize();
@@ -70,17 +73,21 @@ public:
const EmptyFilePoolPartition* getPartition() const;
const efpIdentity_t getIdentity() const;
- std::string takeEmptyFile(const std::string& destDirectory_);
- bool returnEmptyFile(const JournalFile* srcFile_);
+ std::string takeEmptyFile(const std::string& destDirectory);
+ void returnEmptyFile(const std::string& srcFile);
protected:
- void pushEmptyFile(const std::string fqFileName_);
- std::string popEmptyFile();
void createEmptyFile();
- bool validateEmptyFile(const std::string& emptyFileName_) const;
std::string getEfpFileName();
- static efpDataSize_kib_t fileSizeKbFromDirName(const std::string& dirName_,
- const efpPartitionNumber_t partitionNumber_);
+ std::string popEmptyFile();
+ void pushEmptyFile(const std::string fqFileName);
+ void resetEmptyFileHeader(const std::string& fqFileName);
+ bool validateEmptyFile(const std::string& emptyFileName) const;
+
+ static efpDataSize_kib_t fileSizeKbFromDirName(const std::string& dirName,
+ const efpPartitionNumber_t partitionNumber);
+ static int moveEmptyFile(const std::string& fromFqPath,
+ const std::string& toFqPath);
};
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.cpp
index dfee91f6b5..a9b3b12b81 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.cpp
@@ -27,39 +27,37 @@
#include "qpid/linearstore/jrnl/slock.h"
#include <vector>
-// DEBUG
-//#include <iostream>
-
namespace qpid {
namespace qls_jrnl {
-EmptyFilePoolManager::EmptyFilePoolManager(const std::string& qlsStorePath_) :
- qlsStorePath(qlsStorePath_)
+EmptyFilePoolManager::EmptyFilePoolManager(const std::string& qlsStorePath,
+ JournalLog& journalLogRef) :
+ qlsStorePath_(qlsStorePath),
+ journalLogRef_(journalLogRef)
{}
EmptyFilePoolManager::~EmptyFilePoolManager() {
- slock l(partitionMapMutex);
- for (partitionMapItr_t i = partitionMap.begin(); i != partitionMap.end(); ++i) {
+ slock l(partitionMapMutex_);
+ for (partitionMapItr_t i = partitionMap_.begin(); i != partitionMap_.end(); ++i) {
delete i->second;
}
- partitionMap.clear();
+ partitionMap_.clear();
}
-void
-EmptyFilePoolManager::findEfpPartitions() {
+void EmptyFilePoolManager::findEfpPartitions() {
//std::cout << "*** Reading " << qlsStorePath << std::endl; // DEBUG
std::vector<std::string> dirList;
- jdir::read_dir(qlsStorePath, dirList, true, false, true, false);
+ jdir::read_dir(qlsStorePath_, dirList, true, false, true, false);
for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) {
if ((*i)[0] == 'p' && i->length() == 4) { // Filter: look only at names pNNN
efpPartitionNumber_t pn = ::atoi(i->c_str() + 1);
- std::string fullDirPath(qlsStorePath + "/" + (*i));
+ std::string fullDirPath(qlsStorePath_ + "/" + (*i));
EmptyFilePoolPartition* efppp = 0;
try {
- efppp = new EmptyFilePoolPartition(pn, fullDirPath);
+ efppp = new EmptyFilePoolPartition(pn, fullDirPath, journalLogRef_);
{
- slock l(partitionMapMutex);
- partitionMap[pn] = efppp;
+ slock l(partitionMapMutex_);
+ partitionMap_[pn] = efppp;
}
} catch (const std::exception& e) {
if (efppp != 0) {
@@ -72,34 +70,63 @@ EmptyFilePoolManager::findEfpPartitions() {
efppp->findEmptyFilePools();
}
}
+ // TODO: Log results
+/*
+ QLS_LOG(info, "EFP Manager initialization complete");
+ std::vector<qpid::qls_jrnl::EmptyFilePoolPartition*> partitionList;
+ std::vector<qpid::qls_jrnl::EmptyFilePool*> filePoolList;
+ getEfpPartitions(partitionList);
+ if (partitionList.size() == 0) {
+ QLS_LOG(error, "NO EFP PARTITIONS FOUND! No queue creation is possible.")
+ } else {
+ QLS_LOG(info, "> EFP Partitions found: " << partitionList.size());
+ for (std::vector<qpid::qls_jrnl::EmptyFilePoolPartition*>::const_iterator i=partitionList.begin(); i!= partitionList.end(); ++i) {
+ filePoolList.clear();
+ (*i)->getEmptyFilePools(filePoolList);
+ QLS_LOG(info, " * Partition " << (*i)->partitionNumber() << " containing " << filePoolList.size() << " pool" <<
+ (filePoolList.size()>1 ? "s" : "") << " at \'" << (*i)->partitionDirectory() << "\'");
+ for (std::vector<qpid::qls_jrnl::EmptyFilePool*>::const_iterator j=filePoolList.begin(); j!=filePoolList.end(); ++j) {
+ QLS_LOG(info, " - EFP \'" << (*j)->dataSize_kib() << "k\' containing " << (*j)->numEmptyFiles() <<
+ " files of size " << (*j)->dataSize_kib() << " KiB totaling " << (*j)->cumFileSize_kib() << " KiB");
+ }
+ }
+ }
+*/
}
-uint16_t
-EmptyFilePoolManager::getNumEfpPartitions() const {
- return partitionMap.size();
+void EmptyFilePoolManager::getEfpFileSizes(std::vector<efpDataSize_kib_t>& efpFileSizeList,
+ const efpPartitionNumber_t efpPartitionNumber) const {
+ if (efpPartitionNumber == 0) {
+ for (partitionMapConstItr_t i=partitionMap_.begin(); i!=partitionMap_.end(); ++i) {
+ i->second->getEmptyFilePoolSizes_kib(efpFileSizeList);
+ }
+ } else {
+ partitionMapConstItr_t i = partitionMap_.find(efpPartitionNumber);
+ if (i != partitionMap_.end()) {
+ i->second->getEmptyFilePoolSizes_kib(efpFileSizeList);
+ }
+ }
}
-EmptyFilePoolPartition*
-EmptyFilePoolManager::getEfpPartition(const efpPartitionNumber_t partitionNumber) {
- partitionMapItr_t i = partitionMap.find(partitionNumber);
- if (i == partitionMap.end())
+EmptyFilePoolPartition* EmptyFilePoolManager::getEfpPartition(const efpPartitionNumber_t partitionNumber) {
+ partitionMapItr_t i = partitionMap_.find(partitionNumber);
+ if (i == partitionMap_.end())
return 0;
else
return i->second;
}
-void
-EmptyFilePoolManager::getEfpPartitionNumbers(std::vector<efpPartitionNumber_t>& partitionNumberList,
- const efpDataSize_kib_t efpFileSizeKb) const {
- slock l(partitionMapMutex);
- for (partitionMapConstItr_t i=partitionMap.begin(); i!=partitionMap.end(); ++i) {
- if (efpFileSizeKb == 0) {
+void EmptyFilePoolManager::getEfpPartitionNumbers(std::vector<efpPartitionNumber_t>& partitionNumberList,
+ const efpDataSize_kib_t efpDataSize_kib) const {
+ slock l(partitionMapMutex_);
+ for (partitionMapConstItr_t i=partitionMap_.begin(); i!=partitionMap_.end(); ++i) {
+ if (efpDataSize_kib == 0) {
partitionNumberList.push_back(i->first);
} else {
std::vector<efpDataSize_kib_t> efpFileSizeList;
- i->second->getEmptyFilePoolSizesKb(efpFileSizeList);
+ i->second->getEmptyFilePoolSizes_kib(efpFileSizeList);
for (std::vector<efpDataSize_kib_t>::iterator j=efpFileSizeList.begin(); j!=efpFileSizeList.end(); ++j) {
- if (*j == efpFileSizeKb) {
+ if (*j == efpDataSize_kib) {
partitionNumberList.push_back(i->first);
break;
}
@@ -108,18 +135,17 @@ EmptyFilePoolManager::getEfpPartitionNumbers(std::vector<efpPartitionNumber_t>&
}
}
-void
-EmptyFilePoolManager::getEfpPartitions(std::vector<EmptyFilePoolPartition*>& partitionList,
- const efpDataSize_kib_t efpFileSizeKb) {
- slock l(partitionMapMutex);
- for (partitionMapConstItr_t i=partitionMap.begin(); i!=partitionMap.end(); ++i) {
- if (efpFileSizeKb == 0) {
+void EmptyFilePoolManager::getEfpPartitions(std::vector<EmptyFilePoolPartition*>& partitionList,
+ const efpDataSize_kib_t efpDataSize_kib) {
+ slock l(partitionMapMutex_);
+ for (partitionMapConstItr_t i=partitionMap_.begin(); i!=partitionMap_.end(); ++i) {
+ if (efpDataSize_kib == 0) {
partitionList.push_back(i->second);
} else {
std::vector<efpDataSize_kib_t> efpFileSizeList;
- i->second->getEmptyFilePoolSizesKb(efpFileSizeList);
+ i->second->getEmptyFilePoolSizes_kib(efpFileSizeList);
for (std::vector<efpDataSize_kib_t>::iterator j=efpFileSizeList.begin(); j!=efpFileSizeList.end(); ++j) {
- if (*j == efpFileSizeKb) {
+ if (*j == efpDataSize_kib) {
partitionList.push_back(i->second);
break;
}
@@ -128,48 +154,34 @@ EmptyFilePoolManager::getEfpPartitions(std::vector<EmptyFilePoolPartition*>& par
}
}
-void
-EmptyFilePoolManager::getEfpFileSizes(std::vector<efpDataSize_kib_t>& efpFileSizeList,
- const efpPartitionNumber_t efpPartitionNumber) const {
- if (efpPartitionNumber == 0) {
- for (partitionMapConstItr_t i=partitionMap.begin(); i!=partitionMap.end(); ++i) {
- i->second->getEmptyFilePoolSizesKb(efpFileSizeList);
- }
- } else {
- partitionMapConstItr_t i = partitionMap.find(efpPartitionNumber);
- if (i != partitionMap.end()) {
- i->second->getEmptyFilePoolSizesKb(efpFileSizeList);
- }
- }
+EmptyFilePool* EmptyFilePoolManager::getEmptyFilePool(const efpIdentity_t efpIdentity) {
+ return getEmptyFilePool(efpIdentity.first, efpIdentity.second);
+}
+
+EmptyFilePool* EmptyFilePoolManager::getEmptyFilePool(const efpPartitionNumber_t partitionNumber,
+ const efpDataSize_kib_t efpDataSize_kib) {
+ EmptyFilePoolPartition* efppp = getEfpPartition(partitionNumber);
+ if (efppp != 0)
+ return efppp->getEmptyFilePool(efpDataSize_kib);
+ return 0;
}
-void
-EmptyFilePoolManager::getEmptyFilePools(std::vector<EmptyFilePool*>& emptyFilePoolList,
- const efpPartitionNumber_t efpPartitionNumber) {
+void EmptyFilePoolManager::getEmptyFilePools(std::vector<EmptyFilePool*>& emptyFilePoolList,
+ const efpPartitionNumber_t efpPartitionNumber) {
if (efpPartitionNumber == 0) {
- for (partitionMapConstItr_t i=partitionMap.begin(); i!=partitionMap.end(); ++i) {
+ for (partitionMapConstItr_t i=partitionMap_.begin(); i!=partitionMap_.end(); ++i) {
i->second->getEmptyFilePools(emptyFilePoolList);
}
} else {
- partitionMapConstItr_t i = partitionMap.find(efpPartitionNumber);
- if (i != partitionMap.end()) {
+ partitionMapConstItr_t i = partitionMap_.find(efpPartitionNumber);
+ if (i != partitionMap_.end()) {
i->second->getEmptyFilePools(emptyFilePoolList);
}
}
}
-EmptyFilePool*
-EmptyFilePoolManager::getEmptyFilePool(const efpPartitionNumber_t partitionNumber,
- const efpDataSize_kib_t efpFileSizeKib) {
- EmptyFilePoolPartition* efppp = getEfpPartition(partitionNumber);
- if (efppp != 0)
- return efppp->getEmptyFilePool(efpFileSizeKib);
- return 0;
-}
-
-EmptyFilePool*
-EmptyFilePoolManager::getEmptyFilePool(const efpIdentity_t efpIdentity) {
- return getEmptyFilePool(efpIdentity.first, efpIdentity.second);
+uint16_t EmptyFilePoolManager::getNumEfpPartitions() const {
+ return partitionMap_.size();
}
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.h b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.h
index b25f5ac5e5..eab3783d90 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolManager.h
@@ -25,7 +25,6 @@
#include <map>
#include "qpid/linearstore/jrnl/EmptyFilePoolPartition.h"
#include "qpid/linearstore/jrnl/smutex.h"
-#include <string>
namespace qpid {
namespace qls_jrnl {
@@ -37,25 +36,30 @@ protected:
typedef partitionMap_t::iterator partitionMapItr_t;
typedef partitionMap_t::const_iterator partitionMapConstItr_t;
- std::string qlsStorePath;
- partitionMap_t partitionMap;
- smutex partitionMapMutex;
+ std::string qlsStorePath_;
+ JournalLog& journalLogRef_;
+ partitionMap_t partitionMap_;
+ smutex partitionMapMutex_;
public:
- EmptyFilePoolManager(const std::string& qlsStorePath_);
+ EmptyFilePoolManager(const std::string& qlsStorePath_,
+ JournalLog& journalLogRef_);
virtual ~EmptyFilePoolManager();
- void findEfpPartitions();
- uint16_t getNumEfpPartitions() const;
+ void findEfpPartitions();
+ void getEfpFileSizes(std::vector<efpDataSize_kib_t>& efpFileSizeList,
+ const efpPartitionNumber_t efpPartitionNumber = 0) const;
EmptyFilePoolPartition* getEfpPartition(const efpPartitionNumber_t partitionNumber);
- void getEfpPartitionNumbers(std::vector<efpPartitionNumber_t>& partitionNumberList, const efpDataSize_kib_t efpFileSizeKb = 0) const;
- void getEfpPartitions(std::vector<EmptyFilePoolPartition*>& partitionList, const efpDataSize_kib_t efpFileSizeKb = 0);
-
- void getEfpFileSizes(std::vector<efpDataSize_kib_t>& efpFileSizeList, const efpPartitionNumber_t efpPartitionNumber = 0) const;
- void getEmptyFilePools(std::vector<EmptyFilePool*>& emptyFilePoolList, const efpPartitionNumber_t efpPartitionNumber = 0);
-
- EmptyFilePool* getEmptyFilePool(const efpPartitionNumber_t partitionNumber, const efpDataSize_kib_t efpFileSizeKb);
+ void getEfpPartitionNumbers(std::vector<efpPartitionNumber_t>& partitionNumberList,
+ const efpDataSize_kib_t efpDataSize_kib = 0) const;
+ void getEfpPartitions(std::vector<EmptyFilePoolPartition*>& partitionList,
+ const efpDataSize_kib_t efpDataSize_kib = 0);
EmptyFilePool* getEmptyFilePool(const efpIdentity_t efpIdentity);
+ EmptyFilePool* getEmptyFilePool(const efpPartitionNumber_t partitionNumber,
+ const efpDataSize_kib_t efpDataSize_kib);
+ void getEmptyFilePools(std::vector<EmptyFilePool*>& emptyFilePoolList,
+ const efpPartitionNumber_t efpPartitionNumber = 0);
+ uint16_t getNumEfpPartitions() const;
};
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp
index 70840d5e8d..4d8b697b1d 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.cpp
@@ -27,62 +27,54 @@
#include "qpid/linearstore/jrnl/jexception.h"
#include "qpid/linearstore/jrnl/slock.h"
-//#include <iostream> // DEBUG
-
namespace qpid {
namespace qls_jrnl {
-const std::string EmptyFilePoolPartition::efpTopLevelDir("efp"); // Sets the top-level efp dir within a partition
+// static
+const std::string EmptyFilePoolPartition::s_efpTopLevelDir_("efp"); // Sets the top-level efp dir within a partition
-EmptyFilePoolPartition::EmptyFilePoolPartition(const efpPartitionNumber_t partitionNum_, const std::string& partitionDir_) :
- partitionNum(partitionNum_),
- partitionDir(partitionDir_)
+EmptyFilePoolPartition::EmptyFilePoolPartition(const efpPartitionNumber_t partitionNum,
+ const std::string& partitionDir,
+ JournalLog& journalLogRef) :
+ partitionNum_(partitionNum),
+ partitionDir_(partitionDir),
+ journalLogRef_(journalLogRef)
{
validatePartitionDir();
}
EmptyFilePoolPartition::~EmptyFilePoolPartition() {
- slock l(efpMapMutex);
- for (efpMapItr_t i = efpMap.begin(); i != efpMap.end(); ++i) {
+ slock l(efpMapMutex_);
+ for (efpMapItr_t i = efpMap_.begin(); i != efpMap_.end(); ++i) {
delete i->second;
}
- efpMap.clear();
-}
-
-void
-EmptyFilePoolPartition::validatePartitionDir() {
- if (!jdir::is_dir(partitionDir)) {
- std::ostringstream ss;
- ss << "Invalid partition directory: \'" << partitionDir << "\' is not a directory";
- throw jexception(jerrno::JERR_EFP_BADPARTITIONDIR, ss.str(), "EmptyFilePoolPartition", "validatePartitionDir");
- }
- // TODO: other validity checks here
+ efpMap_.clear();
}
void
EmptyFilePoolPartition::findEmptyFilePools() {
//std::cout << "Reading " << partitionDir << std::endl; // DEBUG
std::vector<std::string> dirList;
- jdir::read_dir(partitionDir, dirList, true, false, false, false);
+ jdir::read_dir(partitionDir_, dirList, true, false, false, false);
bool foundEfpDir = false;
for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) {
- if (i->compare(efpTopLevelDir) == 0) {
+ if (i->compare(s_efpTopLevelDir_) == 0) {
foundEfpDir = true;
break;
}
}
if (foundEfpDir) {
- std::string efpDir(partitionDir + "/" + efpTopLevelDir);
+ std::string efpDir(partitionDir_ + "/" + s_efpTopLevelDir_);
//std::cout << "Reading " << efpDir << std::endl; // DEBUG
dirList.clear();
jdir::read_dir(efpDir, dirList, true, false, false, true);
for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) {
EmptyFilePool* efpp = 0;
try {
- efpp = new EmptyFilePool(*i, this);
+ efpp = new EmptyFilePool(*i, this, journalLogRef_);
{
- slock l(efpMapMutex);
- efpMap[efpp->dataSize_kib()] = efpp;
+ slock l(efpMapMutex_);
+ efpMap_[efpp->dataSize_kib()] = efpp;
}
}
catch (const std::exception& e) {
@@ -98,36 +90,42 @@ EmptyFilePoolPartition::findEmptyFilePools() {
}
}
-efpPartitionNumber_t
-EmptyFilePoolPartition::partitionNumber() const {
- return partitionNum;
+EmptyFilePool* EmptyFilePoolPartition::getEmptyFilePool(const efpDataSize_kib_t efpDataSize_kib) {
+ efpMapItr_t i = efpMap_.find(efpDataSize_kib);
+ if (i == efpMap_.end())
+ return 0;
+ return i->second;
}
-std::string
-EmptyFilePoolPartition::partitionDirectory() const {
- return partitionDir;
+void EmptyFilePoolPartition::getEmptyFilePools(std::vector<EmptyFilePool*>& efpList) {
+ for (efpMapItr_t i=efpMap_.begin(); i!=efpMap_.end(); ++i) {
+ efpList.push_back(i->second);
+ }
}
-EmptyFilePool*
-EmptyFilePoolPartition::getEmptyFilePool(const efpDataSize_kib_t efpFileSizeKb) {
- efpMapItr_t i = efpMap.find(efpFileSizeKb);
- if (i == efpMap.end())
- return 0;
- return i->second;
+void EmptyFilePoolPartition::getEmptyFilePoolSizes_kib(std::vector<efpDataSize_kib_t>& efpDataSizesList_kib) const {
+ for (efpMapConstItr_t i=efpMap_.begin(); i!=efpMap_.end(); ++i) {
+ efpDataSizesList_kib.push_back(i->first);
+ }
}
-void
-EmptyFilePoolPartition::getEmptyFilePoolSizesKb(std::vector<efpDataSize_kib_t>& efpFileSizesKbList) const {
- for (efpMapConstItr_t i=efpMap.begin(); i!=efpMap.end(); ++i) {
- efpFileSizesKbList.push_back(i->first);
- }
+std::string EmptyFilePoolPartition::getPartitionDirectory() const {
+ return partitionDir_;
}
-void
-EmptyFilePoolPartition::getEmptyFilePools(std::vector<EmptyFilePool*>& efpList) {
- for (efpMapItr_t i=efpMap.begin(); i!=efpMap.end(); ++i) {
- efpList.push_back(i->second);
+efpPartitionNumber_t EmptyFilePoolPartition::getPartitionNumber() const {
+ return partitionNum_;
+}
+
+// --- protected functions ---
+
+void EmptyFilePoolPartition::validatePartitionDir() {
+ if (!jdir::is_dir(partitionDir_)) {
+ std::ostringstream ss;
+ ss << "Invalid partition directory: \'" << partitionDir_ << "\' is not a directory";
+ throw jexception(jerrno::JERR_EFP_BADPARTITIONDIR, ss.str(), "EmptyFilePoolPartition", "validatePartitionDir");
}
+ // TODO: other validity checks here
}
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.h b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.h
index 1172c84ab7..5b8ba7b4fa 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolPartition.h
@@ -38,33 +38,38 @@ namespace qls_jrnl {
namespace qpid {
namespace qls_jrnl {
+class JournalLog;
class EmptyFilePoolPartition
{
public:
- static const std::string efpTopLevelDir;
+ static const std::string s_efpTopLevelDir_;
protected:
typedef std::map<efpDataSize_kib_t, EmptyFilePool*> efpMap_t;
typedef efpMap_t::iterator efpMapItr_t;
typedef efpMap_t::const_iterator efpMapConstItr_t;
- const efpPartitionNumber_t partitionNum;
- const std::string partitionDir;
- efpMap_t efpMap;
- smutex efpMapMutex;
-
- void validatePartitionDir();
+ const efpPartitionNumber_t partitionNum_;
+ const std::string partitionDir_;
+ JournalLog& journalLogRef_;
+ efpMap_t efpMap_;
+ smutex efpMapMutex_;
public:
- EmptyFilePoolPartition(const efpPartitionNumber_t partitionNum_, const std::string& partitionDir_);
+ EmptyFilePoolPartition(const efpPartitionNumber_t partitionNum,
+ const std::string& partitionDir,
+ JournalLog& journalLogRef);
virtual ~EmptyFilePoolPartition();
- void findEmptyFilePools();
- efpPartitionNumber_t partitionNumber() const;
- std::string partitionDirectory() const;
- EmptyFilePool* getEmptyFilePool(const efpDataSize_kib_t efpFileSizeKb);
- void getEmptyFilePoolSizesKb(std::vector<efpDataSize_kib_t>& efpFileSizesKbList) const;
+ void findEmptyFilePools();
+ EmptyFilePool* getEmptyFilePool(const efpDataSize_kib_t efpDataSize_kib);
void getEmptyFilePools(std::vector<EmptyFilePool*>& efpList);
+ void getEmptyFilePoolSizes_kib(std::vector<efpDataSize_kib_t>& efpDataSizesList) const;
+ std::string getPartitionDirectory() const;
+ efpPartitionNumber_t getPartitionNumber() const;
+
+protected:
+ void validatePartitionDir();
};
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolTypes.h b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolTypes.h
index bdb77d8c8d..4c470e4e40 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolTypes.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/EmptyFilePoolTypes.h
@@ -28,13 +28,13 @@
namespace qpid {
namespace qls_jrnl {
- typedef uint64_t efpDataSize_kib_t; // Size of data part of file (excluding file header) in kib
- typedef uint64_t efpFileSize_kib_t; // Size of file (header + data) in kib
- typedef uint32_t efpDataSize_sblks_t; // Size of data part of file (excluding file header) in sblks
- typedef uint32_t efpFileSize_sblks_t; // Size of file (header + data) in sblks
- typedef uint32_t efpFileCount_t;
- typedef uint16_t efpPartitionNumber_t;
- typedef std::pair<efpPartitionNumber_t, efpDataSize_kib_t> efpIdentity_t;
+ typedef uint64_t efpDataSize_kib_t; ///< Size of data part of file (excluding file header) in kib
+ typedef uint64_t efpFileSize_kib_t; ///< Size of file (header + data) in kib
+ typedef uint32_t efpDataSize_sblks_t; ///< Size of data part of file (excluding file header) in sblks
+ typedef uint32_t efpFileSize_sblks_t; ///< Size of file (header + data) in sblks
+ typedef uint32_t efpFileCount_t; ///< Number of files in a partition or pool
+ typedef uint16_t efpPartitionNumber_t; ///< Number assigned to a partition
+ typedef std::pair<efpPartitionNumber_t, efpDataSize_kib_t> efpIdentity_t; ///< Unique identity of a pool, consisting of the partition number and data size
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.cpp
index af953aa4ef..e02e965823 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.cpp
@@ -31,21 +31,21 @@
namespace qpid {
namespace qls_jrnl {
-JournalFile::JournalFile(const std::string& fqFileName_,
- const uint64_t fileSeqNum_,
- const uint32_t fileSize_kib_) :
- fqFileName(fqFileName_),
- fileSeqNum(fileSeqNum_),
- fileHandle(-1),
- fileCloseFlag(false),
- fileHeaderBasePtr (0),
- fileHeaderPtr(0),
- aioControlBlockPtr(0),
- fileSizeDblks(((fileSize_kib_ * 1024) + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_BYTES)) / JRNL_DBLK_SIZE_BYTES),
- enqueuedRecordCount(0),
- submittedDblkCount(0),
- completedDblkCount(0),
- outstandingAioOpsCount(0)
+JournalFile::JournalFile(const std::string& fqFileName,
+ const uint64_t fileSeqNum,
+ const efpDataSize_kib_t efpDataSize_kib) :
+ fqFileName_(fqFileName),
+ fileSeqNum_(fileSeqNum),
+ fileHandle_(-1),
+ fileCloseFlag_(false),
+ fileHeaderBasePtr_ (0),
+ fileHeaderPtr_(0),
+ aioControlBlockPtr_(0),
+ fileSize_dblks_(((efpDataSize_kib * 1024) + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_BYTES)) / QLS_DBLK_SIZE_BYTES),
+ enqueuedRecordCount_(0),
+ submittedDblkCount_(0),
+ completedDblkCount_(0),
+ outstandingAioOpsCount_(0)
{}
JournalFile::~JournalFile() {
@@ -53,227 +53,223 @@ JournalFile::~JournalFile() {
}
void
-JournalFile::initialize() {
- if (::posix_memalign(&fileHeaderBasePtr, QLS_AIO_ALIGN_BOUNDARY, QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_KIB * 1024))
+JournalFile::initialize(const uint32_t completedDblkCount) {
+ if (::posix_memalign(&fileHeaderBasePtr_, QLS_AIO_ALIGN_BOUNDARY_BYTES, QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024))
{
std::ostringstream oss;
- oss << "posix_memalign(): blksize=" << QLS_AIO_ALIGN_BOUNDARY << " size=" << (QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_KIB * 1024);
+ oss << "posix_memalign(): blksize=" << QLS_AIO_ALIGN_BOUNDARY_BYTES << " size=" << (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024);
oss << FORMAT_SYSERR(errno);
throw jexception(jerrno::JERR__MALLOC, oss.str(), "JournalFile", "initialize");
}
- fileHeaderPtr = (::file_hdr_t*)fileHeaderBasePtr;
- aioControlBlockPtr = new aio_cb;
+ fileHeaderPtr_ = (::file_hdr_t*)fileHeaderBasePtr_;
+ aioControlBlockPtr_ = new aio_cb;
+ if (completedDblkCount > 0UL) {
+ submittedDblkCount_.add(completedDblkCount);
+ completedDblkCount_.add(completedDblkCount);
+ }
}
void
JournalFile::finalize() {
- if (fileHeaderBasePtr != 0) {
- std::free(fileHeaderBasePtr);
- fileHeaderBasePtr = 0;
- fileHeaderPtr = 0;
+ if (fileHeaderBasePtr_ != 0) {
+ std::free(fileHeaderBasePtr_);
+ fileHeaderBasePtr_ = 0;
+ fileHeaderPtr_ = 0;
}
- if (aioControlBlockPtr != 0) {
- std::free(aioControlBlockPtr);
- aioControlBlockPtr = 0;
+ if (aioControlBlockPtr_ != 0) {
+ delete(aioControlBlockPtr_);
+ aioControlBlockPtr_ = 0;
}
}
-const std::string
-JournalFile::getDirectory() const {
- return fqFileName.substr(0, fqFileName.rfind('/'));
+const std::string JournalFile::getDirectory() const {
+ return fqFileName_.substr(0, fqFileName_.rfind('/'));
+}
+
+const std::string JournalFile::getFileName() const {
+ return fqFileName_.substr(fqFileName_.rfind('/')+1);
}
-const std::string
-JournalFile::getFileName() const {
- return fqFileName.substr(fqFileName.rfind('/')+1);
+const std::string JournalFile::getFqFileName() const {
+ return fqFileName_;
}
-const std::string
-JournalFile::getFqFileName() const {
- return fqFileName;
+uint64_t JournalFile::getFileSeqNum() const {
+ return fileSeqNum_;
}
-uint64_t
-JournalFile::getFileSeqNum() const {
- return fileSeqNum;
+bool JournalFile::isOpen() const {
+ return fileHandle_ >= 0;
}
-int
-JournalFile::open() {
- fileHandle = ::open(fqFileName.c_str(), O_WRONLY | O_DIRECT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); // 0644 -rw-r--r--
- if (fileHandle < 0) {
+int JournalFile::open() {
+ fileHandle_ = ::open(fqFileName_.c_str(), O_WRONLY | O_DIRECT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); // 0644 -rw-r--r--
+ if (fileHandle_ < 0) {
std::ostringstream oss;
- oss << "file=\"" << fqFileName << "\"" << FORMAT_SYSERR(errno);
+ oss << "file=\"" << fqFileName_ << "\"" << FORMAT_SYSERR(errno);
throw jexception(jerrno::JERR_JNLF_OPEN, oss.str(), "JournalFile", "open");
}
- return fileHandle;
+ return fileHandle_;
}
-bool
-JournalFile::isOpen() const {
- return fileHandle >= 0;
-}
-
-void
-JournalFile::close() {
- if (fileHandle >= 0) {
+void JournalFile::close() {
+ if (fileHandle_ >= 0) {
if (getOutstandingAioDblks()) {
- fileCloseFlag = true; // Close later when all outstanding AIOs have returned
+ fileCloseFlag_ = true; // Close later when all outstanding AIOs have returned
} else {
- int res = ::close(fileHandle);
- fileHandle = -1;
+ int res = ::close(fileHandle_);
+ fileHandle_ = -1;
if (res != 0) {
std::ostringstream oss;
- oss << "file=\"" << fqFileName << "\"" << FORMAT_SYSERR(errno);
+ oss << "file=\"" << fqFileName_ << "\"" << FORMAT_SYSERR(errno);
throw jexception(jerrno::JERR_JNLF_CLOSE, oss.str(), "JournalFile", "open");
}
}
}
}
-void
-JournalFile::asyncFileHeaderWrite(io_context_t ioContextPtr_,
- const efpPartitionNumber_t efpPartitionNumber_,
- const efpDataSize_kib_t efpDataSize_kib_,
- const uint16_t userFlags_,
- const uint64_t recordId_,
- const uint64_t firstRecordOffset_,
- const std::string queueName_) {
- ::file_hdr_create(fileHeaderPtr, QLS_FILE_MAGIC, QLS_JRNL_VERSION, QLS_JRNL_FHDR_RES_SIZE_SBLKS, efpPartitionNumber_, efpDataSize_kib_);
- ::file_hdr_init(fileHeaderBasePtr, QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_KIB * 1024, userFlags_, recordId_, firstRecordOffset_, fileSeqNum, queueName_.size(), queueName_.data());
- aio::prep_pwrite(aioControlBlockPtr, fileHandle, (void*)fileHeaderBasePtr, QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_KIB * 1024, 0UL);
- if (aio::submit(ioContextPtr_, 1, &aioControlBlockPtr) < 0)
+void JournalFile::asyncFileHeaderWrite(io_context_t ioContextPtr,
+ const efpPartitionNumber_t efpPartitionNumber,
+ const efpDataSize_kib_t efpDataSize_kib,
+ const uint16_t userFlags,
+ const uint64_t recordId,
+ const uint64_t firstRecordOffset,
+ const std::string queueName) {
+ ::file_hdr_create(fileHeaderPtr_, QLS_FILE_MAGIC, QLS_JRNL_VERSION, QLS_JRNL_FHDR_RES_SIZE_SBLKS, efpPartitionNumber, efpDataSize_kib);
+ ::file_hdr_init(fileHeaderBasePtr_,
+ QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024,
+ userFlags,
+ recordId,
+ firstRecordOffset,
+ fileSeqNum_,
+ queueName.size(),
+ queueName.data());
+ aio::prep_pwrite(aioControlBlockPtr_,
+ fileHandle_,
+ (void*)fileHeaderBasePtr_,
+ QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024,
+ 0UL);
+ if (aio::submit(ioContextPtr, 1, &aioControlBlockPtr_) < 0)
throw jexception(jerrno::JERR__AIO, "JournalFile", "asyncPageWrite");
- addSubmittedDblkCount(QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_DBLKS);
+ addSubmittedDblkCount(QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS);
incrOutstandingAioOperationCount();
}
-void
-JournalFile::asyncPageWrite(io_context_t ioContextPtr_,
- aio_cb* aioControlBlockPtr_,
- void* data_,
- uint32_t dataSize_dblks_) {
- aio::prep_pwrite_2(aioControlBlockPtr_, fileHandle, data_, dataSize_dblks_ * JRNL_DBLK_SIZE_BYTES, submittedDblkCount.get() * JRNL_DBLK_SIZE_BYTES);
- pmgr::page_cb* pcbp = (pmgr::page_cb*)(aioControlBlockPtr_->data); // This page's control block (pcb)
- pcbp->_wdblks = dataSize_dblks_;
+void JournalFile::asyncPageWrite(io_context_t ioContextPtr,
+ aio_cb* aioControlBlockPtr,
+ void* data,
+ uint32_t dataSize_dblks) {
+ aio::prep_pwrite_2(aioControlBlockPtr,
+ fileHandle_,
+ data,
+ dataSize_dblks * QLS_DBLK_SIZE_BYTES,
+ submittedDblkCount_.get() * QLS_DBLK_SIZE_BYTES);
+ pmgr::page_cb* pcbp = (pmgr::page_cb*)(aioControlBlockPtr->data); // This page's control block (pcb)
+ pcbp->_wdblks = dataSize_dblks;
pcbp->_jfp = this;
- if (aio::submit(ioContextPtr_, 1, &aioControlBlockPtr_) < 0)
- throw jexception(jerrno::JERR__AIO, "JournalFile", "asyncPageWrite");
- addSubmittedDblkCount(dataSize_dblks_);
+ if (aio::submit(ioContextPtr, 1, &aioControlBlockPtr) < 0) {
+ throw jexception(jerrno::JERR__AIO, "JournalFile", "asyncPageWrite"); // TODO: complete exception details
+ }
+ addSubmittedDblkCount(dataSize_dblks);
incrOutstandingAioOperationCount();
}
-uint32_t
-JournalFile::getEnqueuedRecordCount() const {
- return enqueuedRecordCount.get();
+uint32_t JournalFile::getEnqueuedRecordCount() const {
+ return enqueuedRecordCount_.get();
}
-uint32_t
-JournalFile::incrEnqueuedRecordCount() {
- return enqueuedRecordCount.increment();
+uint32_t JournalFile::incrEnqueuedRecordCount() {
+ return enqueuedRecordCount_.increment();
}
-uint32_t
-JournalFile::addEnqueuedRecordCount(const uint32_t a) {
- return enqueuedRecordCount.add(a);
+uint32_t JournalFile::addEnqueuedRecordCount(const uint32_t a) {
+ return enqueuedRecordCount_.add(a);
}
-uint32_t
-JournalFile::decrEnqueuedRecordCount() {
- return enqueuedRecordCount.decrementLimit();
+uint32_t JournalFile::decrEnqueuedRecordCount() {
+ return enqueuedRecordCount_.decrementLimit();
}
-uint32_t
-JournalFile::subtrEnqueuedRecordCount(const uint32_t s) {
- return enqueuedRecordCount.subtractLimit(s);
+uint32_t JournalFile::subtrEnqueuedRecordCount(const uint32_t s) {
+ return enqueuedRecordCount_.subtractLimit(s);
}
-uint32_t
-JournalFile::getSubmittedDblkCount() const {
- return submittedDblkCount.get();
+uint32_t JournalFile::getSubmittedDblkCount() const {
+ return submittedDblkCount_.get();
}
-uint32_t
-JournalFile::addSubmittedDblkCount(const uint32_t a) {
- return submittedDblkCount.addLimit(a, fileSizeDblks, jerrno::JERR_JNLF_FILEOFFSOVFL);
+uint32_t JournalFile::addSubmittedDblkCount(const uint32_t a) {
+ return submittedDblkCount_.addLimit(a, fileSize_dblks_, jerrno::JERR_JNLF_FILEOFFSOVFL);
}
-uint32_t
-JournalFile::getCompletedDblkCount() const {
- return completedDblkCount.get();
+uint32_t JournalFile::getCompletedDblkCount() const {
+ return completedDblkCount_.get();
}
-uint32_t
-JournalFile::addCompletedDblkCount(const uint32_t a) {
- return completedDblkCount.addLimit(a, submittedDblkCount.get(), jerrno::JERR_JNLF_CMPLOFFSOVFL);
+uint32_t JournalFile::addCompletedDblkCount(const uint32_t a) {
+ return completedDblkCount_.addLimit(a, submittedDblkCount_.get(), jerrno::JERR_JNLF_CMPLOFFSOVFL);
}
uint16_t JournalFile::getOutstandingAioOperationCount() const {
- return outstandingAioOpsCount.get();
+ return outstandingAioOpsCount_.get();
}
uint16_t JournalFile::incrOutstandingAioOperationCount() {
- return outstandingAioOpsCount.increment();
+ return outstandingAioOpsCount_.increment();
}
uint16_t JournalFile::decrOutstandingAioOperationCount() {
- uint16_t r = outstandingAioOpsCount.decrementLimit();
- if (fileCloseFlag && outstandingAioOpsCount == 0) { // Delayed close
+ uint16_t r = outstandingAioOpsCount_.decrementLimit();
+ if (fileCloseFlag_ && outstandingAioOpsCount_ == 0) { // Delayed close
close();
}
return r;
}
-bool
-JournalFile::isEmpty() const {
- return submittedDblkCount == 0;
+// --- Status helper functions ---
+
+bool JournalFile::isEmpty() const {
+ return submittedDblkCount_ == 0;
}
-bool
-JournalFile::isDataEmpty() const {
- return submittedDblkCount <= QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_DBLKS;
+bool JournalFile::isDataEmpty() const {
+ return submittedDblkCount_ <= QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS;
}
-u_int32_t
-JournalFile::dblksRemaining() const {
- return fileSizeDblks - submittedDblkCount;
+u_int32_t JournalFile::dblksRemaining() const {
+ return fileSize_dblks_ - submittedDblkCount_;
}
-bool
-JournalFile::isFull() const {
- return submittedDblkCount == fileSizeDblks;
+bool JournalFile::isFull() const {
+ return submittedDblkCount_ == fileSize_dblks_;
}
-bool
-JournalFile::isFullAndComplete() const {
- return completedDblkCount == fileSizeDblks;
+bool JournalFile::isFullAndComplete() const {
+ return completedDblkCount_ == fileSize_dblks_;
}
-u_int32_t
-JournalFile::getOutstandingAioDblks() const {
- return submittedDblkCount - completedDblkCount;
+u_int32_t JournalFile::getOutstandingAioDblks() const {
+ return submittedDblkCount_ - completedDblkCount_;
}
-bool
-JournalFile::getNextFile() const {
+bool JournalFile::getNextFile() const {
return isFull();
}
-bool
-JournalFile::isNoEnqueuedRecordsRemaining() const {
+bool JournalFile::isNoEnqueuedRecordsRemaining() const {
return !isDataEmpty() && // Must be written to, not empty
- enqueuedRecordCount == 0; // No remaining enqueued records
+ enqueuedRecordCount_ == 0; // No remaining enqueued records
}
-const std::string
-JournalFile::status_str(const uint8_t indentDepth_) const {
- std::string indent((size_t)indentDepth_, '.');
+// debug aid
+const std::string JournalFile::status_str(const uint8_t indentDepth) const {
+ std::string indent((size_t)indentDepth, '.');
std::ostringstream oss;
oss << indent << "JournalFile: fileName=" << getFileName() << std::endl;
oss << indent << " directory=" << getDirectory() << std::endl;
- oss << indent << " fileSizeDblks=" << fileSizeDblks << std::endl;
+ oss << indent << " fileSizeDblks=" << fileSize_dblks_ << std::endl;
oss << indent << " open=" << (isOpen() ? "T" : "F") << std::endl;
- oss << indent << " fileHandle=" << fileHandle << std::endl;
+ oss << indent << " fileHandle=" << fileHandle_ << std::endl;
oss << indent << " enqueuedRecordCount=" << getEnqueuedRecordCount() << std::endl;
oss << indent << " submittedDblkCount=" << getSubmittedDblkCount() << std::endl;
oss << indent << " completedDblkCount=" << getCompletedDblkCount() << std::endl;
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.h b/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.h
index 302c53619c..262d3ce147 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/JournalFile.h
@@ -36,26 +36,27 @@ namespace qls_jrnl {
class JournalFile
{
protected:
- const std::string fqFileName;
- const uint64_t fileSeqNum;
- int fileHandle;
- bool fileCloseFlag;
- void* fileHeaderBasePtr;
- ::file_hdr_t* fileHeaderPtr;
- aio_cb* aioControlBlockPtr;
- uint32_t fileSizeDblks; ///< File size in data blocks, including file header
- AtomicCounter<uint32_t> enqueuedRecordCount; ///< Count of enqueued records
- AtomicCounter<uint32_t> submittedDblkCount; ///< Write file count (data blocks) for submitted AIO
- AtomicCounter<uint32_t> completedDblkCount; ///< Write file count (data blocks) for completed AIO
- AtomicCounter<uint16_t> outstandingAioOpsCount; ///< Outstanding AIO operations on this file
+ const std::string fqFileName_;
+ const uint64_t fileSeqNum_;
+ int fileHandle_;
+ bool fileCloseFlag_;
+ void* fileHeaderBasePtr_;
+ ::file_hdr_t* fileHeaderPtr_;
+ aio_cb* aioControlBlockPtr_;
+ uint32_t fileSize_dblks_; ///< File size in data blocks, including file header
+
+ AtomicCounter<uint32_t> enqueuedRecordCount_; ///< Count of enqueued records
+ AtomicCounter<uint32_t> submittedDblkCount_; ///< Write file count (data blocks) for submitted AIO
+ AtomicCounter<uint32_t> completedDblkCount_; ///< Write file count (data blocks) for completed AIO
+ AtomicCounter<uint16_t> outstandingAioOpsCount_; ///< Outstanding AIO operations on this file
public:
JournalFile(const std::string& fqFileName,
const uint64_t fileSeqNum,
- const uint32_t fileSize_kib);
+ const efpDataSize_kib_t efpDataSize_kib);
virtual ~JournalFile();
- void initialize();
+ void initialize(const uint32_t completedDblkCount);
void finalize();
const std::string getDirectory() const;
@@ -63,8 +64,8 @@ public:
const std::string getFqFileName() const;
uint64_t getFileSeqNum() const;
- int open();
bool isOpen() const;
+ int open();
void close();
void asyncFileHeaderWrite(io_context_t ioContextPtr,
const efpPartitionNumber_t efpPartitionNumber,
@@ -104,7 +105,7 @@ public:
bool getNextFile() const; ///< True when next file is needed
bool isNoEnqueuedRecordsRemaining() const; ///< True when all enqueued records (or parts) have been dequeued
- // Debug aid
+ // debug aid
const std::string status_str(const uint8_t indentDepth) const;
};
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.cpp
index 4cabefec32..15222e3a47 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.cpp
@@ -25,28 +25,27 @@
namespace qpid {
namespace qls_jrnl {
-JournalLog::JournalLog() {}
+JournalLog::JournalLog(log_level_t logLevelThreshold) : logLevelThreshold_(logLevelThreshold) {}
JournalLog::~JournalLog() {}
-void
-JournalLog::log(log_level_t ll, const std::string& jid, const std::string& log_stmt) const {
- log(ll, jid.c_str(), log_stmt.c_str());
+void JournalLog::log(const log_level_t logLevel,
+ const std::string& logStatement) const {
+ if (logLevel >= logLevelThreshold_) {
+ std::cerr << log_level_str(logLevel) << ": " << logStatement << std::endl;
+ }
}
-void
-JournalLog::log(log_level_t ll, const char* jid, const char* const log_stmt) const {
- if (ll > LOG_ERROR) {
- std::cerr << log_level_str(ll) << ": Journal \"" << jid << "\": " << log_stmt << std::endl;
- } else if (ll >= LOG_INFO) {
- std::cout << log_level_str(ll) << ": Journal \"" << jid << "\": " << log_stmt << std::endl;
+void JournalLog::log(log_level_t logLevel,
+ const std::string& journalId,
+ const std::string& logStatement) const {
+ if (logLevel >= logLevelThreshold_) {
+ std::cerr << log_level_str(logLevel) << ": Journal \"" << journalId << "\": " << logStatement << std::endl;
}
-
}
-const char*
-JournalLog::log_level_str(log_level_t ll) {
- switch (ll)
+const char* JournalLog::log_level_str(log_level_t logLevel) {
+ switch (logLevel)
{
case LOG_TRACE: return "TRACE";
case LOG_DEBUG: return "DEBUG";
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.h b/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.h
index 83ba80d0b5..b5fcdbc256 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/JournalLog.h
@@ -30,8 +30,7 @@ namespace qls_jrnl {
class JournalLog
{
public:
- typedef enum _log_level
- {
+ typedef enum _log_level {
LOG_TRACE = 0,
LOG_DEBUG,
LOG_INFO,
@@ -42,13 +41,17 @@ public:
} log_level_t;
protected:
- JournalLog();
- virtual ~JournalLog();
+ const log_level_t logLevelThreshold_;
public:
- virtual void log(log_level_t level, const std::string& jid, const std::string& log_stmt) const;
- virtual void log(log_level_t level, const char* jid, const char* const log_stmt) const;
- static const char* log_level_str(log_level_t ll);
+ JournalLog(log_level_t logLevelThreshold);
+ virtual ~JournalLog();
+ virtual void log(const log_level_t logLevel,
+ const std::string& logStatement) const;
+ virtual void log(const log_level_t logLevel,
+ const std::string& journalId,
+ const std::string& logStatement) const;
+ static const char* log_level_str(const log_level_t logLevel);
};
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.cpp
index 161e6deb3b..925e6597a9 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.cpp
@@ -29,284 +29,265 @@
#include "qpid/linearstore/jrnl/slock.h"
#include "qpid/linearstore/jrnl/utils/file_hdr.h"
-#include <iostream> // DEBUG
-
namespace qpid {
namespace qls_jrnl {
-LinearFileController::LinearFileController(jcntl& jcntlRef_) :
- jcntlRef(jcntlRef_),
- emptyFilePoolPtr(0),
- currentJournalFilePtr(0),
- fileSeqCounter(0),
- recordIdCounter(0)
+LinearFileController::LinearFileController(jcntl& jcntlRef) :
+ jcntlRef_(jcntlRef),
+ emptyFilePoolPtr_(0),
+ currentJournalFilePtr_(0),
+ fileSeqCounter_(0),
+ recordIdCounter_(0)
{}
LinearFileController::~LinearFileController() {}
-void
-LinearFileController::initialize(const std::string& journalDirectory_,
- EmptyFilePool* emptyFilePoolPtr_) {
- journalDirectory.assign(journalDirectory_);
- emptyFilePoolPtr = emptyFilePoolPtr_;
+void LinearFileController::initialize(const std::string& journalDirectory,
+ EmptyFilePool* emptyFilePoolPtr,
+ uint64_t initialFileNumberVal) {
+ journalDirectory_.assign(journalDirectory);
+ emptyFilePoolPtr_ = emptyFilePoolPtr;
+ fileSeqCounter_ = initialFileNumberVal;
}
-void
-LinearFileController::finalize() {
- while (!journalFileList.empty()) {
- delete journalFileList.front();
- journalFileList.pop_front();
+void LinearFileController::finalize() {
+ while (!journalFileList_.empty()) {
+ delete journalFileList_.front();
+ journalFileList_.pop_front();
}
}
-void
-LinearFileController::pullEmptyFileFromEfp() {
- if (currentJournalFilePtr)
- currentJournalFilePtr->close();
- std::string ef = emptyFilePoolPtr->takeEmptyFile(journalDirectory); // Moves file from EFP only, returns new file name
- std::cout << "*** LinearFileController::pullEmptyFileFromEfp() qn=" << jcntlRef.id() << " ef=" << ef << std::endl; // DEBUG
- currentJournalFilePtr = new JournalFile(ef, getNextFileSeqNum(), emptyFilePoolPtr->dataSize_kib());
- currentJournalFilePtr->initialize();
+void LinearFileController::addJournalFile(const std::string& fileName,
+ const uint64_t fileNumber,
+ const uint32_t fileSize_kib,
+ const uint32_t completedDblkCount) {
+ if (currentJournalFilePtr_)
+ currentJournalFilePtr_->close();
+ currentJournalFilePtr_ = new JournalFile(fileName, fileNumber, fileSize_kib);
+ currentJournalFilePtr_->initialize(completedDblkCount);
{
- slock l(journalFileListMutex);
- journalFileList.push_back(currentJournalFilePtr);
+ slock l(journalFileListMutex_);
+ journalFileList_.push_back(currentJournalFilePtr_);
}
- currentJournalFilePtr->open();
+ currentJournalFilePtr_->open();
}
-void
-LinearFileController::purgeFilesToEfp() {
- slock l(journalFileListMutex);
- while (journalFileList.front()->isNoEnqueuedRecordsRemaining()) {
- emptyFilePoolPtr->returnEmptyFile(journalFileList.front());
- delete journalFileList.front();
- journalFileList.pop_front();
- }
+efpDataSize_kib_t LinearFileController::dataSize_kib() const {
+ return emptyFilePoolPtr_->dataSize_kib();
+}
+
+efpDataSize_sblks_t LinearFileController::dataSize_sblks() const {
+ return emptyFilePoolPtr_->dataSize_sblks();
+}
+
+efpFileSize_kib_t LinearFileController::fileSize_kib() const {
+ return emptyFilePoolPtr_->fileSize_kib();
}
-efpDataSize_kib_t
-LinearFileController::dataSize_kib() const {
- return emptyFilePoolPtr->dataSize_kib();
+efpFileSize_sblks_t LinearFileController::fileSize_sblks() const {
+ return emptyFilePoolPtr_->fileSize_sblks();
}
-efpFileSize_kib_t
-LinearFileController::fileSize_kib() const {
- return emptyFilePoolPtr->fileSize_kib();
+uint64_t LinearFileController::getNextRecordId() {
+ return recordIdCounter_.increment();
}
-efpDataSize_sblks_t
-LinearFileController::dataSize_sblks() const {
- return emptyFilePoolPtr->dataSize_sblks();
+void LinearFileController::pullEmptyFileFromEfp() {
+ if (currentJournalFilePtr_)
+ currentJournalFilePtr_->close();
+ std::string ef = emptyFilePoolPtr_->takeEmptyFile(journalDirectory_); // Moves file from EFP only, returns new file name
+//std::cout << "*** LinearFileController::pullEmptyFileFromEfp() qn=" << jcntlRef.id() << " ef=" << ef << std::endl; // DEBUG
+ addJournalFile(ef, getNextFileSeqNum(), emptyFilePoolPtr_->dataSize_kib(), 0);
}
-efpFileSize_sblks_t
-LinearFileController::fileSize_sblks() const {
- return emptyFilePoolPtr->fileSize_sblks();
+void LinearFileController::purgeFilesToEfp() {
+ slock l(journalFileListMutex_);
+ while (journalFileList_.front()->isNoEnqueuedRecordsRemaining()) {
+ emptyFilePoolPtr_->returnEmptyFile(journalFileList_.front()->getFqFileName());
+ delete journalFileList_.front();
+ journalFileList_.pop_front();
+ }
+}
+
+uint32_t LinearFileController::getEnqueuedRecordCount(const efpFileCount_t fileSeqNumber) {
+ slock l(journalFileListMutex_);
+ return find(fileSeqNumber)->getEnqueuedRecordCount();
}
-uint64_t
-LinearFileController::getNextRecordId() {
- return recordIdCounter.increment();
+uint32_t LinearFileController::incrEnqueuedRecordCount(const efpFileCount_t fileSeqNumber) {
+ assertCurrentJournalFileValid("incrEnqueuedRecordCount");
+ return find(fileSeqNumber)->incrEnqueuedRecordCount();
}
-uint32_t
-LinearFileController::decrEnqueuedRecordCount(const efpFileCount_t fileSeqNumber) {
- slock l(journalFileListMutex);
+uint32_t LinearFileController::decrEnqueuedRecordCount(const efpFileCount_t fileSeqNumber) {
+ slock l(journalFileListMutex_);
return find(fileSeqNumber)->decrEnqueuedRecordCount();
}
-uint32_t
-LinearFileController::addWriteCompletedDblkCount(const efpFileCount_t fileSeqNumber, const uint32_t a) {
- slock l(journalFileListMutex);
+uint32_t LinearFileController::addWriteCompletedDblkCount(const efpFileCount_t fileSeqNumber, const uint32_t a) {
+ slock l(journalFileListMutex_);
return find(fileSeqNumber)->addCompletedDblkCount(a);
}
-uint16_t
-LinearFileController::decrOutstandingAioOperationCount(const efpFileCount_t fileSeqNumber) {
- slock l(journalFileListMutex);
+uint16_t LinearFileController::decrOutstandingAioOperationCount(const efpFileCount_t fileSeqNumber) {
+ slock l(journalFileListMutex_);
return find(fileSeqNumber)->decrOutstandingAioOperationCount();
}
-void
-LinearFileController::asyncFileHeaderWrite(io_context_t ioContextPtr,
- const uint16_t userFlags,
- const uint64_t recordId,
- const uint64_t firstRecordOffset) {
- currentJournalFilePtr->asyncFileHeaderWrite(ioContextPtr,
- emptyFilePoolPtr->getPartitionNumber(),
- emptyFilePoolPtr->dataSize_kib(),
- userFlags,
- recordId,
- firstRecordOffset,
- jcntlRef.id());
-}
-
-void
-LinearFileController::asyncPageWrite(io_context_t ioContextPtr,
- aio_cb* aioControlBlockPtr,
- void* data,
- uint32_t dataSize_dblks) {
+void LinearFileController::asyncFileHeaderWrite(io_context_t ioContextPtr,
+ const uint16_t userFlags,
+ const uint64_t recordId,
+ const uint64_t firstRecordOffset) {
+ currentJournalFilePtr_->asyncFileHeaderWrite(ioContextPtr,
+ emptyFilePoolPtr_->getPartitionNumber(),
+ emptyFilePoolPtr_->dataSize_kib(),
+ userFlags,
+ recordId,
+ firstRecordOffset,
+ jcntlRef_.id());
+}
+
+void LinearFileController::asyncPageWrite(io_context_t ioContextPtr,
+ aio_cb* aioControlBlockPtr,
+ void* data,
+ uint32_t dataSize_dblks) {
assertCurrentJournalFileValid("asyncPageWrite");
- currentJournalFilePtr->asyncPageWrite(ioContextPtr, aioControlBlockPtr, data, dataSize_dblks);
+ currentJournalFilePtr_->asyncPageWrite(ioContextPtr, aioControlBlockPtr, data, dataSize_dblks);
}
-uint64_t
-LinearFileController::getCurrentFileSeqNum() const {
+uint64_t LinearFileController::getCurrentFileSeqNum() const {
assertCurrentJournalFileValid("getCurrentFileSeqNum");
- return currentJournalFilePtr->getFileSeqNum();
+ return currentJournalFilePtr_->getFileSeqNum();
}
-uint32_t
-LinearFileController::getEnqueuedRecordCount() const {
+uint32_t LinearFileController::getEnqueuedRecordCount() const {
assertCurrentJournalFileValid("getEnqueuedRecordCount");
- return currentJournalFilePtr->getEnqueuedRecordCount();
+ return currentJournalFilePtr_->getEnqueuedRecordCount();
}
-uint32_t
-LinearFileController::incrEnqueuedRecordCount() {
+uint32_t LinearFileController::incrEnqueuedRecordCount() {
assertCurrentJournalFileValid("incrEnqueuedRecordCount");
- return currentJournalFilePtr->incrEnqueuedRecordCount();
+ return currentJournalFilePtr_->incrEnqueuedRecordCount();
}
-uint32_t
-LinearFileController::addEnqueuedRecordCount(const uint32_t a) {
+uint32_t LinearFileController::addEnqueuedRecordCount(const uint32_t a) {
assertCurrentJournalFileValid("addEnqueuedRecordCount");
- return currentJournalFilePtr->addEnqueuedRecordCount(a);
+ return currentJournalFilePtr_->addEnqueuedRecordCount(a);
}
-uint32_t
-LinearFileController::decrEnqueuedRecordCount() {
+uint32_t LinearFileController::decrEnqueuedRecordCount() {
assertCurrentJournalFileValid("decrEnqueuedRecordCount");
- return currentJournalFilePtr->decrEnqueuedRecordCount();
+ return currentJournalFilePtr_->decrEnqueuedRecordCount();
}
-uint32_t
-LinearFileController::subtrEnqueuedRecordCount(const uint32_t s) {
+uint32_t LinearFileController::subtrEnqueuedRecordCount(const uint32_t s) {
assertCurrentJournalFileValid("subtrEnqueuedRecordCount");
- return currentJournalFilePtr->subtrEnqueuedRecordCount(s);
+ return currentJournalFilePtr_->subtrEnqueuedRecordCount(s);
}
-uint32_t
-LinearFileController::getWriteSubmittedDblkCount() const {
+uint32_t LinearFileController::getWriteSubmittedDblkCount() const {
assertCurrentJournalFileValid("getWriteSubmittedDblkCount");
- return currentJournalFilePtr->getSubmittedDblkCount();
+ return currentJournalFilePtr_->getSubmittedDblkCount();
}
-uint32_t
-LinearFileController::addWriteSubmittedDblkCount(const uint32_t a) {
+uint32_t LinearFileController::addWriteSubmittedDblkCount(const uint32_t a) {
assertCurrentJournalFileValid("addWriteSubmittedDblkCount");
- return currentJournalFilePtr->addSubmittedDblkCount(a);
+ return currentJournalFilePtr_->addSubmittedDblkCount(a);
}
-uint32_t
-LinearFileController::getWriteCompletedDblkCount() const {
+uint32_t LinearFileController::getWriteCompletedDblkCount() const {
assertCurrentJournalFileValid("getWriteCompletedDblkCount");
- return currentJournalFilePtr->getCompletedDblkCount();
+ return currentJournalFilePtr_->getCompletedDblkCount();
}
-uint32_t
-LinearFileController::addWriteCompletedDblkCount(const uint32_t a) {
+uint32_t LinearFileController::addWriteCompletedDblkCount(const uint32_t a) {
assertCurrentJournalFileValid("addWriteCompletedDblkCount");
- return currentJournalFilePtr->addCompletedDblkCount(a);
+ return currentJournalFilePtr_->addCompletedDblkCount(a);
}
-uint16_t
-LinearFileController::getOutstandingAioOperationCount() const {
+uint16_t LinearFileController::getOutstandingAioOperationCount() const {
assertCurrentJournalFileValid("getOutstandingAioOperationCount");
- return currentJournalFilePtr->getOutstandingAioOperationCount();
+ return currentJournalFilePtr_->getOutstandingAioOperationCount();
}
-uint16_t
-LinearFileController::incrOutstandingAioOperationCount() {
+uint16_t LinearFileController::incrOutstandingAioOperationCount() {
assertCurrentJournalFileValid("incrOutstandingAioOperationCount");
- return currentJournalFilePtr->incrOutstandingAioOperationCount();
+ return currentJournalFilePtr_->incrOutstandingAioOperationCount();
}
-uint16_t
-LinearFileController::decrOutstandingAioOperationCount() {
+uint16_t LinearFileController::decrOutstandingAioOperationCount() {
assertCurrentJournalFileValid("decrOutstandingAioOperationCount");
- return currentJournalFilePtr->decrOutstandingAioOperationCount();
+ return currentJournalFilePtr_->decrOutstandingAioOperationCount();
}
-bool
-LinearFileController::isEmpty() const {
+bool LinearFileController::isEmpty() const {
assertCurrentJournalFileValid("isEmpty");
- return currentJournalFilePtr->isEmpty();
+ return currentJournalFilePtr_->isEmpty();
}
-bool
-LinearFileController::isDataEmpty() const {
+bool LinearFileController::isDataEmpty() const {
assertCurrentJournalFileValid("isDataEmpty");
- return currentJournalFilePtr->isDataEmpty();
+ return currentJournalFilePtr_->isDataEmpty();
}
-u_int32_t
-LinearFileController::dblksRemaining() const {
+u_int32_t LinearFileController::dblksRemaining() const {
assertCurrentJournalFileValid("dblksRemaining");
- return currentJournalFilePtr->dblksRemaining();
+ return currentJournalFilePtr_->dblksRemaining();
}
-bool
-LinearFileController::isFull() const {
+bool LinearFileController::isFull() const {
assertCurrentJournalFileValid("isFull");
- return currentJournalFilePtr->isFull();
+ return currentJournalFilePtr_->isFull();
}
-bool
-LinearFileController::isFullAndComplete() const {
+bool LinearFileController::isFullAndComplete() const {
assertCurrentJournalFileValid("isFullAndComplete");
- return currentJournalFilePtr->isFullAndComplete();
+ return currentJournalFilePtr_->isFullAndComplete();
}
-u_int32_t
-LinearFileController::getOutstandingAioDblks() const {
+u_int32_t LinearFileController::getOutstandingAioDblks() const {
assertCurrentJournalFileValid("getOutstandingAioDblks");
- return currentJournalFilePtr->getOutstandingAioDblks();
+ return currentJournalFilePtr_->getOutstandingAioDblks();
}
-bool
-LinearFileController::getNextFile() const {
+bool LinearFileController::needNextFile() const {
assertCurrentJournalFileValid("getNextFile");
- return currentJournalFilePtr->getNextFile();
+ return currentJournalFilePtr_->getNextFile();
}
-const std::string
-LinearFileController::status(const uint8_t indentDepth) const {
+const std::string LinearFileController::status(const uint8_t indentDepth) const {
std::string indent((size_t)indentDepth, '.');
std::ostringstream oss;
- oss << indent << "LinearFileController: queue=" << jcntlRef.id() << std::endl;
- oss << indent << " journalDirectory=" << journalDirectory << std::endl;
- oss << indent << " fileSeqCounter=" << fileSeqCounter.get() << std::endl;
- oss << indent << " recordIdCounter=" << recordIdCounter.get() << std::endl;
- oss << indent << " journalFileList.size=" << journalFileList.size() << std::endl;
+ oss << indent << "LinearFileController: queue=" << jcntlRef_.id() << std::endl;
+ oss << indent << " journalDirectory=" << journalDirectory_ << std::endl;
+ oss << indent << " fileSeqCounter=" << fileSeqCounter_.get() << std::endl;
+ oss << indent << " recordIdCounter=" << recordIdCounter_.get() << std::endl;
+ oss << indent << " journalFileList.size=" << journalFileList_.size() << std::endl;
if (checkCurrentJournalFileValid()) {
- oss << currentJournalFilePtr->status_str(indentDepth+2);
+ oss << currentJournalFilePtr_->status_str(indentDepth+2);
} else {
oss << indent << " <No current journal file>" << std::endl;
}
return oss.str();
}
-// protected
+// --- protected functions ---
-bool
-LinearFileController::checkCurrentJournalFileValid() const {
- return currentJournalFilePtr != 0;
+bool LinearFileController::checkCurrentJournalFileValid() const {
+ return currentJournalFilePtr_ != 0;
}
-void
-LinearFileController::assertCurrentJournalFileValid(const char* const functionName) const {
+void LinearFileController::assertCurrentJournalFileValid(const char* const functionName) const {
if (!checkCurrentJournalFileValid()) {
throw jexception(jerrno::JERR__NULL, "LinearFileController", functionName);
}
}
// NOTE: NOT THREAD SAFE - journalFileList is accessed by multiple threads - use under external lock
-JournalFile*
-LinearFileController::find(const efpFileCount_t fileSeqNumber) {
- if (currentJournalFilePtr != 0 && currentJournalFilePtr->getFileSeqNum() == fileSeqNumber)
- return currentJournalFilePtr;
- for (JournalFileListItr_t i=journalFileList.begin(); i!=journalFileList.end(); ++i) {
+JournalFile* LinearFileController::find(const efpFileCount_t fileSeqNumber) {
+ if (currentJournalFilePtr_ != 0 && currentJournalFilePtr_->getFileSeqNum() == fileSeqNumber)
+ return currentJournalFilePtr_;
+ for (JournalFileListItr_t i=journalFileList_.begin(); i!=journalFileList_.end(); ++i) {
if ((*i)->getFileSeqNum() == fileSeqNumber) {
return *i;
}
@@ -316,9 +297,8 @@ LinearFileController::find(const efpFileCount_t fileSeqNumber) {
throw jexception(jerrno::JERR_LFCR_SEQNUMNOTFOUND, oss.str(), "LinearFileController", "find");
}
-uint64_t
-LinearFileController::getNextFileSeqNum() {
- return fileSeqCounter.increment();
+uint64_t LinearFileController::getNextFileSeqNum() {
+ return fileSeqCounter_.increment();
}
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.h b/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.h
index ff880a2647..7721685196 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/LinearFileController.h
@@ -23,14 +23,16 @@
#define QPID_LINEARSTORE_LINEARFILECONTROLLER_H_
#include <deque>
-#include "qpid/linearstore/jrnl/aio.h"
#include "qpid/linearstore/jrnl/AtomicCounter.h"
#include "qpid/linearstore/jrnl/EmptyFilePoolTypes.h"
-#include "qpid/linearstore/jrnl/smutex.h"
-struct file_hdr_t;
+// libaio forward declares
+typedef struct io_context* io_context_t;
+typedef struct iocb aio_cb;
+
namespace qpid {
namespace qls_jrnl {
+
class EmptyFilePool;
class jcntl;
class JournalFile;
@@ -41,35 +43,44 @@ protected:
typedef std::deque<JournalFile*> JournalFileList_t;
typedef JournalFileList_t::iterator JournalFileListItr_t;
- jcntl& jcntlRef;
- std::string journalDirectory;
- EmptyFilePool* emptyFilePoolPtr;
- JournalFile* currentJournalFilePtr;
- AtomicCounter<uint64_t> fileSeqCounter;
- AtomicCounter<uint64_t> recordIdCounter;
+ jcntl& jcntlRef_;
+ std::string journalDirectory_;
+ EmptyFilePool* emptyFilePoolPtr_;
+ JournalFile* currentJournalFilePtr_;
+ AtomicCounter<uint64_t> fileSeqCounter_;
+ AtomicCounter<uint64_t> recordIdCounter_;
- JournalFileList_t journalFileList;
- smutex journalFileListMutex;
+ JournalFileList_t journalFileList_;
+ smutex journalFileListMutex_;
public:
- LinearFileController(jcntl& jcntlRef_);
+ LinearFileController(jcntl& jcntlRef);
virtual ~LinearFileController();
- void initialize(const std::string& journalDirectory_, EmptyFilePool* emptyFilePoolPtr_);
+ void initialize(const std::string& journalDirectory,
+ EmptyFilePool* emptyFilePoolPtr,
+ uint64_t initialFileNumberVal);
void finalize();
- void pullEmptyFileFromEfp();
- void purgeFilesToEfp();
+ void addJournalFile(const std::string& fileName,
+ const uint64_t fileNumber,
+ const uint32_t fileSize_kib,
+ const uint32_t completedDblkCount);
+
efpDataSize_kib_t dataSize_kib() const;
- efpFileSize_kib_t fileSize_kib() const;
efpDataSize_sblks_t dataSize_sblks() const;
+ efpFileSize_kib_t fileSize_kib() const;
efpFileSize_sblks_t fileSize_sblks() const;
-
uint64_t getNextRecordId();
+ void pullEmptyFileFromEfp();
+ void purgeFilesToEfp();
- // Functions for manipulating counts of non-current JournalFile instances in journalFileList
+ // Functions for manipulating counts of non-current JournalFile instances in journalFileList_
+ uint32_t getEnqueuedRecordCount(const efpFileCount_t fileSeqNumber);
+ uint32_t incrEnqueuedRecordCount(const efpFileCount_t fileSeqNumber);
uint32_t decrEnqueuedRecordCount(const efpFileCount_t fileSeqNumber);
- uint32_t addWriteCompletedDblkCount(const efpFileCount_t fileSeqNumber, const uint32_t a);
+ uint32_t addWriteCompletedDblkCount(const efpFileCount_t fileSeqNumber,
+ const uint32_t a);
uint16_t decrOutstandingAioOperationCount(const efpFileCount_t fileSeqNumber);
// Pass-through functions for JournalFile class
@@ -106,18 +117,20 @@ public:
bool isFull() const; // True if all possible dblks have been submitted (but may not yet have returned from AIO)
bool isFullAndComplete() const; // True if all submitted dblks have returned from AIO
u_int32_t getOutstandingAioDblks() const; // Dblks still to be written
- bool getNextFile() const; // True when next file is needed
+ bool needNextFile() const; // True when next file is needed
// Debug aid
const std::string status(const uint8_t indentDepth) const;
protected:
- bool checkCurrentJournalFileValid() const;
void assertCurrentJournalFileValid(const char* const functionName) const;
- JournalFile* find(const efpFileCount_t fileSeqNumber); // NOT THREAD SAFE - use under external lock
+ bool checkCurrentJournalFileValid() const;
+ JournalFile* find(const efpFileCount_t fileSeqNumber);
uint64_t getNextFileSeqNum();
};
+typedef void (LinearFileController::*lfcAddJournalFileFn)(const std::string&, const uint64_t, const uint32_t, const uint32_t);
+
}} // namespace qpid::qls_jrnl
#endif // QPID_LINEARSTORE_LINEARFILECONTROLLER_H_
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.cpp
index 2c60d70c93..1370949908 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.cpp
@@ -21,79 +21,625 @@
#include "qpid/linearstore/jrnl/RecoveryManager.h"
+#include <algorithm>
+#include <cstdlib>
#include <iomanip>
+#include "qpid/linearstore/jrnl/data_tok.h"
+#include "qpid/linearstore/jrnl/deq_rec.h"
+#include "qpid/linearstore/jrnl/EmptyFilePoolManager.h"
+#include "qpid/linearstore/jrnl/enq_map.h"
+#include "qpid/linearstore/jrnl/enq_rec.h"
#include "qpid/linearstore/jrnl/jcfg.h"
+#include "qpid/linearstore/jrnl/jdir.h"
+#include "qpid/linearstore/jrnl/JournalLog.h"
+#include "qpid/linearstore/jrnl/jrec.h"
+#include "qpid/linearstore/jrnl/LinearFileController.h"
+#include "qpid/linearstore/jrnl/txn_map.h"
+#include "qpid/linearstore/jrnl/txn_rec.h"
+#include "qpid/linearstore/jrnl/utils/enq_hdr.h"
+#include "qpid/linearstore/jrnl/utils/file_hdr.h"
#include <sstream>
+#include <string>
+#include <vector>
namespace qpid {
namespace qls_jrnl
{
-RecoveryManager::RecoveryManager() : _journalFileList(),
- _fileNumberNameMap(),
- _enqueueCountList(),
- _journalEmptyFlag(false),
- _firstRecordOffset(0),
- _endOffset(0),
- _highestRecordId(0ULL),
- _lastFileFullFlag(false),
- _currentRid(0ULL),
- _currentFileNumber(0ULL),
- _currentFileName(),
- _fileSize(0),
- _recordStart(0),
- _inFileStream(),
- _readComplete(false)
+RecoveryManager::RecoveryManager(const std::string& journalDirectory,
+ const std::string& queuename,
+ enq_map& enqueueMapRef,
+ txn_map& transactionMapRef,
+ JournalLog& journalLogRef) :
+ journalDirectory_(journalDirectory),
+ queueName_(queuename),
+ enqueueMapRef_(enqueueMapRef),
+ transactionMapRef_(transactionMapRef),
+ journalLogRef_(journalLogRef),
+ journalEmptyFlag_(false),
+ firstRecordOffset_(0),
+ endOffset_(0),
+ highestRecordId_(0ULL),
+ highestFileNumber_(0ULL),
+ lastFileFullFlag_(false),
+ fileSize_kib_(0)
{}
RecoveryManager::~RecoveryManager() {}
-std::string
-RecoveryManager::toString(const std::string& jid,
- bool compact) {
+void RecoveryManager::analyzeJournals(const std::vector<std::string>* preparedTransactionListPtr,
+ EmptyFilePoolManager* emptyFilePoolManager,
+ EmptyFilePool** emptyFilePoolPtrPtr) {
+ // Analyze file headers of existing journal files
+ efpIdentity_t efpIdentity;
+ analyzeJournalFileHeaders(efpIdentity);
+ *emptyFilePoolPtrPtr = emptyFilePoolManager->getEmptyFilePool(efpIdentity);
+ fileSize_kib_ = (*emptyFilePoolPtrPtr)->fileSize_kib();
+ // Check for file full condition
+ lastFileFullFlag_ = endOffset_ == (std::streamoff)(*emptyFilePoolPtrPtr)->fileSize_kib() * 1024;
+
+ // Restore all read and write pointers and transactions
+ if (!journalEmptyFlag_) {
+ while (getNextRecordHeader()) {
+
+ }
+ if (inFileStream_.is_open()) {
+ inFileStream_.close();
+ }
+ // Remove leading files which have no enqueued records
+ removeEmptyFiles(*emptyFilePoolPtrPtr);
+
+ // Remove all txns from tmap that are not in the prepared list
+ if (preparedTransactionListPtr) {
+ std::vector<std::string> xidList;
+ transactionMapRef_.xid_list(xidList);
+ for (std::vector<std::string>::iterator itr = xidList.begin(); itr != xidList.end(); itr++) {
+ std::vector<std::string>::const_iterator pitr =
+ std::find(preparedTransactionListPtr->begin(), preparedTransactionListPtr->end(), *itr);
+ if (pitr == preparedTransactionListPtr->end()) { // not found in prepared list
+ txn_data_list tdl = transactionMapRef_.get_remove_tdata_list(*itr); // tdl will be empty if xid not found
+ // Unlock any affected enqueues in emap
+ for (tdl_itr i=tdl.begin(); i<tdl.end(); i++) {
+ if (i->_enq_flag) { // enq op - decrement enqueue count
+ enqueueCountList_[i->_pfid]--;
+ } else if (enqueueMapRef_.is_enqueued(i->_drid, true)) { // deq op - unlock enq record
+ int16_t ret = enqueueMapRef_.unlock(i->_drid);
+ if (ret < enq_map::EMAP_OK) { // fail
+ // enq_map::unlock()'s only error is enq_map::EMAP_RID_NOT_FOUND
+ std::ostringstream oss;
+ oss << std::hex << "_emap.unlock(): drid=0x\"" << i->_drid;
+ throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "RecoveryManager", "analyzeJournals");
+ }
+ }
+ }
+ }
+ }
+ }
+ enqueueMapRef_.rid_list(recordIdList_);
+ recordIdListConstItr_ = recordIdList_.begin();
+ }
+}
+
+std::streamoff RecoveryManager::getEndOffset() const {
+ return endOffset_;
+}
+
+uint64_t RecoveryManager::getHighestFileNumber() const {
+ return highestFileNumber_;
+}
+
+uint64_t RecoveryManager::getHighestRecordId() const {
+ return highestRecordId_;
+}
+
+bool RecoveryManager::isLastFileFull() const {
+ return lastFileFullFlag_;
+}
+
+bool RecoveryManager::readNextRemainingRecord(void** const dataPtrPtr,
+ std::size_t& dataSize,
+ void** const xidPtrPtr,
+ std::size_t& xidSize,
+ bool& transient,
+ bool& external,
+ data_tok* const dtokp,
+ bool /*ignore_pending_txns*/) {
+ if (!dtokp->is_readable()) {
+ std::ostringstream oss;
+ oss << std::hex << std::setfill('0') << "dtok_id=0x" << std::setw(8) << dtokp->id();
+ oss << "; dtok_rid=0x" << std::setw(16) << dtokp->rid() << "; dtok_wstate=" << dtokp->wstate_str();
+ throw jexception(jerrno::JERR_JCNTL_ENQSTATE, oss.str(), "RecoveryManager", "readNextRemainingRecord");
+ }
+ if (recordIdListConstItr_ == recordIdList_.end()) {
+ return false;
+ }
+ enq_map::emap_data_struct_t eds;
+ enqueueMapRef_.get_data(*recordIdListConstItr_, eds);
+ uint64_t fileNumber = eds._pfid;
+ currentJournalFileConstItr_ = fileNumberNameMap_.find(fileNumber);
+ getNextFile(false);
+
+ inFileStream_.seekg(eds._file_posn, std::ifstream::beg);
+ if (!inFileStream_.good()) {
+ std::ostringstream oss;
+ oss << "Could not find offset 0x" << std::hex << eds._file_posn << " in file " << getCurrentFileName();
+ throw jexception(jerrno::JERR__FILEIO, oss.str(), "RecoveryManager", "readNextRemainingRecord");
+ }
+ ::enq_hdr_t enqueueHeader;
+ inFileStream_.read((char*)&enqueueHeader, sizeof(::enq_hdr_t));
+ if (inFileStream_.gcount() != sizeof(::enq_hdr_t)) {
+ std::ostringstream oss;
+ oss << "Could not read enqueue header from file " << getCurrentFileName() << " at offset 0x" << std::hex << eds._file_posn;
+ throw jexception(jerrno::JERR__FILEIO, oss.str(), "RecoveryManager", "readNextRemainingRecord");
+ }
+ // check flags
+ transient = ::is_enq_transient(&enqueueHeader);
+ external = ::is_enq_external(&enqueueHeader);
+
+ // read xid
+ xidSize = enqueueHeader._xidsize;
+ *xidPtrPtr = ::malloc(xidSize);
+ if (*xidPtrPtr == 0) {
+ std::ostringstream oss;
+ oss << "xidPtr, size=0x" << std::hex << xidSize;
+ throw jexception(jerrno::JERR__MALLOC, oss.str(), "RecoveryManager", "readNextRemainingRecord");
+ }
+ readJournalData((char*)*xidPtrPtr, xidSize);
+
+ // read data
+ dataSize = enqueueHeader._dsize;
+ *dataPtrPtr = ::malloc(dataSize);
+ if (*xidPtrPtr == 0) {
+ std::ostringstream oss;
+ oss << "dataPtr, size=0x" << std::hex << dataSize;
+ throw jexception(jerrno::JERR__MALLOC, oss.str(), "RecoveryManager", "readNextRemainingRecord");
+ }
+ readJournalData((char*)*dataPtrPtr, dataSize);
+ return true;
+}
+
+void RecoveryManager::setLinearFileControllerJournals(lfcAddJournalFileFn fnPtr,
+ LinearFileController* lfcPtr) {
+//std::cout << "****** RecoveryManager::setLinearFileControllerJournals():" << std::endl; // DEBUG
+ for (fileNumberNameMapConstItr_t i = fileNumberNameMap_.begin(); i != fileNumberNameMap_.end(); ++i) {
+ uint32_t fileDblkCount = i->first == highestFileNumber_ ? // Is this this last file?
+ endOffset_ / QLS_DBLK_SIZE_BYTES : // Last file uses _endOffset
+ fileSize_kib_ * 1024 / QLS_DBLK_SIZE_BYTES; // All others use file size to make them full
+ (lfcPtr->*fnPtr)(i->second, i->first, fileSize_kib_, fileDblkCount);
+//std::cout << " ** f=" << i->second.substr(i->second.rfind('/')+1) << ",fn=" << i->first << ",s=" << _fileSize_kib << ",eo=" << fileDblkCount << "(" << (fileDblkCount * QLS_DBLK_SIZE_BYTES / 1024) << "kiB)" << std::endl; // DEBUG
+ }
+}
+
+std::string RecoveryManager::toString(const std::string& jid,
+ bool compact) {
std::ostringstream oss;
if (compact) {
oss << "Recovery journal analysis (jid=\"" << jid << "\"):";
oss << " jfl=[";
- for (std::map<uint64_t, std::string>::const_iterator i=_fileNumberNameMap.begin(); i!=_fileNumberNameMap.end(); ++i) {
- if (i!=_fileNumberNameMap.begin()) oss << " ";
+ for (fileNumberNameMapConstItr_t i=fileNumberNameMap_.begin(); i!=fileNumberNameMap_.end(); ++i) {
+ if (i!=fileNumberNameMap_.begin()) oss << " ";
oss << i->first << ":" << i->second.substr(i->second.rfind('/')+1);
}
oss << "] ecl=[ ";
- for (std::vector<uint32_t>::const_iterator j = _enqueueCountList.begin(); j!=_enqueueCountList.end(); ++j) {
- if (j != _enqueueCountList.begin()) oss << " ";
+ for (enqueueCountListConstItr_t j = enqueueCountList_.begin(); j!=enqueueCountList_.end(); ++j) {
+ if (j != enqueueCountList_.begin()) oss << " ";
oss << *j;
}
- oss << " ] empty=" << (_journalEmptyFlag ? "T" : "F");
- oss << " fro=0x" << std::hex << _firstRecordOffset << std::dec << " (" << (_firstRecordOffset/JRNL_DBLK_SIZE_BYTES) << " dblks)";
- oss << " eo=0x" << std::hex << _endOffset << std::dec << " (" << (_endOffset/JRNL_DBLK_SIZE_BYTES) << " dblks)";
- oss << " hrid=0x" << std::hex << _highestRecordId << std::dec;
- oss << " lffull=" << (_lastFileFullFlag ? "T" : "F");
+ oss << " ] empty=" << (journalEmptyFlag_ ? "T" : "F");
+ oss << " fro=0x" << std::hex << firstRecordOffset_ << std::dec << " (" << (firstRecordOffset_/QLS_DBLK_SIZE_BYTES) << " dblks)";
+ oss << " eo=0x" << std::hex << endOffset_ << std::dec << " (" << (endOffset_/QLS_DBLK_SIZE_BYTES) << " dblks)";
+ oss << " hrid=0x" << std::hex << highestRecordId_ << std::dec;
+ oss << " hfnum=0x" << std::hex << highestFileNumber_ << std::dec;
+ oss << " lffull=" << (lastFileFullFlag_ ? "T" : "F");
} else {
oss << "Recovery journal analysis (jid=\"" << jid << "\"):" << std::endl;
- oss << " Number of journal files = " << _fileNumberNameMap.size() << std::endl;
+ oss << " Number of journal files = " << fileNumberNameMap_.size() << std::endl;
oss << " Journal File List:" << std::endl;
- for (std::map<uint64_t, std::string>::const_iterator i=_fileNumberNameMap.begin(); i!=_fileNumberNameMap.end(); ++i) {
+ for (fileNumberNameMapConstItr_t i=fileNumberNameMap_.begin(); i!=fileNumberNameMap_.end(); ++i) {
oss << " " << i->first << ": " << i->second.substr(i->second.rfind('/')+1) << std::endl;
}
oss << " Enqueue Counts: [ " << std::endl;
- for (std::vector<uint32_t>::const_iterator j = _enqueueCountList.begin(); j!=_enqueueCountList.end(); ++j) {
- if (j != _enqueueCountList.begin()) oss << ", ";
+ for (enqueueCountListConstItr_t j = enqueueCountList_.begin(); j!=enqueueCountList_.end(); ++j) {
+ if (j != enqueueCountList_.begin()) oss << ", ";
oss << *j;
}
oss << " ]" << std::endl;
- for (unsigned i=0; i<_enqueueCountList.size(); i++)
- oss << " File " << std::setw(2) << i << ": " << _enqueueCountList[i] << std::endl;
- oss << " Journal empty (_jempty) = " << (_journalEmptyFlag ? "TRUE" : "FALSE") << std::endl;
- oss << " First record offset in first fid (_fro) = 0x" << std::hex << _firstRecordOffset <<
- std::dec << " (" << (_firstRecordOffset/JRNL_DBLK_SIZE_BYTES) << " dblks)" << std::endl;
- oss << " End offset (_eo) = 0x" << std::hex << _endOffset << std::dec << " (" <<
- (_endOffset/JRNL_DBLK_SIZE_BYTES) << " dblks)" << std::endl;
- oss << " Highest rid (_h_rid) = 0x" << std::hex << _highestRecordId << std::dec << std::endl;
- oss << " Last file full (_lffull) = " << (_lastFileFullFlag ? "TRUE" : "FALSE") << std::endl;
+ oss << " Journal empty = " << (journalEmptyFlag_ ? "TRUE" : "FALSE") << std::endl;
+ oss << " First record offset in first file = 0x" << std::hex << firstRecordOffset_ <<
+ std::dec << " (" << (firstRecordOffset_/QLS_DBLK_SIZE_BYTES) << " dblks)" << std::endl;
+ oss << " End offset = 0x" << std::hex << endOffset_ << std::dec << " (" <<
+ (endOffset_/QLS_DBLK_SIZE_BYTES) << " dblks)" << std::endl;
+ oss << " Highest rid = 0x" << std::hex << highestRecordId_ << std::dec << std::endl;
+ oss << " Highest file number = 0x" << std::hex << highestFileNumber_ << std::dec << std::endl;
+ oss << " Last file full = " << (lastFileFullFlag_ ? "TRUE" : "FALSE") << std::endl;
oss << " Enqueued records (txn & non-txn):" << std::endl;
}
return oss.str();
}
+// --- protected functions ---
+
+void RecoveryManager::analyzeJournalFileHeaders(efpIdentity_t& efpIdentity) {
+ std::string headerQueueName;
+ ::file_hdr_t fileHeader;
+ directoryList_t directoryList;
+ jdir::read_dir(journalDirectory_, directoryList, false, true, false, true);
+ for (directoryListConstItr_t i = directoryList.begin(); i != directoryList.end(); ++i) {
+ readJournalFileHeader(*i, fileHeader, headerQueueName);
+ if (headerQueueName.compare(queueName_) != 0) {
+ std::ostringstream oss;
+ oss << "Journal file " << (*i) << " belongs to queue \"" << headerQueueName << "\": ignoring";
+ journalLogRef_.log(JournalLog::LOG_WARN, queueName_, oss.str());
+ } else {
+ fileNumberNameMap_[fileHeader._file_number] = *i;
+ if (fileHeader._file_number > highestFileNumber_) {
+ highestFileNumber_ = fileHeader._file_number;
+ }
+ }
+ }
+ efpIdentity.first = fileHeader._efp_partition;
+ efpIdentity.second = fileHeader._file_size_kib;
+ enqueueCountList_.resize(fileNumberNameMap_.size(), 0);
+ currentJournalFileConstItr_ = fileNumberNameMap_.begin();
+}
+
+void RecoveryManager::checkFileStreamOk(bool checkEof) {
+ if (inFileStream_.fail() || inFileStream_.bad() || checkEof ? inFileStream_.eof() : false) {
+ throw jexception("read failure"); // TODO complete exception
+ }
+}
+
+void RecoveryManager::checkJournalAlignment(const std::streampos recordPosition) {
+ std::streampos currentPosn = recordPosition;
+ unsigned sblkOffset = currentPosn % QLS_SBLK_SIZE_BYTES;
+ if (sblkOffset)
+ {
+ std::ostringstream oss1;
+ oss1 << std::hex << "Bad record alignment found at fid=0x" << getCurrentFileNumber();
+ oss1 << " offs=0x" << currentPosn << " (likely journal overwrite boundary); " << std::dec;
+ oss1 << (QLS_SBLK_SIZE_DBLKS - (sblkOffset/QLS_DBLK_SIZE_BYTES)) << " filler record(s) required.";
+ journalLogRef_.log(JournalLog::LOG_WARN, queueName_, oss1.str());
+
+ std::ofstream outFileStream(getCurrentFileName().c_str(), std::ios_base::in | std::ios_base::out | std::ios_base::binary);
+ if (!outFileStream.good()) {
+ throw jexception(jerrno::JERR__FILEIO, getCurrentFileName(), "RecoveryManager", "checkJournalAlignment");
+ }
+ outFileStream.seekp(currentPosn);
+
+ // Prepare write buffer containing a single empty record (1 dblk)
+ void* writeBuffer = std::malloc(QLS_DBLK_SIZE_BYTES);
+ if (writeBuffer == 0) {
+ throw jexception(jerrno::JERR__MALLOC, "RecoveryManager", "checkJournalAlignment");
+ }
+ const uint32_t xmagic = QLS_EMPTY_MAGIC;
+ ::memcpy(writeBuffer, (const void*)&xmagic, sizeof(xmagic));
+ ::memset((char*)writeBuffer + sizeof(xmagic), QLS_CLEAN_CHAR, QLS_DBLK_SIZE_BYTES - sizeof(xmagic));
+
+ // Write as many empty records as are needed to get to sblk boundary
+ while (currentPosn % QLS_SBLK_SIZE_BYTES) {
+ outFileStream.write((const char*)writeBuffer, QLS_DBLK_SIZE_BYTES);
+ if (outFileStream.fail()) {
+ throw jexception(jerrno::JERR_RCVM_WRITE, "RecoveryManager", "checkJournalAlignment");
+ }
+ std::ostringstream oss2;
+ oss2 << std::hex << "Recover phase write: Wrote filler record: fid=0x" << getCurrentFileNumber();
+ oss2 << " offs=0x" << currentPosn;
+ journalLogRef_.log(JournalLog::LOG_NOTICE, queueName_, oss2.str());
+ currentPosn = outFileStream.tellp();
+ }
+ outFileStream.close();
+ std::free(writeBuffer);
+ journalLogRef_.log(JournalLog::LOG_INFO, queueName_, "Bad record alignment fixed.");
+ }
+ endOffset_ = currentPosn;
+}
+
+bool RecoveryManager::decodeRecord(jrec& record,
+ std::size_t& cumulativeSizeRead,
+ ::rec_hdr_t& headerRecord,
+ std::streampos& fileOffset)
+{
+// uint16_t start_fid = getCurrentFileNumber();
+ std::streampos start_file_offs = fileOffset;
+
+ if (highestRecordId_ == 0) {
+ highestRecordId_ = headerRecord._rid;
+ } else if (headerRecord._rid - highestRecordId_ < 0x8000000000000000ULL) { // RFC 1982 comparison for unsigned 64-bit
+ highestRecordId_ = headerRecord._rid;
+ }
+
+ bool done = false;
+ while (!done) {
+ try {
+ done = record.rcv_decode(headerRecord, &inFileStream_, cumulativeSizeRead);
+ }
+ catch (const jexception& e) {
+// TODO - review this logic and tidy up how rd._lfid is assigned. See new jinf.get_end_file() fn.
+// Original
+// if (e.err_code() != jerrno::JERR_JREC_BADRECTAIL ||
+// fid != (rd._ffid ? rd._ffid - 1 : _num_jfiles - 1)) throw;
+// Tried this, but did not work
+// if (e.err_code() != jerrno::JERR_JREC_BADRECTAIL || h._magic != 0) throw;
+ checkJournalAlignment(start_file_offs);
+// rd._lfid = start_fid;
+ return false;
+ }
+ if (!done && !getNextFile(false)) {
+ checkJournalAlignment(start_file_offs);
+ return false;
+ }
+ }
+ return true;
+}
+
+std::string RecoveryManager::getCurrentFileName() const {
+ return currentJournalFileConstItr_->second;
+}
+
+uint64_t RecoveryManager::getCurrentFileNumber() const {
+ return currentJournalFileConstItr_->first;
+}
+
+bool RecoveryManager::getNextFile(bool jumpToFirstRecordOffsetFlag) {
+ if (inFileStream_.is_open()) {
+ if (inFileStream_.eof() || !inFileStream_.good())
+ {
+ inFileStream_.clear();
+ endOffset_ = inFileStream_.tellg(); // remember file offset before closing
+ if (endOffset_ == -1) { throw jexception("tellg() failure"); } // Check for error code -1 TODO: compelete exception
+ inFileStream_.close();
+ if (++currentJournalFileConstItr_ == fileNumberNameMap_.end()) {
+ return false;
+ }
+ }
+ }
+ if (!inFileStream_.is_open())
+ {
+ inFileStream_.clear(); // clear eof flag, req'd for older versions of c++
+ inFileStream_.open(getCurrentFileName().c_str(), std::ios_base::in | std::ios_base::binary);
+ if (!inFileStream_.good()) {
+ throw jexception(jerrno::JERR__FILEIO, getCurrentFileName(), "RecoveryManager", "getNextFile");
+ }
+
+ // Read file header
+//std::cout << " F" << getCurrentFileNumber() << std::flush; // DEBUG
+ file_hdr_t fhdr;
+ inFileStream_.read((char*)&fhdr, sizeof(fhdr));
+ checkFileStreamOk(true);
+ if (fhdr._rhdr._magic == QLS_FILE_MAGIC) {
+ firstRecordOffset_ = fhdr._fro;
+ std::streamoff foffs = jumpToFirstRecordOffsetFlag ? firstRecordOffset_ : QLS_SBLK_SIZE_BYTES;
+ inFileStream_.seekg(foffs);
+ } else {
+ inFileStream_.close();
+ if (currentJournalFileConstItr_ == fileNumberNameMap_.begin()) {
+ journalEmptyFlag_ = true;
+ }
+ return false;
+ }
+ }
+ return true;
+}
+
+bool RecoveryManager::getNextRecordHeader()
+{
+ std::size_t cum_size_read = 0;
+ void* xidp = 0;
+ rec_hdr_t h;
+
+ bool hdr_ok = false;
+ std::streampos file_pos;
+ while (!hdr_ok) {
+ if (!inFileStream_.is_open()) {
+ if (!getNextFile(true)) {
+ return false;
+ }
+ }
+ file_pos = inFileStream_.tellg();
+//std::cout << " 0x" << std::hex << file_pos << std::dec; // DEBUG
+ inFileStream_.read((char*)&h, sizeof(rec_hdr_t));
+ if (inFileStream_.gcount() == sizeof(rec_hdr_t)) {
+ hdr_ok = true;
+ } else {
+ if (!getNextFile(true)) {
+ return false;
+ }
+ }
+ }
+
+ switch(h._magic) {
+ case QLS_ENQ_MAGIC:
+ {
+//std::cout << ".e" << std::flush; // DEBUG
+ enq_rec er;
+ uint64_t start_fid = getCurrentFileNumber(); // fid may increment in decode() if record folds over file boundary
+ if (!decodeRecord(er, cum_size_read, h, file_pos)) {
+ return false;
+ }
+ if (!er.is_transient()) { // Ignore transient msgs
+ enqueueCountList_[start_fid]++;
+ if (er.xid_size()) {
+ er.get_xid(&xidp);
+ if (xidp != 0) { throw jexception("Null xid with non-null xid_size"); } // TODO complete exception
+ std::string xid((char*)xidp, er.xid_size());
+ transactionMapRef_.insert_txn_data(xid, txn_data(h._rid, 0, start_fid, true));
+ if (transactionMapRef_.set_aio_compl(xid, h._rid) < txn_map::TMAP_OK) { // fail - xid or rid not found
+ std::ostringstream oss;
+ oss << std::hex << "_tmap.set_aio_compl: txn_enq xid=\"" << xid << "\" rid=0x" << h._rid;
+ throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "RecoveryManager", "getNextRecordHeader");
+ }
+ std::free(xidp);
+ } else {
+ if (enqueueMapRef_.insert_pfid(h._rid, start_fid, file_pos) < enq_map::EMAP_OK) { // fail
+ // The only error code emap::insert_pfid() returns is enq_map::EMAP_DUP_RID.
+ std::ostringstream oss;
+ oss << std::hex << "rid=0x" << h._rid << " _pfid=0x" << start_fid;
+ throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "RecoveryManager", "getNextRecordHeader");
+ }
+ }
+ }
+ }
+ break;
+ case QLS_DEQ_MAGIC:
+ {
+//std::cout << ".d" << std::flush; // DEBUG
+ deq_rec dr;
+ uint16_t start_fid = getCurrentFileNumber(); // fid may increment in decode() if record folds over file boundary
+ if (!decodeRecord(dr, cum_size_read, h, file_pos)) {
+ return false;
+ }
+ if (dr.xid_size()) {
+ // If the enqueue is part of a pending txn, it will not yet be in emap
+ enqueueMapRef_.lock(dr.deq_rid()); // ignore not found error
+ dr.get_xid(&xidp);
+ if (xidp != 0) { throw jexception("Null xid with non-null xid_size"); } // TODO complete exception
+ std::string xid((char*)xidp, dr.xid_size());
+ transactionMapRef_.insert_txn_data(xid, txn_data(dr.rid(), dr.deq_rid(), start_fid, false,
+ dr.is_txn_coml_commit()));
+ if (transactionMapRef_.set_aio_compl(xid, dr.rid()) < txn_map::TMAP_OK) { // fail - xid or rid not found
+ std::ostringstream oss;
+ oss << std::hex << "_tmap.set_aio_compl: txn_deq xid=\"" << xid << "\" rid=0x" << dr.rid();
+ throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "RecoveryManager", "getNextRecordHeader");
+ }
+ std::free(xidp);
+ } else {
+ uint64_t enq_fid;
+ if (enqueueMapRef_.get_remove_pfid(dr.deq_rid(), enq_fid, true) == enq_map::EMAP_OK) { // ignore not found error
+ enqueueCountList_[enq_fid]--;
+ }
+ }
+ }
+ break;
+ case QLS_TXA_MAGIC:
+ {
+//std::cout << ".a" << std::flush; // DEBUG
+ txn_rec ar;
+ if (!decodeRecord(ar, cum_size_read, h, file_pos)) {
+ return false;
+ }
+ // Delete this txn from tmap, unlock any locked records in emap
+ ar.get_xid(&xidp);
+ if (xidp != 0) {
+ throw jexception("Null xid with non-null xid_size"); // TODO complete exception
+ }
+ std::string xid((char*)xidp, ar.xid_size());
+ txn_data_list tdl = transactionMapRef_.get_remove_tdata_list(xid); // tdl will be empty if xid not found
+ for (tdl_itr itr = tdl.begin(); itr != tdl.end(); itr++) {
+ if (itr->_enq_flag) {
+ enqueueCountList_[itr->_pfid]--;
+ } else {
+ enqueueMapRef_.unlock(itr->_drid); // ignore not found error
+ }
+ }
+ std::free(xidp);
+ }
+ break;
+ case QLS_TXC_MAGIC:
+ {
+//std::cout << ".t" << std::flush; // DEBUG
+ txn_rec cr;
+ if (!decodeRecord(cr, cum_size_read, h, file_pos)) {
+ return false;
+ }
+ // Delete this txn from tmap, process records into emap
+ cr.get_xid(&xidp);
+ if (xidp != 0) {
+ throw jexception("Null xid with non-null xid_size"); // TODO complete exception
+ }
+ std::string xid((char*)xidp, cr.xid_size());
+ txn_data_list tdl = transactionMapRef_.get_remove_tdata_list(xid); // tdl will be empty if xid not found
+ for (tdl_itr itr = tdl.begin(); itr != tdl.end(); itr++) {
+ if (itr->_enq_flag) { // txn enqueue
+ if (enqueueMapRef_.insert_pfid(itr->_rid, itr->_pfid, file_pos) < enq_map::EMAP_OK) { // fail
+ // The only error code emap::insert_pfid() returns is enq_map::EMAP_DUP_RID.
+ std::ostringstream oss;
+ oss << std::hex << "rid=0x" << itr->_rid << " _pfid=0x" << itr->_pfid;
+ throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "RecoveryManager", "getNextRecordHeader");
+ }
+ } else { // txn dequeue
+ uint64_t enq_fid;
+ if (enqueueMapRef_.get_remove_pfid(itr->_drid, enq_fid, true) == enq_map::EMAP_OK) // ignore not found error
+ enqueueCountList_[enq_fid]--;
+ }
+ }
+ std::free(xidp);
+ }
+ break;
+ case QLS_EMPTY_MAGIC:
+ {
+//std::cout << ".x" << std::flush; // DEBUG
+ uint32_t rec_dblks = jrec::size_dblks(sizeof(::rec_hdr_t));
+ inFileStream_.ignore(rec_dblks * QLS_DBLK_SIZE_BYTES - sizeof(::rec_hdr_t));
+ checkFileStreamOk(false);
+ if (!getNextFile(false)) {
+ return false;
+ }
+ }
+ break;
+ case 0:
+//std::cout << ".0" << std::endl << std::flush; // DEBUG
+ checkJournalAlignment(file_pos);
+ return false;
+ default:
+//std::cout << ".?" << std::endl << std::flush; // DEBUG
+ // Stop as this is the overwrite boundary.
+ checkJournalAlignment(file_pos);
+ return false;
+ }
+ return true;
+}
+
+void RecoveryManager::readJournalData(char* target,
+ const std::streamsize readSize) {
+ std::streamoff bytesRead = 0;
+ while (bytesRead < readSize) {
+ if (inFileStream_.eof()) {
+ getNextFile(false);
+ }
+ bool readFitsInFile = inFileStream_.tellg() + readSize <= fileSize_kib_ * 1024;
+ std::streamoff readSize = readFitsInFile ? readSize : (fileSize_kib_ * 1024) - inFileStream_.tellg();
+ inFileStream_.read(target + bytesRead, readSize);
+ if (inFileStream_.gcount() != readSize) {
+ throw jexception(); // TODO - proper exception
+ }
+ bytesRead += readSize;
+ }
+}
+
+// static private
+void RecoveryManager::readJournalFileHeader(const std::string& journalFileName,
+ ::file_hdr_t& fileHeaderRef,
+ std::string& queueName) {
+ const std::size_t headerBlockSize = QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_KIB * 1024;
+ char buffer[headerBlockSize];
+ std::ifstream ifs(journalFileName.c_str(), std::ifstream::in | std::ifstream::binary);
+ if (!ifs.good()) {
+ std::ostringstream oss;
+ oss << "File=" << journalFileName;
+ throw jexception(jerrno::JERR_RCVM_OPENRD, oss.str(), "RecoveryManager", "readJournalFileHeader");
+ }
+ ifs.read(buffer, headerBlockSize);
+ if (!ifs) {
+ std::streamsize s = ifs.gcount();
+ ifs.close();
+ std::ostringstream oss;
+ oss << "File=" << journalFileName << "; attempted_read_size=" << headerBlockSize << "; actual_read_size=" << s;
+ throw jexception(jerrno::JERR_RCVM_READ, oss.str(), "RecoveryManager", "readJournalFileHeader");
+ }
+ ifs.close();
+ ::memcpy(&fileHeaderRef, buffer, sizeof(::file_hdr_t));
+ queueName.assign(buffer + sizeof(::file_hdr_t), fileHeaderRef._queue_name_len);
+
+}
+
+void RecoveryManager::removeEmptyFiles(EmptyFilePool* emptyFilePoolPtr) {
+ while (enqueueCountList_.front() == 0 && enqueueCountList_.size() > 1) {
+ fileNumberNameMapItr_t i = fileNumberNameMap_.begin();
+//std::cout << "*** File " << i->first << ": " << i->second << " is empty." << std::endl;
+ emptyFilePoolPtr->returnEmptyFile(i->second);
+ fileNumberNameMap_.erase(i);
+ enqueueCountList_.pop_front();
+ }
+}
+
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.h b/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.h
index 452b3d144b..91446c6abf 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/RecoveryManager.h
@@ -22,42 +22,111 @@
#ifndef QPID_LINEARSTORE_RECOVERYSTATE_H_
#define QPID_LINEARSTORE_RECOVERYSTATE_H_
+#include <deque>
#include <fstream>
#include <map>
+#include "qpid/linearstore/jrnl/LinearFileController.h"
#include <stdint.h>
#include <vector>
+struct file_hdr_t;
+struct rec_hdr_t;
+
namespace qpid {
namespace qls_jrnl {
+class data_tok;
+class enq_map;
+class EmptyFilePool;
+class EmptyFilePoolManager;
+class JournalLog;
+class jrec;
+class txn_map;
+
class RecoveryManager
{
-private:
+protected:
+ // Types
+ typedef std::vector<std::string> directoryList_t;
+ typedef directoryList_t::const_iterator directoryListConstItr_t;
+ typedef std::map<uint64_t, std::string> fileNumberNameMap_t;
+ typedef fileNumberNameMap_t::iterator fileNumberNameMapItr_t;
+ typedef fileNumberNameMap_t::const_iterator fileNumberNameMapConstItr_t;
+ typedef std::deque<uint32_t> enqueueCountList_t;
+ typedef enqueueCountList_t::const_iterator enqueueCountListConstItr_t;
+ typedef std::vector<uint64_t> recordIdList_t;
+ typedef recordIdList_t::const_iterator recordIdListConstItr_t;
+
+ // Location and identity
+ const std::string journalDirectory_;
+ const std::string queueName_;
+ enq_map& enqueueMapRef_;
+ txn_map& transactionMapRef_;
+ JournalLog& journalLogRef_;
+
// Initial journal analysis data
- std::vector<std::string> _journalFileList; ///< Journal file list
- std::map<uint64_t, std::string> _fileNumberNameMap; ///< File number - name map
- std::vector<uint32_t> _enqueueCountList; ///< Number enqueued records found for each file
- bool _journalEmptyFlag; ///< Journal data files empty
- std::streamoff _firstRecordOffset; ///< First record offset in ffid
- std::streamoff _endOffset; ///< End offset (first byte past last record)
- uint64_t _highestRecordId; ///< Highest rid found
- bool _lastFileFullFlag; ///< Last file is full
+ fileNumberNameMap_t fileNumberNameMap_; ///< File number - name map
+ enqueueCountList_t enqueueCountList_; ///< Number enqueued records found for each file
+ bool journalEmptyFlag_; ///< Journal data files empty
+ std::streamoff firstRecordOffset_; ///< First record offset in ffid
+ std::streamoff endOffset_; ///< End offset (first byte past last record)
+ uint64_t highestRecordId_; ///< Highest rid found
+ uint64_t highestFileNumber_; ///< Highest file number found
+ bool lastFileFullFlag_; ///< Last file is full
// State for recovery of individual enqueued records
- uint64_t _currentRid;
- uint64_t _currentFileNumber;
- std::string _currentFileName;
- std::streamoff _fileSize;
- std::streamoff _recordStart;
- std::ifstream _inFileStream;
- bool _readComplete;
+ uint32_t fileSize_kib_;
+ fileNumberNameMapConstItr_t currentJournalFileConstItr_;
+ std::string currentFileName_;
+ std::ifstream inFileStream_;
+ recordIdList_t recordIdList_;
+ recordIdListConstItr_t recordIdListConstItr_;
public:
- RecoveryManager();
+ RecoveryManager(const std::string& journalDirectory,
+ const std::string& queuename,
+ enq_map& enqueueMapRef,
+ txn_map& transactionMapRef,
+ JournalLog& journalLogRef);
virtual ~RecoveryManager();
+ void analyzeJournals(const std::vector<std::string>* preparedTransactionListPtr,
+ EmptyFilePoolManager* emptyFilePoolManager,
+ EmptyFilePool** emptyFilePoolPtrPtr);
+ std::streamoff getEndOffset() const;
+ uint64_t getHighestFileNumber() const;
+ uint64_t getHighestRecordId() const;
+ bool isLastFileFull() const;
+ bool readNextRemainingRecord(void** const dataPtrPtr,
+ std::size_t& dataSize,
+ void** const xidPtrPtr,
+ std::size_t& xidSize,
+ bool& transient,
+ bool& external,
+ data_tok* const dtokp,
+ bool ignore_pending_txns);
+ void setLinearFileControllerJournals(lfcAddJournalFileFn fnPtr,
+ LinearFileController* lfcPtr);
std::string toString(const std::string& jid,
bool compact = true);
+protected:
+ void analyzeJournalFileHeaders(efpIdentity_t& efpIdentity);
+ void checkFileStreamOk(bool checkEof);
+ void checkJournalAlignment(const std::streampos recordPosition);
+ bool decodeRecord(jrec& record,
+ std::size_t& cumulativeSizeRead,
+ ::rec_hdr_t& recordHeader,
+ std::streampos& fileOffset);
+ std::string getCurrentFileName() const;
+ uint64_t getCurrentFileNumber() const;
+ bool getNextFile(bool jumpToFirstRecordOffsetFlag);
+ bool getNextRecordHeader();
+ void readJournalData(char* target, const std::streamsize size);
+ void removeEmptyFiles(EmptyFilePool* emptyFilePoolPtr);
+
+ static void readJournalFileHeader(const std::string& journalFileName,
+ ::file_hdr_t& fileHeaderRef,
+ std::string& queueName);
};
}} // namespace qpid::qls_jrnl
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.cpp
index e32452b304..5b251b7908 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/deq_rec.cpp
@@ -110,8 +110,8 @@ deq_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks)
if (_xidp == 0)
assert(_deq_hdr._xidsize == 0);
- std::size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE_BYTES;
- std::size_t rem = max_size_dblks * JRNL_DBLK_SIZE_BYTES;
+ std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES;
+ std::size_t rem = max_size_dblks * QLS_DBLK_SIZE_BYTES;
std::size_t wr_cnt = 0;
if (rec_offs_dblks) // Continuation of split dequeue record (over 2 or more pages)
{
@@ -161,10 +161,10 @@ deq_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks)
{
std::memcpy((char*)wptr + wr_cnt, (char*)&_deq_tail + rec_offs, wsize);
wr_cnt += wsize;
-#ifdef RHM_CLEAN
- std::size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE_BYTES;
- std::size_t dblk_rec_size = size_dblks(rec_size() - rec_offs) * JRNL_DBLK_SIZE_BYTES;
- std::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR, dblk_rec_size - wr_cnt);
+#ifdef QLS_CLEAN
+ std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES;
+ std::size_t dblk_rec_size = size_dblks(rec_size() - rec_offs) * QLS_DBLK_SIZE_BYTES;
+ std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt);
#endif
}
rec_offs -= sizeof(_deq_tail) - wsize;
@@ -205,9 +205,9 @@ deq_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks)
std::memcpy((char*)wptr + wr_cnt, (void*)&_deq_tail, sizeof(_deq_tail));
wr_cnt += sizeof(_deq_tail);
}
-#ifdef RHM_CLEAN
- std::size_t dblk_rec_size = size_dblks(rec_size()) * JRNL_DBLK_SIZE_BYTES;
- std::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR, dblk_rec_size - wr_cnt);
+#ifdef QLS_CLEAN
+ std::size_t dblk_rec_size = size_dblks(rec_size()) * QLS_DBLK_SIZE_BYTES;
+ std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt);
#endif
}
}
@@ -226,7 +226,7 @@ deq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
const uint32_t hdr_xid_dblks = size_dblks(sizeof(deq_hdr_t) + _deq_hdr._xidsize);
const uint32_t hdr_xid_tail_dblks = size_dblks(sizeof(deq_hdr_t) + _deq_hdr._xidsize +
sizeof(rec_tail_t));
- const std::size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE_BYTES;
+ const std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES;
if (hdr_xid_tail_dblks - rec_offs_dblks <= max_size_dblks)
{
@@ -259,7 +259,7 @@ deq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
const std::size_t xid_rem = _deq_hdr._xidsize - xid_offs;
std::memcpy((char*)_buff + xid_offs, rptr, xid_rem);
rd_cnt += xid_rem;
- const std::size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
if (tail_rem)
{
std::memcpy((void*)&_deq_tail, ((char*)rptr + xid_rem), tail_rem);
@@ -269,7 +269,7 @@ deq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
else
{
// Remainder of xid split
- const std::size_t xid_cp_size = (max_size_dblks * JRNL_DBLK_SIZE_BYTES);
+ const std::size_t xid_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES);
std::memcpy((char*)_buff + rec_offs - sizeof(deq_hdr_t), rptr, xid_cp_size);
rd_cnt += xid_cp_size;
}
@@ -309,7 +309,7 @@ deq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
// Entire header and xid fit within this page, tail split
std::memcpy(_buff, (char*)rptr + rd_cnt, _deq_hdr._xidsize);
rd_cnt += _deq_hdr._xidsize;
- const std::size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
if (tail_rem)
{
std::memcpy((void*)&_deq_tail, (char*)rptr + rd_cnt, tail_rem);
@@ -319,7 +319,7 @@ deq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
else
{
// Header fits within this page, xid split
- const std::size_t xid_cp_size = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t xid_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
std::memcpy(_buff, (char*)rptr + rd_cnt, xid_cp_size);
rd_cnt += xid_cp_size;
}
@@ -381,7 +381,7 @@ deq_rec::rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs)
return false;
}
}
- ifsp->ignore(rec_size_dblks() * JRNL_DBLK_SIZE_BYTES - rec_size());
+ ifsp->ignore(rec_size_dblks() * QLS_DBLK_SIZE_BYTES - rec_size());
if (_deq_hdr._xidsize)
chk_tail(); // Throws if tail invalid or record incomplete
assert(!ifsp->fail() && !ifsp->bad());
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.cpp
index bc2040ef95..3bc49502c8 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.cpp
@@ -47,13 +47,13 @@ enq_map::~enq_map() {}
short
-enq_map::insert_pfid(const uint64_t rid, const uint16_t pfid, const std::streampos file_posn)
+enq_map::insert_pfid(const uint64_t rid, const uint64_t pfid, const std::streampos file_posn)
{
return insert_pfid(rid, pfid, file_posn, false);
}
short
-enq_map::insert_pfid(const uint64_t rid, const uint16_t pfid, const std::streampos file_posn, const bool locked)
+enq_map::insert_pfid(const uint64_t rid, const uint64_t pfid, const std::streampos file_posn, const bool locked)
{
std::pair<emap_itr, bool> ret;
emap_data_struct_t rec(pfid, file_posn, locked);
@@ -67,7 +67,7 @@ enq_map::insert_pfid(const uint64_t rid, const uint16_t pfid, const std::streamp
}
short
-enq_map::get_pfid(const uint64_t rid, int16_t& pfid)
+enq_map::get_pfid(const uint64_t rid, uint64_t& pfid)
{
slock s(_mutex);
emap_itr itr = _map.find(rid);
@@ -80,7 +80,7 @@ enq_map::get_pfid(const uint64_t rid, int16_t& pfid)
}
short
-enq_map::get_remove_pfid(const uint64_t rid, int16_t& pfid, const bool txn_flag)
+enq_map::get_remove_pfid(const uint64_t rid, uint64_t& pfid, const bool txn_flag)
{
slock s(_mutex);
emap_itr itr = _map.find(rid);
@@ -173,7 +173,7 @@ enq_map::rid_list(std::vector<uint64_t>& rv)
}
void
-enq_map::pfid_list(std::vector<uint16_t>& fv)
+enq_map::pfid_list(std::vector<uint64_t>& fv)
{
fv.clear();
{
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.h b/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.h
index d3be5f4b56..659e3bd052 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/enq_map.h
@@ -72,11 +72,11 @@ namespace qls_jrnl
static short EMAP_TRUE;
typedef struct emap_data_struct_t {
- uint16_t _pfid;
+ uint64_t _pfid;
std::streampos _file_posn;
bool _lock;
emap_data_struct_t() : _pfid(0), _file_posn(0), _lock(false) {}
- emap_data_struct_t(const uint16_t pfid, const std::streampos file_posn, const bool lock) : _pfid(pfid), _file_posn(file_posn), _lock(lock) {}
+ emap_data_struct_t(const uint64_t pfid, const std::streampos file_posn, const bool lock) : _pfid(pfid), _file_posn(file_posn), _lock(lock) {}
} emqp_data_struct_t;
typedef std::pair<uint64_t, emap_data_struct_t> emap_param;
typedef std::map<uint64_t, emap_data_struct_t> emap;
@@ -85,19 +85,15 @@ namespace qls_jrnl
private:
emap _map;
smutex _mutex;
-// std::vector<uint32_t> _pfid_enq_cnt;
public:
enq_map();
virtual ~enq_map();
-// void set_num_jfiles(const uint16_t num_jfiles);
-// inline uint32_t get_enq_cnt(const uint16_t pfid) const { return _pfid_enq_cnt.at(pfid); };
-
- short insert_pfid(const uint64_t rid, const uint16_t pfid, const std::streampos file_posn); // 0=ok; -3=duplicate rid;
- short insert_pfid(const uint64_t rid, const uint16_t pfid, const std::streampos file_posn, const bool locked); // 0=ok; -3=duplicate rid;
- short get_pfid(const uint64_t rid, int16_t& pfid); // >=0=pfid; -1=rid not found; -2=locked
- short get_remove_pfid(const uint64_t rid, int16_t& pfid, const bool txn_flag = false); // >=0=pfid; -1=rid not found; -2=locked
+ short insert_pfid(const uint64_t rid, const uint64_t pfid, const std::streampos file_posn); // 0=ok; -3=duplicate rid;
+ short insert_pfid(const uint64_t rid, const uint64_t pfid, const std::streampos file_posn, const bool locked); // 0=ok; -3=duplicate rid;
+ short get_pfid(const uint64_t rid, uint64_t& pfid); // >=0=pfid; -1=rid not found; -2=locked
+ short get_remove_pfid(const uint64_t rid, uint64_t& pfid, const bool txn_flag = false); // >=0=pfid; -1=rid not found; -2=locked
short get_file_posn(const uint64_t rid, std::streampos& file_posn); // -1=rid not found; -2=locked
short get_data(const uint64_t rid, emap_data_struct_t& eds);
bool is_enqueued(const uint64_t rid, bool ignore_lock = false);
@@ -108,7 +104,7 @@ namespace qls_jrnl
inline bool empty() const { return _map.empty(); }
inline uint32_t size() const { return uint32_t(_map.size()); }
void rid_list(std::vector<uint64_t>& rv);
- void pfid_list(std::vector<uint16_t>& fv);
+ void pfid_list(std::vector<uint64_t>& fv);
};
}}
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.cpp
index c94e552da9..e968320ac6 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/enq_rec.cpp
@@ -109,8 +109,8 @@ enq_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks)
if (_xidp == 0)
assert(_enq_hdr._xidsize == 0);
- std::size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE_BYTES;
- std::size_t rem = max_size_dblks * JRNL_DBLK_SIZE_BYTES;
+ std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES;
+ std::size_t rem = max_size_dblks * QLS_DBLK_SIZE_BYTES;
std::size_t wr_cnt = 0;
if (rec_offs_dblks) // Continuation of split data record (over 2 or more pages)
{
@@ -181,10 +181,10 @@ enq_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks)
{
std::memcpy((char*)wptr + wr_cnt, (char*)&_enq_tail + rec_offs, wsize);
wr_cnt += wsize;
-#ifdef RHM_CLEAN
- std::size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE_BYTES;
- std::size_t dblk_rec_size = size_dblks(rec_size() - rec_offs) * JRNL_DBLK_SIZE_BYTES;
- std::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR, dblk_rec_size - wr_cnt);
+#ifdef QLS_CLEAN
+ std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES;
+ std::size_t dblk_rec_size = size_dblks(rec_size() - rec_offs) * QLS_DBLK_SIZE_BYTES;
+ std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt);
#endif
}
rec_offs -= sizeof(_enq_tail) - wsize;
@@ -237,9 +237,9 @@ enq_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks)
}
std::memcpy((char*)wptr + wr_cnt, (void*)&_enq_tail, sizeof(_enq_tail));
wr_cnt += sizeof(_enq_tail);
-#ifdef RHM_CLEAN
- std::size_t dblk_rec_size = size_dblks(rec_size()) * JRNL_DBLK_SIZE_BYTES;
- std::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR, dblk_rec_size - wr_cnt);
+#ifdef QLS_CLEAN
+ std::size_t dblk_rec_size = size_dblks(rec_size()) * QLS_DBLK_SIZE_BYTES;
+ std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt);
#endif
}
}
@@ -260,7 +260,7 @@ enq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
const uint32_t hdr_xid_data_tail_size = hdr_xid_data_size + sizeof(rec_tail_t);
const uint32_t hdr_data_dblks = size_dblks(hdr_xid_data_size);
const uint32_t hdr_tail_dblks = size_dblks(hdr_xid_data_tail_size);
- const std::size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE_BYTES;
+ const std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES;
const std::size_t offs = rec_offs - sizeof(enq_hdr_t);
if (hdr_tail_dblks - rec_offs_dblks <= max_size_dblks)
@@ -331,7 +331,7 @@ enq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
std::memcpy((char*)_buff + offs, rptr, data_rem);
rd_cnt += data_rem;
}
- const std::size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
if (tail_rem)
{
std::memcpy((void*)&_enq_tail, ((char*)rptr + rd_cnt), tail_rem);
@@ -341,7 +341,7 @@ enq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
else
{
// Since xid and data are contiguous, both fit within current page - copy whole page
- const std::size_t data_cp_size = (max_size_dblks * JRNL_DBLK_SIZE_BYTES);
+ const std::size_t data_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES);
std::memcpy((char*)_buff + offs, rptr, data_cp_size);
rd_cnt += data_cp_size;
}
@@ -405,7 +405,7 @@ enq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
_enq_hdr._dsize);
rd_cnt += _enq_hdr._dsize;
}
- const std::size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
if (tail_rem)
{
std::memcpy((void*)&_enq_tail, (char*)rptr + rd_cnt, tail_rem);
@@ -422,7 +422,7 @@ enq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
}
if (_enq_hdr._dsize && !::is_enq_external(&_enq_hdr))
{
- const std::size_t data_cp_size = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t data_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
std::memcpy((char*)_buff + _enq_hdr._xidsize, (char*)rptr + rd_cnt, data_cp_size);
rd_cnt += data_cp_size;
}
@@ -430,7 +430,7 @@ enq_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
else
{
// Header fits within this page, xid split or separated
- const std::size_t data_cp_size = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t data_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
std::memcpy(_buff, (char*)rptr + rd_cnt, data_cp_size);
rd_cnt += data_cp_size;
}
@@ -516,7 +516,7 @@ enq_rec::rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs)
return false;
}
}
- ifsp->ignore(rec_size_dblks() * JRNL_DBLK_SIZE_BYTES - rec_size());
+ ifsp->ignore(rec_size_dblks() * QLS_DBLK_SIZE_BYTES - rec_size());
chk_tail(); // Throws if tail invalid or record incomplete
assert(!ifsp->fail() && !ifsp->bad());
return true;
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/enums.h b/qpid/cpp/src/qpid/linearstore/jrnl/enums.h
index 824eaa90f4..511a2a41ab 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/enums.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/enums.h
@@ -40,9 +40,9 @@ namespace qls_jrnl
// RHM_IORES_RCINVALID, ///< Read page cache is invalid (ie obsolete or uninitialized)
// RHM_IORES_ENQCAPTHRESH, ///< Enqueue capacity threshold (limit) reached.
// RHM_IORES_FULL, ///< During write operations, the journal files are full.
- RHM_IORES_BUSY, ///< Another blocking operation is in progress.
- RHM_IORES_TXPENDING, ///< Operation blocked by pending transaction.
- RHM_IORES_NOTIMPL ///< Function is not implemented.
+// RHM_IORES_BUSY, ///< Another blocking operation is in progress.
+ RHM_IORES_TXPENDING ///< Operation blocked by pending transaction.
+// RHM_IORES_NOTIMPL ///< Function is not implemented.
};
typedef _iores iores;
@@ -57,9 +57,9 @@ namespace qls_jrnl
// case RHM_IORES_RCINVALID: return "RHM_IORES_RCINVALID";
// case RHM_IORES_ENQCAPTHRESH: return "RHM_IORES_ENQCAPTHRESH";
// case RHM_IORES_FULL: return "RHM_IORES_FULL";
- case RHM_IORES_BUSY: return "RHM_IORES_BUSY";
+// case RHM_IORES_BUSY: return "RHM_IORES_BUSY";
case RHM_IORES_TXPENDING: return "RHM_IORES_TXPENDING";
- case RHM_IORES_NOTIMPL: return "RHM_IORES_NOTIMPL";
+// case RHM_IORES_NOTIMPL: return "RHM_IORES_NOTIMPL";
}
return "<iores unknown>";
}
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jcfg.h b/qpid/cpp/src/qpid/linearstore/jrnl/jcfg.h
index c947ff46db..432887e577 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/jcfg.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/jcfg.h
@@ -22,48 +22,37 @@
#ifndef QPID_LEGACYSTORE_JRNL_JCFG_H
#define QPID_LEGACYSTORE_JRNL_JCFG_H
+#define QLS_SBLK_SIZE_BYTES 4096 /**< Disk softblock size in bytes, should match size used on disk media */
+#define QLS_AIO_ALIGN_BOUNDARY_BYTES QLS_SBLK_SIZE_BYTES /** Memory alignment boundary used for DMA */
/**
-* <b>Rule:</b> Data block size (JRNL_DBLK_SIZE_BYTES) MUST be a power of 2 AND
-* a power of 2 factor of the disk softblock size (JRNL_SBLK_SIZE_BYTES):
+* <b>Rule:</b> Data block size (QLS_DBLK_SIZE_BYTES) MUST be a power of 2 AND
+* a power of 2 factor of the disk softblock size (QLS_SBLK_SIZE_BYTES):
* <pre>
-* n * JRNL_DBLK_SIZE_BYTES == JRNL_SBLK_SIZE_BYTES (n = 1,2,4,8...)
+* n * QLS_DBLK_SIZE_BYTES == QLS_SBLK_SIZE_BYTES (n = 1,2,4,8...)
* </pre>
*/
-#define JRNL_SBLK_SIZE_BYTES 4096 /**< Disk softblock size in bytes */
-#define QLS_AIO_ALIGN_BOUNDARY JRNL_SBLK_SIZE_BYTES
-#define JRNL_DBLK_SIZE_BYTES 128 /**< Data block size in bytes (CANNOT BE LESS THAN 32!) */
-#define JRNL_SBLK_SIZE_DBLKS (JRNL_SBLK_SIZE_BYTES / JRNL_DBLK_SIZE_BYTES) /**< Disk softblock size in multiples of JRNL_DBLK_SIZE */
-#define JRNL_SBLK_SIZE_KIB (JRNL_SBLK_SIZE_BYTES / 1024) /**< Disk softblock size in KiB */
-//#define JRNL_MIN_FILE_SIZE 128 ///< Min. jrnl file size in sblks (excl. file_hdr)
-//#define JRNL_MAX_FILE_SIZE 4194176 ///< Max. jrnl file size in sblks (excl. file_hdr)
-//#define JRNL_MIN_NUM_FILES 4 ///< Min. number of journal files
-//#define JRNL_MAX_NUM_FILES 64 ///< Max. number of journal files
-//#define JRNL_ENQ_THRESHOLD 80 ///< Percent full when enqueue connection will be closed
-//
-//#define JRNL_RMGR_PAGE_SIZE 128 ///< Journal page size in softblocks
-//#define JRNL_RMGR_PAGES 16 ///< Number of pages to use in wmgr
-//
-#define JRNL_WMGR_DEF_PAGE_SIZE_KIB 32
-#define JRNL_WMGR_DEF_PAGE_SIZE_SBLKS (JRNL_WMGR_DEF_PAGE_SIZE_KIB / JRNL_SBLK_SIZE_KIB) ///< Journal write page size in softblocks (default)
-#define JRNL_WMGR_DEF_PAGES 32 ///< Number of pages to use in wmgr (default)
-//
-#define JRNL_WMGR_MAXDTOKPP 1024 ///< Max. dtoks (data blocks) per page in wmgr
-#define JRNL_WMGR_MAXWAITUS 100 ///< Max. wait time (us) before submitting AIO
-//
-//#define JRNL_INFO_EXTENSION "jinf" ///< Extension for journal info files
-//#define JRNL_DATA_EXTENSION "jdat" ///< Extension for journal data files
-#define QLS_JRNL_FILE_EXTENSION ".jrnl" /**< Extension for journal data files */
-#define QLS_TXA_MAGIC 0x61534c51 /**< ("RHMa" in little endian) Magic for dtx abort hdrs */
-#define QLS_TXC_MAGIC 0x63534c51 /**< ("RHMc" in little endian) Magic for dtx commit hdrs */
-#define QLS_DEQ_MAGIC 0x64534c51 /**< ("QLSd" in little endian) Magic for deq rec hdrs */
-#define QLS_ENQ_MAGIC 0x65534c51 /**< ("QLSe" in little endian) Magic for enq rec hdrs */
-#define QLS_FILE_MAGIC 0x66534c51 /**< ("QLSf" in little endian) Magic for file hdrs */
-#define QLS_EMPTY_MAGIC 0x78534c51 /**< ("QLSx" in little endian) Magic for empty dblk */
-#define QLS_JRNL_VERSION 2 /**< Version (of file layout) */
-#define QLS_JRNL_FHDR_RES_SIZE_SBLKS 1 /**< Journal file header reserved size in sblks (as defined by JRNL_SBLK_SIZE_BYTES) */
-#define QLS_CLEAN_CHAR 0xff /**< Char used to clear empty space on disk */
-//
-//#define RHM_LENDIAN_FLAG 0 ///< Value of little endian flag on disk
-//#define RHM_BENDIAN_FLAG 1 ///< Value of big endian flag on disk
+#define QLS_DBLK_SIZE_BYTES 128 /**< Data block size in bytes (CANNOT BE LESS THAN 32!) */
+#define QLS_SBLK_SIZE_DBLKS (QLS_SBLK_SIZE_BYTES / QLS_DBLK_SIZE_BYTES) /**< Disk softblock size in multiples of QLS_DBLK_SIZE_BYTES */
+#define QLS_SBLK_SIZE_KIB (QLS_SBLK_SIZE_BYTES / 1024) /**< Disk softblock size in KiB */
-#endif // ifndef QPID_LEGACYSTORE_JRNL_JCFG_H
+#define QLS_WMGR_DEF_PAGE_SIZE_KIB 32 /**< Journal write page size in KiB (default) */
+#define QLS_WMGR_DEF_PAGE_SIZE_SBLKS (QLS_WMGR_DEF_PAGE_SIZE_KIB / QLS_SBLK_SIZE_KIB) /**< Journal write page size in softblocks (default) */
+#define QLS_WMGR_DEF_PAGES 32 /**< Number of pages to use in wmgr (default) */
+
+#define QLS_WMGR_MAXDTOKPP 1024 /**< Max. dtoks (data blocks) per page in wmgr */
+#define QLS_WMGR_MAXWAITUS 100 /**< Max. wait time (us) before submitting AIO */
+
+#define QLS_JRNL_FILE_EXTENSION ".jrnl" /**< Extension for journal data files */
+#define QLS_TXA_MAGIC 0x61534c51 /**< ("RHMa" in little endian) Magic for dtx abort hdrs */
+#define QLS_TXC_MAGIC 0x63534c51 /**< ("RHMc" in little endian) Magic for dtx commit hdrs */
+#define QLS_DEQ_MAGIC 0x64534c51 /**< ("QLSd" in little endian) Magic for deq rec hdrs */
+#define QLS_ENQ_MAGIC 0x65534c51 /**< ("QLSe" in little endian) Magic for enq rec hdrs */
+#define QLS_FILE_MAGIC 0x66534c51 /**< ("QLSf" in little endian) Magic for file hdrs */
+#define QLS_EMPTY_MAGIC 0x78534c51 /**< ("QLSx" in little endian) Magic for empty dblk */
+#define QLS_JRNL_VERSION 2 /**< Version (of file layout) */
+#define QLS_JRNL_FHDR_RES_SIZE_SBLKS 1 /**< Journal file header reserved size in sblks (as defined by QLS_SBLK_SIZE_BYTES) */
+
+#define QLS_CLEAN /**< If defined, writes QLS_CLEAN_CHAR to all filled areas on disk */
+#define QLS_CLEAN_CHAR 0xff /**< Char used to clear empty space on disk */
+
+#endif /* ifndef QPID_LEGACYSTORE_JRNL_JCFG_H */
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.cpp
index 280d09f886..e8873cddcd 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.cpp
@@ -31,10 +31,8 @@
#include <iostream>
#include <qpid/linearstore/jrnl/EmptyFilePool.h>
#include <qpid/linearstore/jrnl/EmptyFilePoolManager.h>
-//#include "qpid/linearstore/jrnl/file_hdr.h"
#include "qpid/linearstore/jrnl/jerrno.h"
-//#include "qpid/linearstore/jrnl/jinf.h"
-//#include "qpid/linearstore/jrnl/JournalFileController.h"
+#include "qpid/linearstore/jrnl/JournalLog.h"
#include "qpid/linearstore/jrnl/utils/enq_hdr.h"
#include <limits>
#include <sstream>
@@ -66,25 +64,21 @@ bool jcntl::init_statics()
// Functions
-jcntl::jcntl(const std::string& jid, const std::string& jdir/*, const std::string& base_filename*/):
+jcntl::jcntl(const std::string& jid,
+ const std::string& jdir,
+ JournalLog& jrnl_log):
_jid(jid),
- _jdir(jdir/*, base_filename*/),
-// _base_filename(base_filename),
+ _jdir(jdir),
_init_flag(false),
_stop_flag(false),
_readonly_flag(false),
-// _autostop(true),
+ _jrnl_log(jrnl_log),
_linearFileController(*this),
_emptyFilePoolPtr(0),
-// _jfsize_sblks(0),
-// _lpmgr(),
_emap(),
_tmap(),
-// _rrfc(&_lpmgr),
-// _wrfc(&_lpmgr),
-// _rmgr(this, _emap, _tmap/*, _rrfc*/),
- _wmgr(this, _emap, _tmap, _linearFileController/*, _wrfc*/),
- _rcvdat()
+ _wmgr(this, _emap, _tmap, _linearFileController),
+ _recoveryManager(_jdir.dirname(), _jid, _emap, _tmap, jrnl_log)
{}
jcntl::~jcntl()
@@ -92,14 +86,14 @@ jcntl::~jcntl()
if (_init_flag && !_stop_flag)
try { stop(true); }
catch (const jexception& e) { std::cerr << e << std::endl; }
-// _lpmgr.finalize();
_linearFileController.finalize();
}
void
-jcntl::initialize(/*const uint16_t num_jfiles, const bool ae, const uint16_t ae_max_jfiles,
- const uint32_t jfsize_sblks,*/ EmptyFilePool* efpp, const uint16_t wcache_num_pages, const uint32_t wcache_pgsize_sblks,
- aio_callback* const cbp)
+jcntl::initialize(EmptyFilePool* efpp,
+ const uint16_t wcache_num_pages,
+ const uint32_t wcache_pgsize_sblks,
+ aio_callback* const cbp)
{
_init_flag = false;
_stop_flag = false;
@@ -126,14 +120,14 @@ jcntl::initialize(/*const uint16_t num_jfiles, const bool ae, const uint16_t ae_
_jdir.clear_dir();
// _lpmgr.initialize(num_jfiles, ae, ae_max_jfiles, this, &new_fcntl); // Creates new journal files
- _linearFileController.initialize(_jdir.dirname(), efpp);
+ _linearFileController.initialize(_jdir.dirname(), efpp, 0ULL);
_linearFileController.pullEmptyFileFromEfp();
- std::cout << _linearFileController.status(2);
+// std::cout << _linearFileController.status(2);
// _wrfc.initialize(_jfsize_sblks);
// _rrfc.initialize();
// _rrfc.set_findex(0);
// _rmgr.initialize(cbp);
- _wmgr.initialize(cbp, wcache_pgsize_sblks, wcache_num_pages, JRNL_WMGR_MAXDTOKPP, JRNL_WMGR_MAXWAITUS);
+ _wmgr.initialize(cbp, wcache_pgsize_sblks, wcache_num_pages, QLS_WMGR_MAXDTOKPP, QLS_WMGR_MAXWAITUS);
// Write info file (<basename>.jinf) to disk
// write_infofile();
@@ -142,7 +136,7 @@ jcntl::initialize(/*const uint16_t num_jfiles, const bool ae, const uint16_t ae_
}
void
-jcntl::recover(EmptyFilePoolManager* efpm,
+jcntl::recover(EmptyFilePoolManager* efpmp,
const uint16_t wcache_num_pages,
const uint32_t wcache_pgsize_sblks,
aio_callback* const cbp,
@@ -170,21 +164,26 @@ jcntl::recover(EmptyFilePoolManager* efpm,
_jdir.verify_dir();
// _rcvdat.reset(num_jfiles/*, ae, ae_max_jfiles*/);
- rcvr_janalyze(prep_txn_list_ptr, efpm);
- highest_rid = _rcvdat._h_rid;
+// rcvr_janalyze(prep_txn_list_ptr, efpm);
+ efpIdentity_t efpIdentity;
+ _recoveryManager.analyzeJournals(prep_txn_list_ptr, efpmp, &_emptyFilePoolPtr);
+
+ highest_rid = _recoveryManager.getHighestRecordId();
// if (_rcvdat._jfull)
// throw jexception(jerrno::JERR_JCNTL_RECOVERJFULL, "jcntl", "recover");
- this->log(/*LOG_DEBUG*/LOG_INFO, _jid, _rcvdat.to_log(_jid));
+ _jrnl_log.log(/*LOG_DEBUG*/JournalLog::LOG_INFO, _jid, _recoveryManager.toString(_jid, true));
// _lpmgr.recover(_rcvdat, this, &new_fcntl);
- _linearFileController.initialize(_jdir.dirname(), _emptyFilePoolPtr);
+ _linearFileController.initialize(_jdir.dirname(), _emptyFilePoolPtr, _recoveryManager.getHighestFileNumber());
+// _linearFileController.setFileNumberCounter(_recoveryManager.getHighestFileNumber());
+ _recoveryManager.setLinearFileControllerJournals(&qpid::qls_jrnl::LinearFileController::addJournalFile, &_linearFileController);
// _wrfc.initialize(_jfsize_sblks, &_rcvdat);
// _rrfc.initialize();
// _rrfc.set_findex(_rcvdat.ffid());
// _rmgr.initialize(cbp);
- _wmgr.initialize(cbp, wcache_pgsize_sblks, wcache_num_pages, JRNL_WMGR_MAXDTOKPP, JRNL_WMGR_MAXWAITUS,
- (_rcvdat._lffull ? 0 : _rcvdat._eo));
+ _wmgr.initialize(cbp, wcache_pgsize_sblks, wcache_num_pages, QLS_WMGR_MAXDTOKPP, QLS_WMGR_MAXWAITUS,
+ (_recoveryManager.isLastFileFull() ? 0 : _recoveryManager.getEndOffset()));
_readonly_flag = true;
_init_flag = true;
@@ -214,8 +213,11 @@ jcntl::delete_jrnl_files()
iores
-jcntl::enqueue_data_record(const void* const data_buff, const std::size_t tot_data_len,
- const std::size_t this_data_len, data_tok* dtokp, const bool transient)
+jcntl::enqueue_data_record(const void* const data_buff,
+ const std::size_t tot_data_len,
+ const std::size_t this_data_len,
+ data_tok* dtokp,
+ const bool transient)
{
iores r;
check_wstatus("enqueue_data_record");
@@ -228,7 +230,9 @@ jcntl::enqueue_data_record(const void* const data_buff, const std::size_t tot_da
}
iores
-jcntl::enqueue_extern_data_record(const std::size_t tot_data_len, data_tok* dtokp, const bool transient)
+jcntl::enqueue_extern_data_record(const std::size_t tot_data_len,
+ data_tok* dtokp,
+ const bool transient)
{
iores r;
check_wstatus("enqueue_extern_data_record");
@@ -240,9 +244,12 @@ jcntl::enqueue_extern_data_record(const std::size_t tot_data_len, data_tok* dtok
}
iores
-jcntl::enqueue_txn_data_record(const void* const data_buff, const std::size_t tot_data_len,
- const std::size_t this_data_len, data_tok* dtokp, const std::string& xid,
- const bool transient)
+jcntl::enqueue_txn_data_record(const void* const data_buff,
+ const std::size_t tot_data_len,
+ const std::size_t this_data_len,
+ data_tok* dtokp,
+ const std::string& xid,
+ const bool transient)
{
iores r;
check_wstatus("enqueue_tx_data_record");
@@ -255,8 +262,10 @@ jcntl::enqueue_txn_data_record(const void* const data_buff, const std::size_t to
}
iores
-jcntl::enqueue_extern_txn_data_record(const std::size_t tot_data_len, data_tok* dtokp,
- const std::string& xid, const bool transient)
+jcntl::enqueue_extern_txn_data_record(const std::size_t tot_data_len,
+ data_tok* dtokp,
+ const std::string& xid,
+ const bool transient)
{
iores r;
check_wstatus("enqueue_extern_txn_data_record");
@@ -268,27 +277,21 @@ jcntl::enqueue_extern_txn_data_record(const std::size_t tot_data_len, data_tok*
return r;
}
-/* TODO
-iores
-jcntl::get_data_record(const uint64_t& rid, const std::size_t& dsize, const std::size_t& dsize_avail,
- const void** const data, bool auto_discard)
-{
- check_rstatus("get_data_record");
- return _rmgr.get(rid, dsize, dsize_avail, data, auto_discard);
-} */
-
-/* TODO
iores
-jcntl::discard_data_record(data_tok* const dtokp)
-{
- check_rstatus("discard_data_record");
- return _rmgr.discard(dtokp);
-} */
-
-iores
-jcntl::read_data_record(void** const datapp, std::size_t& dsize, void** const xidpp, std::size_t& xidsize,
- bool& transient, bool& external, data_tok* const dtokp, bool /*ignore_pending_txns*/)
+jcntl::read_data_record(void** const datapp,
+ std::size_t& dsize,
+ void** const xidpp,
+ std::size_t& xidsize,
+ bool& transient,
+ bool& external,
+ data_tok* const dtokp,
+ bool ignore_pending_txns)
{
+ check_rstatus("read_data");
+ if (_recoveryManager.readNextRemainingRecord(datapp, dsize, xidpp, xidsize, transient, external, dtokp, ignore_pending_txns))
+ return RHM_IORES_SUCCESS;
+ return RHM_IORES_EMPTY;
+/*
if (!dtokp->is_readable()) {
std::ostringstream oss;
oss << std::hex << std::setfill('0');
@@ -303,18 +306,18 @@ jcntl::read_data_record(void** const datapp, std::size_t& dsize, void** const xi
for (std::vector<uint64_t>::const_iterator i=ridl.begin(); i!=ridl.end(); ++i) {
short res = _emap.get_data(*i, eds);
if (res == enq_map::EMAP_OK) {
- std::ifstream ifs(_rcvdat._fm[eds._pfid].c_str(), std::ifstream::in | std::ifstream::binary);
+ std::ifstream ifs(_recoveryManager._fm[eds._pfid].c_str(), std::ifstream::in | std::ifstream::binary);
if (!ifs.good()) {
std::ostringstream oss;
- oss << "rid=" << (*i) << " pfid=" << eds._pfid << " file=" << _rcvdat._fm[eds._pfid] << " file_posn=" << eds._file_posn;
- throw jexception(jerrno::JERR_JCNTL_OPENRD, oss.str(), "jcntl", "read_data_record");
+ oss << "rid=" << (*i) << " pfid=" << eds._pfid << " file=" << _recoveryManager._fm[eds._pfid] << " file_posn=" << eds._file_posn;
+ throw jexception(jerrno::JERR_RCVM_OPENRD, oss.str(), "jcntl", "read_data_record");
}
ifs.seekg(eds._file_posn, std::ifstream::beg);
::enq_hdr_t eh;
ifs.read((char*)&eh, sizeof(::enq_hdr_t));
if (!::validate_enq_hdr(&eh, QLS_ENQ_MAGIC, QLS_JRNL_VERSION, *i)) {
std::ostringstream oss;
- oss << "rid=" << (*i) << " pfid=" << eds._pfid << " file=" << _rcvdat._fm[eds._pfid] << " file_posn=" << eds._file_posn;
+ oss << "rid=" << (*i) << " pfid=" << eds._pfid << " file=" << _recoveryManager._fm[eds._pfid] << " file_posn=" << eds._file_posn;
throw jexception(jerrno::JERR_JCNTL_INVALIDENQHDR, oss.str(), "jcntl", "read_data_record");
}
dsize = eh._dsize;
@@ -335,6 +338,7 @@ jcntl::read_data_record(void** const datapp, std::size_t& dsize, void** const xi
}
}
}
+*/
/*
check_rstatus("read_data");
iores res = _rmgr.read(datapp, dsize, xidpp, xidsize, transient, external, dtokp, ignore_pending_txns);
@@ -354,7 +358,8 @@ jcntl::read_data_record(void** const datapp, std::size_t& dsize, void** const xi
}
iores
-jcntl::dequeue_data_record(data_tok* const dtokp, const bool txn_coml_commit)
+jcntl::dequeue_data_record(data_tok* const dtokp,
+ const bool txn_coml_commit)
{
iores r;
check_wstatus("dequeue_data");
@@ -366,7 +371,9 @@ jcntl::dequeue_data_record(data_tok* const dtokp, const bool txn_coml_commit)
}
iores
-jcntl::dequeue_txn_data_record(data_tok* const dtokp, const std::string& xid, const bool txn_coml_commit)
+jcntl::dequeue_txn_data_record(data_tok* const dtokp,
+ const std::string& xid,
+ const bool txn_coml_commit)
{
iores r;
check_wstatus("dequeue_data");
@@ -378,7 +385,8 @@ jcntl::dequeue_txn_data_record(data_tok* const dtokp, const std::string& xid, co
}
iores
-jcntl::txn_abort(data_tok* const dtokp, const std::string& xid)
+jcntl::txn_abort(data_tok* const dtokp,
+ const std::string& xid)
{
iores r;
check_wstatus("txn_abort");
@@ -390,7 +398,8 @@ jcntl::txn_abort(data_tok* const dtokp, const std::string& xid)
}
iores
-jcntl::txn_commit(data_tok* const dtokp, const std::string& xid)
+jcntl::txn_commit(data_tok* const dtokp,
+ const std::string& xid)
{
iores r;
check_wstatus("txn_commit");
@@ -415,18 +424,9 @@ jcntl::get_wr_events(timespec* const timeout)
stlock t(_wr_mutex);
if (!t.locked())
return jerrno::LOCK_TAKEN;
- int32_t res = _wmgr.get_events(pmgr::UNUSED, timeout);
- return res;
+ return _wmgr.get_events(timeout, false);
}
-/*
-int32_t
-jcntl::get_rd_events(timespec* const timeout)
-{
- return _rmgr.get_events(pmgr::AIO_COMPLETE, timeout);
-}
-*/
-
void
jcntl::stop(const bool block_till_aio_cmpl)
{
@@ -462,56 +462,6 @@ jcntl::flush(const bool block_till_aio_cmpl)
return res;
}
-/*
-void
-jcntl::log(log_level_t ll, const std::string& log_stmt) const
-{
- log(ll, log_stmt.c_str());
-}
-
-void
-jcntl::log(log_level_t ll, const char* const log_stmt) const
-{
- if (ll > LOG_INFO)
- {
- std::cout << log_level_str(ll) << ": Journal \"" << _jid << "\": " << log_stmt << std::endl;
- }
-}
-*/
-
-/*
-void
-jcntl::chk_wr_frot()
-{
- if (_wrfc.index() == _rrfc.index())
- _rmgr.invalidate();
-}
-*/
-
-void
-jcntl::fhdr_wr_sync(const uint16_t /*lid*/)
-{
-/*
- fcntl* fcntlp = _lpmgr.get_fcntlp(lid);
- while (fcntlp->wr_fhdr_aio_outstanding())
- {
- if (get_wr_events(&_aio_cmpl_timeout) == jerrno::AIO_TIMEOUT)
- throw jexception(jerrno::JERR_JCNTL_AIOCMPLWAIT, "jcntl", "fhdr_wr_sync");
- }
-*/
-}
-
-/*
-fcntl*
-jcntl::new_fcntl(jcntl* const jcp, const uint16_t lid, const uint16_t fid, const rcvdat* const rdp)
-{
- if (!jcp) return 0;
- std::ostringstream oss;
- oss << jcp->jrnl_dir() << "/" << jcp->base_filename();
- return new fcntl(oss.str(), fid, lid, jcp->jfsize_sblks(), rdp);
-}
-*/
-
// Protected/Private functions
void
@@ -561,11 +511,15 @@ jcntl::handle_aio_wait(const iores res, iores& resout, const data_tok* dtp)
{
while (_wmgr.curr_pg_blocked())
{
- if (_wmgr.get_events(pmgr::UNUSED, &_aio_cmpl_timeout) == jerrno::AIO_TIMEOUT)
+ if (_wmgr.get_aio_evt_rem() == 0) {
+std::cout << "&&&&&& jcntl::handle_aio_wait() " << _wmgr.status_str() << std::endl; // DEBUG
+ throw jexception("_wmgr.curr_pg_blocked() with no events remaining"); // TODO - complete exception
+ }
+ if (_wmgr.get_events(&_aio_cmpl_timeout, false) == jerrno::AIO_TIMEOUT)
{
std::ostringstream oss;
oss << "get_events() returned JERR_JCNTL_AIOCMPLWAIT; wmgr_status: " << _wmgr.status_str();
- this->log(LOG_CRITICAL, _jid, oss.str());
+ _jrnl_log.log(JournalLog::LOG_CRITICAL, _jid, oss.str());
throw jexception(jerrno::JERR_JCNTL_AIOCMPLWAIT, "jcntl", "handle_aio_wait");
}
}
@@ -592,417 +546,4 @@ jcntl::handle_aio_wait(const iores res, iores& resout, const data_tok* dtp)
return false;
}
-
-// static
-void
-jcntl::rcvr_read_jfile(const std::string& jfn, ::file_hdr_t* fh, std::string& queueName) {
- const std::size_t headerBlockSize = QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_KIB * 1024;
- char buffer[headerBlockSize];
- std::ifstream ifs(jfn.c_str(), std::ifstream::in | std::ifstream::binary);
- if (!ifs.good()) {
- std::ostringstream oss;
- oss << "File=" << jfn;
- throw jexception(jerrno::JERR_JCNTL_OPENRD, oss.str(), "jcntl", "rcvr_read_jfile");
- }
- ifs.read(buffer, headerBlockSize);
- if (!ifs) {
- std::streamsize s = ifs.gcount();
- ifs.close();
- std::ostringstream oss;
- oss << "File=" << jfn << "; attempted_read_size=" << headerBlockSize << "; actual_read_size=" << s;
- throw jexception(jerrno::JERR_JCNTL_READ, oss.str(), "jcntl", "rcvr_read_jfile");
- }
- ifs.close();
- ::memcpy(fh, buffer, sizeof(::file_hdr_t));
- queueName.assign(buffer + sizeof(::file_hdr_t), fh->_queue_name_len);
-}
-
-
-void jcntl::rcvr_analyze_fhdrs(EmptyFilePoolManager* efpmp) {
- std::string headerQueueName;
- ::file_hdr_t fh;
- efpIdentity_t efpid;
-// std::map<uint64_t, std::string> fileMap;
- std::vector<std::string> dirList;
- jdir::read_dir(_jdir.dirname(), dirList, false, true, false, true);
- for (std::vector<std::string>::iterator i = dirList.begin(); i != dirList.end(); ++i) {
- rcvr_read_jfile(*i, &fh, headerQueueName);
- if (headerQueueName.compare(_jid) != 0) {
- std::ostringstream oss;
- oss << "Journal file " << (*i) << " belongs to queue \"" << headerQueueName << "\": ignoring";
- log(LOG_WARN, _jid, oss.str());
- } else {
- _rcvdat._fm[fh._file_number] = *i;
- efpid.first = fh._efp_partition;
- efpid.second = fh._file_size_kib;
- }
- }
- _rcvdat._jfl.clear();
- for (std::map<uint64_t, std::string>::iterator i=_rcvdat._fm.begin(); i!=_rcvdat._fm.end(); ++i) {
- _rcvdat._jfl.push_back(i->second);
- }
- _rcvdat._enq_cnt_list.resize(_rcvdat._jfl.size(), 0);
- _emptyFilePoolPtr = efpmp->getEmptyFilePool(efpid);
-}
-
-
-void jcntl::rcvr_janalyze(const std::vector<std::string>* prep_txn_list_ptr, EmptyFilePoolManager* efpmp) {
- // Analyze file headers of existing journal files
- rcvr_analyze_fhdrs(efpmp);
-
- // Restore all read and write pointers and transactions
- if (!_rcvdat._jempty)
- {
- uint16_t fid = 0;
- std::ifstream ifs;
- //bool lowi = rd._owi; // local copy of owi to be used during analysis
- while (rcvr_get_next_record(fid, &ifs)) ;
- if (ifs.is_open()) ifs.close();
-
- // Remove all txns from tmap that are not in the prepared list
- if (prep_txn_list_ptr)
- {
- std::vector<std::string> xid_list;
- _tmap.xid_list(xid_list);
- for (std::vector<std::string>::iterator itr = xid_list.begin(); itr != xid_list.end(); itr++)
- {
- std::vector<std::string>::const_iterator pitr =
- std::find(prep_txn_list_ptr->begin(), prep_txn_list_ptr->end(), *itr);
- if (pitr == prep_txn_list_ptr->end()) // not found in prepared list
- {
- txn_data_list tdl = _tmap.get_remove_tdata_list(*itr); // tdl will be empty if xid not found
- // Unlock any affected enqueues in emap
- for (tdl_itr i=tdl.begin(); i<tdl.end(); i++)
- {
- if (i->_enq_flag) // enq op - decrement enqueue count
- _rcvdat._enq_cnt_list[i->_pfid]--;
- else if (_emap.is_enqueued(i->_drid, true)) // deq op - unlock enq record
- {
- int16_t ret = _emap.unlock(i->_drid);
- if (ret < enq_map::EMAP_OK) // fail
- {
- // enq_map::unlock()'s only error is enq_map::EMAP_RID_NOT_FOUND
- std::ostringstream oss;
- oss << std::hex << "_emap.unlock(): drid=0x\"" << i->_drid;
- throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "jcntl", "rcvr_janalyze");
- }
- }
- }
- }
- }
- }
-
- // Check for file full condition
- _rcvdat._lffull = _rcvdat._eo == _emptyFilePoolPtr->fileSize_kib() * 1024;
- }
-}
-
-
-bool
-jcntl::rcvr_get_next_record(uint16_t& fid, std::ifstream* ifsp)
-{
- std::size_t cum_size_read = 0;
- void* xidp = 0;
- rec_hdr_t h;
-
- bool hdr_ok = false;
- std::streampos file_pos;
- while (!hdr_ok)
- {
- if (!ifsp->is_open())
- {
- if (!jfile_cycle(fid, ifsp, true))
- return false;
- }
- file_pos = ifsp->tellg();
- ifsp->read((char*)&h, sizeof(rec_hdr_t));
- if (ifsp->gcount() == sizeof(rec_hdr_t))
- hdr_ok = true;
- else
- {
- if (!jfile_cycle(fid, ifsp, true))
- return false;
- }
- }
-
- switch(h._magic)
- {
- case QLS_ENQ_MAGIC:
- {
- std::cout << " e" << std::flush;
- enq_rec er;
- uint16_t start_fid = fid; // fid may increment in decode() if record folds over file boundary
- if (!decode(er, fid, ifsp, cum_size_read, h, file_pos))
- return false;
- if (!er.is_transient()) // Ignore transient msgs
- {
- _rcvdat._enq_cnt_list[start_fid]++;
- if (er.xid_size())
- {
- er.get_xid(&xidp);
- assert(xidp != 0);
- std::string xid((char*)xidp, er.xid_size());
- _tmap.insert_txn_data(xid, txn_data(h._rid, 0, start_fid, true));
- if (_tmap.set_aio_compl(xid, h._rid) < txn_map::TMAP_OK) // fail - xid or rid not found
- {
- std::ostringstream oss;
- oss << std::hex << "_tmap.set_aio_compl: txn_enq xid=\"" << xid << "\" rid=0x" << h._rid;
- throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "jcntl", "rcvr_get_next_record");
- }
- std::free(xidp);
- }
- else
- {
- if (_emap.insert_pfid(h._rid, start_fid, file_pos) < enq_map::EMAP_OK) // fail
- {
- // The only error code emap::insert_pfid() returns is enq_map::EMAP_DUP_RID.
- std::ostringstream oss;
- oss << std::hex << "rid=0x" << h._rid << " _pfid=0x" << start_fid;
- throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "jcntl", "rcvr_get_next_record");
- }
- }
- }
- }
- break;
- case QLS_DEQ_MAGIC:
- {
- std::cout << " d" << std::flush;
- deq_rec dr;
- uint16_t start_fid = fid; // fid may increment in decode() if record folds over file boundary
- if (!decode(dr, fid, ifsp, cum_size_read, h, file_pos))
- return false;
- if (dr.xid_size())
- {
- // If the enqueue is part of a pending txn, it will not yet be in emap
- _emap.lock(dr.deq_rid()); // ignore not found error
- dr.get_xid(&xidp);
- assert(xidp != 0);
- std::string xid((char*)xidp, dr.xid_size());
- _tmap.insert_txn_data(xid, txn_data(dr.rid(), dr.deq_rid(), start_fid, false,
- dr.is_txn_coml_commit()));
- if (_tmap.set_aio_compl(xid, dr.rid()) < txn_map::TMAP_OK) // fail - xid or rid not found
- {
- std::ostringstream oss;
- oss << std::hex << "_tmap.set_aio_compl: txn_deq xid=\"" << xid << "\" rid=0x" << dr.rid();
- throw jexception(jerrno::JERR_MAP_NOTFOUND, oss.str(), "jcntl", "rcvr_get_next_record");
- }
- std::free(xidp);
- }
- else
- {
- int16_t enq_fid;
- if (_emap.get_remove_pfid(dr.deq_rid(), enq_fid, true) == enq_map::EMAP_OK) // ignore not found error
- _rcvdat._enq_cnt_list[enq_fid]--;
- }
- }
- break;
- case QLS_TXA_MAGIC:
- {
- std::cout << " a" << std::flush;
- txn_rec ar;
- if (!decode(ar, fid, ifsp, cum_size_read, h, file_pos))
- return false;
- // Delete this txn from tmap, unlock any locked records in emap
- ar.get_xid(&xidp);
- assert(xidp != 0);
- std::string xid((char*)xidp, ar.xid_size());
- txn_data_list tdl = _tmap.get_remove_tdata_list(xid); // tdl will be empty if xid not found
- for (tdl_itr itr = tdl.begin(); itr != tdl.end(); itr++)
- {
- if (itr->_enq_flag)
- _rcvdat._enq_cnt_list[itr->_pfid]--;
- else
- _emap.unlock(itr->_drid); // ignore not found error
- }
- std::free(xidp);
- }
- break;
- case QLS_TXC_MAGIC:
- {
- std::cout << " t" << std::flush;
- txn_rec cr;
- if (!decode(cr, fid, ifsp, cum_size_read, h, file_pos))
- return false;
- // Delete this txn from tmap, process records into emap
- cr.get_xid(&xidp);
- assert(xidp != 0);
- std::string xid((char*)xidp, cr.xid_size());
- txn_data_list tdl = _tmap.get_remove_tdata_list(xid); // tdl will be empty if xid not found
- for (tdl_itr itr = tdl.begin(); itr != tdl.end(); itr++)
- {
- if (itr->_enq_flag) // txn enqueue
- {
- if (_emap.insert_pfid(itr->_rid, itr->_pfid, file_pos) < enq_map::EMAP_OK) // fail
- {
- // The only error code emap::insert_pfid() returns is enq_map::EMAP_DUP_RID.
- std::ostringstream oss;
- oss << std::hex << "rid=0x" << itr->_rid << " _pfid=0x" << itr->_pfid;
- throw jexception(jerrno::JERR_MAP_DUPLICATE, oss.str(), "jcntl", "rcvr_get_next_record");
- }
- }
- else // txn dequeue
- {
- int16_t enq_fid;
- if (_emap.get_remove_pfid(itr->_drid, enq_fid, true) == enq_map::EMAP_OK) // ignore not found error
- _rcvdat._enq_cnt_list[enq_fid]--;
- }
- }
- std::free(xidp);
- }
- break;
- case QLS_EMPTY_MAGIC:
- {
- std::cout << " x" << std::flush;
- uint32_t rec_dblks = jrec::size_dblks(sizeof(rec_hdr_t));
- ifsp->ignore(rec_dblks * JRNL_DBLK_SIZE_BYTES - sizeof(rec_hdr_t));
- assert(!ifsp->fail() && !ifsp->bad());
- if (!jfile_cycle(fid, ifsp, false))
- return false;
- }
- break;
- case 0:
- std::cout << " 0" << std::endl << std::flush;
- check_journal_alignment(fid, file_pos);
- return false;
- default:
- std::cout << " ?" << std::endl << std::flush;
- // Stop as this is the overwrite boundary.
- check_journal_alignment(fid, file_pos);
- return false;
- }
- return true;
-}
-
-
-bool
-jcntl::decode(jrec& rec, uint16_t& fid, std::ifstream* ifsp, std::size_t& cum_size_read,
- rec_hdr_t& h, std::streampos& file_offs)
-{
- uint16_t start_fid = fid;
- std::streampos start_file_offs = file_offs;
-
- if (_rcvdat._h_rid == 0)
- _rcvdat._h_rid = h._rid;
- else if (h._rid - _rcvdat._h_rid < 0x8000000000000000ULL) // RFC 1982 comparison for unsigned 64-bit
- _rcvdat._h_rid = h._rid;
-
- bool done = false;
- while (!done)
- {
- try { done = rec.rcv_decode(h, ifsp, cum_size_read); }
- catch (const jexception& e)
- {
-// TODO - review this logic and tidy up how rd._lfid is assigned. See new jinf.get_end_file() fn.
-// Original
-// if (e.err_code() != jerrno::JERR_JREC_BADRECTAIL ||
-// fid != (rd._ffid ? rd._ffid - 1 : _num_jfiles - 1)) throw;
-// Tried this, but did not work
-// if (e.err_code() != jerrno::JERR_JREC_BADRECTAIL || h._magic != 0) throw;
- check_journal_alignment(start_fid, start_file_offs);
-// rd._lfid = start_fid;
- return false;
- }
- if (!done && !jfile_cycle(fid, ifsp, /*lowi, rd,*/ false))
- {
- check_journal_alignment(start_fid, start_file_offs);
- return false;
- }
- }
- return true;
-}
-
-
-bool
-jcntl::jfile_cycle(uint16_t& fid, std::ifstream* ifsp, const bool jump_fro)
-{
- if (ifsp->is_open())
- {
- if (ifsp->eof() || !ifsp->good())
- {
- ifsp->clear();
- _rcvdat._eo = ifsp->tellg(); // remember file offset before closing
- assert(_rcvdat._eo != std::numeric_limits<std::size_t>::max()); // Check for error code -1
- ifsp->close();
- if (++fid == _rcvdat._jfl.size()) // used up all known journal files
- return false;
- }
- }
- if (!ifsp->is_open())
- {
- ifsp->clear(); // clear eof flag, req'd for older versions of c++
- ifsp->open(_rcvdat._jfl[fid].c_str(), std::ios_base::in | std::ios_base::binary);
- if (!ifsp->good())
- throw jexception(jerrno::JERR__FILEIO, _rcvdat._jfl[fid], "jcntl", "jfile_cycle");
-
- // Read file header
- std::cout << " F" << fid << std::flush;
- file_hdr_t fhdr;
- ifsp->read((char*)&fhdr, sizeof(fhdr));
- assert(ifsp->good());
- if (fhdr._rhdr._magic == QLS_FILE_MAGIC)
- {
- if (!_rcvdat._fro)
- _rcvdat._fro = fhdr._fro;
- std::streamoff foffs = jump_fro ? fhdr._fro : JRNL_SBLK_SIZE_BYTES;
- ifsp->seekg(foffs);
- }
- else
- {
- ifsp->close();
- if (fid == 0) {
- _rcvdat._jempty = true;
- }
- return false;
- }
- }
- return true;
-}
-
-
-void
-jcntl::check_journal_alignment(const uint16_t fid, std::streampos& file_pos/*, rcvdat& rd*/)
-{
- unsigned sblk_offs = file_pos % JRNL_SBLK_SIZE_BYTES;
- if (sblk_offs)
- {
- {
- std::ostringstream oss;
- oss << std::hex << "Bad record alignment found at fid=0x" << fid;
- oss << " offs=0x" << file_pos << " (likely journal overwrite boundary); " << std::dec;
- oss << (JRNL_SBLK_SIZE_DBLKS - (sblk_offs/JRNL_DBLK_SIZE_BYTES)) << " filler record(s) required.";
- this->log(LOG_WARN, _jid, oss.str());
- }
- const uint32_t xmagic = QLS_EMPTY_MAGIC;
- std::ostringstream oss;
- oss << _jdir.dirname() << "/" /*<< _base_filename*/ << "."; // TODO linear journal name
- oss << std::hex << std::setfill('0') << std::setw(4) << fid << QLS_JRNL_FILE_EXTENSION;
- std::ofstream ofsp(oss.str().c_str(),
- std::ios_base::in | std::ios_base::out | std::ios_base::binary);
- if (!ofsp.good())
- throw jexception(jerrno::JERR__FILEIO, oss.str(), "jcntl", "check_journal_alignment");
- ofsp.seekp(file_pos);
- void* buff = std::malloc(JRNL_DBLK_SIZE_BYTES);
- assert(buff != 0);
- std::memcpy(buff, (const void*)&xmagic, sizeof(xmagic));
- // Normally, RHM_CLEAN must be set before these fills are done, but this is a recover
- // situation (i.e. performance is not an issue), and it makes the location of the write
- // clear should inspection of the file be required.
- std::memset((char*)buff + sizeof(xmagic), QLS_CLEAN_CHAR, JRNL_DBLK_SIZE_BYTES - sizeof(xmagic));
-
- while (file_pos % JRNL_SBLK_SIZE_BYTES)
- {
- ofsp.write((const char*)buff, JRNL_DBLK_SIZE_BYTES);
- assert(!ofsp.fail());
- std::ostringstream oss;
- oss << std::hex << "Recover phase write: Wrote filler record: fid=0x" << fid << " offs=0x" << file_pos;
- this->log(LOG_NOTICE, _jid, oss.str());
- file_pos = ofsp.tellp();
- }
- ofsp.close();
- std::free(buff);
- this->log(LOG_INFO, _jid, "Bad record alignment fixed.");
- }
- _rcvdat._eo = file_pos;
-}
-
}}
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.h b/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.h
index d875ab55c6..bb91eac569 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/jcntl.h
@@ -32,16 +32,11 @@ namespace qls_jrnl
#include <cstddef>
#include <deque>
#include <qpid/linearstore/jrnl/LinearFileController.h>
-#include <qpid/linearstore/jrnl/JournalLog.h>
#include "qpid/linearstore/jrnl/jdir.h"
-//#include "qpid/linearstore/jrnl/fcntl.h"
-//#include "qpid/linearstore/jrnl/lpmgr.h"
-#include "qpid/linearstore/jrnl/rcvdat.h"
+#include "qpid/linearstore/jrnl/RecoveryManager.h"
#include "qpid/linearstore/jrnl/slock.h"
#include "qpid/linearstore/jrnl/smutex.h"
-#include "qpid/linearstore/jrnl/rmgr.h"
#include "qpid/linearstore/jrnl/wmgr.h"
-//#include "qpid/linearstore/jrnl/wrfc.h"
namespace qpid
{
@@ -60,7 +55,7 @@ namespace qls_jrnl
* which is used per data block written to the journal, and is used to track its status through
* the AIO enqueue, read and dequeue process.
*/
- class jcntl : public JournalLog
+ class jcntl
{
protected:
/**
@@ -82,16 +77,6 @@ namespace qls_jrnl
jdir _jdir;
/**
- * \brief Base filename
- *
- * This string contains the base filename used for the journal files. The filenames will
- * start with this base, and have various sections added to it to derive the final file names
- * that will be written to disk. No file separator characters should be included here, but
- * all other legal filename characters are valid.
- */
-// std::string _base_filename;
-
- /**
* \brief Initialized flag
*
* This flag starts out set to false, is set to true once this object has been initialized,
@@ -120,21 +105,14 @@ namespace qls_jrnl
*/
bool _readonly_flag;
- /**
- * \brief If set, calls stop() if the jouranl write pointer overruns dequeue low water
- * marker. If not set, then attempts to write will throw exceptions until the journal
- * file low water marker moves to the next journal file.
- */
- //bool _autostop; ///< Autostop flag - stops journal when overrun occurs
-
// Journal control structures
+ JournalLog& _jrnl_log; ///< Ref to Journal Log instance
LinearFileController _linearFileController; ///< Linear File Controller
EmptyFilePool* _emptyFilePoolPtr; ///< Pointer to Empty File Pool for this queue
enq_map _emap; ///< Enqueue map for low water mark management
txn_map _tmap; ///< Transaction map open transactions
- //rmgr _rmgr; ///< Read page manager which manages AIO
wmgr _wmgr; ///< Write page manager which manages AIO
- rcvdat _rcvdat; ///< Recovery data used for recovery
+ RecoveryManager _recoveryManager; ///< Recovery data used for recovery
smutex _wr_mutex; ///< Mutex for journal writes
public:
@@ -150,7 +128,9 @@ namespace qls_jrnl
* \param jdir The directory which will contain the journal files.
* \param base_filename The string which will be used to start all journal filenames.
*/
- jcntl(const std::string& jid, const std::string& jdir/*, const std::string& base_filename*/);
+ jcntl(const std::string& jid,
+ const std::string& jdir,
+ JournalLog& jrnl_log);
/**
* \brief Destructor.
@@ -158,6 +138,7 @@ namespace qls_jrnl
virtual ~jcntl();
inline const std::string& id() const { return _jid; }
+
inline const std::string& jrnl_dir() const { return _jdir.dirname(); }
/**
@@ -191,9 +172,10 @@ namespace qls_jrnl
*
* \exception TODO
*/
- void initialize(/*const uint16_t num_jfiles, const bool auto_expand, const uint16_t ae_max_jfiles,
- const uint32_t jfsize_sblks,*/EmptyFilePool* efpp, const uint16_t wcache_num_pages, const uint32_t wcache_pgsize_sblks,
- aio_callback* const cbp);
+ void initialize(EmptyFilePool* efpp,
+ const uint16_t wcache_num_pages,
+ const uint32_t wcache_pgsize_sblks,
+ aio_callback* const cbp);
/**
* /brief Initialize journal by recovering state from previously written journal.
@@ -294,11 +276,15 @@ namespace qls_jrnl
*
* \exception TODO
*/
- iores enqueue_data_record(const void* const data_buff, const std::size_t tot_data_len,
- const std::size_t this_data_len, data_tok* dtokp, const bool transient = false);
+ iores enqueue_data_record(const void* const data_buff,
+ const std::size_t tot_data_len,
+ const std::size_t this_data_len,
+ data_tok* dtokp,
+ const bool transient = false);
- iores enqueue_extern_data_record(const std::size_t tot_data_len, data_tok* dtokp,
- const bool transient = false);
+ iores enqueue_extern_data_record(const std::size_t tot_data_len,
+ data_tok* dtokp,
+ const bool transient = false);
/**
* \brief Enqueue data.
@@ -313,84 +299,17 @@ namespace qls_jrnl
*
* \exception TODO
*/
- iores enqueue_txn_data_record(const void* const data_buff, const std::size_t tot_data_len,
- const std::size_t this_data_len, data_tok* dtokp, const std::string& xid,
- const bool transient = false);
- iores enqueue_extern_txn_data_record(const std::size_t tot_data_len, data_tok* dtokp,
- const std::string& xid, const bool transient = false);
-
- /* TODO
- **
- * \brief Retrieve details of next record to be read without consuming the record.
- *
- * Retrieve information about current read record. A pointer to the data is returned, along
- * with the data size and available data size. Data is considered "available" when the AIO
- * operations to fill page-cache pages from disk have returned, and is ready for consumption.
- *
- * If <i>dsize_avail</i> &lt; <i>dsize</i>, then not all of the data is available or part of
- * the data is in non-contiguous memory, and a subsequent call will update both the pointer
- * and <i>dsize_avail</i> if more pages have returned from AIO.
- *
- * The <i>dsize_avail</i> parameter will return the amount of data from this record that is
- * available in the page cache as contiguous memory, even if it spans page cache boundaries.
- * However, if a record spans the end of the page cache and continues at the beginning, even
- * if both parts are ready for consumption, then this must be divided into at least two
- * get_data_record() operations, as the data is contained in at least two non-contiguous
- * segments of the page cache.
- *
- * Once all the available data for a record is exposed, it can not be read again using
- * this function. It must be consumed prior to getting the next record. This can be done by
- * calling discard_data_record() or read_data_record(). However, if parameter
- * <i>auto_discard</i> is set to <b><i>true</i></b>, then this record will be automatically
- * consumed when the entire record has become available without having to explicitly call
- * discard_next_data_record() or read_data_record().
- *
- * If the current record is an open transactional record, then it cannot be read until it is
- * committed. If it is aborted, it can never be read. Under this condition, get_data_record()
- * will return RHM_IORES_TXPENDING, the data pointer will be set to NULL and all data
- * lengths will be set to 0.
- *
- * Example: Read a record of 30k. Assume a read page cache of 10 pages of size 10k starting
- * at address base_ptr (page0 = base_ptr, page1 = page_ptr+10k, etc.). The first 15k of
- * the record falls at the end of the page cache, the remaining 15k folded to the beginning.
- * The current page (page 8) containing 5k is available, the remaining pages which contain
- * this record are pending AIO return:
- * <pre>
- * call dsize
- * no. dsize avail data ptr Return Comment
- * ----+-----+-----+------------+--------+--------------------------------------------------
- * 1 30k 5k base_ptr+85k SUCCESS Initial call, read first 5k
- * 2 30k 0k base_ptr+90k AIO_WAIT AIO still pending; no further pages avail
- * 3 30k 10k base_ptr+90k SUCCESS AIO now returned; now read till end of page cache
- * 4 30k 15k base_ptr SUCCESS data_ptr now pointing to start of page cache
- * </pre>
- *
- * \param rid Reference that returns the record ID (rid)
- * \param dsize Reference that returns the total data size of the record data .
- * \param dsize_avail Reference that returns the amount of the data that is available for
- * consumption.
- * \param data Pointer to data pointer which will point to the first byte of the next record
- * data.
- * \param auto_discard If <b><i>true</i></b>, automatically discard the record being read if
- * the entire record is available (i.e. dsize == dsize_avail). Otherwise
- * discard_next_data_record() must be explicitly called.
- *
- * \exception TODO
- *
- // *** NOT YET IMPLEMENTED ***
- iores get_data_record(const uint64_t& rid, const std::size_t& dsize,
- const std::size_t& dsize_avail, const void** const data, bool auto_discard = false);
- */
+ iores enqueue_txn_data_record(const void* const data_buff,
+ const std::size_t tot_data_len,
+ const std::size_t this_data_len,
+ data_tok* dtokp,
+ const std::string& xid,
+ const bool transient = false);
- /* TODO
- **
- * \brief Discard (skip) next record to be read without reading or retrieving it.
- *
- * \exception TODO
- *
- // *** NOT YET IMPLEMENTED ***
- iores discard_data_record(data_tok* const dtokp);
- */
+ iores enqueue_extern_txn_data_record(const std::size_t tot_data_len,
+ data_tok* dtokp,
+ const std::string& xid,
+ const bool transient = false);
/**
* \brief Reads data from the journal. It is the responsibility of the reader to free
@@ -434,9 +353,14 @@ namespace qls_jrnl
*
* \exception TODO
*/
- iores read_data_record(void** const datapp, std::size_t& dsize, void** const xidpp,
- std::size_t& xidsize, bool& transient, bool& external, data_tok* const dtokp,
- bool ignore_pending_txns = false);
+ iores read_data_record(void** const datapp,
+ std::size_t& dsize,
+ void** const xidpp,
+ std::size_t& xidsize,
+ bool& transient,
+ bool& external,
+ data_tok* const dtokp,
+ bool ignore_pending_txns = false);
/**
* \brief Dequeues (marks as no longer needed) data record in journal.
@@ -455,7 +379,8 @@ namespace qls_jrnl
*
* \exception TODO
*/
- iores dequeue_data_record(data_tok* const dtokp, const bool txn_coml_commit = false);
+ iores dequeue_data_record(data_tok* const dtokp,
+ const bool txn_coml_commit = false);
/**
* \brief Dequeues (marks as no longer needed) data record in journal.
@@ -476,7 +401,9 @@ namespace qls_jrnl
*
* \exception TODO
*/
- iores dequeue_txn_data_record(data_tok* const dtokp, const std::string& xid, const bool txn_coml_commit = false);
+ iores dequeue_txn_data_record(data_tok* const dtokp,
+ const std::string& xid,
+ const bool txn_coml_commit = false);
/**
* \brief Abort the transaction for all records enqueued or dequeued with the matching xid.
@@ -491,7 +418,8 @@ namespace qls_jrnl
*
* \exception TODO
*/
- iores txn_abort(data_tok* const dtokp, const std::string& xid);
+ iores txn_abort(data_tok* const dtokp,
+ const std::string& xid);
/**
* \brief Commit the transaction for all records enqueued or dequeued with the matching xid.
@@ -506,7 +434,8 @@ namespace qls_jrnl
*
* \exception TODO
*/
- iores txn_commit(data_tok* const dtokp, const std::string& xid);
+ iores txn_commit(data_tok* const dtokp,
+ const std::string& xid);
/**
* \brief Check whether all the enqueue records for the given xid have reached disk.
@@ -527,15 +456,6 @@ namespace qls_jrnl
int32_t get_wr_events(timespec* const timeout);
/**
- * \brief Forces a check for returned AIO read events.
- *
- * Forces a check for returned AIO read events. This is normally performed by read_data()
- * operations, but if these operations cease, then this call needs to be made to force the
- * processing of any outstanding AIO operations.
- */
-// int32_t get_rd_events(timespec* const timeout);
-
- /**
* \brief Stop the journal from accepting any further requests to read or write data.
*
* This operation is used to stop the journal. This is the normal mechanism for bringing the
@@ -555,27 +475,14 @@ namespace qls_jrnl
*/
iores flush(const bool block_till_aio_cmpl = false);
- inline uint32_t get_enq_cnt() const { return _emap.size(); } // TODO: Thread safe?
+ inline uint32_t get_enq_cnt() const { return _emap.size(); } // TODO: _emap: Thread safe?
inline uint32_t get_wr_aio_evt_rem() const { slock l(_wr_mutex); return _wmgr.get_aio_evt_rem(); }
-// inline uint32_t get_rd_aio_evt_rem() const { return _rmgr.get_aio_evt_rem(); }
-
- inline uint32_t get_wr_outstanding_aio_dblks() const;
- /*{ return _wrfc.aio_outstanding_dblks(); }*/
-
-// inline uint32_t get_wr_outstanding_aio_dblks(uint16_t lfid) const;
-// { return _lpmgr.get_fcntlp(lfid)->wr_aio_outstanding_dblks(); }
+ uint32_t get_wr_outstanding_aio_dblks() const;
- inline uint32_t get_rd_outstanding_aio_dblks() const;
-// { return _rrfc.aio_outstanding_dblks(); }
+ uint32_t get_rd_outstanding_aio_dblks() const;
-// inline uint32_t get_rd_outstanding_aio_dblks(uint16_t lfid) const;
-// { return _lpmgr.get_fcntlp(lfid)->rd_aio_outstanding_dblks(); }
-
-// inline uint16_t get_rd_fid() const { return _rrfc.index(); }
-// inline uint16_t get_wr_fid() const { return _wrfc.index(); }
-// uint16_t get_earliest_fid();
LinearFileController& getLinearFileControllerRef();
/**
@@ -583,13 +490,20 @@ namespace qls_jrnl
* false if the rid is transactionally enqueued and is not committed, or if it is
* locked (i.e. transactionally dequeued, but the dequeue has not been committed).
*/
- inline bool is_enqueued(const uint64_t rid, bool ignore_lock = false)
- { return _emap.is_enqueued(rid, ignore_lock); }
- inline bool is_locked(const uint64_t rid)
- { if (_emap.is_enqueued(rid, true) < enq_map::EMAP_OK) return false; return _emap.is_locked(rid) == enq_map::EMAP_TRUE; }
+ inline bool is_enqueued(const uint64_t rid, bool ignore_lock = false) { return _emap.is_enqueued(rid, ignore_lock); }
+
+ inline bool is_locked(const uint64_t rid) {
+ if (_emap.is_enqueued(rid, true) < enq_map::EMAP_OK)
+ return false;
+ return _emap.is_locked(rid) == enq_map::EMAP_TRUE;
+ }
+
inline void enq_rid_list(std::vector<uint64_t>& rids) { _emap.rid_list(rids); }
+
inline void enq_xid_list(std::vector<std::string>& xids) { _tmap.xid_list(xids); }
+
inline uint32_t get_open_txn_cnt() const { return _tmap.size(); }
+
// TODO Make this a const, but txn_map must support const first.
inline txn_map& get_txn_map() { return _tmap; }
@@ -626,41 +540,10 @@ namespace qls_jrnl
*/
inline const std::string& dirname() const { return _jdir.dirname(); }
- /**
- * \brief Get the journal base filename.
- *
- * Get the journal base filename as set during initialization. This is the prefix used in all
- * journal files of this instance. Note that if more than one instance of the journal shares
- * the same directory, their base filenames <b>MUST</b> be different or else the instances
- * will overwrite one another.
- */
-// inline const std::string& base_filename() const { return _base_filename; }
-
-// inline uint16_t num_jfiles() const; { return _lpmgr.num_jfiles(); }
-
-// inline fcntl* get_fcntlp(const uint16_t lfid) const { return _lpmgr.get_fcntlp(lfid); }
-
-// inline uint32_t jfsize_sblks() const { return _jfsize_sblks; }
-
- // Logging
-// virtual void log(log_level_t level, const std::string& log_stmt) const;
-// virtual void log(log_level_t level, const char* const log_stmt) const;
-
- // FIXME these are _rmgr to _wmgr interactions, remove when _rmgr contains ref to _wmgr:
- //void chk_wr_frot();
- inline uint32_t unflushed_dblks() { return _wmgr.unflushed_dblks(); }
- void fhdr_wr_sync(const uint16_t lid);
- inline uint32_t wr_subm_cnt_dblks(const uint16_t lfid) const; /*{ return _lpmgr.get_fcntlp(lfid)->wr_subm_cnt_dblks(); }*/
-
// Management instrumentation callbacks
inline virtual void instr_incr_outstanding_aio_cnt() {}
inline virtual void instr_decr_outstanding_aio_cnt() {}
- /**
- * /brief Static function for creating new fcntl objects for use with obj_arr.
- */
-// static fcntl* new_fcntl(jcntl* const jcp, const uint16_t lid, const uint16_t fid, const rcvdat* const rdp);
-
protected:
static bool _init;
static bool init_statics();
@@ -676,11 +559,6 @@ namespace qls_jrnl
void check_rstatus(const char* fn_name) const;
/**
- * \brief Write info file &lt;basefilename&gt;.jinf to disk
- */
-// void write_infofile() const;
-
- /**
* \brief Call that blocks while waiting for all outstanding AIOs to complete
*/
void aio_cmpl_wait();
@@ -690,27 +568,6 @@ namespace qls_jrnl
* AIO wait conditions to clear.
*/
bool handle_aio_wait(const iores res, iores& resout, const data_tok* dtp);
-
- /**
- * \brief Analyze journal for recovery.
- */
- static void rcvr_read_jfile(const std::string& jfn, ::file_hdr_t* fh, std::string& queueName);
-
- void rcvr_analyze_fhdrs(EmptyFilePoolManager* efpmp);
-
- void rcvr_janalyze(const std::vector<std::string>* prep_txn_list_ptr, EmptyFilePoolManager* efpmp);
-
- bool rcvr_get_next_record(uint16_t& fid, std::ifstream* ifsp/*, bool& lowi, rcvdat& rd*/);
-
- bool decode(jrec& rec, uint16_t& fid, std::ifstream* ifsp, std::size_t& cum_size_read,
- rec_hdr_t& h, /*bool& lowi, rcvdat& rd,*/ std::streampos& rec_offset);
-
- bool jfile_cycle(uint16_t& fid, std::ifstream* ifsp, /*bool& lowi, rcvdat& rd,*/ const bool jump_fro);
-
- //bool check_owi(const uint16_t fid, rec_hdr_t& h, bool& lowi, rcvdat& rd,
- // std::streampos& read_pos);
-
- void check_journal_alignment(const uint16_t fid, std::streampos& rec_offset/*, rcvdat& rd*/);
};
}}
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.cpp
index 8793a882f7..7c70be1ed7 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.cpp
@@ -50,8 +50,6 @@ const uint32_t jerrno::JERR_JCNTL_READONLY = 0x0201;
const uint32_t jerrno::JERR_JCNTL_AIOCMPLWAIT = 0x0202;
const uint32_t jerrno::JERR_JCNTL_UNKNOWNMAGIC = 0x0203;
const uint32_t jerrno::JERR_JCNTL_NOTRECOVERED = 0x0204;
-const uint32_t jerrno::JERR_JCNTL_OPENRD = 0x0205;
-const uint32_t jerrno::JERR_JCNTL_READ = 0x0206;
const uint32_t jerrno::JERR_JCNTL_ENQSTATE = 0x0207;
const uint32_t jerrno::JERR_JCNTL_INVALIDENQHDR = 0x0208;
@@ -90,6 +88,11 @@ const uint32_t jerrno::JERR_WMGR_DEQDISCONT = 0x0804;
const uint32_t jerrno::JERR_WMGR_DEQRIDNOTENQ = 0x0805;
const uint32_t jerrno::JERR_WMGR_BADFH = 0x0806;
+// class RecoveryManager
+const uint32_t jerrno::JERR_RCVM_OPENRD = 0x0900;
+const uint32_t jerrno::JERR_RCVM_READ = 0x0901;
+const uint32_t jerrno::JERR_RCVM_WRITE = 0x0902;
+
//// class rmgr
//const uint32_t jerrno::JERR_RMGR_UNKNOWNMAGIC = 0x0900;
//const uint32_t jerrno::JERR_RMGR_RIDMISMATCH = 0x0901;
@@ -143,8 +146,6 @@ jerrno::__init()
_err_map[JERR_JCNTL_AIOCMPLWAIT] = "JERR_JCNTL_AIOCMPLWAIT: Timeout waiting for AIOs to complete.";
_err_map[JERR_JCNTL_UNKNOWNMAGIC] = "JERR_JCNTL_UNKNOWNMAGIC: Found record with unknown magic.";
_err_map[JERR_JCNTL_NOTRECOVERED] = "JERR_JCNTL_NOTRECOVERED: Operation requires recover() to be run first.";
- _err_map[JERR_JCNTL_OPENRD] = "JERR_JCNTL_OPENRD: Unable to open file for write";
- _err_map[JERR_JCNTL_READ] = "JERR_JCNTL_READ: Read error: no or insufficient data to read";
_err_map[JERR_JCNTL_ENQSTATE] = "JERR_JCNTL_ENQSTATE: Read error: Record not in ENQ state";
_err_map[JERR_JCNTL_INVALIDENQHDR] = "JERR_JCNTL_INVALIDENQHDR: Invalid ENQ header";
@@ -182,6 +183,10 @@ jerrno::__init()
_err_map[JERR_WMGR_DEQRIDNOTENQ] = "JERR_WMGR_DEQRIDNOTENQ: Dequeue rid is not enqueued.";
_err_map[JERR_WMGR_BADFH] = "JERR_WMGR_BADFH: Bad file handle.";
+ // class RecoveryManager
+ _err_map[JERR_RCVM_OPENRD] = "JERR_JCNTL_OPENRD: Unable to open file for write";
+ _err_map[JERR_RCVM_READ] = "JERR_JCNTL_READ: Read error: no or insufficient data to read";
+ _err_map[JERR_RCVM_WRITE] = "JERR_RCVM_WRITE: Write error";
// // class rmgr
// _err_map[JERR_RMGR_UNKNOWNMAGIC] = "JERR_RMGR_UNKNOWNMAGIC: Found record with unknown magic.";
// _err_map[JERR_RMGR_RIDMISMATCH] = "JERR_RMGR_RIDMISMATCH: RID mismatch between current record and dtok RID";
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.h b/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.h
index ecf969a344..7bd521ecbf 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/jerrno.h
@@ -69,8 +69,6 @@ namespace qls_jrnl
static const uint32_t JERR_JCNTL_AIOCMPLWAIT; ///< Timeout waiting for AIOs to complete
static const uint32_t JERR_JCNTL_UNKNOWNMAGIC; ///< Found record with unknown magic
static const uint32_t JERR_JCNTL_NOTRECOVERED; ///< Req' recover() to be called first
- static const uint32_t JERR_JCNTL_OPENRD; ///< Unable to open file for read
- static const uint32_t JERR_JCNTL_READ; ///< Read error: no or insufficient data to read
static const uint32_t JERR_JCNTL_ENQSTATE; ///< Read error: Record not in ENQ state
static const uint32_t JERR_JCNTL_INVALIDENQHDR;///< Invalid ENQ header
@@ -108,6 +106,11 @@ namespace qls_jrnl
static const uint32_t JERR_WMGR_DEQRIDNOTENQ; ///< Deq. rid not enqueued
static const uint32_t JERR_WMGR_BADFH; ///< Bad file handle
+ // class RecoveryManager
+ static const uint32_t JERR_RCVM_OPENRD; ///< Unable to open file for read
+ static const uint32_t JERR_RCVM_READ; ///< Read error: no or insufficient data to read
+ static const uint32_t JERR_RCVM_WRITE; ///< Write error
+
// // class rmgr
// static const uint32_t JERR_RMGR_UNKNOWNMAGIC; ///< Found record with unknown magic
// static const uint32_t JERR_RMGR_RIDMISMATCH; ///< RID mismatch between rec and dtok
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/jrec.h b/qpid/cpp/src/qpid/linearstore/jrnl/jrec.h
index c77daa0cb4..619cd0bdec 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/jrec.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/jrec.h
@@ -148,9 +148,9 @@ namespace qls_jrnl
virtual std::size_t rec_size() const = 0;
inline virtual uint32_t rec_size_dblks() const { return size_dblks(rec_size()); }
static inline uint32_t size_dblks(const std::size_t size)
- { return size_blks(size, JRNL_DBLK_SIZE_BYTES); }
+ { return size_blks(size, QLS_DBLK_SIZE_BYTES); }
static inline uint32_t size_sblks(const std::size_t size)
- { return size_blks(size, JRNL_SBLK_SIZE_BYTES); }
+ { return size_blks(size, QLS_SBLK_SIZE_BYTES); }
static inline uint32_t size_blks(const std::size_t size, const std::size_t blksize)
{ return (size + blksize - 1)/blksize; }
virtual uint64_t rid() const = 0;
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.cpp
index f61a2bb970..32b024a09d 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.cpp
@@ -41,12 +41,11 @@ pmgr::page_cb::page_cb(uint16_t index):
_wdblks(0),
_rdblks(0),
_pdtokl(0),
-// _wfh(0),
-// _rfh(0),
_jfp(0),
_pbuff(0)
{}
+// TODO: almost identical to pmgr::page_state_str() below - resolve
const char*
pmgr::page_cb::state_str() const
{
@@ -58,14 +57,12 @@ pmgr::page_cb::state_str() const
return "IN_USE";
case AIO_PENDING:
return "AIO_PENDING";
- case AIO_COMPLETE:
- return "AIO_COMPLETE";
}
return "<unknown>";
}
// static
-const uint32_t pmgr::_sblkSizeBytes = JRNL_SBLK_SIZE_BYTES;
+const uint32_t pmgr::_sblkSizeBytes = QLS_SBLK_SIZE_BYTES;
pmgr::pmgr(jcntl* jc, enq_map& emap, txn_map& tmap):
_cache_pgsize_sblks(0),
@@ -109,11 +106,11 @@ pmgr::initialize(aio_callback* const cbp, const uint32_t cache_pgsize_sblks, con
// 1. Allocate page memory (as a single block)
std::size_t cache_pgsize = _cache_num_pages * _cache_pgsize_sblks * _sblkSizeBytes;
- if (::posix_memalign(&_page_base_ptr, QLS_AIO_ALIGN_BOUNDARY, cache_pgsize))
+ if (::posix_memalign(&_page_base_ptr, QLS_AIO_ALIGN_BOUNDARY_BYTES, cache_pgsize))
{
clean();
std::ostringstream oss;
- oss << "posix_memalign(): alignment=" << QLS_AIO_ALIGN_BOUNDARY << " size=" << cache_pgsize;
+ oss << "posix_memalign(): alignment=" << QLS_AIO_ALIGN_BOUNDARY_BYTES << " size=" << cache_pgsize;
oss << FORMAT_SYSERR(errno);
throw jexception(jerrno::JERR__MALLOC, oss.str(), "pmgr", "initialize");
}
@@ -186,6 +183,7 @@ pmgr::clean()
_aio_event_arr = 0;
}
+// TODO: almost identical to pmgr::page_cb::state_str() above - resolve
const char*
pmgr::page_state_str(page_state ps)
{
@@ -197,8 +195,6 @@ pmgr::page_state_str(page_state ps)
return "IN_USE";
case AIO_PENDING:
return "AIO_PENDING";
- case AIO_COMPLETE:
- return "AIO_COMPLETE";
}
return "<page_state unknown>";
}
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.h b/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.h
index e9764aeeac..64143a9e6e 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/pmgr.h
@@ -37,7 +37,6 @@ namespace qls_jrnl
#include "qpid/linearstore/jrnl/deq_rec.h"
#include "qpid/linearstore/jrnl/enq_map.h"
#include "qpid/linearstore/jrnl/enq_rec.h"
-//#include "qpid/linearstore/jrnl/fcntl.h"
#include "qpid/linearstore/jrnl/txn_map.h"
#include "qpid/linearstore/jrnl/txn_rec.h"
@@ -61,8 +60,7 @@ class JournalFile;
{
UNUSED, ///< A page is uninitialized, contains no data.
IN_USE, ///< Page is in use.
- AIO_PENDING, ///< An AIO request outstanding.
- AIO_COMPLETE ///< An AIO request is complete.
+ AIO_PENDING ///< An AIO request outstanding.
};
/**
@@ -77,8 +75,6 @@ class JournalFile;
uint32_t _wdblks; ///< Total number of dblks in page so far
uint32_t _rdblks; ///< Total number of dblks in page
std::deque<data_tok*>* _pdtokl; ///< Page message tokens list
- //fcntl* _wfh; ///< File handle for incrementing write compl counts
- //fcntl* _rfh; ///< File handle for incrementing read compl counts
JournalFile* _jfp; ///< Journal file for incrementing compl counts
void* _pbuff; ///< Page buffer
@@ -113,7 +109,7 @@ class JournalFile;
pmgr(jcntl* jc, enq_map& emap, txn_map& tmap);
virtual ~pmgr();
- virtual int32_t get_events(page_state state, timespec* const timeout, bool flush = false) = 0;
+ virtual int32_t get_events(timespec* const timeout, bool flush) = 0;
inline uint32_t get_aio_evt_rem() const { return _aio_evt_rem; }
static const char* page_state_str(page_state ps);
inline uint32_t cache_pgsize_sblks() const { return _cache_pgsize_sblks; }
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/rcvdat.h b/qpid/cpp/src/qpid/linearstore/jrnl/rcvdat.h
deleted file mode 100644
index 46541e7f31..0000000000
--- a/qpid/cpp/src/qpid/linearstore/jrnl/rcvdat.h
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-#ifndef QPID_LEGACYSTORE_JRNL_RCVDAT_H
-#define QPID_LEGACYSTORE_JRNL_RCVDAT_H
-
-#include <cstddef>
-#include <iomanip>
-#include <map>
-#include "qpid/linearstore/jrnl/jcfg.h"
-#include <sstream>
-#include <stdint.h>
-#include <vector>
-
-namespace qpid
-{
-namespace qls_jrnl
-{
-
- struct rcvdat
- {
- std::vector<std::string> _jfl; ///< Journal file list
- std::map<uint64_t, std::string> _fm; ///< File number - name map
- std::vector<uint32_t> _enq_cnt_list; ///< Number enqueued records found for each file
- bool _jempty; ///< Journal data files empty
- std::size_t _fro; ///< First record offset in ffid
- std::size_t _eo; ///< End offset (first byte past last record)
- uint64_t _h_rid; ///< Highest rid found
- bool _lffull; ///< Last file is full
-
- rcvdat():
- _jfl(),
- _fm(),
- _enq_cnt_list(),
- _jempty(false),
- _fro(0),
- _eo(0),
- _h_rid(0),
- _lffull(false)
- {}
-
- std::string to_string(const std::string& jid)
- {
- std::ostringstream oss;
- oss << "Recover file analysis (jid=\"" << jid << "\"):" << std::endl;
- oss << " Number of journal files = " << _fm.size() << std::endl;
- oss << " Journal File List (_jfl):";
- for (std::map<uint64_t, std::string>::const_iterator i=_fm.begin(); i!=_fm.end(); ++i) {
- oss << " " << i->first << ": " << i->second.substr(i->second.rfind('/')+1) << std::endl;
- }
- oss << " Journal empty (_jempty) = " << (_jempty ? "TRUE" : "FALSE") << std::endl;
- oss << " First record offset in first fid (_fro) = 0x" << std::hex << _fro <<
- std::dec << " (" << (_fro/JRNL_DBLK_SIZE_BYTES) << " dblks)" << std::endl;
- oss << " End offset (_eo) = 0x" << std::hex << _eo << std::dec << " (" <<
- (_eo/JRNL_DBLK_SIZE_BYTES) << " dblks)" << std::endl;
- oss << " Highest rid (_h_rid) = 0x" << std::hex << _h_rid << std::dec << std::endl;
- oss << " Last file full (_lffull) = " << (_lffull ? "TRUE" : "FALSE") << std::endl;
- oss << " Enqueued records (txn & non-txn):" << std::endl;
- for (unsigned i=0; i<_enq_cnt_list.size(); i++)
- oss << " File " << std::setw(2) << i << ": " << _enq_cnt_list[i] <<
- std::endl;
- return oss.str();
- }
-
- std::string to_log(const std::string& jid)
- {
- std::ostringstream oss;
- oss << "Recover file analysis (jid=\"" << jid << "\"):";
- oss << " jfl=[";
- for (std::map<uint64_t, std::string>::const_iterator i=_fm.begin(); i!=_fm.end(); ++i) {
- if (i!=_fm.begin()) oss << " ";
- oss << i->first << ":" << i->second.substr(i->second.rfind('/')+1);
- }
- oss << "]";
- oss << " _enq_cnt_list: [ ";
- for (unsigned i=0; i<_enq_cnt_list.size(); i++) {
- if (i) oss << " ";
- oss << _enq_cnt_list[i];
- }
- oss << " ]";
- oss << " jempty=" << (_jempty ? "T" : "F");
- oss << " fro=0x" << std::hex << _fro << std::dec << " (" <<
- (_fro/JRNL_DBLK_SIZE_BYTES) << " dblks)";
- oss << " eo=0x" << std::hex << _eo << std::dec << " (" <<
- (_eo/JRNL_DBLK_SIZE_BYTES) << " dblks)";
- oss << " h_rid=0x" << std::hex << _h_rid << std::dec;
- oss << " lffull=" << (_lffull ? "T" : "F");
- return oss.str();
- }
- };
-}}
-
-#endif // ifndef QPID_LEGACYSTORE_JRNL_RCVDAT_H
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/rmgr.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/rmgr.cpp
deleted file mode 100644
index f0ab4fe5cf..0000000000
--- a/qpid/cpp/src/qpid/linearstore/jrnl/rmgr.cpp
+++ /dev/null
@@ -1,707 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-#include "qpid/linearstore/jrnl/rmgr.h"
-
-#include <cassert>
-#include <cerrno>
-#include <cstdlib>
-#include "qpid/linearstore/jrnl/jcntl.h"
-#include "qpid/linearstore/jrnl/jerrno.h"
-#include <sstream>
-
-namespace qpid
-{
-namespace qls_jrnl
-{
-
-rmgr::rmgr(jcntl* jc, enq_map& emap, txn_map& tmap/*, rrfc& rrfc*/):
- pmgr(jc, emap, tmap),
-// _rrfc(rrfc),
- _hdr(),
- _fhdr_buffer(0),
- _fhdr_aio_cb_ptr(0),
- _fhdr_rd_outstanding(false)
-{}
-
-rmgr::~rmgr()
-{
- rmgr::clean();
-}
-
-void
-rmgr::initialize(aio_callback* const cbp)
-{
- pmgr::initialize(cbp, JRNL_RMGR_PAGE_SIZE, JRNL_RMGR_PAGES);
- clean();
- // Allocate memory for reading file header
- if (::posix_memalign(&_fhdr_buffer, _sblkSizeBytes, _sblkSizeBytes))
- {
- std::ostringstream oss;
- oss << "posix_memalign(): blksize=" << _sblkSizeBytes << " size=" << _sblkSizeBytes;
- oss << FORMAT_SYSERR(errno);
- throw jexception(jerrno::JERR__MALLOC, oss.str(), "rmgr", "initialize");
- }
- _fhdr_aio_cb_ptr = new aio_cb;
- std::memset(_fhdr_aio_cb_ptr, 0, sizeof(aio_cb));
-}
-
-void
-rmgr::clean()
-{
- std::free(_fhdr_buffer);
- _fhdr_buffer = 0;
-
- if (_fhdr_aio_cb_ptr)
- {
- delete _fhdr_aio_cb_ptr;
- _fhdr_aio_cb_ptr = 0;
- }
-}
-
-iores
-rmgr::read(void** const datapp, std::size_t& dsize, void** const xidpp, std::size_t& xidsize,
- bool& transient, bool& external, data_tok* dtokp, bool ignore_pending_txns)
-{
- iores res = pre_read_check(dtokp);
- if (res != RHM_IORES_SUCCESS)
- {
- set_params_null(datapp, dsize, xidpp, xidsize);
- return res;
- }
-
- if (dtokp->rstate() == data_tok::SKIP_PART)
- {
- if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
- {
- aio_cycle(); // check if rd AIOs returned; initiate new reads if possible
- return RHM_IORES_PAGE_AIOWAIT;
- }
- const iores res = skip(dtokp);
- if (res != RHM_IORES_SUCCESS)
- {
- set_params_null(datapp, dsize, xidpp, xidsize);
- return res;
- }
- }
- if (dtokp->rstate() == data_tok::READ_PART)
- {
- assert(dtokp->rid() == _hdr._rid);
- void* rptr = (void*)((char*)_page_ptr_arr[_pg_index] + (_pg_offset_dblks * JRNL_DBLK_SIZE));
- const iores res = read_enq(_hdr, rptr, dtokp);
- dsize = _enq_rec.get_data(datapp);
- xidsize = _enq_rec.get_xid(xidpp);
- transient = _enq_rec.is_transient();
- external = _enq_rec.is_external();
- return res;
- }
-
- set_params_null(datapp, dsize, xidpp, xidsize);
- //_hdr.reset();
- ::rec_hdr_init(&_hdr, 0, 0, 0, 0);
- // Read header, determine next record type
- while (true)
- {
- if(dblks_rem() == 0 /*&& _rrfc.is_compl() && !_rrfc.is_wr_aio_outstanding()*/) // TODO: replace for linear store
- {
- aio_cycle(); // check if rd AIOs returned; initiate new reads if possible
- if(dblks_rem() == 0 /*&& _rrfc.is_compl() && !_rrfc.is_wr_aio_outstanding()*/) // TODO: replace for linear store
- {
- if (_jc->unflushed_dblks() > 0)
- _jc->flush();
- else if (!_aio_evt_rem)
- return RHM_IORES_EMPTY;
- }
- }
- if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
- {
- aio_cycle();
- return RHM_IORES_PAGE_AIOWAIT;
- }
- void* rptr = (void*)((char*)_page_ptr_arr[_pg_index] + (_pg_offset_dblks * JRNL_DBLK_SIZE));
- std::memcpy(&_hdr, rptr, sizeof(rec_hdr_t));
- switch (_hdr._magic)
- {
- case QLS_ENQ_MAGIC:
- {
- _enq_rec.reset(); // sets enqueue rec size
- // Check if RID of this rec is still enqueued, if so read it, else skip
- bool is_enq = false;
- int16_t fid = _emap.get_pfid(_hdr._rid);
- if (fid < enq_map::EMAP_OK)
- {
- bool enforce_txns = !_jc->is_read_only() && !ignore_pending_txns;
- // Block read for transactionally locked record (only when not recovering)
- if (fid == enq_map::EMAP_LOCKED && enforce_txns)
- return RHM_IORES_TXPENDING;
-
- // (Recover mode only) Ok, not in emap - now search tmap, if present then read
- is_enq = _tmap.is_enq(_hdr._rid);
- if (enforce_txns && is_enq)
- return RHM_IORES_TXPENDING;
- }
- else
- is_enq = true;
-
- if (is_enq) // ok, this record is enqueued, check it, then read it...
- {
- if (dtokp->rid())
- {
- if (_hdr._rid != dtokp->rid())
- {
- std::ostringstream oss;
- oss << std::hex << "rid=0x" << _hdr._rid << "; dtok_rid=0x" << dtokp->rid()
- << "; dtok_id=0x" << dtokp->id();
- throw jexception(jerrno::JERR_RMGR_RIDMISMATCH, oss.str(), "rmgr", "read");
- }
- }
- else
- dtokp->set_rid(_hdr._rid);
-
-// TODO: Add member _fid to pmgr::page_cb which indicates the fid from which this page was
-// populated. When this value is set in wmgr::flush() somewehere, then uncomment the following
-// check:
-// if (fid != _page_cb_arr[_pg_index]._fid)
-// {
-// std::ostringstream oss;
-// oss << std::hex << std::setfill('0');
-// oss << "rid=0x" << std::setw(16) << _hdr._rid;
-// oss << "; emap_fid=0x" << std::setw(4) << fid;
-// oss << "; current_fid=" << _rrfc.fid();
-// throw jexception(jerrno::JERR_RMGR_FIDMISMATCH, oss.str(), "rmgr",
-// "read");
-// }
-
- const iores res = read_enq(_hdr, rptr, dtokp);
- dsize = _enq_rec.get_data(datapp);
- xidsize = _enq_rec.get_xid(xidpp);
- transient = _enq_rec.is_transient();
- external = _enq_rec.is_external();
- return res;
- }
- else // skip this record, it is already dequeued
- consume_xid_rec(_hdr, rptr, dtokp);
- break;
- }
- case QLS_DEQ_MAGIC:
- consume_xid_rec(_hdr, rptr, dtokp);
- break;
- case QLS_TXA_MAGIC:
- consume_xid_rec(_hdr, rptr, dtokp);
- break;
- case QLS_TXC_MAGIC:
- consume_xid_rec(_hdr, rptr, dtokp);
- break;
- case QLS_EMPTY_MAGIC:
- consume_filler();
- break;
- default:
- return RHM_IORES_EMPTY;
- }
- }
-}
-
-int32_t
-rmgr::get_events(page_state /*state*/, timespec* const timeout, bool flush)
-{
- if (_aio_evt_rem == 0) // no events to get
- return 0;
-
- int32_t ret;
- if ((ret = aio::getevents(_ioctx, flush ? _aio_evt_rem : 1, _aio_evt_rem/*_cache_num_pages + _jc->num_jfiles()*/, _aio_event_arr, timeout)) < 0)
- {
- if (ret == -EINTR) // Interrupted by signal
- return 0;
- std::ostringstream oss;
- oss << "io_getevents() failed: " << std::strerror(-ret) << " (" << ret << ")";
- throw jexception(jerrno::JERR__AIO, oss.str(), "rmgr", "get_events");
- }
- if (ret == 0 && timeout)
- return jerrno::AIO_TIMEOUT;
-
- std::vector<uint16_t> pil;
- pil.reserve(ret);
- for (int i=0; i<ret; i++) // Index of returned AIOs
- {
- if (_aio_evt_rem == 0)
- {
- std::ostringstream oss;
- oss << "_aio_evt_rem; evt " << (i + 1) << " of " << ret;
- throw jexception(jerrno::JERR__UNDERFLOW, oss.str(), "rmgr", "get_events");
- }
- _aio_evt_rem--;
- aio_cb* aiocbp = _aio_event_arr[i].obj; // This I/O control block (iocb)
- page_cb* pcbp = (page_cb*)(aiocbp->data); // This page control block (pcb)
- long aioret = (long)_aio_event_arr[i].res;
- if (aioret < 0)
- {
- std::ostringstream oss;
- oss << "AIO read operation failed: " << std::strerror(-aioret) << " (" << aioret << ")";
- oss << " [pg=" << pcbp->_index << " buf=" << aiocbp->u.c.buf;
- oss << " rsize=0x" << std::hex << aiocbp->u.c.nbytes;
- oss << " offset=0x" << aiocbp->u.c.offset << std::dec;
- oss << " fh=" << aiocbp->aio_fildes << "]";
- throw jexception(jerrno::JERR__AIO, oss.str(), "rmgr", "get_events");
- }
-
- if (pcbp) // Page reads have pcb
- {
- // TODO: replace for linear store: _rfh
-/*
- if (pcbp->_rfh->rd_subm_cnt_dblks() >= JRNL_SBLK_SIZE_DBLKS) // Detects if write reset of this fcntl obj has occurred.
- {
- // Increment the completed read offset
- // NOTE: We cannot use _rrfc here, as it may have rotated since submitting count.
- // Use stored pointer to fcntl in the pcb instead.
- pcbp->_rdblks = aiocbp->u.c.nbytes / JRNL_DBLK_SIZE;
- pcbp->_rfh->add_rd_cmpl_cnt_dblks(pcbp->_rdblks);
- pcbp->_state = state;
- pil[i] = pcbp->_index;
- }
-*/
- }
- else // File header reads have no pcb
- {
- std::memcpy(&_fhdr, _fhdr_buffer, sizeof(file_hdr_t));
- /*_rrfc.add_cmpl_cnt_dblks(JRNL_SBLK_SIZE_DBLKS);*/ // TODO: replace for linear store: _rrfc
-
- uint32_t fro_dblks = (_fhdr._fro / JRNL_DBLK_SIZE) - JRNL_SBLK_SIZE_DBLKS;
- // Check fro_dblks does not exceed the write pointers which can happen in some corrupted journal recoveries
- // TODO: replace for linear store: _fhdr._pfid, _rrfc
-// if (fro_dblks > _jc->wr_subm_cnt_dblks(_fhdr._pfid) - JRNL_SBLK_SIZE_DBLKS)
-// fro_dblks = _jc->wr_subm_cnt_dblks(_fhdr._pfid) - JRNL_SBLK_SIZE_DBLKS;
- _pg_cntr = fro_dblks / (JRNL_RMGR_PAGE_SIZE * JRNL_SBLK_SIZE_DBLKS);
- uint32_t tot_pg_offs_dblks = _pg_cntr * JRNL_RMGR_PAGE_SIZE * JRNL_SBLK_SIZE_DBLKS;
- _pg_index = _pg_cntr % JRNL_RMGR_PAGES;
- _pg_offset_dblks = fro_dblks - tot_pg_offs_dblks;
-// _rrfc.add_subm_cnt_dblks(tot_pg_offs_dblks);
-// _rrfc.add_cmpl_cnt_dblks(tot_pg_offs_dblks);
-
- _fhdr_rd_outstanding = false;
-// _rrfc.set_valid();
- }
- }
-
- // Perform AIO return callback
- if (_cbp && ret)
- _cbp->rd_aio_cb(pil);
- return ret;
-}
-
-void
-rmgr::recover_complete()
-{}
-
-void
-rmgr::invalidate()
-{
- // TODO: replace for linear store: _rrfc
-// if (_rrfc.is_valid())
-// _rrfc.set_invalid();
-}
-
-void
-rmgr::flush(timespec* timeout)
-{
- // Wait for any outstanding AIO read operations to complete before synchronizing
- while (_aio_evt_rem)
- {
- if (get_events(AIO_COMPLETE, timeout) == jerrno::AIO_TIMEOUT) // timed out, nothing returned
- {
- throw jexception(jerrno::JERR__TIMEOUT,
- "Timed out waiting for outstanding read aio to return", "rmgr", "init_validation");
- }
- }
-
- // Reset all read states and pointers
- for (int i=0; i<_cache_num_pages; i++)
- _page_cb_arr[i]._state = UNUSED;
- // TODO: replace for linear store: _rrfc
-// _rrfc.unset_findex();
- _pg_index = 0;
- _pg_offset_dblks = 0;
-}
-
-/*
-bool
-rmgr::wait_for_validity(timespec* timeout, const bool throw_on_timeout)
-{
- bool timed_out = false;
- while (!_rrfc.is_valid() && !timed_out)
- {
- timed_out = get_events(AIO_COMPLETE, timeout) == jerrno::AIO_TIMEOUT;
- if (timed_out && throw_on_timeout)
- throw jexception(jerrno::JERR__TIMEOUT, "Timed out waiting for read validity", "rmgr", "wait_for_validity");
- }
- return _rrfc.is_valid();
-}
-*/
-
-iores
-rmgr::pre_read_check(data_tok* dtokp)
-{
- if (_aio_evt_rem)
- get_events(AIO_COMPLETE, 0);
-
- // TODO: replace for linear store: _rrfc
-// if (!_rrfc.is_valid())
-// return RHM_IORES_RCINVALID;
-
- // block reads until outstanding file header read completes as fro is needed to read
- if (_fhdr_rd_outstanding)
- return RHM_IORES_PAGE_AIOWAIT;
-
- if(dblks_rem() == 0 /*&& _rrfc.is_compl() && !_rrfc.is_wr_aio_outstanding()*/)// TODO: replace for linear store: _rrfc
- {
- aio_cycle(); // check if any AIOs have returned
- if(dblks_rem() == 0 /*&& _rrfc.is_compl() && !_rrfc.is_wr_aio_outstanding()*/)// TODO: replace for linear store: _rrfc
- {
- if (_jc->unflushed_dblks() > 0)
- _jc->flush();
- else if (!_aio_evt_rem)
- return RHM_IORES_EMPTY;
- }
- }
-
- // Check write state of this token is ENQ - required for read
- if (dtokp)
- {
- if (!dtokp->is_readable())
- {
- std::ostringstream oss;
- oss << std::hex << std::setfill('0');
- oss << "dtok_id=0x" << std::setw(8) << dtokp->id();
- oss << "; dtok_rid=0x" << std::setw(16) << dtokp->rid();
- oss << "; dtok_wstate=" << dtokp->wstate_str();
- throw jexception(jerrno::JERR_RMGR_ENQSTATE, oss.str(), "rmgr", "pre_read_check");
- }
- }
-
- return RHM_IORES_SUCCESS;
-}
-
-iores
-rmgr::read_enq(rec_hdr_t& h, void* rptr, data_tok* dtokp)
-{
- if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
- {
- aio_cycle(); // check if any AIOs have returned
- return RHM_IORES_PAGE_AIOWAIT;
- }
-
- // Read data from this page, first block will have header and data size.
- uint32_t dblks_rd = _enq_rec.decode(h, rptr, dtokp->dblocks_read(), dblks_rem());
- dtokp->incr_dblocks_read(dblks_rd);
-
- _pg_offset_dblks += dblks_rd;
-
- // If data still incomplete, move to next page and decode again
- while (dtokp->dblocks_read() < _enq_rec.rec_size_dblks())
- {
- rotate_page();
- if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
- {
- dtokp->set_rstate(data_tok::READ_PART);
- dtokp->set_dsize(_enq_rec.data_size());
- return RHM_IORES_PAGE_AIOWAIT;
- }
-
- rptr = (void*)((char*)_page_ptr_arr[_pg_index]);
- dblks_rd = _enq_rec.decode(h, rptr, dtokp->dblocks_read(), dblks_rem());
- dtokp->incr_dblocks_read(dblks_rd);
- _pg_offset_dblks += dblks_rd;
- }
-
- // If we have finished with this page, rotate it
- if (dblks_rem() == 0)
- rotate_page();
-
- // Set the record size in dtokp
- dtokp->set_rstate(data_tok::READ);
- dtokp->set_dsize(_enq_rec.data_size());
- return RHM_IORES_SUCCESS;
-}
-
-void
-rmgr::consume_xid_rec(rec_hdr_t& h, void* rptr, data_tok* dtokp)
-{
- if (h._magic == QLS_ENQ_MAGIC)
- {
- enq_hdr_t ehdr;
- std::memcpy(&ehdr, rptr, sizeof(enq_hdr_t));
- if (::is_enq_external(&ehdr))
- dtokp->set_dsize(ehdr._xidsize + sizeof(enq_hdr_t) + sizeof(rec_tail_t));
- else
- dtokp->set_dsize(ehdr._xidsize + ehdr._dsize + sizeof(enq_hdr_t) + sizeof(rec_tail_t));
- }
- else if (h._magic == QLS_DEQ_MAGIC)
- {
- deq_hdr_t dhdr;
- std::memcpy(&dhdr, rptr, sizeof(deq_hdr_t));
- if (dhdr._xidsize)
- dtokp->set_dsize(dhdr._xidsize + sizeof(deq_hdr_t) + sizeof(rec_tail_t));
- else
- dtokp->set_dsize(sizeof(deq_hdr_t));
- }
- else if (h._magic == QLS_TXA_MAGIC || h._magic == QLS_TXC_MAGIC)
- {
- txn_hdr_t thdr;
- std::memcpy(&thdr, rptr, sizeof(txn_hdr_t));
- dtokp->set_dsize(thdr._xidsize + sizeof(txn_hdr_t) + sizeof(rec_tail_t));
- }
- else
- {
- std::ostringstream oss;
- oss << "Record type found = \"" << (char*)&h._magic << "\"";
- throw jexception(jerrno::JERR_RMGR_BADRECTYPE, oss.str(), "rmgr", "consume_xid_rec");
- }
- dtokp->set_dblocks_read(0);
- skip(dtokp);
-}
-
-void
-rmgr::consume_filler()
-{
- // Filler (Magic "RHMx") is one dblk by definition
- _pg_offset_dblks++;
- if (dblks_rem() == 0)
- rotate_page();
-}
-
-iores
-rmgr::skip(data_tok* dtokp)
-{
- uint32_t dsize_dblks = jrec::size_dblks(dtokp->dsize());
- uint32_t tot_dblk_cnt = dtokp->dblocks_read();
- while (true)
- {
- uint32_t this_dblk_cnt = 0;
- if (dsize_dblks - tot_dblk_cnt > dblks_rem())
- this_dblk_cnt = dblks_rem();
- else
- this_dblk_cnt = dsize_dblks - tot_dblk_cnt;
- if (this_dblk_cnt)
- {
- dtokp->incr_dblocks_read(this_dblk_cnt);
- _pg_offset_dblks += this_dblk_cnt;
- tot_dblk_cnt += this_dblk_cnt;
- }
- // If skip still incomplete, move to next page and decode again
- if (tot_dblk_cnt < dsize_dblks)
- {
- if (dblks_rem() == 0)
- rotate_page();
- if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
- {
- dtokp->set_rstate(data_tok::SKIP_PART);
- return RHM_IORES_PAGE_AIOWAIT;
- }
- }
- else
- {
- // Skip complete, put state back to unread
- dtokp->set_rstate(data_tok::UNREAD);
- dtokp->set_dsize(0);
- dtokp->set_dblocks_read(0);
-
- // If we have finished with this page, rotate it
- if (dblks_rem() == 0)
- rotate_page();
- return RHM_IORES_SUCCESS;
- }
- }
-}
-
-iores
-rmgr::aio_cycle()
-{
- // Perform validity checks
- if (_fhdr_rd_outstanding) // read of file header still outstanding in aio
- return RHM_IORES_SUCCESS;
- // TODO: replace for linear store: _rrfc
-/*
- if (!_rrfc.is_valid())
- {
- // Flush and reset all read states and pointers
- flush(&jcntl::_aio_cmpl_timeout);
-
- _jc->get_earliest_fid(); // determine initial file to read; calls _rrfc.set_findex() to set value
- // If this file has not yet been written to, return RHM_IORES_EMPTY
- if (_rrfc.is_void() && !_rrfc.is_wr_aio_outstanding())
- return RHM_IORES_EMPTY;
- init_file_header_read(); // send off AIO read request for file header
- return RHM_IORES_SUCCESS;
- }
-*/
-
- int16_t first_uninit = -1;
- uint16_t num_uninit = 0;
- uint16_t num_compl = 0;
- bool outstanding = false;
- // Index must start with current buffer and cycle around so that first
- // uninitialized buffer is initialized first
- for (uint16_t i=_pg_index; i<_pg_index+_cache_num_pages; i++)
- {
- int16_t ci = i % _cache_num_pages;
- switch (_page_cb_arr[ci]._state)
- {
- case UNUSED:
- if (first_uninit < 0)
- first_uninit = ci;
- num_uninit++;
- break;
- case IN_USE:
- break;
- case AIO_PENDING:
- outstanding = true;
- break;
- case AIO_COMPLETE:
- num_compl++;
- break;
- default:;
- }
- }
- iores res = RHM_IORES_SUCCESS;
- if (num_uninit)
- res = init_aio_reads(first_uninit, num_uninit);
- else if (num_compl == _cache_num_pages) // This condition exists after invalidation
- res = init_aio_reads(0, _cache_num_pages);
- if (outstanding)
- get_events(AIO_COMPLETE, 0);
- return res;
-}
-
-iores
-rmgr::init_aio_reads(const int16_t /*first_uninit*/, const uint16_t /*num_uninit*/)
-{
- // TODO: replace for linear store: _rrfc
-/*
- for (int16_t i=0; i<num_uninit; i++)
- {
- if (_rrfc.is_void()) // Nothing to do; this file not yet written to
- break;
-
- if (_rrfc.subm_offs() == 0)
- {
- _rrfc.add_subm_cnt_dblks(JRNL_SBLK_SIZE_DBLKS);
- _rrfc.add_cmpl_cnt_dblks(JRNL_SBLK_SIZE_DBLKS);
- }
-
- // TODO: Future perf improvement: Do a single AIO read for all available file
- // space into all contiguous empty pages in one AIO operation.
-
- uint32_t file_rem_dblks = _rrfc.remaining_dblks();
- file_rem_dblks -= file_rem_dblks % JRNL_SBLK_SIZE_DBLKS; // round down to closest sblk boundary
- uint32_t pg_size_dblks = JRNL_RMGR_PAGE_SIZE * JRNL_SBLK_SIZE_DBLKS;
- uint32_t rd_size = file_rem_dblks > pg_size_dblks ? pg_size_dblks : file_rem_dblks;
- if (rd_size)
- {
- int16_t pi = (i + first_uninit) % _cache_num_pages;
- // TODO: For perf, combine contiguous pages into single read
- // 1 or 2 AIOs needed depending on whether read block folds
- aio_cb* aiocbp = &_aio_cb_arr[pi];
- aio::prep_pread_2(aiocbp, _rrfc.fh(), _page_ptr_arr[pi], rd_size * JRNL_DBLK_SIZE_DBLKS, _rrfc.subm_offs());
- if (aio::submit(_ioctx, 1, &aiocbp) < 0)
- throw jexception(jerrno::JERR__AIO, "rmgr", "init_aio_reads");
- _rrfc.add_subm_cnt_dblks(rd_size);
- _aio_evt_rem++;
- _page_cb_arr[pi]._state = AIO_PENDING;
- _page_cb_arr[pi]._rfh = _rrfc.file_controller();
- }
- else // If there is nothing to read for this page, neither will there be for the others...
- break;
- if (_rrfc.file_rotate())
- _rrfc.rotate();
- }
-*/
- return RHM_IORES_SUCCESS;
-}
-
-void
-rmgr::rotate_page()
-{
-/*
- _page_cb_arr[_pg_index]._rdblks = 0;
- _page_cb_arr[_pg_index]._state = UNUSED;
- if (_pg_offset_dblks >= JRNL_RMGR_PAGE_SIZE * JRNL_SBLK_SIZE_DBLKS)
- {
- _pg_offset_dblks = 0;
- _pg_cntr++;
- }
- if (++_pg_index >= _cache_num_pages)
- _pg_index = 0;
- aio_cycle();
- _pg_offset_dblks = 0;
- // This counter is for bookkeeping only, page rotates are handled directly in init_aio_reads()
- // FIXME: _pg_cntr should be sync'd with aio ops, not use of page as it is now...
- // Need to move reset into if (_rrfc.file_rotate()) above.
- if (_pg_cntr >= (_jc->jfsize_sblks() / JRNL_RMGR_PAGE_SIZE))
- _pg_cntr = 0;
-*/
-}
-
-uint32_t
-rmgr::dblks_rem() const
-{
- return _page_cb_arr[_pg_index]._rdblks - _pg_offset_dblks;
-}
-
-void
-rmgr::set_params_null(void** const datapp, std::size_t& dsize, void** const xidpp, std::size_t& xidsize)
-{
- *datapp = 0;
- dsize = 0;
- *xidpp = 0;
- xidsize = 0;
-}
-
-void
-rmgr::init_file_header_read()
-{
- // TODO: replace for linear store: _rrfc
-/*
- _jc->fhdr_wr_sync(_rrfc.index()); // wait if the file header write is outstanding
- int rfh = _rrfc.fh();
- aio::prep_pread_2(_fhdr_aio_cb_ptr, rfh, _fhdr_buffer, _sblksize, 0);
- if (aio::submit(_ioctx, 1, &_fhdr_aio_cb_ptr) < 0)
- throw jexception(jerrno::JERR__AIO, "rmgr", "init_file_header_read");
- _aio_evt_rem++;
- _rrfc.add_subm_cnt_dblks(JRNL_SBLK_SIZE_DBLKS);
- _fhdr_rd_outstanding = true;
-*/
-}
-
-/* TODO (sometime in the future)
-const iores
-rmgr::get(const uint64_t& rid, const std::size_t& dsize, const std::size_t& dsize_avail,
- const void** const data, bool auto_discard)
-{
- return RHM_IORES_SUCCESS;
-}
-
-const iores
-rmgr::discard(data_tok* dtokp)
-{
- return RHM_IORES_SUCCESS;
-}
-*/
-
-}}
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/rmgr.h b/qpid/cpp/src/qpid/linearstore/jrnl/rmgr.h
deleted file mode 100644
index d7d38624fa..0000000000
--- a/qpid/cpp/src/qpid/linearstore/jrnl/rmgr.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-#ifndef QPID_LEGACYSTORE_JRNL_RMGR_H
-#define QPID_LEGACYSTORE_JRNL_RMGR_H
-
-namespace qpid
-{
-namespace qls_jrnl
-{
-class rmgr;
-}}
-
-#include <cstring>
-#include "qpid/linearstore/jrnl/enums.h"
-//#include "qpid/linearstore/jrnl/file_hdr.h"
-#include "qpid/linearstore/jrnl/pmgr.h"
-//#include "qpid/linearstore/jrnl/rec_hdr.h"
-//#include "qpid/linearstore/jrnl/rrfc.h"
-#include "qpid/linearstore/jrnl/utils/file_hdr.h"
-
-namespace qpid
-{
-namespace qls_jrnl
-{
-
- /**
- * \brief Class for managing a read page cache of arbitrary size and number of pages.
- *
- * The read page cache works on the principle of filling as many pages as possilbe in advance of
- * reading the data. This ensures that delays caused by AIO operations are minimized.
- */
- class rmgr : public pmgr
- {
- private:
- //rrfc& _rrfc; ///< Ref to read rotating file controller
- rec_hdr_t _hdr; ///< Header used to determind record type
-
- void* _fhdr_buffer; ///< Buffer used for fhdr reads
- aio_cb* _fhdr_aio_cb_ptr; ///< iocb pointer for fhdr reads
- file_hdr_t _fhdr; ///< file header instance for reading file headers
- bool _fhdr_rd_outstanding; ///< true if a fhdr read is outstanding
-
- public:
- rmgr(jcntl* jc, enq_map& emap, txn_map& tmap/*, rrfc& rrfc*/);
- virtual ~rmgr();
-
- using pmgr::initialize;
- void initialize(aio_callback* const cbp);
- iores read(void** const datapp, std::size_t& dsize, void** const xidpp,
- std::size_t& xidsize, bool& transient, bool& external, data_tok* dtokp,
- bool ignore_pending_txns);
- int32_t get_events(page_state state, timespec* const timeout, bool flush = false);
- void recover_complete();
- inline iores synchronize() { /*if (_rrfc.is_valid()) return RHM_IORES_SUCCESS;*/ return aio_cycle(); }
- void invalidate();
-// bool wait_for_validity(timespec* const timeout, const bool throw_on_timeout = false);
-
- /* TODO (if required)
- const iores get(const uint64_t& rid, const std::size_t& dsize, const std::size_t& dsize_avail,
- const void** const data, bool auto_discard);
- const iores discard(data_tok* dtok);
- */
-
- private:
- void clean();
- void flush(timespec* timeout);
- iores pre_read_check(data_tok* dtokp);
- iores read_enq(rec_hdr_t& h, void* rptr, data_tok* dtokp);
- void consume_xid_rec(rec_hdr_t& h, void* rptr, data_tok* dtokp);
- void consume_filler();
- iores skip(data_tok* dtokp);
- iores aio_cycle();
- iores init_aio_reads(const int16_t first_uninit, const uint16_t num_uninit);
- void rotate_page();
- uint32_t dblks_rem() const;
- void set_params_null(void** const datapp, std::size_t& dsize, void** const xidpp,
- std::size_t& xidsize);
- void init_file_header_read();
- };
-
-}}
-
-#endif // ifndef QPID_LEGACYSTORE_JRNL_RMGR_H
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.cpp
index e6f6f238f1..3c78903520 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/txn_rec.cpp
@@ -94,8 +94,8 @@ txn_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks)
assert(max_size_dblks > 0);
assert(_xidp != 0 && _txn_hdr._xidsize > 0);
- std::size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE_BYTES;
- std::size_t rem = max_size_dblks * JRNL_DBLK_SIZE_BYTES;
+ std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES;
+ std::size_t rem = max_size_dblks * QLS_DBLK_SIZE_BYTES;
std::size_t wr_cnt = 0;
if (rec_offs_dblks) // Continuation of split dequeue record (over 2 or more pages)
{
@@ -145,10 +145,10 @@ txn_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks)
{
std::memcpy((char*)wptr + wr_cnt, (char*)&_txn_tail + rec_offs, wsize);
wr_cnt += wsize;
-#ifdef RHM_CLEAN
- std::size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE_BYTES;
- std::size_t dblk_rec_size = size_dblks(rec_size() - rec_offs) * JRNL_DBLK_SIZE_BYTES;
- std::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR, dblk_rec_size - wr_cnt);
+#ifdef QLS_CLEAN
+ std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES;
+ std::size_t dblk_rec_size = size_dblks(rec_size() - rec_offs) * QLS_DBLK_SIZE_BYTES;
+ std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt);
#endif
}
rec_offs -= sizeof(_txn_tail) - wsize;
@@ -186,9 +186,9 @@ txn_rec::encode(void* wptr, uint32_t rec_offs_dblks, uint32_t max_size_dblks)
wr_cnt += _txn_hdr._xidsize;
std::memcpy((char*)wptr + wr_cnt, (void*)&_txn_tail, sizeof(_txn_tail));
wr_cnt += sizeof(_txn_tail);
-#ifdef RHM_CLEAN
- std::size_t dblk_rec_size = size_dblks(rec_size()) * JRNL_DBLK_SIZE_BYTES;
- std::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR, dblk_rec_size - wr_cnt);
+#ifdef QLS_CLEAN
+ std::size_t dblk_rec_size = size_dblks(rec_size()) * QLS_DBLK_SIZE_BYTES;
+ std::memset((char*)wptr + wr_cnt, QLS_CLEAN_CHAR, dblk_rec_size - wr_cnt);
#endif
}
}
@@ -206,7 +206,7 @@ txn_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
{
const uint32_t hdr_xid_dblks = size_dblks(sizeof(txn_hdr_t) + _txn_hdr._xidsize);
const uint32_t hdr_xid_tail_dblks = size_dblks(sizeof(txn_hdr_t) + _txn_hdr._xidsize + sizeof(rec_tail_t));
- const std::size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE_BYTES;
+ const std::size_t rec_offs = rec_offs_dblks * QLS_DBLK_SIZE_BYTES;
if (hdr_xid_tail_dblks - rec_offs_dblks <= max_size_dblks)
{
@@ -239,7 +239,7 @@ txn_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
const std::size_t xid_rem = _txn_hdr._xidsize - xid_offs;
std::memcpy((char*)_buff + xid_offs, rptr, xid_rem);
rd_cnt += xid_rem;
- const std::size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
if (tail_rem)
{
std::memcpy((void*)&_txn_tail, ((char*)rptr + xid_rem), tail_rem);
@@ -249,7 +249,7 @@ txn_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
else
{
// Remainder of xid split
- const std::size_t xid_cp_size = (max_size_dblks * JRNL_DBLK_SIZE_BYTES);
+ const std::size_t xid_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES);
std::memcpy((char*)_buff + rec_offs - sizeof(txn_hdr_t), rptr, xid_cp_size);
rd_cnt += xid_cp_size;
}
@@ -288,7 +288,7 @@ txn_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
// Entire header and xid fit within this page, tail split
std::memcpy(_buff, (char*)rptr + rd_cnt, _txn_hdr._xidsize);
rd_cnt += _txn_hdr._xidsize;
- const std::size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t tail_rem = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
if (tail_rem)
{
std::memcpy((void*)&_txn_tail, (char*)rptr + rd_cnt, tail_rem);
@@ -298,7 +298,7 @@ txn_rec::decode(rec_hdr_t& h, void* rptr, uint32_t rec_offs_dblks, uint32_t max_
else
{
// Header fits within this page, xid split
- const std::size_t xid_cp_size = (max_size_dblks * JRNL_DBLK_SIZE_BYTES) - rd_cnt;
+ const std::size_t xid_cp_size = (max_size_dblks * QLS_DBLK_SIZE_BYTES) - rd_cnt;
std::memcpy(_buff, (char*)rptr + rd_cnt, xid_cp_size);
rd_cnt += xid_cp_size;
}
@@ -357,7 +357,7 @@ txn_rec::rcv_decode(rec_hdr_t h, std::ifstream* ifsp, std::size_t& rec_offs)
return false;
}
}
- ifsp->ignore(rec_size_dblks() * JRNL_DBLK_SIZE_BYTES - rec_size());
+ ifsp->ignore(rec_size_dblks() * QLS_DBLK_SIZE_BYTES - rec_size());
chk_tail(); // Throws if tail invalid or record incomplete
assert(!ifsp->fail() && !ifsp->bad());
return true;
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.c b/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.c
index 8689ca3097..7e53be5e18 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.c
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/utils/file_hdr.c
@@ -73,7 +73,6 @@ void file_hdr_reset(file_hdr_t* target) {
target->_ts_nsec = 0;
target->_file_number = 0;
target->_queue_name_len = 0;
- memset(target + sizeof(file_hdr_t), 0, MAX_FILE_HDR_LEN - sizeof(file_hdr_t));
}
int is_file_hdr_reset(file_hdr_t* target) {
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.cpp b/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.cpp
index f61da49a1b..a4ed6cbf19 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.cpp
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.cpp
@@ -26,20 +26,24 @@
#include <cstdlib>
#include <cstring>
#include "qpid/linearstore/jrnl/utils/file_hdr.h"
+#include "qpid/linearstore/jrnl/jcfg.h"
#include "qpid/linearstore/jrnl/jcntl.h"
#include "qpid/linearstore/jrnl/jerrno.h"
#include "qpid/linearstore/jrnl/JournalFile.h"
#include <sstream>
#include <stdint.h>
-//#include <iostream> // DEBUG
+#include <iostream> // DEBUG
namespace qpid
{
namespace qls_jrnl
{
-wmgr::wmgr(jcntl* jc, enq_map& emap, txn_map& tmap, LinearFileController& lfc):
+wmgr::wmgr(jcntl* jc,
+ enq_map& emap,
+ txn_map& tmap,
+ LinearFileController& lfc):
pmgr(jc, emap, tmap),
_lfc(lfc),
_max_dtokpp(0),
@@ -52,8 +56,13 @@ wmgr::wmgr(jcntl* jc, enq_map& emap, txn_map& tmap, LinearFileController& lfc):
_txn_pending_set()
{}
-wmgr::wmgr(jcntl* jc, enq_map& emap, txn_map& tmap, LinearFileController& lfc, const uint32_t max_dtokpp, const uint32_t max_iowait_us):
- pmgr(jc, emap, tmap /* , dtoklp */),
+wmgr::wmgr(jcntl* jc,
+ enq_map& emap,
+ txn_map& tmap,
+ LinearFileController& lfc,
+ const uint32_t max_dtokpp,
+ const uint32_t max_iowait_us):
+ pmgr(jc, emap, tmap),
_lfc(lfc),
_max_dtokpp(max_dtokpp),
_max_io_wait_us(max_iowait_us),
@@ -71,9 +80,12 @@ wmgr::~wmgr()
}
void
-wmgr::initialize(aio_callback* const cbp, const uint32_t wcache_pgsize_sblks,
- const uint16_t wcache_num_pages, const uint32_t max_dtokpp, const uint32_t max_iowait_us,
- std::size_t eo)
+wmgr::initialize(aio_callback* const cbp,
+ const uint32_t wcache_pgsize_sblks,
+ const uint16_t wcache_num_pages,
+ const uint32_t max_dtokpp,
+ const uint32_t max_iowait_us,
+ std::size_t eo)
{
_enq_busy = false;
_deq_busy = false;
@@ -86,26 +98,38 @@ wmgr::initialize(aio_callback* const cbp, const uint32_t wcache_pgsize_sblks,
if (eo)
{
- const uint32_t wr_pg_size_dblks = _cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS;
- uint32_t data_dblks = (eo / JRNL_DBLK_SIZE_BYTES) - 4; // 4 dblks for file hdr
+ const uint32_t wr_pg_size_dblks = _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS;
+ uint32_t data_dblks = (eo / QLS_DBLK_SIZE_BYTES) - 4; // 4 dblks for file hdr
_pg_cntr = data_dblks / wr_pg_size_dblks;
_pg_offset_dblks = data_dblks - (_pg_cntr * wr_pg_size_dblks);
}
}
iores
-wmgr::enqueue(const void* const data_buff, const std::size_t tot_data_len,
- const std::size_t this_data_len, data_tok* dtokp, const void* const xid_ptr,
- const std::size_t xid_len, const bool transient, const bool external)
+wmgr::enqueue(const void* const data_buff,
+ const std::size_t tot_data_len,
+ const std::size_t this_data_len,
+ data_tok* dtokp,
+ const void* const xid_ptr,
+ const std::size_t xid_len,
+ const bool transient,
+ const bool external)
{
if (xid_len)
assert(xid_ptr != 0);
- if (_deq_busy || _abort_busy || _commit_busy)
- return RHM_IORES_BUSY;
+ if (_deq_busy || _abort_busy || _commit_busy) {
+ std::ostringstream oss;
+ oss << "RHM_IORES_BUSY: enqueue while part way through another op:";
+ oss << " _deq_busy=" << (_deq_busy?"T":"F");
+ oss << " _abort_busy=" << (_abort_busy?"T":"F");
+ oss << " _commit_busy=" << (_commit_busy?"T":"F");
+ throw jexception(oss.str()); // TODO: complete exception
+ }
- if (this_data_len != tot_data_len && !external)
- return RHM_IORES_NOTIMPL;
+ if (this_data_len != tot_data_len && !external) {
+ throw jexception("RHM_IORES_NOTIMPL: partial enqueues not implemented"); // TODO: complete exception;
+ }
iores res = pre_write_check(WMGR_ENQUEUE, dtokp, xid_len, tot_data_len, external);
if (res != RHM_IORES_SUCCESS)
@@ -124,9 +148,8 @@ wmgr::enqueue(const void* const data_buff, const std::size_t tot_data_len,
}
}
- uint64_t rid = (dtokp->external_rid() | cont) ? dtokp->rid() : _lfc.getNextRecordId();/*_wrfc.get_incr_rid()*/
- _enq_rec.reset(rid, data_buff, tot_data_len, xid_ptr, xid_len/*, _wrfc.owi()*/, transient,
- external);
+ uint64_t rid = (dtokp->external_rid() | cont) ? dtokp->rid() : _lfc.getNextRecordId();
+ _enq_rec.reset(rid, data_buff, tot_data_len, xid_ptr, xid_len, transient, external);
if (!cont)
{
dtokp->set_rid(rid);
@@ -137,14 +160,16 @@ wmgr::enqueue(const void* const data_buff, const std::size_t tot_data_len,
dtokp->clear_xid();
_enq_busy = true;
}
+//std::cout << "---+++ wmgr::enqueue() ENQ rid=0x" << std::hex << rid << " " << std::dec << std::flush; // DEBUG
bool done = false;
while (!done)
{
- assert(_pg_offset_dblks < _cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS);
- void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE_BYTES);
+//std::cout << "*" << std::flush; // DEBUG
+ assert(_pg_offset_dblks < _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS);
+ void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES);
uint32_t data_offs_dblks = dtokp->dblocks_written();
uint32_t ret = _enq_rec.encode(wptr, data_offs_dblks,
- (_cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS) - _pg_offset_dblks);
+ (_cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) - _pg_offset_dblks);
// Remember fid which contains the record header in case record is split over several files
if (data_offs_dblks == 0) {
@@ -159,6 +184,7 @@ wmgr::enqueue(const void* const data_buff, const std::size_t tot_data_len,
// Is the encoding of this record complete?
if (dtokp->dblocks_written() >= _enq_rec.rec_size_dblks())
{
+//std::cout << "!" << std::flush; // DEBUG
// TODO: Incorrect - must set state to ENQ_CACHED; ENQ_SUBM is set when AIO returns.
dtokp->set_wstate(data_tok::ENQ_SUBM);
dtokp->set_dsize(tot_data_len);
@@ -166,7 +192,8 @@ wmgr::enqueue(const void* const data_buff, const std::size_t tot_data_len,
// long multi-page messages have their token on the page containing the END of the
// message. AIO callbacks will then only process this token when entire message is
// enqueued.
- _lfc.incrEnqueuedRecordCount();
+ _lfc.incrEnqueuedRecordCount(dtokp->fid());
+//std::cout << "[0x" << std::hex << _lfc.getEnqueuedRecordCount() << std::dec << std::flush; // DEBUG
if (xid_len) // If part of transaction, add to transaction map
{
@@ -185,26 +212,37 @@ wmgr::enqueue(const void* const data_buff, const std::size_t tot_data_len,
}
done = true;
- }
- else
+ } else {
+//std::cout << "$" << std::endl << std::flush; // DEBUG
dtokp->set_wstate(data_tok::ENQ_PART);
+ }
file_header_check(rid, cont, _enq_rec.rec_size_dblks() - data_offs_dblks);
- flush_check(res, cont, done);
+ flush_check(res, cont, done, rid);
}
if (dtokp->wstate() >= data_tok::ENQ_SUBM)
_enq_busy = false;
+//std::cout << " res=" << iores_str(res) << " _enq_busy=" << (_enq_busy?"T":"F") << std::endl << std::flush; // DEBUG
return res;
}
iores
-wmgr::dequeue(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_len, const bool txn_coml_commit)
+wmgr::dequeue(data_tok* dtokp,
+ const void* const xid_ptr,
+ const std::size_t xid_len,
+ const bool txn_coml_commit)
{
if (xid_len)
assert(xid_ptr != 0);
- if (_enq_busy || _abort_busy || _commit_busy)
- return RHM_IORES_BUSY;
+ if (_enq_busy || _abort_busy || _commit_busy) {
+ std::ostringstream oss;
+ oss << "RHM_IORES_BUSY: dequeue while part way through another op:";
+ oss << " _enq_busy=" << (_enq_busy?"T":"F");
+ oss << " _abort_busy=" << (_abort_busy?"T":"F");
+ oss << " _commit_busy=" << (_commit_busy?"T":"F");
+ throw jexception(oss.str()); // TODO: complete exception
+ }
iores res = pre_write_check(WMGR_DEQUEUE, dtokp);
if (res != RHM_IORES_SUCCESS)
@@ -224,7 +262,7 @@ wmgr::dequeue(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_
}
const bool ext_rid = dtokp->external_rid();
- uint64_t rid = (ext_rid | cont) ? dtokp->rid() : /*_wrfc.get_incr_rid()*/0; // TODO: replace for linearstore: _wrfc
+ uint64_t rid = (ext_rid | cont) ? dtokp->rid() : _lfc.getNextRecordId();
uint64_t dequeue_rid = (ext_rid | cont) ? dtokp->dequeue_rid() : dtokp->rid();
_deq_rec.reset(rid, dequeue_rid, xid_ptr, xid_len/*, _wrfc.owi()*/, txn_coml_commit);
if (!cont)
@@ -242,14 +280,16 @@ wmgr::dequeue(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_
dtokp->set_dblocks_written(0); // Reset dblks_written from previous op
_deq_busy = true;
}
+//std::cout << "---+++ wmgr::dequeue() DEQ rid=0x" << std::hex << rid << " drid=0x" << dequeue_rid << " " << std::dec << std::flush; // DEBUG
bool done = false;
while (!done)
{
- assert(_pg_offset_dblks < _cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS);
- void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE_BYTES);
+//std::cout << "*" << std::flush; // DEBUG
+ assert(_pg_offset_dblks < _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS);
+ void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES);
uint32_t data_offs_dblks = dtokp->dblocks_written();
uint32_t ret = _deq_rec.encode(wptr, data_offs_dblks,
- (_cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS) - _pg_offset_dblks);
+ (_cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) - _pg_offset_dblks);
// Remember fid which contains the record header in case record is split over several files
if (data_offs_dblks == 0) {
@@ -264,6 +304,7 @@ wmgr::dequeue(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_
// Is the encoding of this record complete?
if (dtokp->dblocks_written() >= _deq_rec.rec_size_dblks())
{
+//std::cout << "!" << std::flush; // DEBUG
// TODO: Incorrect - must set state to ENQ_CACHED; ENQ_SUBM is set when AIO returns.
dtokp->set_wstate(data_tok::DEQ_SUBM);
@@ -276,7 +317,7 @@ wmgr::dequeue(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_
}
else
{
- int16_t fid;
+ uint64_t fid;
short eres = _emap.get_remove_pfid(dtokp->dequeue_rid(), fid);
if (eres < enq_map::EMAP_OK) // fail
{
@@ -293,30 +334,43 @@ wmgr::dequeue(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_
throw jexception(jerrno::JERR_MAP_LOCKED, oss.str(), "wmgr", "dequeue");
}
}
- _lfc.decrEnqueuedRecordCount();
+//std::cout << "[0x" << std::hex << _lfc.getEnqueuedRecordCount(fid) << std::dec << std::flush; // DEBUG
+//try {
+ _lfc.decrEnqueuedRecordCount(fid);
+//} catch (std::exception& e) { std::cout << "***OOPS*** " << e.what() << " cfid=" << _lfc.getCurrentFileSeqNum() << " fid=" << fid << std::flush; throw; }
}
done = true;
- }
- else
+ } else {
+//std::cout << "$" << std::flush; // DEBUG
dtokp->set_wstate(data_tok::DEQ_PART);
+ }
file_header_check(rid, cont, _deq_rec.rec_size_dblks() - data_offs_dblks);
- flush_check(res, cont, done);
+ flush_check(res, cont, done, rid);
}
if (dtokp->wstate() >= data_tok::DEQ_SUBM)
_deq_busy = false;
+//std::cout << " res=" << iores_str(res) << " _deq_busy=" << (_deq_busy?"T":"F") << std::endl << std::flush; // DEBUG
return res;
}
iores
-wmgr::abort(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_len)
+wmgr::abort(data_tok* dtokp,
+ const void* const xid_ptr,
+ const std::size_t xid_len)
{
// commit and abort MUST have a valid xid
assert(xid_ptr != 0 && xid_len > 0);
- if (_enq_busy || _deq_busy || _commit_busy)
- return RHM_IORES_BUSY;
+ if (_enq_busy || _deq_busy || _commit_busy) {
+ std::ostringstream oss;
+ oss << "RHM_IORES_BUSY: abort while part way through another op:";
+ oss << " _enq_busy=" << (_enq_busy?"T":"F");
+ oss << " _deq_busy=" << (_deq_busy?"T":"F");
+ oss << " _commit_busy=" << (_commit_busy?"T":"F");
+ throw jexception(oss.str()); // TODO: complete exception
+ }
iores res = pre_write_check(WMGR_ABORT, dtokp);
if (res != RHM_IORES_SUCCESS)
@@ -348,11 +402,11 @@ wmgr::abort(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_le
bool done = false;
while (!done)
{
- assert(_pg_offset_dblks < _cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS);
- void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE_BYTES);
+ assert(_pg_offset_dblks < _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS);
+ void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES);
uint32_t data_offs_dblks = dtokp->dblocks_written();
uint32_t ret = _txn_rec.encode(wptr, data_offs_dblks,
- (_cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS) - _pg_offset_dblks);
+ (_cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) - _pg_offset_dblks);
// Remember fid which contains the record header in case record is split over several files
if (data_offs_dblks == 0)
@@ -392,7 +446,7 @@ wmgr::abort(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_le
dtokp->set_wstate(data_tok::ABORT_PART);
file_header_check(rid, cont, _txn_rec.rec_size_dblks() - data_offs_dblks);
- flush_check(res, cont, done);
+ flush_check(res, cont, done, rid);
}
if (dtokp->wstate() >= data_tok::ABORT_SUBM)
_abort_busy = false;
@@ -400,13 +454,21 @@ wmgr::abort(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_le
}
iores
-wmgr::commit(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_len)
+wmgr::commit(data_tok* dtokp,
+ const void* const xid_ptr,
+ const std::size_t xid_len)
{
// commit and abort MUST have a valid xid
assert(xid_ptr != 0 && xid_len > 0);
- if (_enq_busy || _deq_busy || _abort_busy)
- return RHM_IORES_BUSY;
+ if (_enq_busy || _deq_busy || _abort_busy) {
+ std::ostringstream oss;
+ oss << "RHM_IORES_BUSY: commit while part way through another op:";
+ oss << " _enq_busy=" << (_enq_busy?"T":"F");
+ oss << " _deq_busy=" << (_deq_busy?"T":"F");
+ oss << " _abort_busy=" << (_abort_busy?"T":"F");
+ throw jexception(oss.str()); // TODO: complete exception
+ }
iores res = pre_write_check(WMGR_COMMIT, dtokp);
if (res != RHM_IORES_SUCCESS)
@@ -438,11 +500,11 @@ wmgr::commit(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_l
bool done = false;
while (!done)
{
- assert(_pg_offset_dblks < _cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS);
- void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE_BYTES);
+ assert(_pg_offset_dblks < _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS);
+ void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES);
uint32_t data_offs_dblks = dtokp->dblocks_written();
uint32_t ret = _txn_rec.encode(wptr, data_offs_dblks,
- (_cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS) - _pg_offset_dblks);
+ (_cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS) - _pg_offset_dblks);
// Remember fid which contains the record header in case record is split over several files
if (data_offs_dblks == 0)
@@ -475,7 +537,7 @@ wmgr::commit(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_l
}
else // txn dequeue
{
- int16_t fid;
+ uint64_t fid;
short eres = _emap.get_remove_pfid(itr->_drid, fid, true);
if (eres < enq_map::EMAP_OK) // fail
{
@@ -509,7 +571,7 @@ wmgr::commit(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_l
dtokp->set_wstate(data_tok::COMMIT_PART);
file_header_check(rid, cont, _txn_rec.rec_size_dblks() - data_offs_dblks);
- flush_check(res, cont, done);
+ flush_check(res, cont, done, rid);
}
if (dtokp->wstate() >= data_tok::COMMIT_SUBM)
_commit_busy = false;
@@ -517,29 +579,34 @@ wmgr::commit(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_l
}
void
-wmgr::file_header_check(const uint64_t rid, const bool cont, const uint32_t rec_dblks_rem)
+wmgr::file_header_check(const uint64_t rid,
+ const bool cont,
+ const uint32_t rec_dblks_rem)
{
if (_lfc.isEmpty()) // File never written (i.e. no header or data)
{
std::size_t fro = 0;
if (cont) {
- bool file_fit = rec_dblks_rem <= _lfc.dataSize_sblks() * JRNL_SBLK_SIZE_DBLKS; // Will fit within this journal file
- bool file_full = rec_dblks_rem == _lfc.dataSize_sblks() * JRNL_SBLK_SIZE_DBLKS; // Will exactly fill this journal file
+ bool file_fit = rec_dblks_rem <= _lfc.dataSize_sblks() * QLS_SBLK_SIZE_DBLKS; // Will fit within this journal file
+ bool file_full = rec_dblks_rem == _lfc.dataSize_sblks() * QLS_SBLK_SIZE_DBLKS; // Will exactly fill this journal file
if (file_fit && !file_full) {
- fro = (rec_dblks_rem + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_DBLKS)) * JRNL_DBLK_SIZE_BYTES;
+ fro = (rec_dblks_rem + (QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS)) * QLS_DBLK_SIZE_BYTES;
}
} else {
- fro = QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_BYTES;
+ fro = QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_BYTES;
}
_lfc.asyncFileHeaderWrite(_ioctx, 0, rid, fro);
+ _aio_evt_rem++;
}
}
void
-wmgr::flush_check(iores& res, bool& cont, bool& done)
+wmgr::flush_check(iores& res,
+ bool& cont,
+ bool& done, const uint64_t /*rid*/) // DEBUG
{
// Is page is full, flush
- if (_pg_offset_dblks >= _cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS)
+ if (_pg_offset_dblks >= _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS)
{
res = write_flush();
assert(res == RHM_IORES_SUCCESS);
@@ -558,6 +625,7 @@ wmgr::flush_check(iores& res, bool& cont, bool& done)
if (!done) {
cont = true;
}
+//std::cout << "***** wmgr::flush_check(): GET NEXT FILE: rid=0x" << std::hex << rid << std::dec << " res=" << iores_str(res) << " cont=" << (cont?"T":"F") << " done=" << (done?"T":"F") << std::endl; // DEBUG
}
}
}
@@ -580,28 +648,28 @@ wmgr::write_flush()
// Don't bother flushing an empty page or one that is still in state AIO_PENDING
if (_cached_offset_dblks)
{
- if (_page_cb_arr[_pg_index]._state == AIO_PENDING)
+ if (_page_cb_arr[_pg_index]._state == AIO_PENDING) {
+//std::cout << "#"; // DEBUG
res = RHM_IORES_PAGE_AIOWAIT;
- else
- {
+ } else {
if (_page_cb_arr[_pg_index]._state != IN_USE)
{
std::ostringstream oss;
oss << "pg_index=" << _pg_index << " state=" << _page_cb_arr[_pg_index].state_str();
- throw jexception(jerrno::JERR_WMGR_BADPGSTATE, oss.str(), "wmgr",
- "write_flush");
+ throw jexception(jerrno::JERR_WMGR_BADPGSTATE, oss.str(), "wmgr", "write_flush");
}
// Send current page using AIO
- // In manual flushes, dblks may not coincide with sblks, add filler records ("RHMx")
- // if necessary.
+ // In manual flushes, dblks may not coincide with sblks, add filler records ("RHMx") if necessary.
dblk_roundup();
- std::size_t pg_offs = (_pg_offset_dblks - _cached_offset_dblks) * JRNL_DBLK_SIZE_BYTES;
+ std::size_t pg_offs = (_pg_offset_dblks - _cached_offset_dblks) * QLS_DBLK_SIZE_BYTES;
aio_cb* aiocbp = &_aio_cb_arr[_pg_index];
_lfc.asyncPageWrite(_ioctx, aiocbp, (char*)_page_ptr_arr[_pg_index] + pg_offs, _cached_offset_dblks);
+ _page_cb_arr[_pg_index]._state = AIO_PENDING;
_aio_evt_rem++;
+//std::cout << "." << _aio_evt_rem << std::flush; // DEBUG
_cached_offset_dblks = 0;
_jc->instr_incr_outstanding_aio_cnt();
@@ -610,7 +678,7 @@ wmgr::write_flush()
_page_cb_arr[_pg_index]._state = IN_USE;
}
}
- get_events(UNUSED, 0);
+ get_events(0, false);
if (_page_cb_arr[_pg_index]._state == UNUSED)
_page_cb_arr[_pg_index]._state = IN_USE;
return res;
@@ -620,17 +688,19 @@ void
wmgr::get_next_file()
{
_pg_cntr = 0;
+//std::cout << "&&&&& wmgr::get_next_file(): " << status_str() << std::endl; // DEBUG
_lfc.pullEmptyFileFromEfp();
}
int32_t
-wmgr::get_events(page_state state, timespec* const timeout, bool flush)
+wmgr::get_events(timespec* const timeout,
+ bool flush)
{
if (_aio_evt_rem == 0) // no events to get
return 0;
int ret = 0;
- if ((ret = aio::getevents(_ioctx, flush ? _aio_evt_rem : 1, _aio_evt_rem/*_cache_num_pages + _jc->num_jfiles()*/, _aio_event_arr, timeout)) < 0)
+ if ((ret = aio::getevents(_ioctx, flush ? _aio_evt_rem : 1, _aio_evt_rem, _aio_event_arr, timeout)) < 0)
{
if (ret == -EINTR) // Interrupted by signal
return 0;
@@ -652,6 +722,7 @@ wmgr::get_events(page_state state, timespec* const timeout, bool flush)
throw jexception(jerrno::JERR__UNDERFLOW, oss.str(), "wmgr", "get_events");
}
_aio_evt_rem--;
+//std::cout << "'" << _aio_evt_rem; // DEBUG
aio_cb* aiocbp = _aio_event_arr[i].obj; // This I/O control block (iocb)
page_cb* pcbp = (page_cb*)(aiocbp->data); // This page control block (pcb)
long aioret = (long)_aio_event_arr[i].res;
@@ -671,6 +742,7 @@ wmgr::get_events(page_state state, timespec* const timeout, bool flush)
}
if (pcbp) // Page writes have pcb
{
+//std::cout << "p"; // DEBUG
uint32_t s = pcbp->_pdtokl->size();
std::vector<data_tok*> dtokl;
dtokl.reserve(s);
@@ -754,7 +826,8 @@ wmgr::get_events(page_state state, timespec* const timeout, bool flush)
// Clean up this pcb's data_tok list
pcbp->_pdtokl->clear();
- pcbp->_state = state;
+ pcbp->_state = UNUSED;
+//std::cout << "c" << pcbp->_index << pcbp->state_str(); // DEBUG
// Perform AIO return callback
if (_cbp && tot_data_toks)
@@ -762,10 +835,10 @@ wmgr::get_events(page_state state, timespec* const timeout, bool flush)
}
else // File header writes have no pcb
{
+//std::cout << "f"; // DEBUG
file_hdr_t* fhp = (file_hdr_t*)aiocbp->u.c.buf;
- _lfc.addWriteCompletedDblkCount(fhp->_file_number, QLS_JRNL_FHDR_RES_SIZE_SBLKS * JRNL_SBLK_SIZE_DBLKS);
+ _lfc.addWriteCompletedDblkCount(fhp->_file_number, QLS_JRNL_FHDR_RES_SIZE_SBLKS * QLS_SBLK_SIZE_DBLKS);
_lfc.decrOutstandingAioOperationCount(fhp->_file_number);
- //fcntlp->set_wr_fhdr_aio_outstanding(false); // TODO: Do we need this?
}
}
@@ -784,7 +857,9 @@ wmgr::is_txn_synced(const std::string& xid)
}
void
-wmgr::initialize(aio_callback* const cbp, const uint32_t wcache_pgsize_sblks, const uint16_t wcache_num_pages)
+wmgr::initialize(aio_callback* const cbp,
+ const uint32_t wcache_pgsize_sblks,
+ const uint16_t wcache_num_pages)
{
pmgr::initialize(cbp, wcache_pgsize_sblks, wcache_num_pages);
@@ -796,9 +871,11 @@ wmgr::initialize(aio_callback* const cbp, const uint32_t wcache_pgsize_sblks, co
}
iores
-wmgr::pre_write_check(const _op_type op, const data_tok* const dtokp,
- const std::size_t /*xidsize*/, const std::size_t /*dsize*/, const bool /*external*/
- ) const
+wmgr::pre_write_check(const _op_type op,
+ const data_tok* const dtokp,
+ const std::size_t /*xidsize*/,
+ const std::size_t /*dsize*/,
+ const bool /*external*/) const
{
// Check status of current file
// TODO: Replace for LFC
@@ -861,11 +938,12 @@ wmgr::pre_write_check(const _op_type op, const data_tok* const dtokp,
}
void
-wmgr::dequeue_check(const std::string& xid, const uint64_t drid)
+wmgr::dequeue_check(const std::string& xid,
+ const uint64_t drid)
{
// First check emap
bool found = false;
- int16_t fid;
+ uint64_t fid;
short eres = _emap.get_pfid(drid, fid);
if (eres < enq_map::EMAP_OK) { // fail
if (eres == enq_map::EMAP_RID_NOT_FOUND) {
@@ -891,13 +969,13 @@ void
wmgr::dblk_roundup()
{
const uint32_t xmagic = QLS_EMPTY_MAGIC;
- uint32_t wdblks = jrec::size_blks(_cached_offset_dblks, JRNL_SBLK_SIZE_DBLKS) * JRNL_SBLK_SIZE_DBLKS;
+ uint32_t wdblks = jrec::size_blks(_cached_offset_dblks, QLS_SBLK_SIZE_DBLKS) * QLS_SBLK_SIZE_DBLKS;
while (_cached_offset_dblks < wdblks)
{
- void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE_BYTES);
+ void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * QLS_DBLK_SIZE_BYTES);
std::memcpy(wptr, (const void*)&xmagic, sizeof(xmagic));
-#ifdef RHM_CLEAN
- std::memset((char*)wptr + sizeof(xmagic), RHM_CLEAN_CHAR, JRNL_DBLK_SIZE_BYTES - sizeof(xmagic));
+#ifdef QLS_CLEAN
+ std::memset((char*)wptr + sizeof(xmagic), QLS_CLEAN_CHAR, QLS_DBLK_SIZE_BYTES - sizeof(xmagic));
#endif
_pg_offset_dblks++;
_cached_offset_dblks++;
@@ -907,14 +985,15 @@ wmgr::dblk_roundup()
void
wmgr::rotate_page()
{
- _page_cb_arr[_pg_index]._state = AIO_PENDING;
- if (_pg_offset_dblks >= _cache_pgsize_sblks * JRNL_SBLK_SIZE_DBLKS)
+//std::cout << "^^^^^ wmgr::rotate_page() " << status_str() << " pi=" << _pg_index; // DEBUG
+ if (_pg_offset_dblks >= _cache_pgsize_sblks * QLS_SBLK_SIZE_DBLKS)
{
_pg_offset_dblks = 0;
_pg_cntr++;
}
if (++_pg_index >= _cache_num_pages)
_pg_index = 0;
+//std::cout << "->" << _pg_index << std::endl; // DEBUG
}
void
@@ -928,7 +1007,7 @@ wmgr::status_str() const
std::ostringstream oss;
oss << "wmgr: pi=" << _pg_index << " pc=" << _pg_cntr;
oss << " po=" << _pg_offset_dblks << " aer=" << _aio_evt_rem;
- oss << " edac:" << (_enq_busy?"T":"F") << (_deq_busy?"T":"F");
+ oss << " edac=" << (_enq_busy?"T":"F") << (_deq_busy?"T":"F");
oss << (_abort_busy?"T":"F") << (_commit_busy?"T":"F");
oss << " ps=[";
for (int i=0; i<_cache_num_pages; i++)
@@ -938,11 +1017,10 @@ wmgr::status_str() const
case UNUSED: oss << "-"; break;
case IN_USE: oss << "U"; break;
case AIO_PENDING: oss << "A"; break;
- case AIO_COMPLETE: oss << "*"; break;
default: oss << _page_cb_arr[i]._state;
}
}
- oss << "] " << _lfc.status(0);
+ oss << "] ";
return oss.str();
}
diff --git a/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.h b/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.h
index e859f17063..6b455febd9 100644
--- a/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.h
+++ b/qpid/cpp/src/qpid/linearstore/jrnl/wmgr.h
@@ -84,22 +84,45 @@ namespace qls_jrnl
std::set<std::string> _txn_pending_set; ///< Set containing xids of pending commits/aborts
public:
- wmgr(jcntl* jc, enq_map& emap, txn_map& tmap, LinearFileController& lfc);
- wmgr(jcntl* jc, enq_map& emap, txn_map& tmap, LinearFileController& lfc, const uint32_t max_dtokpp, const uint32_t max_iowait_us);
+ wmgr(jcntl* jc,
+ enq_map& emap,
+ txn_map& tmap,
+ LinearFileController& lfc);
+ wmgr(jcntl* jc,
+ enq_map& emap,
+ txn_map& tmap,
+ LinearFileController& lfc,
+ const uint32_t max_dtokpp,
+ const uint32_t max_iowait_us);
virtual ~wmgr();
- void initialize(aio_callback* const cbp, const uint32_t wcache_pgsize_sblks,
- const uint16_t wcache_num_pages, const uint32_t max_dtokpp,
- const uint32_t max_iowait_us, std::size_t eo = 0);
- iores enqueue(const void* const data_buff, const std::size_t tot_data_len,
- const std::size_t this_data_len, data_tok* dtokp, const void* const xid_ptr,
- const std::size_t xid_len, const bool transient, const bool external);
- iores dequeue(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_len,
- const bool txn_coml_commit);
- iores abort(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_len);
- iores commit(data_tok* dtokp, const void* const xid_ptr, const std::size_t xid_len);
+ void initialize(aio_callback* const cbp,
+ const uint32_t wcache_pgsize_sblks,
+ const uint16_t wcache_num_pages,
+ const uint32_t max_dtokpp,
+ const uint32_t max_iowait_us,
+ std::size_t eo = 0);
+ iores enqueue(const void* const data_buff,
+ const std::size_t tot_data_len,
+ const std::size_t this_data_len,
+ data_tok* dtokp,
+ const void* const xid_ptr,
+ const std::size_t xid_len,
+ const bool transient,
+ const bool external);
+ iores dequeue(data_tok* dtokp,
+ const void* const xid_ptr,
+ const std::size_t xid_len,
+ const bool txn_coml_commit);
+ iores abort(data_tok* dtokp,
+ const void* const xid_ptr,
+ const std::size_t xid_len);
+ iores commit(data_tok* dtokp,
+ const void* const xid_ptr,
+ const std::size_t xid_len);
iores flush();
- int32_t get_events(page_state state, timespec* const timeout, bool flush = false);
+ int32_t get_events(timespec* const timeout,
+ bool flush);
bool is_txn_synced(const std::string& xid);
inline bool curr_pg_blocked() const { return _page_cb_arr[_pg_index]._state != UNUSED; }
inline uint32_t unflushed_dblks() { return _cached_offset_dblks; }
@@ -108,14 +131,22 @@ namespace qls_jrnl
const std::string status_str() const;
private:
- void initialize(aio_callback* const cbp, const uint32_t wcache_pgsize_sblks,
- const uint16_t wcache_num_pages);
- iores pre_write_check(const _op_type op, const data_tok* const dtokp,
- const std::size_t xidsize = 0, const std::size_t dsize = 0, const bool external = false)
- const;
- void dequeue_check(const std::string& xid, const uint64_t drid);
- void file_header_check(const uint64_t rid, const bool cont, const uint32_t rec_dblks_rem);
- void flush_check(iores& res, bool& cont, bool& done);
+ void initialize(aio_callback* const cbp,
+ const uint32_t wcache_pgsize_sblks,
+ const uint16_t wcache_num_pages);
+ iores pre_write_check(const _op_type op,
+ const data_tok* const dtokp,
+ const std::size_t xidsize = 0,
+ const std::size_t dsize = 0,
+ const bool external = false) const;
+ void dequeue_check(const std::string& xid,
+ const uint64_t drid);
+ void file_header_check(const uint64_t rid,
+ const bool cont,
+ const uint32_t rec_dblks_rem);
+ void flush_check(iores& res,
+ bool& cont,
+ bool& done, const uint64_t rid);
iores write_flush();
void get_next_file();
void dblk_roundup();