diff options
author | Gordon Sim <gsim@apache.org> | 2012-08-10 12:04:27 +0000 |
---|---|---|
committer | Gordon Sim <gsim@apache.org> | 2012-08-10 12:04:27 +0000 |
commit | df36b35eb7ca20c3b354d6895004fb201346482b (patch) | |
tree | 357d90752f44304284639014f3b9db0cae1f2b2b | |
parent | 798cebf0e4f41953eb542d6358e5f0eea33d85a7 (diff) | |
download | qpid-python-df36b35eb7ca20c3b354d6895004fb201346482b.tar.gz |
QPID-4178: broker refactoring
git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1371676 13f79535-47bb-0310-9956-ffa450edef68
138 files changed, 4331 insertions, 7268 deletions
diff --git a/qpid/cpp/src/CMakeLists.txt b/qpid/cpp/src/CMakeLists.txt index 9ea7bd4024..681f095b09 100644 --- a/qpid/cpp/src/CMakeLists.txt +++ b/qpid/cpp/src/CMakeLists.txt @@ -1110,7 +1110,6 @@ set (qpidbroker_SOURCES qpid/broker/Exchange.cpp qpid/broker/ExpiryPolicy.cpp qpid/broker/Fairshare.cpp - qpid/broker/LegacyLVQ.cpp qpid/broker/MessageDeque.cpp qpid/broker/MessageMap.cpp qpid/broker/PriorityQueue.cpp @@ -1137,6 +1136,8 @@ set (qpidbroker_SOURCES qpid/broker/HeadersExchange.cpp qpid/broker/Link.cpp qpid/broker/LinkRegistry.cpp + qpid/broker/LossyQueue.cpp + qpid/broker/Lvq.cpp qpid/broker/Message.cpp qpid/broker/MessageAdapter.cpp qpid/broker/MessageBuilder.cpp @@ -1145,9 +1146,11 @@ set (qpidbroker_SOURCES qpid/broker/NullMessageStore.cpp qpid/broker/QueueBindings.cpp qpid/broker/QueuedMessage.cpp - qpid/broker/QueueEvents.cpp - qpid/broker/QueuePolicy.cpp + qpid/broker/QueueCursor.cpp + qpid/broker/QueueDepth.cpp + qpid/broker/QueueFactory.cpp qpid/broker/QueueRegistry.cpp + qpid/broker/QueueSettings.cpp qpid/broker/QueueFlowLimit.cpp qpid/broker/RecoveryManagerImpl.cpp qpid/broker/RecoveredEnqueue.cpp @@ -1170,8 +1173,8 @@ set (qpidbroker_SOURCES qpid/broker/TopicExchange.cpp qpid/broker/TxAccept.cpp qpid/broker/TxBuffer.cpp - qpid/broker/TxPublish.cpp qpid/broker/Vhost.cpp + qpid/broker/amqp_0_10/MessageTransfer.cpp qpid/management/ManagementAgent.cpp qpid/management/ManagementDirectExchange.cpp qpid/management/ManagementTopicExchange.cpp @@ -1419,45 +1422,6 @@ install (FILES ${qmfconsole_HEADERS} COMPONENT ${QPID_COMPONENT_QMF}) install_pdb (qmfconsole ${QPID_COMPONENT_QMF}) -# A queue event listener plugin that creates messages on a replication -# queue corresponding to enqueue and dequeue events: -set (replicating_listener_SOURCES - qpid/replication/constants.h - qpid/replication/ReplicatingEventListener.cpp - qpid/replication/ReplicatingEventListener.h - ) -add_msvc_version (replicating_listener library dll) -add_library (replicating_listener MODULE ${replicating_listener_SOURCES}) -target_link_libraries (replicating_listener qpidbroker ${Boost_PROGRAM_OPTIONS_LIBRARY}) -set_target_properties (replicating_listener PROPERTIES PREFIX "") -if (CMAKE_COMPILER_IS_GNUCXX) - set_target_properties(replicating_listener PROPERTIES - LINK_FLAGS "${GCC_CATCH_UNDEFINED}") -endif (CMAKE_COMPILER_IS_GNUCXX) -install (TARGETS replicating_listener - DESTINATION ${QPIDD_MODULE_DIR} - COMPONENT ${QPID_COMPONENT_BROKER}) - -# A custom exchange plugin that allows an exchange to be created that -# can process the messages from a replication queue (populated on the -# source system by the replicating listener plugin above) and take the -# corresponding action on the local queues -set (replication_exchange_SOURCES - qpid/replication/constants.h - qpid/replication/ReplicationExchange.cpp - qpid/replication/ReplicationExchange.h - ) -add_msvc_version (replication_exchange library dll) -add_library (replication_exchange MODULE ${replication_exchange_SOURCES}) -target_link_libraries (replication_exchange qpidbroker) -set_target_properties (replication_exchange PROPERTIES PREFIX "") -if (CMAKE_COMPILER_IS_GNUCXX) - set_target_properties(replication_exchange PROPERTIES - LINK_FLAGS "${GCC_CATCH_UNDEFINED}") -endif (CMAKE_COMPILER_IS_GNUCXX) -install (TARGETS replication_exchange - DESTINATION ${QPIDD_MODULE_DIR} - COMPONENT ${QPID_COMPONENT_BROKER}) # This is only really needed until all the trunk builds (Linux, UNIX, Windows) # are all on cmake only. This is because cmake builds always have a config.h diff --git a/qpid/cpp/src/Makefile.am b/qpid/cpp/src/Makefile.am index cbfeffd3ee..350351ab9c 100644 --- a/qpid/cpp/src/Makefile.am +++ b/qpid/cpp/src/Makefile.am @@ -223,7 +223,6 @@ include qmfc.mk if HAVE_XML include xml.mk endif -include replication.mk if RDMA @@ -334,6 +333,7 @@ libqpidcommon_la_LIBADD = \ -lboost_program_options \ -lboost_filesystem \ -luuid \ + -lpthread \ $(LIB_DLOPEN) \ $(LIB_CLOCK_GETTIME) @@ -553,7 +553,6 @@ libqpidbroker_la_SOURCES = \ qpid/broker/Deliverable.h \ qpid/broker/DeliverableMessage.cpp \ qpid/broker/DeliverableMessage.h \ - qpid/broker/DeliveryAdapter.h \ qpid/broker/DeliveryId.h \ qpid/broker/DeliveryRecord.cpp \ qpid/broker/DeliveryRecord.h \ @@ -584,12 +583,14 @@ libqpidbroker_la_SOURCES = \ qpid/broker/HeadersExchange.cpp \ qpid/broker/HeadersExchange.h \ qpid/broker/AsyncCompletion.h \ - qpid/broker/LegacyLVQ.h \ - qpid/broker/LegacyLVQ.cpp \ + qpid/broker/IndexedDeque.h \ qpid/broker/Link.cpp \ qpid/broker/Link.h \ qpid/broker/LinkRegistry.cpp \ qpid/broker/LinkRegistry.h \ + qpid/broker/Lvq.h \ + qpid/broker/Lvq.cpp \ + qpid/broker/MapHandler.h \ qpid/broker/Message.cpp \ qpid/broker/Message.h \ qpid/broker/MessageAdapter.cpp \ @@ -624,19 +625,25 @@ libqpidbroker_la_SOURCES = \ qpid/broker/QueueBindings.h \ qpid/broker/QueueCleaner.cpp \ qpid/broker/QueueCleaner.h \ - qpid/broker/QueueEvents.cpp \ - qpid/broker/QueueEvents.h \ + qpid/broker/QueueCursor.h \ + qpid/broker/QueueCursor.cpp \ + qpid/broker/QueueDepth.h \ + qpid/broker/QueueDepth.cpp \ + qpid/broker/QueueFactory.h \ + qpid/broker/QueueFactory.cpp \ + qpid/broker/QueueSettings.h \ + qpid/broker/QueueSettings.cpp \ qpid/broker/QueueListeners.cpp \ qpid/broker/QueueListeners.h \ qpid/broker/QueueObserver.h \ - qpid/broker/QueuePolicy.cpp \ - qpid/broker/QueuePolicy.h \ qpid/broker/QueueRegistry.cpp \ qpid/broker/QueueRegistry.h \ qpid/broker/QueuedMessage.cpp \ qpid/broker/QueuedMessage.h \ qpid/broker/QueueFlowLimit.h \ qpid/broker/QueueFlowLimit.cpp \ + qpid/broker/LossyQueue.h \ + qpid/broker/LossyQueue.cpp \ qpid/broker/RecoverableConfig.h \ qpid/broker/RecoverableExchange.h \ qpid/broker/RecoverableMessage.h \ @@ -687,9 +694,6 @@ libqpidbroker_la_SOURCES = \ qpid/broker/TxBuffer.cpp \ qpid/broker/TxBuffer.h \ qpid/broker/TxOp.h \ - qpid/broker/TxOpVisitor.h \ - qpid/broker/TxPublish.cpp \ - qpid/broker/TxPublish.h \ qpid/broker/Vhost.cpp \ qpid/broker/Vhost.h \ qpid/broker/MessageDistributor.h \ @@ -697,6 +701,8 @@ libqpidbroker_la_SOURCES = \ qpid/broker/FifoDistributor.cpp \ qpid/broker/MessageGroupManager.cpp \ qpid/broker/MessageGroupManager.h \ + qpid/broker/amqp_0_10/MessageTransfer.h \ + qpid/broker/amqp_0_10/MessageTransfer.cpp \ qpid/management/ManagementAgent.cpp \ qpid/management/ManagementAgent.h \ qpid/management/ManagementDirectExchange.cpp \ diff --git a/qpid/cpp/src/qpid/broker/AsyncCompletion.h b/qpid/cpp/src/qpid/broker/AsyncCompletion.h index fef994438f..0cf2856584 100644 --- a/qpid/cpp/src/qpid/broker/AsyncCompletion.h +++ b/qpid/cpp/src/qpid/broker/AsyncCompletion.h @@ -22,6 +22,7 @@ * */ +#include "qpid/RefCounted.h" #include <boost/intrusive_ptr.hpp> #include "qpid/broker/BrokerImportExport.h" @@ -77,7 +78,7 @@ namespace broker { * assuming no need for synchronization with Completer threads. */ -class AsyncCompletion +class AsyncCompletion : public virtual RefCounted { public: diff --git a/qpid/cpp/src/qpid/broker/Broker.cpp b/qpid/cpp/src/qpid/broker/Broker.cpp index fba5299169..02a6886f90 100644 --- a/qpid/cpp/src/qpid/broker/Broker.cpp +++ b/qpid/cpp/src/qpid/broker/Broker.cpp @@ -33,6 +33,7 @@ #include "qpid/broker/Link.h" #include "qpid/broker/ExpiryPolicy.h" #include "qpid/broker/QueueFlowLimit.h" +#include "qpid/broker/QueueSettings.h" #include "qpid/broker/MessageGroupManager.h" #include "qmf/org/apache/qpid/broker/Package.h" @@ -120,7 +121,6 @@ Broker::Options::Options(const std::string& name) : queueLimit(100*1048576/*100M default limit*/), tcpNoDelay(false), requireEncrypted(false), - asyncQueueEvents(false), // Must be false in a cluster. qmf2Support(true), qmf1Support(true), queueFlowStopRatio(80), @@ -164,7 +164,6 @@ Broker::Options::Options(const std::string& name) : ("require-encryption", optValue(requireEncrypted), "Only accept connections that are encrypted") ("known-hosts-url", optValue(knownHosts, "URL or 'none'"), "URL to send as 'known-hosts' to clients ('none' implies empty list)") ("sasl-config", optValue(saslConfigPath, "DIR"), "gets sasl config info from nonstandard location") - ("async-queue-events", optValue(asyncQueueEvents, "yes|no"), "Set Queue Events async, used for services like replication") ("default-flow-stop-threshold", optValue(queueFlowStopRatio, "PERCENT"), "Percent of queue's maximum capacity at which flow control is activated.") ("default-flow-resume-threshold", optValue(queueFlowResumeRatio, "PERCENT"), "Percent of queue's maximum capacity at which flow control is de-activated.") ("default-event-threshold-ratio", optValue(queueThresholdEventRatio, "%age of limit"), "The ratio of any specified queue limit at which an event will be raised") @@ -206,7 +205,6 @@ Broker::Broker(const Broker::Options& conf) : *this), mgmtObject(0), queueCleaner(queues, &timer), - queueEvents(poller,!conf.asyncQueueEvents), recovery(true), inCluster(false), clusterUpdatee(false), @@ -265,8 +263,6 @@ Broker::Broker(const Broker::Options& conf) : federationTag = conf.fedTag; } - QueuePolicy::setDefaultMaxSize(conf.queueLimit); - // Early-Initialize plugins Plugin::earlyInitAll(*this); @@ -425,7 +421,6 @@ void Broker::shutdown() { Broker::~Broker() { shutdown(); - queueEvents.shutdown(); finalize(); // Finalize any plugins. if (config.auth) SaslAuthenticator::fini(); @@ -689,11 +684,15 @@ void Broker::createObject(const std::string& type, const std::string& name, //treat everything else as extension properties else extensions[i->first] = i->second; } - framing::FieldTable arguments; - amqp_0_10::translate(extensions, arguments); + QueueSettings settings(durable, autodelete); + Variant::Map unused; + settings.populate(extensions, unused); + qpid::amqp_0_10::translate(unused, settings.storeSettings); + //TODO: unused doesn't take store settings into account... so can't yet implement strict + QPID_LOG(debug, "Broker did not use the following settings (store module may): " << unused); std::pair<boost::shared_ptr<Queue>, bool> result = - createQueue(name, durable, autodelete, 0, alternateExchange, arguments, userId, connectionId); + createQueue(name, settings, 0, alternateExchange, userId, connectionId); if (!result.second) { throw ObjectAlreadyExists(name); } @@ -1041,8 +1040,7 @@ Broker::getKnownBrokersImpl() return knownBrokers; } -bool Broker::deferDeliveryImpl(const std::string& , - const boost::intrusive_ptr<Message>& ) +bool Broker::deferDeliveryImpl(const std::string&, const Message&) { return false; } void Broker::setClusterTimer(std::auto_ptr<sys::Timer> t) { @@ -1056,23 +1054,21 @@ const std::string Broker::TCP_TRANSPORT("tcp"); std::pair<boost::shared_ptr<Queue>, bool> Broker::createQueue( const std::string& name, - bool durable, - bool autodelete, + const QueueSettings& settings, const OwnershipToken* owner, const std::string& alternateExchange, - const qpid::framing::FieldTable& arguments, const std::string& userId, const std::string& connectionId) { if (acl) { std::map<acl::Property, std::string> params; params.insert(make_pair(acl::PROP_ALTERNATE, alternateExchange)); - params.insert(make_pair(acl::PROP_DURABLE, durable ? _TRUE : _FALSE)); + params.insert(make_pair(acl::PROP_DURABLE, settings.durable ? _TRUE : _FALSE)); params.insert(make_pair(acl::PROP_EXCLUSIVE, owner ? _TRUE : _FALSE)); - params.insert(make_pair(acl::PROP_AUTODELETE, autodelete ? _TRUE : _FALSE)); - params.insert(make_pair(acl::PROP_POLICYTYPE, arguments.getAsString("qpid.policy_type"))); - params.insert(make_pair(acl::PROP_MAXQUEUECOUNT, boost::lexical_cast<string>(arguments.getAsInt("qpid.max_count")))); - params.insert(make_pair(acl::PROP_MAXQUEUESIZE, boost::lexical_cast<string>(arguments.getAsInt64("qpid.max_size")))); + params.insert(make_pair(acl::PROP_AUTODELETE, settings.autodelete ? _TRUE : _FALSE)); + params.insert(make_pair(acl::PROP_POLICYTYPE, settings.dropMessagesAtLimit ? "ring" : "reject")); + params.insert(make_pair(acl::PROP_MAXQUEUECOUNT, boost::lexical_cast<string>(settings.maxDepth.getCount()))); + params.insert(make_pair(acl::PROP_MAXQUEUESIZE, boost::lexical_cast<string>(settings.maxDepth.getSize()))); if (!acl->authorise(userId,acl::ACT_CREATE,acl::OBJ_QUEUE,name,¶ms) ) throw framing::UnauthorizedAccessException(QPID_MSG("ACL denied queue create request from " << userId)); @@ -1084,7 +1080,7 @@ std::pair<boost::shared_ptr<Queue>, bool> Broker::createQueue( if (!alternate) throw framing::NotFoundException(QPID_MSG("Alternate exchange does not exist: " << alternateExchange)); } - std::pair<Queue::shared_ptr, bool> result = queues.declare(name, durable, autodelete, owner, alternate, arguments); + std::pair<Queue::shared_ptr, bool> result = queues.declare(name, settings, alternate); if (result.second) { //add default binding: result.first->bind(exchanges.getDefault(), name); @@ -1095,16 +1091,16 @@ std::pair<boost::shared_ptr<Queue>, bool> Broker::createQueue( //event instead? managementAgent->raiseEvent( _qmf::EventQueueDeclare(connectionId, userId, name, - durable, owner, autodelete, alternateExchange, - ManagementAgent::toMap(arguments), + settings.durable, owner, settings.autodelete, alternateExchange, + settings.asMap(), "created")); } QPID_LOG_CAT(debug, model, "Create queue. name:" << name << " user:" << userId << " rhost:" << connectionId - << " durable:" << (durable ? "T" : "F") + << " durable:" << (settings.durable ? "T" : "F") << " owner:" << owner - << " autodelete:" << (autodelete ? "T" : "F") + << " autodelete:" << (settings.autodelete ? "T" : "F") << " alternateExchange:" << alternateExchange ); } return result; diff --git a/qpid/cpp/src/qpid/broker/Broker.h b/qpid/cpp/src/qpid/broker/Broker.h index c385a3ec56..823ed54ddb 100644 --- a/qpid/cpp/src/qpid/broker/Broker.h +++ b/qpid/cpp/src/qpid/broker/Broker.h @@ -33,7 +33,6 @@ #include "qpid/broker/LinkRegistry.h" #include "qpid/broker/SessionManager.h" #include "qpid/broker/QueueCleaner.h" -#include "qpid/broker/QueueEvents.h" #include "qpid/broker/Vhost.h" #include "qpid/broker/System.h" #include "qpid/broker/ExpiryPolicy.h" @@ -75,7 +74,7 @@ namespace broker { class ConnectionState; class ExpiryPolicy; class Message; - +struct QueueSettings; static const uint16_t DEFAULT_PORT=5672; struct NoSuchTransportException : qpid::Exception @@ -117,7 +116,6 @@ class Broker : public sys::Runnable, public Plugin::Target, bool requireEncrypted; std::string knownHosts; std::string saslConfigPath; - bool asyncQueueEvents; bool qmf2Support; bool qmf1Support; uint queueFlowStopRatio; // producer flow control: on @@ -177,11 +175,10 @@ class Broker : public sys::Runnable, public Plugin::Target, Vhost::shared_ptr vhostObject; System::shared_ptr systemObject; QueueCleaner queueCleaner; - QueueEvents queueEvents; std::vector<Url> knownBrokers; std::vector<Url> getKnownBrokersImpl(); bool deferDeliveryImpl(const std::string& queue, - const boost::intrusive_ptr<Message>& msg); + const Message& msg); std::string federationTag; bool recovery; bool inCluster, clusterUpdatee; @@ -225,7 +222,6 @@ class Broker : public sys::Runnable, public Plugin::Target, DtxManager& getDtxManager() { return dtxManager; } DataDir& getDataDir() { return dataDir; } Options& getOptions() { return config; } - QueueEvents& getQueueEvents() { return queueEvents; } void setExpiryPolicy(const boost::intrusive_ptr<ExpiryPolicy>& e) { expiryPolicy = e; } boost::intrusive_ptr<ExpiryPolicy> getExpiryPolicy() { return expiryPolicy; } @@ -307,7 +303,8 @@ class Broker : public sys::Runnable, public Plugin::Target, * context. *@return true if delivery of a message should be deferred. */ - boost::function<bool (const std::string& queue, const boost::intrusive_ptr<Message>& msg)> deferDelivery; + boost::function<bool (const std::string& queue, + const Message& msg)> deferDelivery; bool isAuthenticating ( ) { return config.auth; } bool isTimestamping() { return config.timestampRcvMsgs; } @@ -316,11 +313,9 @@ class Broker : public sys::Runnable, public Plugin::Target, QPID_BROKER_EXTERN std::pair<boost::shared_ptr<Queue>, bool> createQueue( const std::string& name, - bool durable, - bool autodelete, + const QueueSettings& settings, const OwnershipToken* owner, const std::string& alternateExchange, - const qpid::framing::FieldTable& arguments, const std::string& userId, const std::string& connectionId); diff --git a/qpid/cpp/src/qpid/broker/Consumer.h b/qpid/cpp/src/qpid/broker/Consumer.h index 64073621be..64fc4288af 100644 --- a/qpid/cpp/src/qpid/broker/Consumer.h +++ b/qpid/cpp/src/qpid/broker/Consumer.h @@ -21,21 +21,23 @@ #ifndef _Consumer_ #define _Consumer_ -#include "qpid/broker/Message.h" -#include "qpid/broker/QueuedMessage.h" +#include "qpid/broker/QueueCursor.h" #include "qpid/broker/OwnershipToken.h" +#include <boost/shared_ptr.hpp> +#include <string> namespace qpid { namespace broker { +class DeliveryRecord; +class Message; class Queue; class QueueListeners; /** * Base class for consumers which represent a subscription to a queue. */ -class Consumer -{ +class Consumer : public QueueCursor { const bool acquires; // inListeners allows QueueListeners to efficiently track if this // instance is registered for notifications without having to @@ -47,22 +49,17 @@ class Consumer public: typedef boost::shared_ptr<Consumer> shared_ptr; - Consumer(const std::string& _name, bool preAcquires = true) - : acquires(preAcquires), inListeners(false), name(_name), position(0) {} + Consumer(const std::string& _name, SubscriptionType type) + : QueueCursor(type), acquires(type == CONSUMER), inListeners(false), name(_name) {} virtual ~Consumer(){} bool preAcquires() const { return acquires; } const std::string& getName() const { return name; } - /**@return the position of the last message seen by this consumer */ - virtual framing::SequenceNumber getPosition() const { return position; } - - virtual void setPosition(framing::SequenceNumber pos) { position = pos; } - - virtual bool deliver(QueuedMessage& msg) = 0; + virtual bool deliver(const QueueCursor& cursor, const Message& msg) = 0; virtual void notify() = 0; - virtual bool filter(boost::intrusive_ptr<Message>) { return true; } - virtual bool accept(boost::intrusive_ptr<Message>) { return true; } + virtual bool filter(const Message&) { return true; } + virtual bool accept(const Message&) { return true; } virtual OwnershipToken* getSession() = 0; virtual void cancel() = 0; @@ -75,7 +72,7 @@ class Consumer * Not to be confused with accept() above, which is asking if * this consumer will consume/browse the message. */ - virtual void acknowledged(const QueuedMessage&) = 0; + virtual void acknowledged(const DeliveryRecord&) = 0; /** Called if queue has been deleted, if true suppress the error message. * Used by HA ReplicatingSubscriptions where such errors are normal. @@ -83,7 +80,7 @@ class Consumer virtual bool hideDeletedError() { return false; } protected: - framing::SequenceNumber position; + //framing::SequenceNumber position; private: friend class QueueListeners; diff --git a/qpid/cpp/src/qpid/broker/Deliverable.h b/qpid/cpp/src/qpid/broker/Deliverable.h index ffb5a77bca..e08d0e1b20 100644 --- a/qpid/cpp/src/qpid/broker/Deliverable.h +++ b/qpid/cpp/src/qpid/broker/Deliverable.h @@ -21,17 +21,22 @@ #ifndef _Deliverable_ #define _Deliverable_ -#include "qpid/broker/Message.h" +#include "qpid/broker/AsyncCompletion.h" +#include "qpid/sys/IntegerTypes.h" +#include <boost/shared_ptr.hpp> namespace qpid { namespace broker { - class Deliverable{ + class Message; + class Queue; + + class Deliverable : public AsyncCompletion { public: bool delivered; Deliverable() : delivered(false) {} virtual Message& getMessage() = 0; - + virtual void deliverTo(const boost::shared_ptr<Queue>& queue) = 0; virtual uint64_t contentSize() { return 0; } virtual ~Deliverable(){} diff --git a/qpid/cpp/src/qpid/broker/DeliverableMessage.cpp b/qpid/cpp/src/qpid/broker/DeliverableMessage.cpp index 3ebb12461c..be4b7f0796 100644 --- a/qpid/cpp/src/qpid/broker/DeliverableMessage.cpp +++ b/qpid/cpp/src/qpid/broker/DeliverableMessage.cpp @@ -24,22 +24,20 @@ using namespace qpid::broker; -DeliverableMessage::DeliverableMessage(const boost::intrusive_ptr<Message>& _msg) : msg(_msg) -{ -} +DeliverableMessage::DeliverableMessage(const Message& _msg, TxBuffer* _txn) : msg(_msg), txn(_txn) {} void DeliverableMessage::deliverTo(const boost::shared_ptr<Queue>& queue) { - queue->deliver(msg); + queue->deliver(msg, txn); delivered = true; } Message& DeliverableMessage::getMessage() { - return *msg; + return msg; } -uint64_t DeliverableMessage::contentSize () +uint64_t DeliverableMessage::contentSize() { - return msg->contentSize (); + return msg.getContentSize(); } diff --git a/qpid/cpp/src/qpid/broker/DeliverableMessage.h b/qpid/cpp/src/qpid/broker/DeliverableMessage.h index c8d21001eb..d6d6bf5265 100644 --- a/qpid/cpp/src/qpid/broker/DeliverableMessage.h +++ b/qpid/cpp/src/qpid/broker/DeliverableMessage.h @@ -25,14 +25,15 @@ #include "qpid/broker/Deliverable.h" #include "qpid/broker/Message.h" -#include <boost/intrusive_ptr.hpp> - namespace qpid { namespace broker { - class QPID_BROKER_CLASS_EXTERN DeliverableMessage : public Deliverable{ - boost::intrusive_ptr<Message> msg; + class TxBuffer; + class QPID_BROKER_CLASS_EXTERN DeliverableMessage : public Deliverable + { + Message msg; + TxBuffer* txn; public: - QPID_BROKER_EXTERN DeliverableMessage(const boost::intrusive_ptr<Message>& msg); + QPID_BROKER_EXTERN DeliverableMessage(const Message& msg, TxBuffer* txn); QPID_BROKER_EXTERN virtual void deliverTo(const boost::shared_ptr<Queue>& queue); QPID_BROKER_EXTERN Message& getMessage(); QPID_BROKER_EXTERN uint64_t contentSize(); diff --git a/qpid/cpp/src/qpid/broker/DeliveryAdapter.h b/qpid/cpp/src/qpid/broker/DeliveryAdapter.h index b0bec60890..e69de29bb2 100644 --- a/qpid/cpp/src/qpid/broker/DeliveryAdapter.h +++ b/qpid/cpp/src/qpid/broker/DeliveryAdapter.h @@ -1,53 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#ifndef _DeliveryAdapter_ -#define _DeliveryAdapter_ - -#include "qpid/broker/DeliveryId.h" -#include "qpid/broker/Message.h" -#include "qpid/framing/amqp_types.h" - -namespace qpid { -namespace broker { - -class DeliveryRecord; - -/** - * The intention behind this interface is to separate the generic - * handling of some form of message delivery to clients that is - * contained in the version independent Channel class from the - * details required for a particular situation or - * version. i.e. where the existing adapters allow (through - * supporting the generated interface for a version of the - * protocol) inputs of a channel to be adapted to the version - * independent part, this does the same for the outputs. - */ -class DeliveryAdapter -{ - public: - virtual void deliver(DeliveryRecord&, bool sync) = 0; - virtual ~DeliveryAdapter(){} -}; - -}} - - -#endif diff --git a/qpid/cpp/src/qpid/broker/DeliveryRecord.cpp b/qpid/cpp/src/qpid/broker/DeliveryRecord.cpp index 5d6aee9045..f547ee54c9 100644 --- a/qpid/cpp/src/qpid/broker/DeliveryRecord.cpp +++ b/qpid/cpp/src/qpid/broker/DeliveryRecord.cpp @@ -24,6 +24,7 @@ #include "qpid/broker/Consumer.h" #include "qpid/broker/Exchange.h" #include "qpid/broker/Queue.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/log/Statement.h" #include "qpid/framing/FrameHandler.h" #include "qpid/framing/MessageTransferBody.h" @@ -32,77 +33,46 @@ using namespace qpid; using namespace qpid::broker; using std::string; -DeliveryRecord::DeliveryRecord(const QueuedMessage& _msg, +DeliveryRecord::DeliveryRecord(const QueueCursor& _msg, + framing::SequenceNumber _msgId, const Queue::shared_ptr& _queue, const std::string& _tag, const boost::shared_ptr<Consumer>& _consumer, bool _acquired, bool accepted, bool _windowing, - uint32_t _credit): - msg(_msg), - queue(_queue), - tag(_tag), - consumer(_consumer), - acquired(_acquired), - acceptExpected(!accepted), - cancelled(false), - completed(false), - ended(accepted && acquired), - windowing(_windowing), - credit(msg.payload ? msg.payload->getRequiredCredit() : _credit) + uint32_t _credit) : msg(_msg), + queue(_queue), + tag(_tag), + consumer(_consumer), + acquired(_acquired), + acceptExpected(!accepted), + cancelled(false), + completed(false), + ended(accepted && acquired), + windowing(_windowing), + credit(_credit), + msgId(_msgId) {} bool DeliveryRecord::setEnded() { ended = true; - //reset msg pointer, don't need to hold on to it anymore - msg.payload = boost::intrusive_ptr<Message>(); QPID_LOG(debug, "DeliveryRecord::setEnded() id=" << id); return isRedundant(); } -void DeliveryRecord::redeliver(SemanticState* const session) { - if (!ended) { - if(cancelled){ - //if subscription was cancelled, requeue it (waiting for - //final confirmation for AMQP WG on this case) - requeue(); - }else{ - msg.payload->redeliver();//mark as redelivered - session->deliver(*this, false); - } - } -} - -void DeliveryRecord::deliver(framing::FrameHandler& h, DeliveryId deliveryId, uint16_t framesize) -{ - id = deliveryId; - if (msg.payload->getRedelivered()){ - msg.payload->setRedelivered(); - } - msg.payload->adjustTtl(); - - framing::AMQFrame method((framing::MessageTransferBody(framing::ProtocolVersion(), tag, acceptExpected ? 0 : 1, acquired ? 0 : 1))); - method.setEof(false); - h.handle(method); - msg.payload->sendHeader(h, framesize); - msg.payload->sendContent(*queue, h, framesize); -} - -void DeliveryRecord::requeue() const +void DeliveryRecord::requeue() { if (acquired && !ended) { - msg.payload->redeliver(); - queue->requeue(msg); + queue->release(msg); } } void DeliveryRecord::release(bool setRedelivered) { if (acquired && !ended) { - if (setRedelivered) msg.payload->redeliver(); - queue->requeue(msg); + queue->release(msg, setRedelivered); acquired = false; setEnded(); } else { @@ -110,13 +80,14 @@ void DeliveryRecord::release(bool setRedelivered) } } -void DeliveryRecord::complete() { +void DeliveryRecord::complete() +{ completed = true; } bool DeliveryRecord::accept(TransactionContext* ctxt) { if (!ended) { - if (consumer) consumer->acknowledged(getMessage()); + if (consumer) consumer->acknowledged(*this); if (acquired) queue->dequeue(ctxt, msg); setEnded(); QPID_LOG(debug, "Accepted " << id); @@ -124,31 +95,22 @@ bool DeliveryRecord::accept(TransactionContext* ctxt) { return isRedundant(); } -void DeliveryRecord::dequeue(TransactionContext* ctxt) const{ +void DeliveryRecord::dequeue(TransactionContext* ctxt) const +{ if (acquired && !ended) { queue->dequeue(ctxt, msg); } } -void DeliveryRecord::committed() const{ +void DeliveryRecord::committed() const +{ queue->dequeueCommitted(msg); } void DeliveryRecord::reject() { if (acquired && !ended) { - Exchange::shared_ptr alternate = queue->getAlternateExchange(); - if (alternate) { - DeliverableMessage delivery(msg.payload); - alternate->routeWithAlternate(delivery); - QPID_LOG(info, "Routed rejected message from " << queue->getName() << " to " - << alternate->getName()); - } else { - //just drop it - QPID_LOG(info, "Dropping rejected message from " << queue->getName()); - } - queue->countRejected(); - dequeue(); + queue->reject(msg); setEnded(); } } diff --git a/qpid/cpp/src/qpid/broker/DeliveryRecord.h b/qpid/cpp/src/qpid/broker/DeliveryRecord.h index 21074d4274..10436f3fa0 100644 --- a/qpid/cpp/src/qpid/broker/DeliveryRecord.h +++ b/qpid/cpp/src/qpid/broker/DeliveryRecord.h @@ -26,15 +26,17 @@ #include <deque> #include <vector> #include <ostream> +#include "qpid/framing/FrameHandler.h" #include "qpid/framing/SequenceSet.h" #include "qpid/broker/BrokerImportExport.h" -#include "qpid/broker/QueuedMessage.h" +#include "qpid/broker/QueueCursor.h" #include "qpid/broker/DeliveryId.h" #include "qpid/broker/Message.h" namespace qpid { namespace broker { +class Queue; class TransactionContext; class SemanticState; struct AckRange; @@ -45,7 +47,7 @@ class Consumer; */ class DeliveryRecord { - QueuedMessage msg; + QueueCursor msg; mutable boost::shared_ptr<Queue> queue; std::string tag; // name of consumer boost::shared_ptr<Consumer> consumer; @@ -65,9 +67,10 @@ class DeliveryRecord * after that). */ uint32_t credit; + framing::SequenceNumber msgId; public: - QPID_BROKER_EXTERN DeliveryRecord(const QueuedMessage& msg, + QPID_BROKER_EXTERN DeliveryRecord(const QueueCursor& msgCursor, framing::SequenceNumber msgId, const boost::shared_ptr<Queue>& queue, const std::string& tag, const boost::shared_ptr<Consumer>& consumer, @@ -80,11 +83,10 @@ class DeliveryRecord bool coveredBy(const framing::SequenceSet* const range) const { return range->contains(id); } void dequeue(TransactionContext* ctxt = 0) const; - void requeue() const; + void requeue(); void release(bool setRedelivered); void reject(); void cancel(const std::string& tag); - void redeliver(SemanticState* const); void acquire(DeliveryIds& results); void complete(); bool accept(TransactionContext* ctxt); // Returns isRedundant() @@ -102,13 +104,13 @@ class DeliveryRecord uint32_t getCredit() const; const std::string& getTag() const { return tag; } - void deliver(framing::FrameHandler& h, DeliveryId deliveryId, uint16_t framesize); void setId(DeliveryId _id) { id = _id; } typedef std::deque<DeliveryRecord> DeliveryRecords; static AckRange findRange(DeliveryRecords& records, DeliveryId first, DeliveryId last); - const QueuedMessage& getMessage() const { return msg; } + const QueueCursor& getMessage() const { return msg; } framing::SequenceNumber getId() const { return id; } + framing::SequenceNumber getMessageId() const { return msgId; } boost::shared_ptr<Queue> getQueue() const { return queue; } friend std::ostream& operator<<(std::ostream&, const DeliveryRecord&); diff --git a/qpid/cpp/src/qpid/broker/DtxAck.h b/qpid/cpp/src/qpid/broker/DtxAck.h index 16c3ff8ba0..10d63f5b0c 100644 --- a/qpid/cpp/src/qpid/broker/DtxAck.h +++ b/qpid/cpp/src/qpid/broker/DtxAck.h @@ -40,7 +40,6 @@ class DtxAck : public TxOp{ virtual void commit() throw(); virtual void rollback() throw(); virtual ~DtxAck(){} - virtual void accept(TxOpConstVisitor& visitor) const { visitor(*this); } const DeliveryRecords& getPending() const { return pending; } }; diff --git a/qpid/cpp/src/qpid/broker/Exchange.cpp b/qpid/cpp/src/qpid/broker/Exchange.cpp index 82d4b4df15..bb5dc2b807 100644 --- a/qpid/cpp/src/qpid/broker/Exchange.cpp +++ b/qpid/cpp/src/qpid/broker/Exchange.cpp @@ -25,6 +25,7 @@ #include "qpid/broker/ExchangeRegistry.h" #include "qpid/broker/FedOps.h" #include "qpid/broker/Queue.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/framing/MessageProperties.h" #include "qpid/framing/reply_exceptions.h" #include "qpid/log/Statement.h" @@ -62,10 +63,10 @@ Exchange::PreRoute::PreRoute(Deliverable& msg, Exchange* _p):parent(_p) { if (parent->sequence){ parent->sequenceNo++; - msg.getMessage().insertCustomProperty(qpidMsgSequence,parent->sequenceNo); + msg.getMessage().addAnnotation(qpidMsgSequence,parent->sequenceNo); } if (parent->ive) { - parent->lastMsg = &( msg.getMessage()); + parent->lastMsg = msg.getMessage(); } } } @@ -111,12 +112,6 @@ void Exchange::doRoute(Deliverable& msg, ConstBindingList b) int count = 0; if (b.get()) { - // Block the content release if the message is transient AND there is more than one binding - if (!msg.getMessage().isPersistent() && b->size() > 1) { - msg.getMessage().blockContentRelease(); - } - - ExInfo error(getName()); // Save exception to throw at the end. for(std::vector<Binding::shared_ptr>::const_iterator i = b->begin(); i != b->end(); i++, count++) { try { @@ -161,8 +156,8 @@ void Exchange::doRoute(Deliverable& msg, ConstBindingList b) } void Exchange::routeIVE(){ - if (ive && lastMsg.get()){ - DeliverableMessage dmsg(lastMsg); + if (ive && lastMsg){ + DeliverableMessage dmsg(lastMsg, 0); route(dmsg); } } @@ -400,9 +395,9 @@ bool Exchange::MatchQueue::operator()(Exchange::Binding::shared_ptr b) return b->queue == queue; } -void Exchange::setProperties(const boost::intrusive_ptr<Message>& msg) { - msg->setExchange(getName()); -} +//void Exchange::setProperties(Message& msg) { +// qpid::broker::amqp_0_10::MessageTransfer::setExchange(msg, getName()); +//} bool Exchange::routeWithAlternate(Deliverable& msg) { diff --git a/qpid/cpp/src/qpid/broker/Exchange.h b/qpid/cpp/src/qpid/broker/Exchange.h index fba752210f..2b2f7db934 100644 --- a/qpid/cpp/src/qpid/broker/Exchange.h +++ b/qpid/cpp/src/qpid/broker/Exchange.h @@ -25,6 +25,7 @@ #include <boost/shared_ptr.hpp> #include "qpid/broker/BrokerImportExport.h" #include "qpid/broker/Deliverable.h" +#include "qpid/broker/Message.h" #include "qpid/broker/MessageStore.h" #include "qpid/broker/PersistableExchange.h" #include "qpid/framing/FieldTable.h" @@ -74,7 +75,7 @@ protected: mutable qpid::sys::Mutex sequenceLock; int64_t sequenceNo; bool ive; - boost::intrusive_ptr<Message> lastMsg; + Message lastMsg; class PreRoute{ public: @@ -196,7 +197,7 @@ public: virtual bool bind(boost::shared_ptr<Queue> queue, const std::string& routingKey, const qpid::framing::FieldTable* args) = 0; virtual bool unbind(boost::shared_ptr<Queue> queue, const std::string& routingKey, const qpid::framing::FieldTable* args) = 0; virtual bool isBound(boost::shared_ptr<Queue> queue, const std::string* const routingKey, const qpid::framing::FieldTable* const args) = 0; - QPID_BROKER_EXTERN virtual void setProperties(const boost::intrusive_ptr<Message>&); + //QPID_BROKER_EXTERN virtual void setProperties(Message&); virtual void route(Deliverable& msg) = 0; //PersistableExchange: diff --git a/qpid/cpp/src/qpid/broker/ExpiryPolicy.cpp b/qpid/cpp/src/qpid/broker/ExpiryPolicy.cpp index 62cb3fc116..687eac7817 100644 --- a/qpid/cpp/src/qpid/broker/ExpiryPolicy.cpp +++ b/qpid/cpp/src/qpid/broker/ExpiryPolicy.cpp @@ -27,7 +27,7 @@ namespace broker { ExpiryPolicy::~ExpiryPolicy() {} -bool ExpiryPolicy::hasExpired(Message& m) { +bool ExpiryPolicy::hasExpired(const Message& m) { return m.getExpiration() < sys::AbsTime::now(); } diff --git a/qpid/cpp/src/qpid/broker/ExpiryPolicy.h b/qpid/cpp/src/qpid/broker/ExpiryPolicy.h index 2caf00ce00..1fb41ccd29 100644 --- a/qpid/cpp/src/qpid/broker/ExpiryPolicy.h +++ b/qpid/cpp/src/qpid/broker/ExpiryPolicy.h @@ -42,7 +42,7 @@ class QPID_BROKER_CLASS_EXTERN ExpiryPolicy : public RefCounted { public: QPID_BROKER_EXTERN virtual ~ExpiryPolicy(); - QPID_BROKER_EXTERN virtual bool hasExpired(Message&); + QPID_BROKER_EXTERN virtual bool hasExpired(const Message&); QPID_BROKER_EXTERN virtual qpid::sys::AbsTime getCurrentTime(); }; }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/Fairshare.cpp b/qpid/cpp/src/qpid/broker/Fairshare.cpp index 7cdad1a44f..ec8ae9a037 100644 --- a/qpid/cpp/src/qpid/broker/Fairshare.cpp +++ b/qpid/cpp/src/qpid/broker/Fairshare.cpp @@ -19,7 +19,8 @@ * */ #include "qpid/broker/Fairshare.h" -#include "qpid/broker/QueuedMessage.h" +#include "qpid/broker/Message.h" +#include "qpid/broker/QueueSettings.h" #include "qpid/framing/FieldTable.h" #include "qpid/framing/FieldValue.h" #include "qpid/log/Statement.h" @@ -83,17 +84,6 @@ bool Fairshare::setState(uint p, uint c) return true; } -bool Fairshare::findFrontLevel(uint& p, PriorityLevels& messages) -{ - const uint start = p = currentLevel(); - do { - if (!messages[p].empty()) return true; - } while ((p = nextLevel()) != start); - return false; -} - - - bool Fairshare::getState(const Messages& m, uint& priority, uint& count) { const Fairshare* fairshare = dynamic_cast<const Fairshare*>(&m); @@ -106,82 +96,30 @@ bool Fairshare::setState(Messages& m, uint priority, uint count) return fairshare && fairshare->setState(priority, count); } -int getIntegerSetting(const qpid::framing::FieldTable& settings, const std::vector<std::string>& keys) -{ - qpid::framing::FieldTable::ValuePtr v; - std::vector<std::string>::const_iterator i = keys.begin(); - while (!v && i != keys.end()) { - v = settings.get(*i++); - } - - if (!v) { - return 0; - } else if (v->convertsTo<int>()) { - return v->get<int>(); - } else if (v->convertsTo<std::string>()){ - std::string s = v->get<std::string>(); - try { - return boost::lexical_cast<int>(s); - } catch(const boost::bad_lexical_cast&) { - QPID_LOG(warning, "Ignoring invalid integer value for " << *i << ": " << s); - return 0; - } - } else { - QPID_LOG(warning, "Ignoring invalid integer value for " << *i << ": " << *v); - return 0; - } -} - -int getIntegerSettingForKey(const qpid::framing::FieldTable& settings, const std::string& key) -{ - return getIntegerSetting(settings, boost::assign::list_of<std::string>(key)); -} - -int getSetting(const qpid::framing::FieldTable& settings, const std::vector<std::string>& keys, int minvalue, int maxvalue) +PriorityQueue::Priority Fairshare::firstLevel() { - return std::max(minvalue,std::min(getIntegerSetting(settings, keys), maxvalue)); + return Priority(currentLevel()); } -std::auto_ptr<Fairshare> getFairshareForKey(const qpid::framing::FieldTable& settings, uint levels, const std::string& key) +bool Fairshare::nextLevel(Priority& p) { - uint defaultLimit = getIntegerSettingForKey(settings, key); - std::auto_ptr<Fairshare> fairshare(new Fairshare(levels, defaultLimit)); - for (uint i = 0; i < levels; i++) { - std::string levelKey = (boost::format("%1%-%2%") % key % i).str(); - if(settings.isSet(levelKey)) { - fairshare->setLimit(i, getIntegerSettingForKey(settings, levelKey)); - } - } - if (!fairshare->isNull()) { - return fairshare; + int next = nextLevel(); + if (next == p.start) { + return false; } else { - return std::auto_ptr<Fairshare>(); - } -} - -std::auto_ptr<Fairshare> getFairshare(const qpid::framing::FieldTable& settings, - uint levels, - const std::vector<std::string>& keys) -{ - std::auto_ptr<Fairshare> fairshare; - for (std::vector<std::string>::const_iterator i = keys.begin(); i != keys.end() && !fairshare.get(); ++i) { - fairshare = getFairshareForKey(settings, levels, *i); + p.current = next; + return true; } - return fairshare; } -std::auto_ptr<Messages> Fairshare::create(const qpid::framing::FieldTable& settings) +std::auto_ptr<Messages> Fairshare::create(const QueueSettings& settings) { - using boost::assign::list_of; - std::auto_ptr<Messages> result; - size_t levels = getSetting(settings, list_of<std::string>("qpid.priorities")("x-qpid-priorities"), 0, 100); - if (levels) { - std::auto_ptr<Fairshare> fairshare = - getFairshare(settings, levels, list_of<std::string>("qpid.fairshare")("x-qpid-fairshare")); - if (fairshare.get()) result = fairshare; - else result = std::auto_ptr<Messages>(new PriorityQueue(levels)); + std::auto_ptr<Fairshare> fairshare(new Fairshare(settings.priorities, settings.defaultFairshare)); + for (uint i = 0; i < settings.priorities; i++) { + std::map<uint32_t,uint32_t>::const_iterator l = settings.fairshare.find(i); + if (l != settings.fairshare.end()) fairshare->setLimit(i, l->second); } - return result; + return std::auto_ptr<Messages>(fairshare.release()); } }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/Fairshare.h b/qpid/cpp/src/qpid/broker/Fairshare.h index 1b25721e0c..e7c8b04ecf 100644 --- a/qpid/cpp/src/qpid/broker/Fairshare.h +++ b/qpid/cpp/src/qpid/broker/Fairshare.h @@ -24,13 +24,11 @@ #include "qpid/broker/PriorityQueue.h" namespace qpid { -namespace framing { -class FieldTable; -} namespace broker { +struct QueueSettings; /** - * Modifies a basic prioirty queue by limiting the number of messages + * Modifies a basic priority queue by limiting the number of messages * from each priority level that are dispatched before allowing * dispatch from the next level. */ @@ -42,7 +40,7 @@ class Fairshare : public PriorityQueue bool setState(uint priority, uint count); void setLimit(size_t level, uint limit); bool isNull(); - static std::auto_ptr<Messages> create(const qpid::framing::FieldTable& settings); + static std::auto_ptr<Messages> create(const QueueSettings& settings); static bool getState(const Messages&, uint& priority, uint& count); static bool setState(Messages&, uint priority, uint count); private: @@ -54,7 +52,8 @@ class Fairshare : public PriorityQueue uint currentLevel(); uint nextLevel(); bool limitReached(); - bool findFrontLevel(uint& p, PriorityLevels&); + Priority firstLevel(); + bool nextLevel(Priority& ); }; }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/FifoDistributor.cpp b/qpid/cpp/src/qpid/broker/FifoDistributor.cpp index c9ba894297..e1c0d268ce 100644 --- a/qpid/cpp/src/qpid/broker/FifoDistributor.cpp +++ b/qpid/cpp/src/qpid/broker/FifoDistributor.cpp @@ -28,21 +28,14 @@ using namespace qpid::broker; FifoDistributor::FifoDistributor(Messages& container) : messages(container) {} -bool FifoDistributor::nextConsumableMessage( Consumer::shared_ptr&, QueuedMessage& next ) +bool FifoDistributor::acquire(const std::string&, Message& msg) { - return messages.consume(next); -} - -bool FifoDistributor::allocate(const std::string&, const QueuedMessage& ) -{ - // by default, all messages present on the queue may be allocated as they have yet to - // be acquired. - return true; -} - -bool FifoDistributor::nextBrowsableMessage( Consumer::shared_ptr& c, QueuedMessage& next ) -{ - return messages.browse(c->getPosition(), next, !c->browseAcquired()); + if (msg.getState() == AVAILABLE) { + msg.setState(ACQUIRED); + return true; + } else { + return false; + } } void FifoDistributor::query(qpid::types::Variant::Map&) const diff --git a/qpid/cpp/src/qpid/broker/FifoDistributor.h b/qpid/cpp/src/qpid/broker/FifoDistributor.h index 245537ed12..aa5ebe28c5 100644 --- a/qpid/cpp/src/qpid/broker/FifoDistributor.h +++ b/qpid/cpp/src/qpid/broker/FifoDistributor.h @@ -38,15 +38,7 @@ class FifoDistributor : public MessageDistributor public: FifoDistributor(Messages& container); - /** Locking Note: all methods assume the caller is holding the Queue::messageLock - * during the method call. - */ - - /** MessageDistributor interface */ - - bool nextConsumableMessage( Consumer::shared_ptr& consumer, QueuedMessage& next ); - bool allocate(const std::string& consumer, const QueuedMessage& target); - bool nextBrowsableMessage( Consumer::shared_ptr& consumer, QueuedMessage& next ); + bool acquire(const std::string& consumer, Message& target); void query(qpid::types::Variant::Map&) const; private: diff --git a/qpid/cpp/src/qpid/broker/HeadersExchange.cpp b/qpid/cpp/src/qpid/broker/HeadersExchange.cpp index 9975d26c72..02c05852ff 100644 --- a/qpid/cpp/src/qpid/broker/HeadersExchange.cpp +++ b/qpid/cpp/src/qpid/broker/HeadersExchange.cpp @@ -19,6 +19,7 @@ * */ #include "qpid/broker/HeadersExchange.h" +#include "qpid/broker/MapHandler.h" #include "qpid/framing/FieldValue.h" #include "qpid/framing/reply_exceptions.h" #include "qpid/log/Statement.h" @@ -55,6 +56,100 @@ namespace { const std::string fedOpUnbind("U"); const std::string fedOpReorigin("R"); const std::string fedOpHello("H"); + +std::string getMatch(const FieldTable* args) +{ + if (!args) { + throw InternalErrorException(QPID_MSG("No arguments given.")); + } + FieldTable::ValuePtr what = args->get(x_match); + if (!what) { + return empty; + } + if (!what->convertsTo<std::string>()) { + throw InternalErrorException(QPID_MSG("Invalid x-match binding format to headers exchange. Must be a string [\"all\" or \"any\"]")); + } + return what->get<std::string>(); +} +class Matcher : public MapHandler +{ + public: + Matcher(const FieldTable& b) : binding(b), matched(0) {} + void handleUint8(const MapHandler::CharSequence& key, uint8_t value) { processUint(std::string(key.data, key.size), value); } + void handleUint16(const MapHandler::CharSequence& key, uint16_t value) { processUint(std::string(key.data, key.size), value); } + void handleUint32(const MapHandler::CharSequence& key, uint32_t value) { processUint(std::string(key.data, key.size), value); } + void handleUint64(const MapHandler::CharSequence& key, uint64_t value) { processUint(std::string(key.data, key.size), value); } + void handleInt8(const MapHandler::CharSequence& key, int8_t value) { processInt(std::string(key.data, key.size), value); } + void handleInt16(const MapHandler::CharSequence& key, int16_t value) { processInt(std::string(key.data, key.size), value); } + void handleInt32(const MapHandler::CharSequence& key, int32_t value) { processInt(std::string(key.data, key.size), value); } + void handleInt64(const MapHandler::CharSequence& key, int64_t value) { processInt(std::string(key.data, key.size), value); } + void handleFloat(const MapHandler::CharSequence& key, float value) { processFloat(std::string(key.data, key.size), value); } + void handleDouble(const MapHandler::CharSequence& key, double value) { processFloat(std::string(key.data, key.size), value); } + void handleString(const MapHandler::CharSequence& key, const MapHandler::CharSequence& value, const MapHandler::CharSequence& /*encoding*/) + { + processString(std::string(key.data, key.size), std::string(value.data, value.size)); + } + void handleVoid(const MapHandler::CharSequence& key) + { + valueCheckRequired(std::string(key.data, key.size)); + } + bool matches() + { + std::string what = getMatch(&binding); + if (what == all) { + //must match all entries in the binding, except the match mode indicator + return matched == binding.size() - 1; + } else if (what == any) { + //match any of the entries in the binding + return matched > 0; + } else { + return false; + } + } + private: + bool valueCheckRequired(const std::string& key) + { + FieldTable::ValuePtr v = binding.get(key); + if (v) { + if (v->getType() == 0xf0/*VOID*/) { + ++matched; + return false; + } else { + return true; + } + } else { + return false; + } + } + + void processString(const std::string& key, const std::string& actual) + { + if (valueCheckRequired(key) && binding.getAsString(key) == actual) { + ++matched; + } + } + void processFloat(const std::string& key, double actual) + { + double bound; + if (valueCheckRequired(key) && binding.getDouble(key, bound) && bound == actual) { + ++matched; + } + } + void processInt(const std::string& key, int64_t actual) + { + if (valueCheckRequired(key) && binding.getAsInt64(key) == actual) { + ++matched; + } + } + void processUint(const std::string& key, uint64_t actual) + { + if (valueCheckRequired(key) && binding.getAsUInt64(key) == actual) { + ++matched; + } + } + const FieldTable& binding; + size_t matched; +}; } HeadersExchange::HeadersExchange(const string& _name, Manageable* _parent, Broker* b) : @@ -72,21 +167,6 @@ HeadersExchange::HeadersExchange(const std::string& _name, bool _durable, mgmtExchange->set_type (typeName); } -std::string HeadersExchange::getMatch(const FieldTable* args) -{ - if (!args) { - throw InternalErrorException(QPID_MSG("No arguments given.")); - } - FieldTable::ValuePtr what = args->get(x_match); - if (!what) { - return empty; - } - if (!what->convertsTo<std::string>()) { - throw InternalErrorException(QPID_MSG("Invalid x-match binding format to headers exchange. Must be a string [\"all\" or \"any\"]")); - } - return what->get<std::string>(); -} - bool HeadersExchange::bind(Queue::shared_ptr queue, const string& bindingKey, const FieldTable* args) { string fedOp(fedOpBind); @@ -196,28 +276,16 @@ bool HeadersExchange::unbind(Queue::shared_ptr queue, const string& bindingKey, void HeadersExchange::route(Deliverable& msg) { - const FieldTable* args = msg.getMessage().getApplicationHeaders(); - if (!args) { - //can't match if there were no headers passed in - if (mgmtExchange != 0) { - mgmtExchange->inc_msgReceives(); - mgmtExchange->inc_byteReceives(msg.contentSize()); - mgmtExchange->inc_msgDrops(); - mgmtExchange->inc_byteDrops(msg.contentSize()); - if (brokerMgmtObject) - brokerMgmtObject->inc_discardsNoRoute(); - } - return; - } - PreRoute pr(msg, this); BindingList b(new std::vector<boost::shared_ptr<qpid::broker::Exchange::Binding> >); Bindings::ConstPtr p = bindings.snapshot(); if (p.get()) { for (std::vector<BoundKey>::const_iterator i = p->begin(); i != p->end(); ++i) { - if (match((*i).binding->args, *args)) { - b->push_back((*i).binding); + Matcher matcher(i->binding->args); + msg.getMessage().processProperties(matcher); + if (matcher.matches()) { + b->push_back(i->binding); } } } diff --git a/qpid/cpp/src/qpid/broker/HeadersExchange.h b/qpid/cpp/src/qpid/broker/HeadersExchange.h index d10892b9cc..2e4669a018 100644 --- a/qpid/cpp/src/qpid/broker/HeadersExchange.h +++ b/qpid/cpp/src/qpid/broker/HeadersExchange.h @@ -73,9 +73,6 @@ class HeadersExchange : public virtual Exchange { Bindings bindings; qpid::sys::Mutex lock; - - static std::string getMatch(const framing::FieldTable* args); - protected: void getNonFedArgs(const framing::FieldTable* args, framing::FieldTable& nonFedArgs); diff --git a/qpid/cpp/src/qpid/broker/IndexedDeque.h b/qpid/cpp/src/qpid/broker/IndexedDeque.h new file mode 100644 index 0000000000..c64f321789 --- /dev/null +++ b/qpid/cpp/src/qpid/broker/IndexedDeque.h @@ -0,0 +1,226 @@ +#ifndef QPID_BROKER_INDEXEDDEQUE_H +#define QPID_BROKER_INDEXEDDEQUE_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/framing/SequenceNumber.h" +#include "qpid/broker/Message.h" +#include "qpid/broker/Messages.h" +#include "qpid/broker/QueueCursor.h" +#include "qpid/log/Statement.h" +#include <deque> + +namespace qpid { +namespace broker { + +/** + * Template for a deque whose contents can be refered to by + * QueueCursor + */ +template <typename T> class IndexedDeque +{ + public: + typedef boost::function1<T, qpid::framing::SequenceNumber> Padding; + IndexedDeque(Padding p) : head(0), version(0), padding(p) {} + + bool index(const QueueCursor& cursor, size_t& result) + { + return cursor.valid && index(qpid::framing::SequenceNumber(cursor.position + 1), result); + } + + /** + * Finds the index for the message with the specified sequence number. + * + * @returns true if a message was found with the specified sequence, + * in which case the second parameter will be set to the index of that + * message; false if no message with that sequence exists, in which + * case the second parameter will be 0 if the sequence is less than + * that of the first message and non-zero if it is greater than that + * of the last message + */ + bool index(const qpid::framing::SequenceNumber& position, size_t& i) + { + //assuming a monotonic sequence, with no messages removed except + //from the ends of the deque, we can use the position to determine + //an index into the deque + if (messages.size()) { + qpid::framing::SequenceNumber front(messages.front().getSequence()); + if (position < front) { + i = 0; + } else { + i = position - front; + return i < messages.size(); + } + } + return false; + } + + bool deleted(const QueueCursor& cursor) + { + size_t i; + if (cursor.valid && index(cursor.position, i)) { + messages[i].setState(DELETED); + clean(); + return true; + } else { + return false; + } + } + + T& publish(const T& added) + { + // QPID-4046: let producer help clean the backlog of deleted messages + clean(); + //for ha replication, the queue can sometimes be reset by + //removing some of the more recent messages, in this case we + //need to ensure the DELETED records at the tail do not interfere with indexing + while (messages.size() && added.getSequence() <= messages.back().getSequence() && messages.back().getState() == DELETED) + messages.pop_back(); + if (messages.size() && added.getSequence() <= messages.back().getSequence()) throw qpid::Exception(QPID_MSG("Index out of sequence!")); + + //add padding to prevent gaps in sequence, which break the index + //calculation (needed for queue replication) + while (messages.size() && (added.getSequence() - messages.back().getSequence()) > 1) + messages.push_back(padding(messages.back().getSequence() + 1)); + + messages.push_back(added); + T& m = messages.back(); + m.setState(AVAILABLE); + if (head >= messages.size()) head = messages.size() - 1; + QPID_LOG(debug, "Message " << &m << " published, state is " << m.getState() << " (head is now " << head << ")"); + return m; + } + + T* release(const QueueCursor& cursor) + { + size_t i; + if (cursor.valid && index(cursor.position, i)) { + messages[i].setState(AVAILABLE); + ++version; + QPID_LOG(debug, "Released message at position " << cursor.position << ", index " << i); + return &messages[i]; + } else { + if (!cursor.valid) { QPID_LOG(debug, "Could not release message; cursor was invalid");} + else { QPID_LOG(debug, "Could not release message at position " << cursor.position); } + return 0; + } + } + + bool reset(const QueueCursor& cursor) + { + return !cursor.valid || (cursor.type == CONSUMER && cursor.version != version); + } + + T* next(QueueCursor& cursor) + { + size_t i; + if (reset(cursor)) i = head; //start from head + else index(cursor, i); //get first message that is greater than position + + if (cursor.valid) { + QPID_LOG(debug, "next() called for cursor at " << cursor.position << ", index set to " << i << " (of " << messages.size() << ")"); + } else { + QPID_LOG(debug, "next() called for invalid cursor, index started at " << i << " (of " << messages.size() << ")"); + } + while (i < messages.size()) { + T& m = messages[i++]; + if (m.getState() == DELETED) continue; + cursor.setPosition(m.getSequence(), version); + QPID_LOG(debug, "in next(), cursor set to " << cursor.position); + + if (cursor.check(m)) { + QPID_LOG(debug, "in next(), returning message at " << cursor.position); + return &m; + } + } + QPID_LOG(debug, "no message to return from next"); + return 0; + } + + size_t size() + { + size_t count(0); + for (size_t i = head; i < messages.size(); ++i) { + if (messages[i].getState() == AVAILABLE) ++count; + } + return count; + } + + T* find(const qpid::framing::SequenceNumber& position, QueueCursor* cursor) + { + size_t i; + if (index(position, i)){ + T& m = messages[i]; + if (cursor) cursor->setPosition(position, version); + if (m.getState() == AVAILABLE || m.getState() == ACQUIRED) { + return &m; + } + } else if (cursor) { + if (i >= messages.size()) cursor->setPosition(position, version);//haven't yet got a message with that seq no + else if (i == 0) cursor->valid = false;//reset + } + return 0; + } + + T* find(const QueueCursor& cursor) + { + if (cursor.valid) return find(cursor.position, 0); + else return 0; + } + + void clean() + { + // QPID-4046: If a queue has multiple consumers, then it is possible for a large + // collection of deleted messages to build up. Limit the number of messages cleaned + // up on each call to clean(). + size_t count = 0; + while (messages.size() && messages.front().getState() == DELETED && count < 10) { + messages.pop_front(); + count += 1; + } + head = (head > count) ? head - count : 0; + QPID_LOG(debug, "clean(): " << messages.size() << " messages remain; head is now " << head); + } + + void foreach(Messages::Functor f) + { + for (typename Deque::iterator i = messages.begin(); i != messages.end(); ++i) { + if (i->getState() == AVAILABLE) { + f(*i); + } + } + clean(); + } + + void resetCursors() + { + ++version; + } + + typedef std::deque<T> Deque; + Deque messages; + size_t head; + int32_t version; + Padding padding; +}; +}} // namespace qpid::broker + +#endif /*!QPID_BROKER_INDEXEDDEQUE_H*/ diff --git a/qpid/cpp/src/qpid/broker/LegacyLVQ.cpp b/qpid/cpp/src/qpid/broker/LegacyLVQ.cpp index f1deddf4c8..e69de29bb2 100644 --- a/qpid/cpp/src/qpid/broker/LegacyLVQ.cpp +++ b/qpid/cpp/src/qpid/broker/LegacyLVQ.cpp @@ -1,127 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/broker/LegacyLVQ.h" -#include "qpid/broker/Broker.h" -#include "qpid/broker/QueuedMessage.h" - -namespace qpid { -namespace broker { - -LegacyLVQ::LegacyLVQ(const std::string& k, bool b, Broker* br) : MessageMap(k), noBrowse(b), broker(br) {} - -void LegacyLVQ::setNoBrowse(bool b) -{ - noBrowse = b; -} -bool LegacyLVQ::deleted(const QueuedMessage& message) -{ - Ordering::iterator i = messages.find(message.position); - if (i != messages.end() && i->second.payload == message.payload) { - erase(i); - return true; - } else { - return false; - } -} - -bool LegacyLVQ::acquire(const framing::SequenceNumber& position, QueuedMessage& message) -{ - Ordering::iterator i = messages.find(position); - if (i != messages.end() && i->second.payload == message.payload && i->second.status == QueuedMessage::AVAILABLE) { - i->second.status = QueuedMessage::ACQUIRED; - message = i->second; - return true; - } else { - return false; - } -} - -bool LegacyLVQ::browse(const framing::SequenceNumber& position, QueuedMessage& message, bool unacquired) -{ - if (MessageMap::browse(position, message, unacquired)) { - if (!noBrowse) index.erase(getKey(message)); - return true; - } else { - return false; - } -} - -bool LegacyLVQ::push(const QueuedMessage& added, QueuedMessage& removed) -{ - //Hack to disable LVQ behaviour on cluster update: - if (broker && broker->isClusterUpdatee()) { - messages[added.position] = added; - return false; - } else { - return MessageMap::push(added, removed); - } -} - -const QueuedMessage& LegacyLVQ::replace(const QueuedMessage& original, const QueuedMessage& update) -{ - //add the new message into the original position of the replaced message - Ordering::iterator i = messages.find(original.position); - if (i != messages.end()) { - i->second = update; - i->second.position = original.position; - return i->second; - } else { - QPID_LOG(error, "Failed to replace message at " << original.position); - return update; - } -} - -void LegacyLVQ::removeIf(Predicate p) -{ - //Note: This method is currently called periodically on the timer - //thread to expire messages. In a clustered broker this means that - //the purging does not occur on the cluster event dispatch thread - //and consequently that is not totally ordered w.r.t other events - //(including publication of messages). The cluster does ensure - //that the actual expiration of messages (as distinct from the - //removing of those expired messages from the queue) *is* - //consistently ordered w.r.t. cluster events. This means that - //delivery of messages is in general consistent across the cluster - //inspite of any non-determinism in the triggering of a - //purge. However at present purging a last value queue (of the - //legacy sort) could potentially cause inconsistencies in the - //cluster (as the order w.r.t publications can affect the order in - //which messages appear in the queue). Consequently periodic - //purging of an LVQ is not enabled if the broker is clustered - //(expired messages will be removed on delivery and consolidated - //by key as part of normal LVQ operation). - if (!broker || !broker->isInCluster()) - MessageMap::removeIf(p); -} - -std::auto_ptr<Messages> LegacyLVQ::updateOrReplace(std::auto_ptr<Messages> current, - const std::string& key, bool noBrowse, Broker* broker) -{ - LegacyLVQ* lvq = dynamic_cast<LegacyLVQ*>(current.get()); - if (lvq) { - lvq->setNoBrowse(noBrowse); - return current; - } else { - return std::auto_ptr<Messages>(new LegacyLVQ(key, noBrowse, broker)); - } -} - -}} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/LegacyLVQ.h b/qpid/cpp/src/qpid/broker/LegacyLVQ.h index 9355069f37..e69de29bb2 100644 --- a/qpid/cpp/src/qpid/broker/LegacyLVQ.h +++ b/qpid/cpp/src/qpid/broker/LegacyLVQ.h @@ -1,60 +0,0 @@ -#ifndef QPID_BROKER_LEGACYLVQ_H -#define QPID_BROKER_LEGACYLVQ_H - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/broker/MessageMap.h" -#include <memory> - -namespace qpid { -namespace broker { -class Broker; - -/** - * This class encapsulates the behaviour of the old style LVQ where a - * message replacing another messages for the given key will use the - * position in the queue of the previous message. This however causes - * problems for browsing. Either browsers stop the coalescing of - * messages by key (default) or they may mis updates (if the no-browse - * option is specified). - */ -class LegacyLVQ : public MessageMap -{ - public: - LegacyLVQ(const std::string& key, bool noBrowse = false, Broker* broker = 0); - bool deleted(const QueuedMessage&); - bool acquire(const framing::SequenceNumber&, QueuedMessage&); - bool browse(const framing::SequenceNumber&, QueuedMessage&, bool); - bool push(const QueuedMessage& added, QueuedMessage& removed); - void removeIf(Predicate); - void setNoBrowse(bool); - static std::auto_ptr<Messages> updateOrReplace(std::auto_ptr<Messages> current, - const std::string& key, bool noBrowse, - Broker* broker); - protected: - bool noBrowse; - Broker* broker; - - const QueuedMessage& replace(const QueuedMessage&, const QueuedMessage&); -}; -}} // namespace qpid::broker - -#endif /*!QPID_BROKER_LEGACYLVQ_H*/ diff --git a/qpid/cpp/src/qpid/broker/Link.cpp b/qpid/cpp/src/qpid/broker/Link.cpp index 84dd163ac3..6479e47799 100644 --- a/qpid/cpp/src/qpid/broker/Link.cpp +++ b/qpid/cpp/src/qpid/broker/Link.cpp @@ -92,10 +92,10 @@ public: // Process messages sent from the remote's amq.failover exchange by extracting the failover URLs // and saving them should the Link need to reconnect. - void route(broker::Deliverable& msg) + void route(broker::Deliverable& /*msg*/) { if (!link) return; - const framing::FieldTable* headers = msg.getMessage().getApplicationHeaders(); + const framing::FieldTable* headers = 0;//TODO: msg.getMessage().getApplicationHeaders(); framing::Array addresses; if (headers && headers->getArray(FAILOVER_HEADER_KEY, addresses)) { // convert the Array of addresses to a single Url container for used with setUrl(): diff --git a/qpid/cpp/src/qpid/broker/LossyQueue.cpp b/qpid/cpp/src/qpid/broker/LossyQueue.cpp new file mode 100644 index 0000000000..ee2c3ca794 --- /dev/null +++ b/qpid/cpp/src/qpid/broker/LossyQueue.cpp @@ -0,0 +1,86 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "LossyQueue.h" +#include "QueueDepth.h" +#include "qpid/framing/reply_exceptions.h" +#include "qpid/log/Statement.h" + +namespace qpid { +namespace broker { + +namespace { +bool isLowerPriorityThan(uint8_t priority, const Message& m) +{ + return m.getPriority() <= priority; +} +} + +LossyQueue::LossyQueue(const std::string& n, const QueueSettings& s, MessageStore* const ms, management::Manageable* p, Broker* b) + : Queue(n, s, ms, p, b) {} + +bool LossyQueue::checkDepth(const QueueDepth& increment, const Message& message) +{ + if (increment.getSize() > settings.maxDepth.getSize()) { + if (mgmtObject) { + mgmtObject->inc_discardsOverflow(); + if (brokerMgmtObject) + brokerMgmtObject->inc_discardsOverflow(); + } + throw qpid::framing::ResourceLimitExceededException(QPID_MSG("Message larger than configured maximum depth on " + << name << ": size=" << increment.getSize() << ", max-size=" << settings.maxDepth.getSize())); + } + + while (settings.maxDepth && (current + increment > settings.maxDepth)) { + QPID_LOG(debug, "purging " << name << ": current depth is [" << current << "], max depth is [" << settings.maxDepth << "], new message has size " << increment.getSize()); + qpid::sys::Mutex::ScopedUnlock u(messageLock); + //TODO: arguably we should try and purge expired messages first but that is potentially expensive + if (remove(1, settings.priorities ? boost::bind(&isLowerPriorityThan, message.getPriority(), _1) : MessagePredicate(), MessageFunctor(), PURGE)) { + if (mgmtObject) { + mgmtObject->inc_discardsRing(1); + if (brokerMgmtObject) + brokerMgmtObject->inc_discardsRing(1); + } + } else { + //should only be the case for a non-empty queue if we are + //testing priority and there was no lower (or equal) + //priority message available to purge + break; + } + } + if (settings.maxDepth && (current + increment > settings.maxDepth)) { + //will only be the case where we were unable to purge another + //message, which should only be the case if we are purging + //based on priority and there was no message with a lower (or + //equal) priority than this one, meaning that we drop this + //current message + if (mgmtObject) { + mgmtObject->inc_discardsRing(1); + if (brokerMgmtObject) + brokerMgmtObject->inc_discardsRing(1); + } + return false; + } else { + //have sufficient space for this message + current += increment; + return true; + } +} +}} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/replication/constants.h b/qpid/cpp/src/qpid/broker/LossyQueue.h index c5ba7d3d6a..3e62151d6f 100644 --- a/qpid/cpp/src/qpid/replication/constants.h +++ b/qpid/cpp/src/qpid/broker/LossyQueue.h @@ -1,3 +1,6 @@ +#ifndef QPID_BROKER_LOSSYQUEUE_H +#define QPID_BROKER_LOSSYQUEUE_H + /* * * Licensed to the Apache Software Foundation (ASF) under one @@ -7,9 +10,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,17 +21,21 @@ * under the License. * */ -namespace qpid { -namespace replication { -namespace constants { +#include "qpid/broker/Queue.h" -const std::string REPLICATION_EVENT_TYPE("qpid.replication.type"); -const std::string REPLICATION_EVENT_SEQNO("qpid.replication.seqno"); -const std::string REPLICATION_TARGET_QUEUE("qpid.replication.target_queue"); -const std::string DEQUEUED_MESSAGE_POSITION("qpid.replication.message"); -const std::string QUEUE_MESSAGE_POSITION("qpid.replication.queue.position"); +namespace qpid { +namespace broker { -const int ENQUEUE(1); -const int DEQUEUE(2); +/** + * Drops messages to prevent a breach of any configured maximum depth. + */ +class LossyQueue : public Queue +{ + public: + LossyQueue(const std::string&, const QueueSettings&, MessageStore* const, management::Manageable*, Broker*); + bool checkDepth(const QueueDepth& increment, const Message&); + private: +}; +}} // namespace qpid::broker -}}} +#endif /*!QPID_BROKER_LOSSYQUEUE_H*/ diff --git a/qpid/cpp/src/qpid/broker/Lvq.cpp b/qpid/cpp/src/qpid/broker/Lvq.cpp new file mode 100644 index 0000000000..d053616c8a --- /dev/null +++ b/qpid/cpp/src/qpid/broker/Lvq.cpp @@ -0,0 +1,64 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "Lvq.h" +#include "MessageMap.h" +#include "qpid/sys/ClusterSafe.h" +#include "qpid/sys/Monitor.h" + +namespace qpid { +namespace broker { +Lvq::Lvq(const std::string& n, std::auto_ptr<MessageMap> m, const QueueSettings& s, MessageStore* const ms, management::Manageable* p, Broker* b) + : Queue(n, s, ms, p, b), messageMap(*m) +{ + messages = m; +} + +void Lvq::push(Message& message, bool isRecovery) +{ + qpid::sys::assertClusterSafe(); + QueueListeners::NotificationSet copy; + Message old; + bool removed; + { + qpid::sys::Mutex::ScopedLock locker(messageLock); + message.setSequence(++sequence); + removed = messageMap.update(message, old); + listeners.populate(copy); + observeEnqueue(message, locker); + if (removed) { + if (mgmtObject) { + mgmtObject->inc_acquires(); + mgmtObject->inc_discardsLvq(); + if (brokerMgmtObject) { + brokerMgmtObject->inc_acquires(); + brokerMgmtObject->inc_discardsLvq(); + } + } + observeDequeue(old, locker); + } + } + copy.notify(); + if (removed) { + if (isRecovery) pendingDequeues.push_back(old); + else dequeueFromStore(old.getPersistentContext());//do outside of lock + } +} +}} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/Lvq.h b/qpid/cpp/src/qpid/broker/Lvq.h new file mode 100644 index 0000000000..335270a073 --- /dev/null +++ b/qpid/cpp/src/qpid/broker/Lvq.h @@ -0,0 +1,45 @@ +#ifndef QPID_BROKER_LVQ_H +#define QPID_BROKER_LVQ_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/broker/Queue.h" + +namespace qpid { +namespace broker { +class MessageMap; + +/** + * Subclass of queue that handles last-value-queue semantics in + * conjunction with the MessageMap class. This requires an existing + * message to be 'replaced' by a newer message with the same key. + */ +class Lvq : public Queue +{ + public: + Lvq(const std::string&, std::auto_ptr<MessageMap>, const QueueSettings&, MessageStore* const, management::Manageable*, Broker*); + void push(Message& msg, bool isRecovery=false); + private: + MessageMap& messageMap; +}; +}} // namespace qpid::broker + +#endif /*!QPID_BROKER_LVQ_H*/ diff --git a/qpid/cpp/src/qpid/broker/MapHandler.h b/qpid/cpp/src/qpid/broker/MapHandler.h new file mode 100644 index 0000000000..200eba4f7a --- /dev/null +++ b/qpid/cpp/src/qpid/broker/MapHandler.h @@ -0,0 +1,57 @@ +#ifndef QPID_BROKER_MAPHANDLER_H +#define QPID_BROKER_MAPHANDLER_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/IntegerTypes.h" + +namespace qpid { +namespace broker { + +/** + * Interface for processing entries in some map-like object + */ +class MapHandler +{ + public: + typedef struct { + const char* data; + size_t size; + } CharSequence; + + virtual ~MapHandler() {} + virtual void handleVoid(const CharSequence& key) = 0; + virtual void handleUint8(const CharSequence& key, uint8_t value) = 0; + virtual void handleUint16(const CharSequence& key, uint16_t value) = 0; + virtual void handleUint32(const CharSequence& key, uint32_t value) = 0; + virtual void handleUint64(const CharSequence& key, uint64_t value) = 0; + virtual void handleInt8(const CharSequence& key, int8_t value) = 0; + virtual void handleInt16(const CharSequence& key, int16_t value) = 0; + virtual void handleInt32(const CharSequence& key, int32_t value) = 0; + virtual void handleInt64(const CharSequence& key, int64_t value) = 0; + virtual void handleFloat(const CharSequence& key, float value) = 0; + virtual void handleDouble(const CharSequence& key, double value) = 0; + virtual void handleString(const CharSequence& key, const CharSequence& value, const CharSequence& encoding) = 0; + private: +}; +}} // namespace qpid::broker + +#endif /*!QPID_BROKER_MAPHANDLER_H*/ diff --git a/qpid/cpp/src/qpid/broker/Message.cpp b/qpid/cpp/src/qpid/broker/Message.cpp index 4dd8a349dd..c48e9bcfa4 100644 --- a/qpid/cpp/src/qpid/broker/Message.cpp +++ b/qpid/cpp/src/qpid/broker/Message.cpp @@ -20,19 +20,12 @@ */ #include "qpid/broker/Message.h" -#include "qpid/broker/Queue.h" -#include "qpid/broker/ExchangeRegistry.h" -#include "qpid/broker/ExpiryPolicy.h" +#include "qpid/broker/MapHandler.h" #include "qpid/StringUtils.h" -#include "qpid/framing/frame_functors.h" -#include "qpid/framing/FieldTable.h" -#include "qpid/framing/MessageTransferBody.h" -#include "qpid/framing/SendContent.h" -#include "qpid/framing/SequenceNumber.h" -#include "qpid/framing/TypeFilter.h" -#include "qpid/framing/reply_exceptions.h" #include "qpid/log/Statement.h" +#include <algorithm> +#include <string.h> #include <time.h> using boost::intrusive_ptr; @@ -41,492 +34,261 @@ using qpid::sys::Duration; using qpid::sys::TIME_MSEC; using qpid::sys::FAR_FUTURE; using std::string; -using namespace qpid::framing; namespace qpid { namespace broker { -TransferAdapter Message::TRANSFER; - -Message::Message(const framing::SequenceNumber& id) : - frames(id), persistenceId(0), redelivered(false), loaded(false), - staged(false), forcePersistentPolicy(false), publisher(0), adapter(0), - expiration(FAR_FUTURE), dequeueCallback(0), - inCallback(false), requiredCredit(0), isManagementMessage(false), copyHeaderOnWrite(false) -{} - -Message::~Message() {} - -void Message::forcePersistent() +Message::Message() : deliveryCount(0), publisher(0), expiration(FAR_FUTURE), timestamp(0), isManagementMessage(false) {} +Message::Message(boost::intrusive_ptr<Encoding> e, boost::intrusive_ptr<PersistableMessage> p) + : encoding(e), persistentContext(p), deliveryCount(0), publisher(0), expiration(FAR_FUTURE), timestamp(0), isManagementMessage(false) { - sys::Mutex::ScopedLock l(lock); - // only set forced bit if we actually need to force. - if (! getAdapter().isPersistent(frames) ){ - forcePersistentPolicy = true; - } + if (persistentContext) persistentContext->setIngressCompletion(e); } +Message::~Message() {} -bool Message::isForcedPersistent() -{ - return forcePersistentPolicy; -} std::string Message::getRoutingKey() const { - return getAdapter().getRoutingKey(frames); -} - -std::string Message::getExchangeName() const -{ - return getAdapter().getExchange(frames); -} - -const boost::shared_ptr<Exchange> Message::getExchange(ExchangeRegistry& registry) const -{ - if (!exchange) { - exchange = registry.get(getExchangeName()); - } - return exchange; -} - -bool Message::isImmediate() const -{ - return getAdapter().isImmediate(frames); -} - -const FieldTable* Message::getApplicationHeaders() const -{ - sys::Mutex::ScopedLock l(lock); - return getAdapter().getApplicationHeaders(frames); -} - -std::string Message::getAppId() const -{ - sys::Mutex::ScopedLock l(lock); - return getAdapter().getAppId(frames); + return getEncoding().getRoutingKey(); } bool Message::isPersistent() const { - sys::Mutex::ScopedLock l(lock); - return (getAdapter().isPersistent(frames) || forcePersistentPolicy); + return getEncoding().isPersistent(); } -bool Message::requiresAccept() +uint64_t Message::getContentSize() const { - return getAdapter().requiresAccept(frames); + return getEncoding().getContentSize(); } -uint32_t Message::getRequiredCredit() +boost::intrusive_ptr<AsyncCompletion> Message::getIngressCompletion() const { - sys::Mutex::ScopedLock l(lock); - if (!requiredCredit) { - //add up payload for all header and content frames in the frameset - SumBodySize sum; - frames.map_if(sum, TypeFilter2<HEADER_BODY, CONTENT_BODY>()); - requiredCredit = sum.getSize(); - } - return requiredCredit; + return encoding; } -void Message::encode(framing::Buffer& buffer) const -{ - sys::Mutex::ScopedLock l(lock); - //encode method and header frames - EncodeFrame f1(buffer); - frames.map_if(f1, TypeFilter2<METHOD_BODY, HEADER_BODY>()); - - //then encode the payload of each content frame - framing::EncodeBody f2(buffer); - frames.map_if(f2, TypeFilter<CONTENT_BODY>()); -} - -void Message::encodeContent(framing::Buffer& buffer) const -{ - sys::Mutex::ScopedLock l(lock); - //encode the payload of each content frame - EncodeBody f2(buffer); - frames.map_if(f2, TypeFilter<CONTENT_BODY>()); -} - -uint32_t Message::encodedSize() const -{ - return encodedHeaderSize() + encodedContentSize(); -} - -uint32_t Message::encodedContentSize() const -{ - sys::Mutex::ScopedLock l(lock); - return frames.getContentSize(); -} - -uint32_t Message::encodedHeaderSize() const -{ - sys::Mutex::ScopedLock l(lock); // prevent modifications while computing size - //add up the size for all method and header frames in the frameset - SumFrameSize sum; - frames.map_if(sum, TypeFilter2<METHOD_BODY, HEADER_BODY>()); - return sum.getSize(); -} - -void Message::decodeHeader(framing::Buffer& buffer) +namespace { - AMQFrame method; - method.decode(buffer); - frames.append(method); - - AMQFrame header; - header.decode(buffer); - frames.append(header); +const std::string X_QPID_TRACE("x-qpid.trace"); } -void Message::decodeContent(framing::Buffer& buffer) +bool Message::isExcluded(const std::vector<std::string>& excludes) const { - if (buffer.available()) { - //get the data as a string and set that as the content - //body on a frame then add that frame to the frameset - AMQFrame frame((AMQContentBody())); - frame.castBody<AMQContentBody>()->decode(buffer, buffer.available()); - frame.setFirstSegment(false); - frames.append(frame); - } else { - //adjust header flags - MarkLastSegment f; - frames.map_if(f, TypeFilter<HEADER_BODY>()); + std::string traceStr = getEncoding().getAnnotationAsString(X_QPID_TRACE); + if (traceStr.size()) { + std::vector<std::string> trace = split(traceStr, ", "); + for (std::vector<std::string>::const_iterator i = excludes.begin(); i != excludes.end(); i++) { + for (std::vector<std::string>::const_iterator j = trace.begin(); j != trace.end(); j++) { + if (*i == *j) { + return true; + } + } + } } - //mark content loaded - loaded = true; + return false; } -// Used for testing only -void Message::tryReleaseContent() +void Message::addTraceId(const std::string& id) { - if (checkContentReleasable()) { - releaseContent(); + std::string trace = getEncoding().getAnnotationAsString(X_QPID_TRACE); + if (trace.empty()) { + annotations[X_QPID_TRACE] = id; + } else if (trace.find(id) == std::string::npos) { + trace += ","; + trace += id; + annotations[X_QPID_TRACE] = trace; } + annotationsChanged(); } -void Message::releaseContent(MessageStore* s) +void Message::clearTrace() { - //deprecated, use setStore(store); releaseContent(); instead - if (!store) setStore(s); - releaseContent(); + annotations[X_QPID_TRACE] = std::string(); + annotationsChanged(); } -void Message::releaseContent() +void Message::setTimestamp() { - sys::Mutex::ScopedLock l(lock); - if (store) { - if (!getPersistenceId()) { - intrusive_ptr<PersistableMessage> pmsg(this); - store->stage(pmsg); - staged = true; - } - //ensure required credit and size is cached before content frames are released - getRequiredCredit(); - contentSize(); - //remove any content frames from the frameset - frames.remove(TypeFilter<CONTENT_BODY>()); - setContentReleased(); - } + timestamp = ::time(0); // AMQP-0.10: posix time_t - secs since Epoch } -void Message::destroy() +uint64_t Message::getTimestamp() const { - if (staged) { - if (store) { - store->destroy(*this); - } else { - QPID_LOG(error, "Message content was staged but no store is set so it can't be destroyed"); - } - } + return timestamp; } -bool Message::getContentFrame(const Queue& queue, AMQFrame& frame, uint16_t maxContentSize, uint64_t offset) const +uint64_t Message::getTtl() const { - intrusive_ptr<const PersistableMessage> pmsg(this); - - bool done = false; - string& data = frame.castBody<AMQContentBody>()->getData(); - store->loadContent(queue, pmsg, data, offset, maxContentSize); - done = data.size() < maxContentSize; - frame.setBof(false); - frame.setEof(true); - QPID_LOG(debug, "loaded frame" << frame); - if (offset > 0) { - frame.setBos(false); - } - if (!done) { - frame.setEos(false); - } else return false; - return true; -} - -void Message::sendContent(const Queue& queue, framing::FrameHandler& out, uint16_t maxFrameSize) const -{ - sys::Mutex::ScopedLock l(lock); - if (isContentReleased() && !frames.isComplete()) { - sys::Mutex::ScopedUnlock u(lock); - uint16_t maxContentSize = maxFrameSize - AMQFrame::frameOverhead(); - bool morecontent = true; - for (uint64_t offset = 0; morecontent; offset += maxContentSize) - { - AMQFrame frame((AMQContentBody())); - morecontent = getContentFrame(queue, frame, maxContentSize, offset); - out.handle(frame); - } - queue.countLoadedFromDisk(contentSize()); + uint64_t ttl; + if (encoding->getTtl(ttl) && expiration < FAR_FUTURE) { + sys::AbsTime current( + expiryPolicy ? expiryPolicy->getCurrentTime() : sys::AbsTime::now()); + sys::Duration ttl(current, getExpiration()); + // convert from ns to ms; set to 1 if expired + return (int64_t(ttl) >= 1000000 ? int64_t(ttl)/1000000 : 1); } else { - Count c; - frames.map_if(c, TypeFilter<CONTENT_BODY>()); - - SendContent f(out, maxFrameSize, c.getCount()); - frames.map_if(f, TypeFilter<CONTENT_BODY>()); + return 0; } } -void Message::sendHeader(framing::FrameHandler& out, uint16_t /*maxFrameSize*/) const -{ - sys::Mutex::ScopedLock l(lock); - Relay f(out); - frames.map_if(f, TypeFilter<HEADER_BODY>()); - //as frame (and pointer to body) has now been passed to handler, - //subsequent modifications should use a copy - copyHeaderOnWrite = true; -} - -// TODO aconway 2007-11-09: Obsolete, remove. Was used to cover over -// 0-8/0-9 message differences. -MessageAdapter& Message::getAdapter() const +void Message::computeExpiration(const boost::intrusive_ptr<ExpiryPolicy>& e) { - if (!adapter) { - if(frames.isA<MessageTransferBody>()) { - adapter = &TRANSFER; - } else { - const AMQMethodBody* method = frames.getMethod(); - if (!method) throw Exception("Can't adapt message with no method"); - else throw Exception(QPID_MSG("Can't adapt message based on " << *method)); + //TODO: this is still quite 0-10 specific... + uint64_t ttl; + if (getEncoding().getTtl(ttl)) { + if (e) { + // Use higher resolution time for the internal expiry calculation. + // Prevent overflow as a signed int64_t + Duration duration(std::min(ttl * TIME_MSEC, + (uint64_t) std::numeric_limits<int64_t>::max())); + expiration = AbsTime(e->getCurrentTime(), duration); + setExpiryPolicy(e); } } - return *adapter; } -uint64_t Message::contentSize() const +void Message::addAnnotation(const std::string& key, const qpid::types::Variant& value) { - return frames.getContentSize(); + annotations[key] = value; + annotationsChanged(); } -bool Message::isContentLoaded() const +void Message::annotationsChanged() { - return loaded; + if (persistentContext) { + persistentContext = persistentContext->merge(annotations); + persistentContext->setIngressCompletion(encoding); + } } - -namespace -{ -const std::string X_QPID_TRACE("x-qpid.trace"); +void Message::setExpiryPolicy(const boost::intrusive_ptr<ExpiryPolicy>& e) { + expiryPolicy = e; } -bool Message::isExcluded(const std::vector<std::string>& excludes) const +bool Message::hasExpired() const { - sys::Mutex::ScopedLock l(lock); - const FieldTable* headers = getApplicationHeaders(); - if (headers) { - std::string traceStr = headers->getAsString(X_QPID_TRACE); - if (traceStr.size()) { - std::vector<std::string> trace = split(traceStr, ", "); - - for (std::vector<std::string>::const_iterator i = excludes.begin(); i != excludes.end(); i++) { - for (std::vector<std::string>::const_iterator j = trace.begin(); j != trace.end(); j++) { - if (*i == *j) { - return true; - } - } - } - } - } - return false; + return expiryPolicy && expiryPolicy->hasExpired(*this); } -class CloneHeaderBody -{ -public: - void operator()(AMQFrame& f) - { - f.cloneBody(); - } -}; - -AMQHeaderBody* Message::getHeaderBody() +uint8_t Message::getPriority() const { - // expects lock to be held - if (copyHeaderOnWrite) { - CloneHeaderBody f; - frames.map_if(f, TypeFilter<HEADER_BODY>()); - copyHeaderOnWrite = false; - } - return frames.getHeaders(); + return getEncoding().getPriority(); } -void Message::addTraceId(const std::string& id) +bool Message::getIsManagementMessage() const { return isManagementMessage; } +void Message::setIsManagementMessage(bool b) { isManagementMessage = b; } +qpid::framing::SequenceNumber Message::getSequence() const { - sys::Mutex::ScopedLock l(lock); - if (isA<MessageTransferBody>()) { - FieldTable& headers = getModifiableProperties<MessageProperties>()->getApplicationHeaders(); - std::string trace = headers.getAsString(X_QPID_TRACE); - if (trace.empty()) { - headers.setString(X_QPID_TRACE, id); - } else if (trace.find(id) == std::string::npos) { - trace += ","; - trace += id; - headers.setString(X_QPID_TRACE, trace); - } - } + return sequence; } - -void Message::clearTrace() +void Message::setSequence(const qpid::framing::SequenceNumber& s) { - sys::Mutex::ScopedLock l(lock); - if (isA<MessageTransferBody>()) { - FieldTable& headers = getModifiableProperties<MessageProperties>()->getApplicationHeaders(); - std::string trace = headers.getAsString(X_QPID_TRACE); - if (!trace.empty()) { - headers.setString(X_QPID_TRACE, ""); - } - } + sequence = s; } -void Message::setTimestamp() +MessageState Message::getState() const { - sys::Mutex::ScopedLock l(lock); - DeliveryProperties* props = getModifiableProperties<DeliveryProperties>(); - time_t now = ::time(0); - props->setTimestamp(now); // AMQP-0.10: posix time_t - secs since Epoch + return state; } - -void Message::computeExpiration(const boost::intrusive_ptr<ExpiryPolicy>& e) +void Message::setState(MessageState s) { - sys::Mutex::ScopedLock l(lock); - DeliveryProperties* props = getModifiableProperties<DeliveryProperties>(); - if (props->getTtl()) { - // AMQP requires setting the expiration property to be posix - // time_t in seconds. TTL is in milliseconds - if (!props->getExpiration()) { - //only set expiration in delivery properties if not already set - time_t now = ::time(0); - props->setExpiration(now + (props->getTtl()/1000)); - } - if (e) { - // Use higher resolution time for the internal expiry calculation. - // Prevent overflow as a signed int64_t - Duration ttl(std::min(props->getTtl() * TIME_MSEC, - (uint64_t) std::numeric_limits<int64_t>::max())); - expiration = AbsTime(e->getCurrentTime(), ttl); - setExpiryPolicy(e); - } - } + state = s; } -void Message::adjustTtl() +const qpid::types::Variant::Map& Message::getAnnotations() const { - sys::Mutex::ScopedLock l(lock); - DeliveryProperties* props = getModifiableProperties<DeliveryProperties>(); - if (props->getTtl()) { - if (expiration < FAR_FUTURE) { - sys::AbsTime current( - expiryPolicy ? expiryPolicy->getCurrentTime() : sys::AbsTime::now()); - sys::Duration ttl(current, getExpiration()); - // convert from ns to ms; set to 1 if expired - props->setTtl(int64_t(ttl) >= 1000000 ? int64_t(ttl)/1000000 : 1); - } - } + return annotations; } -void Message::setRedelivered() +qpid::types::Variant Message::getAnnotation(const std::string& key) const { - sys::Mutex::ScopedLock l(lock); - getModifiableProperties<framing::DeliveryProperties>()->setRedelivered(true); + qpid::types::Variant::Map::const_iterator i = annotations.find(key); + if (i != annotations.end()) return i->second; + //FIXME: modify Encoding interface to allow retrieval of + //annotations of different types from the message data as received + //off the wire + return qpid::types::Variant(getEncoding().getAnnotationAsString(key)); } -void Message::insertCustomProperty(const std::string& key, int64_t value) +std::string Message::getUserId() const { - sys::Mutex::ScopedLock l(lock); - getModifiableProperties<MessageProperties>()->getApplicationHeaders().setInt64(key,value); + return encoding->getUserId(); } -void Message::insertCustomProperty(const std::string& key, const std::string& value) +Message::Encoding& Message::getEncoding() { - sys::Mutex::ScopedLock l(lock); - getModifiableProperties<MessageProperties>()->getApplicationHeaders().setString(key,value); + return *encoding; } - -void Message::removeCustomProperty(const std::string& key) +const Message::Encoding& Message::getEncoding() const { - sys::Mutex::ScopedLock l(lock); - getModifiableProperties<MessageProperties>()->getApplicationHeaders().erase(key); + return *encoding; } - -void Message::setExchange(const std::string& exchange) +Message::operator bool() const { - sys::Mutex::ScopedLock l(lock); - getModifiableProperties<DeliveryProperties>()->setExchange(exchange); + return encoding; } -void Message::clearApplicationHeadersFlag() +std::string Message::getContent() const { - sys::Mutex::ScopedLock l(lock); - getModifiableProperties<MessageProperties>()->clearApplicationHeadersFlag(); -} - -void Message::setExpiryPolicy(const boost::intrusive_ptr<ExpiryPolicy>& e) { - expiryPolicy = e; + return encoding->getContent(); } -bool Message::hasExpired() +std::string Message::getPropertyAsString(const std::string& key) const { - return expiryPolicy && expiryPolicy->hasExpired(*this); + return encoding->getPropertyAsString(key); } - namespace { -struct ScopedSet { - sys::Monitor& lock; - bool& flag; - ScopedSet(sys::Monitor& l, bool& f) : lock(l), flag(f) { - sys::Monitor::ScopedLock sl(lock); - flag = true; +class PropertyRetriever : public MapHandler +{ + public: + PropertyRetriever(const std::string& key) : name(key) {} + void handleVoid(const CharSequence&) {} + void handleUint8(const CharSequence& key, uint8_t value) { handle(key, value); } + void handleUint16(const CharSequence& key, uint16_t value) { handle(key, value); } + void handleUint32(const CharSequence& key, uint32_t value) { handle(key, value); } + void handleUint64(const CharSequence& key, uint64_t value) { handle(key, value); } + void handleInt8(const CharSequence& key, int8_t value) { handle(key, value); } + void handleInt16(const CharSequence& key, int16_t value) { handle(key, value); } + void handleInt32(const CharSequence& key, int32_t value) { handle(key, value); } + void handleInt64(const CharSequence& key, int64_t value) { handle(key, value); } + void handleFloat(const CharSequence& key, float value) { handle(key, value); } + void handleDouble(const CharSequence& key, double value) { handle(key, value); } + void handleString(const CharSequence& key, const CharSequence& value, const CharSequence& /*encoding*/) + { + if (matches(key)) result = std::string(value.data, value.size); } - ~ScopedSet(){ - sys::Monitor::ScopedLock sl(lock); - flag = false; - lock.notifyAll(); + qpid::types::Variant getResult() { return result; } + + private: + std::string name; + qpid::types::Variant result; + + bool matches(const CharSequence& key) + { + return ::strncmp(key.data, name.data(), std::min(key.size, name.size())) == 0; } -}; -} -void Message::allDequeuesComplete() { - ScopedSet ss(callbackLock, inCallback); - MessageCallback* cb = dequeueCallback; - if (cb && *cb) (*cb)(intrusive_ptr<Message>(this)); + template <typename T> void handle(const CharSequence& key, T value) + { + if (matches(key)) result = value; + } +}; } - -void Message::setDequeueCompleteCallback(MessageCallback& cb) { - sys::Mutex::ScopedLock l(callbackLock); - while (inCallback) callbackLock.wait(); - dequeueCallback = &cb; +qpid::types::Variant Message::getProperty(const std::string& key) const +{ + PropertyRetriever r(key); + encoding->processProperties(r); + return r.getResult(); } -void Message::resetDequeueCompleteCallback() { - sys::Mutex::ScopedLock l(callbackLock); - while (inCallback) callbackLock.wait(); - dequeueCallback = 0; +boost::intrusive_ptr<PersistableMessage> Message::getPersistentContext() const +{ + return persistentContext; } -uint8_t Message::getPriority() const { - sys::Mutex::ScopedLock l(lock); - return getAdapter().getPriority(frames); +void Message::processProperties(MapHandler& handler) const +{ + encoding->processProperties(handler); } -bool Message::getIsManagementMessage() const { return isManagementMessage; } -void Message::setIsManagementMessage(bool b) { isManagementMessage = b; } - }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/Message.h b/qpid/cpp/src/qpid/broker/Message.h index 90e4eec889..599819d7b6 100644 --- a/qpid/cpp/src/qpid/broker/Message.h +++ b/qpid/cpp/src/qpid/broker/Message.h @@ -23,194 +23,131 @@ */ #include "qpid/broker/BrokerImportExport.h" -#include "qpid/broker/PersistableMessage.h" -#include "qpid/broker/MessageAdapter.h" -#include "qpid/framing/amqp_types.h" -#include "qpid/sys/Monitor.h" #include "qpid/sys/Time.h" -#include <boost/function.hpp> -#include <boost/intrusive_ptr.hpp> -#include <boost/shared_ptr.hpp> -#include <memory> +#include "qpid/types/Variant.h" +//TODO: move the following out of framing or replace it +#include "qpid/framing/SequenceNumber.h" #include <string> #include <vector> -namespace qpid { - -namespace framing { -class AMQBody; -class AMQHeaderBody; -class FieldTable; -class SequenceNumber; -} +#include "qpid/RefCounted.h" +#include <boost/intrusive_ptr.hpp> +#include "qpid/broker/ExpiryPolicy.h" +#include "qpid/broker/PersistableMessage.h" +namespace qpid { namespace broker { class ConnectionToken; -class Exchange; -class ExchangeRegistry; -class MessageStore; -class Queue; -class ExpiryPolicy; +class MapHandler; + +enum MessageState +{ + AVAILABLE=1, + ACQUIRED=2, + DELETED=4, + UNAVAILABLE=8 +}; -class Message : public PersistableMessage { +class Message { public: - typedef boost::function<void (const boost::intrusive_ptr<Message>&)> MessageCallback; - - QPID_BROKER_EXTERN Message(const framing::SequenceNumber& id = framing::SequenceNumber()); + class Encoding : public AsyncCompletion + { + public: + virtual ~Encoding() {} + virtual std::string getRoutingKey() const = 0; + virtual bool isPersistent() const = 0; + virtual uint8_t getPriority() const = 0; + virtual uint64_t getContentSize() const = 0; + virtual std::string getPropertyAsString(const std::string& key) const = 0; + virtual std::string getAnnotationAsString(const std::string& key) const = 0; + virtual bool getTtl(uint64_t&) const = 0; + virtual bool hasExpiration() const = 0; + virtual std::string getContent() const = 0; + virtual void processProperties(MapHandler&) const = 0; + virtual std::string getUserId() const = 0; + }; + + QPID_BROKER_EXTERN Message(boost::intrusive_ptr<Encoding>, boost::intrusive_ptr<PersistableMessage>); + QPID_BROKER_EXTERN Message(); QPID_BROKER_EXTERN ~Message(); - uint64_t getPersistenceId() const { return persistenceId; } - void setPersistenceId(uint64_t _persistenceId) const { persistenceId = _persistenceId; } - - bool getRedelivered() const { return redelivered; } - void redeliver() { redelivered = true; } + bool isRedelivered() const { return deliveryCount > 1; } + void deliver() { ++deliveryCount; } + void undeliver() { --deliveryCount; } + int getDeliveryCount() const { return deliveryCount; } + void resetDeliveryCount() { deliveryCount = 0; } const ConnectionToken* getPublisher() const { return publisher; } void setPublisher(ConnectionToken* p) { publisher = p; } - const framing::SequenceNumber& getCommandId() { return frames.getId(); } - - QPID_BROKER_EXTERN uint64_t contentSize() const; QPID_BROKER_EXTERN std::string getRoutingKey() const; - const boost::shared_ptr<Exchange> getExchange(ExchangeRegistry&) const; - QPID_BROKER_EXTERN std::string getExchangeName() const; - bool isImmediate() const; - QPID_BROKER_EXTERN const framing::FieldTable* getApplicationHeaders() const; - QPID_BROKER_EXTERN std::string getAppId() const; QPID_BROKER_EXTERN bool isPersistent() const; - bool requiresAccept(); /** determine msg expiration time using the TTL value if present */ QPID_BROKER_EXTERN void computeExpiration(const boost::intrusive_ptr<ExpiryPolicy>& e); void setExpiryPolicy(const boost::intrusive_ptr<ExpiryPolicy>& e); - bool hasExpired(); + + bool hasExpired() const; sys::AbsTime getExpiration() const { return expiration; } void setExpiration(sys::AbsTime exp) { expiration = exp; } - void adjustTtl(); - void setRedelivered(); - QPID_BROKER_EXTERN void insertCustomProperty(const std::string& key, int64_t value); - QPID_BROKER_EXTERN void insertCustomProperty(const std::string& key, const std::string& value); - QPID_BROKER_EXTERN void removeCustomProperty(const std::string& key); - void setExchange(const std::string&); - void clearApplicationHeadersFlag(); + uint64_t getTtl() const; + /** set the timestamp delivery property to the current time-of-day */ QPID_BROKER_EXTERN void setTimestamp(); + QPID_BROKER_EXTERN uint64_t getTimestamp() const; - framing::FrameSet& getFrames() { return frames; } - const framing::FrameSet& getFrames() const { return frames; } - - template <class T> const T* getProperties() const { - const qpid::framing::AMQHeaderBody* p = frames.getHeaders(); - return p->get<T>(); - } - - template <class T> const T* hasProperties() const { - const qpid::framing::AMQHeaderBody* p = frames.getHeaders(); - return p->get<T>(); - } - - template <class T> void eraseProperties() { - qpid::framing::AMQHeaderBody* p = frames.getHeaders(); - p->erase<T>(); - } + QPID_BROKER_EXTERN void addAnnotation(const std::string& key, const qpid::types::Variant& value); + QPID_BROKER_EXTERN bool isExcluded(const std::vector<std::string>& excludes) const; + QPID_BROKER_EXTERN void addTraceId(const std::string& id); + QPID_BROKER_EXTERN void clearTrace(); + QPID_BROKER_EXTERN uint8_t getPriority() const; + QPID_BROKER_EXTERN std::string getPropertyAsString(const std::string& key) const; + QPID_BROKER_EXTERN qpid::types::Variant getProperty(const std::string& key) const; + void processProperties(MapHandler&) const; - template <class T> const T* getMethod() const { - return frames.as<T>(); - } + QPID_BROKER_EXTERN uint64_t getContentSize() const; - template <class T> T* getMethod() { - return frames.as<T>(); - } + Encoding& getEncoding(); + const Encoding& getEncoding() const; + QPID_BROKER_EXTERN operator bool() const; - template <class T> bool isA() const { - return frames.isA<T>(); - } - - uint32_t getRequiredCredit(); - - void encode(framing::Buffer& buffer) const; - void encodeContent(framing::Buffer& buffer) const; - - /** - * @returns the size of the buffer needed to encode this - * message in its entirety - */ - uint32_t encodedSize() const; - /** - * @returns the size of the buffer needed to encode the - * 'header' of this message (not just the header frame, - * but other meta data e.g.routing key and exchange) - */ - uint32_t encodedHeaderSize() const; - uint32_t encodedContentSize() const; - - QPID_BROKER_EXTERN void decodeHeader(framing::Buffer& buffer); - QPID_BROKER_EXTERN void decodeContent(framing::Buffer& buffer); - - void QPID_BROKER_EXTERN tryReleaseContent(); - void releaseContent(); - void releaseContent(MessageStore* s);//deprecated, use 'setStore(store); releaseContent();' instead - void destroy(); - - bool getContentFrame(const Queue& queue, framing::AMQFrame& frame, uint16_t maxContentSize, uint64_t offset) const; - QPID_BROKER_EXTERN void sendContent(const Queue& queue, framing::FrameHandler& out, uint16_t maxFrameSize) const; - void sendHeader(framing::FrameHandler& out, uint16_t maxFrameSize) const; + bool getIsManagementMessage() const; + void setIsManagementMessage(bool b); - QPID_BROKER_EXTERN bool isContentLoaded() const; + QPID_BROKER_EXTERN qpid::framing::SequenceNumber getSequence() const; + QPID_BROKER_EXTERN void setSequence(const qpid::framing::SequenceNumber&); - bool isExcluded(const std::vector<std::string>& excludes) const; - void addTraceId(const std::string& id); - void clearTrace(); + MessageState getState() const; + void setState(MessageState); - void forcePersistent(); - bool isForcedPersistent(); + QPID_BROKER_EXTERN qpid::types::Variant getAnnotation(const std::string& key) const; + QPID_BROKER_EXTERN const qpid::types::Variant::Map& getAnnotations() const; + std::string getUserId() const; - /** Call cb when dequeue is complete, may call immediately. Holds cb by reference. */ - void setDequeueCompleteCallback(MessageCallback& cb); - void resetDequeueCompleteCallback(); + QPID_BROKER_EXTERN std::string getContent() const;//TODO: may be better to get rid of this... - uint8_t getPriority() const; - bool getIsManagementMessage() const; - void setIsManagementMessage(bool b); + QPID_BROKER_EXTERN boost::intrusive_ptr<AsyncCompletion> getIngressCompletion() const; + QPID_BROKER_EXTERN boost::intrusive_ptr<PersistableMessage> getPersistentContext() const; private: - MessageAdapter& getAdapter() const; - void allDequeuesComplete(); - - mutable sys::Mutex lock; - framing::FrameSet frames; - mutable boost::shared_ptr<Exchange> exchange; - mutable uint64_t persistenceId; - bool redelivered; - bool loaded; - bool staged; - bool forcePersistentPolicy; // used to force message as durable, via a broker policy + boost::intrusive_ptr<Encoding> encoding; + boost::intrusive_ptr<PersistableMessage> persistentContext; + int deliveryCount; ConnectionToken* publisher; - mutable MessageAdapter* adapter; qpid::sys::AbsTime expiration; boost::intrusive_ptr<ExpiryPolicy> expiryPolicy; - - static TransferAdapter TRANSFER; - - mutable boost::intrusive_ptr<Message> empty; - - sys::Monitor callbackLock; - MessageCallback* dequeueCallback; - bool inCallback; - - uint32_t requiredCredit; + uint64_t timestamp; + qpid::types::Variant::Map annotations; bool isManagementMessage; - mutable bool copyHeaderOnWrite; - - /** - * Expects lock to be held - */ - template <class T> T* getModifiableProperties() { - return getHeaderBody()->get<T>(true); - } - qpid::framing::AMQHeaderBody* getHeaderBody(); + MessageState state; + qpid::framing::SequenceNumber sequence; + + void annotationsChanged(); }; +QPID_BROKER_EXTERN void encode(const Message&, std::string&); +QPID_BROKER_EXTERN void decode(const std::string&, Message&); + }} diff --git a/qpid/cpp/src/qpid/broker/MessageBuilder.cpp b/qpid/cpp/src/qpid/broker/MessageBuilder.cpp index a6d605c296..7cb99514d5 100644 --- a/qpid/cpp/src/qpid/broker/MessageBuilder.cpp +++ b/qpid/cpp/src/qpid/broker/MessageBuilder.cpp @@ -21,10 +21,11 @@ #include "qpid/broker/MessageBuilder.h" #include "qpid/broker/Message.h" -#include "qpid/broker/MessageStore.h" -#include "qpid/broker/NullMessageStore.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/framing/AMQFrame.h" +#include "qpid/framing/MessageTransferBody.h" #include "qpid/framing/reply_exceptions.h" +#include "qpid/log/Statement.h" using boost::intrusive_ptr; using namespace qpid::broker; @@ -36,8 +37,7 @@ namespace const std::string QPID_MANAGEMENT("qpid.management"); } -MessageBuilder::MessageBuilder(MessageStore* const _store) : - state(DORMANT), store(_store) {} +MessageBuilder::MessageBuilder() : state(DORMANT) {} void MessageBuilder::handle(AMQFrame& frame) { @@ -45,6 +45,7 @@ void MessageBuilder::handle(AMQFrame& frame) switch(state) { case METHOD: checkType(METHOD_BODY, type); + exchange = frame.castBody<qpid::framing::MessageTransferBody>()->getDestination(); state = HEADER; break; case HEADER: @@ -55,7 +56,9 @@ void MessageBuilder::handle(AMQFrame& frame) header.setBof(false); header.setEof(false); message->getFrames().append(header); - } else if (type != HEADER_BODY) { + } else if (type == HEADER_BODY) { + frame.castBody<AMQHeaderBody>()->get<DeliveryProperties>(true)->setExchange(exchange); + } else { throw CommandInvalidException( QPID_MSG("Invalid frame sequence for message, expected header or content got " << type_str(type) << ")")); @@ -73,14 +76,14 @@ void MessageBuilder::handle(AMQFrame& frame) void MessageBuilder::end() { + message->computeRequiredCredit(); message = 0; state = DORMANT; } void MessageBuilder::start(const SequenceNumber& id) { - message = intrusive_ptr<Message>(new Message(id)); - message->setStore(store); + message = intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer>(new qpid::broker::amqp_0_10::MessageTransfer(id)); state = METHOD; } @@ -112,3 +115,5 @@ void MessageBuilder::checkType(uint8_t expected, uint8_t actual) << type_str(expected) << " got " << type_str(actual) << ")")); } } + +boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> MessageBuilder::getMessage() { return message; } diff --git a/qpid/cpp/src/qpid/broker/MessageBuilder.h b/qpid/cpp/src/qpid/broker/MessageBuilder.h index b99b8efee6..5ca6fa0c66 100644 --- a/qpid/cpp/src/qpid/broker/MessageBuilder.h +++ b/qpid/cpp/src/qpid/broker/MessageBuilder.h @@ -30,21 +30,22 @@ namespace qpid { namespace broker { - class Message; - class MessageStore; + namespace amqp_0_10 { + class MessageTransfer; + } class QPID_BROKER_CLASS_EXTERN MessageBuilder : public framing::FrameHandler{ public: - QPID_BROKER_EXTERN MessageBuilder(MessageStore* const store); + QPID_BROKER_EXTERN MessageBuilder(); QPID_BROKER_EXTERN void handle(framing::AMQFrame& frame); - boost::intrusive_ptr<Message> getMessage() { return message; } + boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> getMessage(); QPID_BROKER_EXTERN void start(const framing::SequenceNumber& id); void end(); private: enum State {DORMANT, METHOD, HEADER, CONTENT}; State state; - boost::intrusive_ptr<Message> message; - MessageStore* const store; + boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> message; + std::string exchange; void checkType(uint8_t expected, uint8_t actual); }; diff --git a/qpid/cpp/src/qpid/broker/MessageDeque.cpp b/qpid/cpp/src/qpid/broker/MessageDeque.cpp index 83c8ca6868..1529d4ac94 100644 --- a/qpid/cpp/src/qpid/broker/MessageDeque.cpp +++ b/qpid/cpp/src/qpid/broker/MessageDeque.cpp @@ -19,218 +19,71 @@ * */ #include "qpid/broker/MessageDeque.h" -#include "qpid/broker/QueuedMessage.h" -#include "qpid/log/Statement.h" #include "assert.h" +#include "qpid/broker/Message.h" +#include "qpid/broker/QueueCursor.h" +#include "qpid/framing/SequenceNumber.h" +#include "qpid/log/Statement.h" namespace qpid { namespace broker { - -MessageDeque::MessageDeque() : available(0), head(0) {} - -size_t MessageDeque::index(const framing::SequenceNumber& position) -{ - //assuming a monotonic sequence, with no messages removed except - //from the ends of the deque, we can use the position to determin - //an index into the deque - if (messages.empty() || position < messages.front().position) return 0; - return position - messages.front().position; +namespace { +Message padding(qpid::framing::SequenceNumber id) { + Message m; + m.setState(DELETED); + m.setSequence(id); + return m; } - -bool MessageDeque::deleted(const QueuedMessage& m) -{ - size_t i = index(m.position); - if (i < messages.size()) { - QueuedMessage *qm = &messages[i]; - if (qm->status != QueuedMessage::DELETED) { - qm->status = QueuedMessage::DELETED; - qm->payload = 0; // message no longer needed - clean(); - return true; - } - } - return false; } -size_t MessageDeque::size() -{ - return available; -} +using qpid::framing::SequenceNumber; -QueuedMessage* MessageDeque::releasePtr(const QueuedMessage& message) -{ - size_t i = index(message.position); - if (i < messages.size()) { - QueuedMessage& m = messages[i]; - if (m.status == QueuedMessage::ACQUIRED) { - if (head > i) head = i; - m.status = QueuedMessage::AVAILABLE; - ++available; - return &messages[i]; - } - } else { - assert(0); - QPID_LOG(error, "Failed to release message at " << message.position << " " << message.payload->getFrames().getContent() << "; no such message (index=" << i << ", size=" << messages.size() << ")"); - } - return 0; -} +MessageDeque::MessageDeque() : messages(&padding) {} -void MessageDeque::release(const QueuedMessage& message) { releasePtr(message); } -bool MessageDeque::acquire(const framing::SequenceNumber& position, QueuedMessage& message) +bool MessageDeque::deleted(const QueueCursor& cursor) { - if (position < messages.front().position) return false; - size_t i = index(position); - if (i < messages.size()) { - QueuedMessage& temp = messages[i]; - if (temp.status == QueuedMessage::AVAILABLE) { - temp.status = QueuedMessage::ACQUIRED; - --available; - message = temp; - return true; - } - } - return false; + return messages.deleted(cursor); } -bool MessageDeque::find(const framing::SequenceNumber& position, QueuedMessage& message) +void MessageDeque::publish(const Message& added) { - size_t i = index(position); - if (i < messages.size()) { - message = messages[i]; - return true; - } else { - return false; - } + messages.publish(added); } -bool MessageDeque::browse(const framing::SequenceNumber& position, QueuedMessage& message, bool unacquired) +Message* MessageDeque::release(const QueueCursor& cursor) { - //get first message that is greater than position - size_t i = index(position + 1); - while (i < messages.size()) { - QueuedMessage& m = messages[i++]; - if (m.status == QueuedMessage::AVAILABLE || (!unacquired && m.status == QueuedMessage::ACQUIRED)) { - message = m; - return true; - } - } - return false; + return messages.release(cursor); } -bool MessageDeque::consume(QueuedMessage& message) +Message* MessageDeque::next(QueueCursor& cursor) { - while (head < messages.size()) { - QueuedMessage& i = messages[head++]; - if (i.status == QueuedMessage::AVAILABLE) { - i.status = QueuedMessage::ACQUIRED; - --available; - message = i; - return true; - } - } - return false; + return messages.next(cursor); } -namespace { -QueuedMessage padding(uint32_t pos) { - return QueuedMessage(0, 0, pos, QueuedMessage::DELETED); -} -} // namespace - -QueuedMessage* MessageDeque::pushPtr(const QueuedMessage& added) { - //add padding to prevent gaps in sequence, which break the index - //calculation (needed for queue replication) - while (messages.size() && (added.position - messages.back().position) > 1) - messages.push_back(padding(messages.back().position + 1)); - messages.push_back(added); - messages.back().status = QueuedMessage::AVAILABLE; - if (head >= messages.size()) head = messages.size() - 1; - ++available; - clean(); // QPID-4046: let producer help clean the backlog of deleted messages - return &messages.back(); -} - -bool MessageDeque::push(const QueuedMessage& added, QueuedMessage& /*not needed*/) { - pushPtr(added); - return false; // adding a message never causes one to be removed for deque -} - -void MessageDeque::updateAcquired(const QueuedMessage& acquired) +size_t MessageDeque::size() { - // Pad the front of the queue if necessary - while (messages.size() && (acquired.position < messages.front().position)) - messages.push_front(padding(uint32_t(messages.front().position) - 1)); - size_t i = index(acquired.position); - if (i < messages.size()) { // Replace an existing padding message - assert(messages[i].status == QueuedMessage::DELETED); - messages[i] = acquired; - messages[i].status = QueuedMessage::ACQUIRED; - } - else { // Push to the back - // Pad the back of the queue if necessary - while (messages.size() && (acquired.position - messages.back().position) > 1) - messages.push_back(padding(messages.back().position + 1)); - assert(!messages.size() || (acquired.position - messages.back().position) == 1); - messages.push_back(acquired); - messages.back().status = QueuedMessage::ACQUIRED; - } + return messages.size(); } -namespace { -bool isNotDeleted(const QueuedMessage& qm) { return qm.status != QueuedMessage::DELETED; } -} // namespace - -void MessageDeque::setPosition(const framing::SequenceNumber& n) { - size_t i = index(n+1); - if (i >= messages.size()) return; // Nothing to do. - - // Assertion to verify the precondition: no messaages after n. - assert(std::find_if(messages.begin()+i, messages.end(), &isNotDeleted) == - messages.end()); - messages.erase(messages.begin()+i, messages.end()); - if (head >= messages.size()) head = messages.size() - 1; - // Re-count the available messages - available = 0; - for (Deque::iterator i = messages.begin(); i != messages.end(); ++i) { - if (i->status == QueuedMessage::AVAILABLE) ++available; - } +Message* MessageDeque::find(const framing::SequenceNumber& position, QueueCursor* cursor) +{ + return messages.find(position, cursor); } -void MessageDeque::clean() +Message* MessageDeque::find(const QueueCursor& cursor) { - // QPID-4046: If a queue has multiple consumers, then it is possible for a large - // collection of deleted messages to build up. Limit the number of messages cleaned - // up on each call to clean(). - size_t count = 0; - while (messages.size() && messages.front().status == QueuedMessage::DELETED && count < 10) { - messages.pop_front(); - count += 1; - } - head = (head > count) ? head - count : 0; + return messages.find(cursor); } void MessageDeque::foreach(Functor f) { - for (Deque::iterator i = messages.begin(); i != messages.end(); ++i) { - if (i->status == QueuedMessage::AVAILABLE) { - f(*i); - } - } + messages.foreach(f); } -void MessageDeque::removeIf(Predicate p) +void MessageDeque::resetCursors() { - for (Deque::iterator i = messages.begin(); i != messages.end(); ++i) { - if (i->status == QueuedMessage::AVAILABLE && p(*i)) { - //Use special status for this as messages are not yet - //dequeued, but should not be considered on the queue - //either (used for purging and moving) - i->status = QueuedMessage::REMOVED; - --available; - } - } - clean(); + messages.resetCursors(); } }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/MessageDeque.h b/qpid/cpp/src/qpid/broker/MessageDeque.h index c5670b2a72..ec67476926 100644 --- a/qpid/cpp/src/qpid/broker/MessageDeque.h +++ b/qpid/cpp/src/qpid/broker/MessageDeque.h @@ -22,8 +22,7 @@ * */ #include "qpid/broker/Messages.h" -#include "qpid/broker/QueuedMessage.h" -#include <deque> +#include "qpid/broker/IndexedDeque.h" namespace qpid { namespace broker { @@ -36,31 +35,20 @@ class MessageDeque : public Messages public: MessageDeque(); size_t size(); - bool deleted(const QueuedMessage&); - void release(const QueuedMessage&); - bool acquire(const framing::SequenceNumber&, QueuedMessage&); - bool find(const framing::SequenceNumber&, QueuedMessage&); - bool browse(const framing::SequenceNumber&, QueuedMessage&, bool); - bool consume(QueuedMessage&); - bool push(const QueuedMessage& added, QueuedMessage& removed); - void updateAcquired(const QueuedMessage& acquired); - void setPosition(const framing::SequenceNumber&); + bool deleted(const QueueCursor&); + void publish(const Message& added); + Message* next(QueueCursor&); + Message* release(const QueueCursor& cursor); + Message* find(const QueueCursor&); + Message* find(const framing::SequenceNumber&, QueueCursor*); + void foreach(Functor); - void removeIf(Predicate); - // For use by other Messages implementations that use MessageDeque as a FIFO index - // and keep pointers to its elements in their own indexing strctures. - void clean(); - QueuedMessage* releasePtr(const QueuedMessage&); - QueuedMessage* pushPtr(const QueuedMessage& added); + void resetCursors(); private: - typedef std::deque<QueuedMessage> Deque; + typedef IndexedDeque<Message> Deque; Deque messages; - size_t available; - size_t head; - - size_t index(const framing::SequenceNumber&); }; }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/MessageDistributor.h b/qpid/cpp/src/qpid/broker/MessageDistributor.h index 090393c160..c11e4495a1 100644 --- a/qpid/cpp/src/qpid/broker/MessageDistributor.h +++ b/qpid/cpp/src/qpid/broker/MessageDistributor.h @@ -21,51 +21,28 @@ * under the License. * */ - +#include "qpid/types/Variant.h" /** Abstraction used by Queue to determine the next "most desirable" message to provide to * a particular consuming client */ - -#include "qpid/broker/Consumer.h" - namespace qpid { namespace broker { -struct QueuedMessage; +class Message; class MessageDistributor { public: virtual ~MessageDistributor() {}; - /** Locking Note: all methods assume the caller is holding the Queue::messageLock - * during the method call. - */ - - /** Determine the next message available for consumption by the consumer - * @param consumer the consumer that needs a message to consume - * @param next set to the next message that the consumer may consume. - * @return true if message is available and next is set - */ - virtual bool nextConsumableMessage( Consumer::shared_ptr& consumer, - QueuedMessage& next ) = 0; - - /** Allow the comsumer to take ownership of the given message. + /** + * Determine whether the named consumer can take ownership of the specified message. * @param consumer the name of the consumer that is attempting to acquire the message - * @param qm the message to be acquired, previously returned from nextConsumableMessage() + * @param target the message to be acquired * @return true if ownership is permitted, false if ownership cannot be assigned. */ - virtual bool allocate( const std::string& consumer, - const QueuedMessage& target) = 0; - - /** Determine the next message available for browsing by the consumer - * @param consumer the consumer that is browsing the queue - * @param next set to the next message that the consumer may browse. - * @return true if a message is available and next is returned - */ - virtual bool nextBrowsableMessage( Consumer::shared_ptr& consumer, - QueuedMessage& next ) = 0; + virtual bool acquire(const std::string& consumer, Message& target) = 0; /** hook to add any interesting management state to the status map */ virtual void query(qpid::types::Variant::Map&) const = 0; diff --git a/qpid/cpp/src/qpid/broker/MessageGroupManager.cpp b/qpid/cpp/src/qpid/broker/MessageGroupManager.cpp index 15cd56a676..47e40a4794 100644 --- a/qpid/cpp/src/qpid/broker/MessageGroupManager.cpp +++ b/qpid/cpp/src/qpid/broker/MessageGroupManager.cpp @@ -1,4 +1,4 @@ -/* + /* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -20,10 +20,16 @@ */ #include "qpid/broker/MessageGroupManager.h" - -#include "qpid/broker/Queue.h" +#include "qpid/broker/Message.h" +#include "qpid/broker/Messages.h" +#include "qpid/broker/MessageDeque.h" +#include "qpid/broker/QueueSettings.h" +#include "qpid/framing/Array.h" +#include "qpid/framing/DeliveryProperties.h" #include "qpid/framing/FieldTable.h" #include "qpid/framing/FieldValue.h" +#include "qpid/framing/TypeCode.h" +#include "qpid/types/Variant.h" #include "qpid/log/Statement.h" #include "qpid/types/Variant.h" @@ -75,24 +81,16 @@ void MessageGroupManager::disown( GroupState& state ) freeGroups[state.members.front().position] = &state; } -MessageGroupManager::GroupState& MessageGroupManager::findGroup( const QueuedMessage& qm ) +MessageGroupManager::GroupState& MessageGroupManager::findGroup( const Message& m ) { - uint32_t thisMsg = qm.position.getValue(); + uint32_t thisMsg = m.getSequence().getValue(); if (cachedGroup && lastMsg == thisMsg) { hits++; return *cachedGroup; } - std::string group = defaultGroupId; - const qpid::framing::FieldTable* headers = qm.payload->getApplicationHeaders(); - if (headers) { - qpid::framing::FieldTable::ValuePtr id = headers->get( groupIdHeader ); - if (id && id->convertsTo<std::string>()) { - std::string tmp = id->get<std::string>(); - if (!tmp.empty()) // empty group is reserved - group = tmp; - } - } + std::string group = m.getPropertyAsString(groupIdHeader); + if (group.empty()) group = defaultGroupId; //empty group is reserved if (cachedGroup && group == lastGroup) { hits++; @@ -112,48 +110,48 @@ MessageGroupManager::GroupState& MessageGroupManager::findGroup( const QueuedMes } -void MessageGroupManager::enqueued( const QueuedMessage& qm ) +void MessageGroupManager::enqueued( const Message& m ) { // @todo KAG optimization - store reference to group state in QueuedMessage // issue: const-ness?? - GroupState& state = findGroup(qm); - GroupState::MessageState mState(qm.position); + GroupState& state = findGroup(m); + GroupState::MessageState mState(m.getSequence()); state.members.push_back(mState); uint32_t total = state.members.size(); QPID_LOG( trace, "group queue " << qName << ": added message to group id=" << state.group << " total=" << total ); if (total == 1) { // newly created group, no owner - assert(freeGroups.find(qm.position) == freeGroups.end()); - freeGroups[qm.position] = &state; + assert(freeGroups.find(m.getSequence()) == freeGroups.end()); + freeGroups[m.getSequence()] = &state; } } -void MessageGroupManager::acquired( const QueuedMessage& qm ) +void MessageGroupManager::acquired( const Message& m ) { // @todo KAG avoid lookup: retrieve direct reference to group state from QueuedMessage // issue: const-ness?? - GroupState& state = findGroup(qm); - GroupState::MessageFifo::iterator m = state.findMsg(qm.position); - assert(m != state.members.end()); - m->acquired = true; + GroupState& state = findGroup(m); + GroupState::MessageFifo::iterator gm = state.findMsg(m.getSequence()); + assert(gm != state.members.end()); + gm->acquired = true; state.acquired += 1; QPID_LOG( trace, "group queue " << qName << ": acquired message in group id=" << state.group << " acquired=" << state.acquired ); } -void MessageGroupManager::requeued( const QueuedMessage& qm ) +void MessageGroupManager::requeued( const Message& m ) { // @todo KAG avoid lookup: retrieve direct reference to group state from QueuedMessage // issue: const-ness?? - GroupState& state = findGroup(qm); + GroupState& state = findGroup(m); assert( state.acquired != 0 ); state.acquired -= 1; - GroupState::MessageFifo::iterator m = state.findMsg(qm.position); - assert(m != state.members.end()); - m->acquired = false; + GroupState::MessageFifo::iterator i = state.findMsg(m.getSequence()); + assert(i != state.members.end()); + i->acquired = false; if (state.acquired == 0 && state.owned()) { QPID_LOG( trace, "group queue " << qName << ": consumer name=" << state.owner << " released group id=" << state.group); @@ -164,14 +162,14 @@ void MessageGroupManager::requeued( const QueuedMessage& qm ) } -void MessageGroupManager::dequeued( const QueuedMessage& qm ) +void MessageGroupManager::dequeued( const Message& m ) { // @todo KAG avoid lookup: retrieve direct reference to group state from QueuedMessage // issue: const-ness?? - GroupState& state = findGroup(qm); - GroupState::MessageFifo::iterator m = state.findMsg(qm.position); - assert(m != state.members.end()); - if (m->acquired) { + GroupState& state = findGroup(m); + GroupState::MessageFifo::iterator i = state.findMsg(m.getSequence()); + assert(i != state.members.end()); + if (i->acquired) { assert( state.acquired != 0 ); state.acquired -= 1; } @@ -179,7 +177,7 @@ void MessageGroupManager::dequeued( const QueuedMessage& qm ) // special case if qm is first (oldest) message in the group: // may need to re-insert it back on the freeGroups list, as the index will change bool reFreeNeeded = false; - if (m == state.members.begin()) { + if (i == state.members.begin()) { if (!state.owned()) { // will be on the freeGroups list if mgmt is dequeueing rather than a consumer! // if on freelist, it is indexed by first member, which is about to be removed! @@ -188,7 +186,7 @@ void MessageGroupManager::dequeued( const QueuedMessage& qm ) } state.members.pop_front(); } else { - state.members.erase(m); + state.members.erase(i); } uint32_t total = state.members.size(); @@ -206,6 +204,12 @@ void MessageGroupManager::dequeued( const QueuedMessage& qm ) QPID_LOG( trace, "group queue " << qName << ": consumer name=" << state.owner << " released group id=" << state.group); disown(state); + MessageDeque* md = dynamic_cast<MessageDeque*>(&messages); + if (md) { + md->resetCursors(); + } else { + QPID_LOG(warning, "Could not reset cursors for message group, unexpected container type"); + } } else if (reFreeNeeded) { disown(state); } @@ -215,55 +219,27 @@ MessageGroupManager::~MessageGroupManager() { QPID_LOG( debug, "group queue " << qName << " cache results: hits=" << hits << " misses=" << misses ); } -bool MessageGroupManager::nextConsumableMessage( Consumer::shared_ptr& c, QueuedMessage& next ) + +bool MessageGroupManager::acquire(const std::string& consumer, Message& m) { - if (!messages.size()) - return false; + if (m.getState() == AVAILABLE) { + // @todo KAG avoid lookup: retrieve direct reference to group state from QueuedMessage + GroupState& state = findGroup(m); - next.position = c->getPosition(); - if (!freeGroups.empty()) { - const framing::SequenceNumber& nextFree = freeGroups.begin()->first; - if (nextFree <= next.position) { // take oldest free - next.position = nextFree; - --next.position; + if (!state.owned()) { + own( state, consumer ); + QPID_LOG( trace, "group queue " << qName << + ": consumer name=" << consumer << " has acquired group id=" << state.group); } - } - - while (messages.browse( next.position, next, true )) { - GroupState& group = findGroup(next); - if (!group.owned()) { - //TODO: make acquire more efficient when we already have the message in question - if (group.members.front().position == next.position && messages.acquire(next.position, next)) { // only take from head! - return true; - } - QPID_LOG(debug, "Skipping " << next.position << " since group " << group.group - << "'s head message still pending. pos=" << group.members.front().position); - } else if (group.owner == c->getName() && messages.acquire(next.position, next)) { + if (state.owner == consumer) { + m.setState(ACQUIRED); return true; + } else { + return false; } + } else { + return false; } - return false; -} - - -bool MessageGroupManager::allocate(const std::string& consumer, const QueuedMessage& qm) -{ - // @todo KAG avoid lookup: retrieve direct reference to group state from QueuedMessage - GroupState& state = findGroup(qm); - - if (!state.owned()) { - own( state, consumer ); - QPID_LOG( trace, "group queue " << qName << - ": consumer name=" << consumer << " has acquired group id=" << state.group); - return true; - } - return state.owner == consumer; -} - -bool MessageGroupManager::nextBrowsableMessage( Consumer::shared_ptr& c, QueuedMessage& next ) -{ - // browse: allow access to any available msg, regardless of group ownership (?ok?) - return messages.browse(c->getPosition(), next, false); } void MessageGroupManager::query(qpid::types::Variant::Map& status) const @@ -296,11 +272,9 @@ void MessageGroupManager::query(qpid::types::Variant::Map& status) const // set the timestamp to the arrival timestamp of the oldest (HEAD) message, if present info[GROUP_TIMESTAMP] = 0; if (g->second.members.size() != 0) { - QueuedMessage qm; - if (messages.find(g->second.members.front().position, qm) && - qm.payload && - qm.payload->hasProperties<framing::DeliveryProperties>()) { - info[GROUP_TIMESTAMP] = qm.payload->getProperties<framing::DeliveryProperties>()->getTimestamp(); + Message* m = messages.find(g->second.members.front().position, 0); + if (m && m->getTimestamp()) { + info[GROUP_TIMESTAMP] = m->getTimestamp(); } } info[GROUP_CONSUMER] = g->second.owner; @@ -313,33 +287,13 @@ void MessageGroupManager::query(qpid::types::Variant::Map& status) const boost::shared_ptr<MessageGroupManager> MessageGroupManager::create( const std::string& qName, Messages& messages, - const qpid::framing::FieldTable& settings ) + const QueueSettings& settings ) { - boost::shared_ptr<MessageGroupManager> empty; - - if (settings.isSet(qpidMessageGroupKey)) { - - // @todo: remove once "sticky" consumers are supported - see QPID-3347 - if (!settings.isSet(qpidSharedGroup)) { - QPID_LOG( error, "Only shared groups are supported in this version of the broker. Use '--shared-groups' in qpid-config." ); - return empty; - } - - std::string headerKey = settings.getAsString(qpidMessageGroupKey); - if (headerKey.empty()) { - QPID_LOG( error, "A Message Group header key must be configured, queue=" << qName); - return empty; - } - unsigned int timestamp = settings.getAsInt(qpidMessageGroupTimestamp); - - boost::shared_ptr<MessageGroupManager> manager( new MessageGroupManager( headerKey, qName, messages, timestamp ) ); - - QPID_LOG( debug, "Configured Queue '" << qName << - "' for message grouping using header key '" << headerKey << "'" << - " (timestamp=" << timestamp << ")"); - return manager; - } - return empty; + boost::shared_ptr<MessageGroupManager> manager( new MessageGroupManager( settings.groupKey, qName, messages, settings.addTimestamp ) ); + QPID_LOG( debug, "Configured Queue '" << qName << + "' for message grouping using header key '" << settings.groupKey << "'" << + " (timestamp=" << settings.addTimestamp << ")"); + return manager; } std::string MessageGroupManager::defaultGroupId; diff --git a/qpid/cpp/src/qpid/broker/MessageGroupManager.h b/qpid/cpp/src/qpid/broker/MessageGroupManager.h index 2dd97ea2ff..fe39e007b5 100644 --- a/qpid/cpp/src/qpid/broker/MessageGroupManager.h +++ b/qpid/cpp/src/qpid/broker/MessageGroupManager.h @@ -24,8 +24,10 @@ /* for managing message grouping on Queues */ +#include "qpid/broker/BrokerImportExport.h" #include "qpid/broker/StatefulQueueObserver.h" #include "qpid/broker/MessageDistributor.h" +#include "qpid/framing/SequenceNumber.h" #include "qpid/sys/unordered_map.h" #include <deque> @@ -34,6 +36,7 @@ namespace qpid { namespace broker { class QueueObserver; +struct QueueSettings; class MessageDistributor; class Messages; @@ -76,11 +79,7 @@ class MessageGroupManager : public StatefulQueueObserver, public MessageDistribu GroupFifo freeGroups; // ordered by oldest free msg //Consumers consumers; // index: consumer name - static const std::string qpidMessageGroupKey; - static const std::string qpidSharedGroup; // if specified, one group can be consumed by multiple receivers - static const std::string qpidMessageGroupTimestamp; - - GroupState& findGroup( const QueuedMessage& qm ); + GroupState& findGroup( const Message& m ); unsigned long hits, misses; // for debug uint32_t lastMsg; std::string lastGroup; @@ -91,11 +90,14 @@ class MessageGroupManager : public StatefulQueueObserver, public MessageDistribu void disown( GroupState& state ); public: + static const std::string qpidMessageGroupKey; + static const std::string qpidSharedGroup; // if specified, one group can be consumed by multiple receivers + static const std::string qpidMessageGroupTimestamp; static QPID_BROKER_EXTERN void setDefaults(const std::string& groupId); static boost::shared_ptr<MessageGroupManager> create( const std::string& qName, Messages& messages, - const qpid::framing::FieldTable& settings ); + const QueueSettings& settings ); MessageGroupManager(const std::string& header, const std::string& _qName, Messages& container, unsigned int _timestamp=0 ) @@ -106,22 +108,20 @@ class MessageGroupManager : public StatefulQueueObserver, public MessageDistribu virtual ~MessageGroupManager(); // QueueObserver iface - void enqueued( const QueuedMessage& qm ); - void acquired( const QueuedMessage& qm ); - void requeued( const QueuedMessage& qm ); - void dequeued( const QueuedMessage& qm ); + void enqueued( const Message& qm ); + void acquired( const Message& qm ); + void requeued( const Message& qm ); + void dequeued( const Message& qm ); void consumerAdded( const Consumer& ) {}; void consumerRemoved( const Consumer& ) {}; void getState(qpid::framing::FieldTable& state ) const; void setState(const qpid::framing::FieldTable&); // MessageDistributor iface - bool nextConsumableMessage(Consumer::shared_ptr& c, QueuedMessage& next); - bool allocate(const std::string& c, const QueuedMessage& qm); - bool nextBrowsableMessage(Consumer::shared_ptr& c, QueuedMessage& next); + bool acquire(const std::string& c, Message& ); void query(qpid::types::Variant::Map&) const; - bool match(const qpid::types::Variant::Map*, const QueuedMessage&) const; + bool match(const qpid::types::Variant::Map*, const Message&) const; }; }} diff --git a/qpid/cpp/src/qpid/broker/MessageMap.cpp b/qpid/cpp/src/qpid/broker/MessageMap.cpp index 592f3fefde..4cdd83c9aa 100644 --- a/qpid/cpp/src/qpid/broker/MessageMap.cpp +++ b/qpid/cpp/src/qpid/broker/MessageMap.cpp @@ -19,7 +19,8 @@ * */ #include "qpid/broker/MessageMap.h" -#include "qpid/broker/QueuedMessage.h" +#include "qpid/broker/Message.h" +#include "qpid/broker/QueueCursor.h" #include "qpid/log/Statement.h" #include <algorithm> @@ -29,29 +30,17 @@ namespace { const std::string EMPTY; } -bool MessageMap::deleted(const QueuedMessage& message) -{ - Ordering::iterator i = messages.find(message.position); - if (i != messages.end()) { - erase(i); - return true; - } else { - return false; - } -} -std::string MessageMap::getKey(const QueuedMessage& message) +std::string MessageMap::getKey(const Message& message) { - const framing::FieldTable* ft = message.payload->getApplicationHeaders(); - if (ft) return ft->getAsString(key); - else return EMPTY; + return message.getPropertyAsString(key); } size_t MessageMap::size() { size_t count(0); for (Ordering::iterator i = messages.begin(); i != messages.end(); ++i) { - if (i->second.status == QueuedMessage::AVAILABLE) ++count; + if (i->second.getState() == AVAILABLE) ++count; } return count; } @@ -61,116 +50,103 @@ bool MessageMap::empty() return size() == 0;//TODO: more efficient implementation } -void MessageMap::release(const QueuedMessage& message) +bool MessageMap::deleted(const QueueCursor& cursor) { - Ordering::iterator i = messages.find(message.position); - if (i != messages.end() && i->second.status == QueuedMessage::ACQUIRED) { - i->second.status = QueuedMessage::AVAILABLE; - } -} - -bool MessageMap::acquire(const framing::SequenceNumber& position, QueuedMessage& message) -{ - Ordering::iterator i = messages.find(position); - if (i != messages.end() && i->second.status == QueuedMessage::AVAILABLE) { - i->second.status = QueuedMessage::ACQUIRED; - message = i->second; + Ordering::iterator i = messages.find(cursor.position); + if (i != messages.end()) { + erase(i); return true; } else { return false; } } -bool MessageMap::find(const framing::SequenceNumber& position, QueuedMessage& message) +Message* MessageMap::find(const QueueCursor& cursor) { - Ordering::iterator i = messages.find(position); - if (i != messages.end() && i->second.status == QueuedMessage::AVAILABLE) { - message = i->second; - return true; - } else { - return false; - } + if (cursor.valid) return find(cursor.position, 0); + else return 0; } -bool MessageMap::browse(const framing::SequenceNumber& position, QueuedMessage& message, bool unacquired) +Message* MessageMap::find(const framing::SequenceNumber& position, QueueCursor* cursor) { - Ordering::iterator i = messages.lower_bound(position+1); - if (i != messages.end() && (i->second.status == QueuedMessage::AVAILABLE || (!unacquired && i->second.status == QueuedMessage::ACQUIRED))) { - message = i->second; - return true; + Ordering::iterator i = messages.lower_bound(position); + if (i != messages.end()) { + if (cursor) cursor->setPosition(i->first, version); + if (i->first == position) return &(i->second); + else return 0; } else { - return false; + //there is no message whose sequence is greater than position, + //i.e. haven't got there yet + if (cursor) cursor->setPosition(position, version); + return 0; } } -bool MessageMap::consume(QueuedMessage& message) +Message* MessageMap::next(QueueCursor& cursor) { - for (Ordering::iterator i = messages.begin(); i != messages.end(); ++i) { - if (i->second.status == QueuedMessage::AVAILABLE) { - i->second.status = QueuedMessage::ACQUIRED; - message = i->second; - return true; + Ordering::iterator i; + if (!cursor.valid) i = messages.begin(); //start with oldest message + else i = messages.upper_bound(cursor.position); //get first message that is greater than position + + while (i != messages.end()) { + Message& m = i->second; + cursor.setPosition(m.getSequence(), version); + if (cursor.check(m)) { + return &m; + } else { + ++i; } } - return false; + return 0; } -const QueuedMessage& MessageMap::replace(const QueuedMessage& original, const QueuedMessage& update) +const Message& MessageMap::replace(const Message& original, const Message& update) { - messages.erase(original.position); - messages[update.position] = update; - return update; + messages.erase(original.getSequence()); + std::pair<Ordering::iterator, bool> i = messages.insert(Ordering::value_type(update.getSequence(), update)); + i.first->second.setState(AVAILABLE); + return i.first->second; } -bool MessageMap::push(const QueuedMessage& added, QueuedMessage& removed) +void MessageMap::publish(const Message& added) +{ + Message dummy; + update(added, dummy); +} + +bool MessageMap::update(const Message& added, Message& removed) { std::pair<Index::iterator, bool> result = index.insert(Index::value_type(getKey(added), added)); if (result.second) { //there was no previous message for this key; nothing needs to //be removed, just add the message into its correct position - QueuedMessage& a = messages[added.position]; - a = added; - a.status = QueuedMessage::AVAILABLE; - QPID_LOG(debug, "Added message " << a); + messages.insert(Ordering::value_type(added.getSequence(), added)).first->second.setState(AVAILABLE); return false; } else { //there is already a message with that key which needs to be replaced removed = result.first->second; result.first->second = replace(result.first->second, added); - result.first->second.status = QueuedMessage::AVAILABLE; - QPID_LOG(debug, "Displaced message " << removed << " with " << result.first->second << ": " << result.first->first); + result.first->second.setState(AVAILABLE); + QPID_LOG(debug, "Displaced message at " << removed.getSequence() << " with " << result.first->second.getSequence() << ": " << result.first->first); return true; } } -void MessageMap::setPosition(const framing::SequenceNumber& seq) { - // Nothing to do, just assert that the precondition is respected and there - // are no undeleted messages after seq. - (void) seq; assert(messages.empty() || (--messages.end())->first <= seq); -} - -void MessageMap::foreach(Functor f) +Message* MessageMap::release(const QueueCursor& cursor) { - for (Ordering::iterator i = messages.begin(); i != messages.end(); ++i) { - if (i->second.status == QueuedMessage::AVAILABLE) f(i->second); + Ordering::iterator i = messages.find(cursor.position); + if (i != messages.end()) { + i->second.setState(AVAILABLE); + return &i->second; + } else { + return 0; } } -void MessageMap::removeIf(Predicate p) +void MessageMap::foreach(Functor f) { - for (Ordering::iterator i = messages.begin(); i != messages.end();) { - if (i->second.status == QueuedMessage::AVAILABLE && p(i->second)) { - index.erase(getKey(i->second)); - //Note: Removing from messages means that the subsequent - //call to deleted() for the same message will return - //false. At present that is not a problem. If this were - //changed to hold onto the message until dequeued - //(e.g. with REMOVED state), then the erase() below would - //need to take that into account. - messages.erase(i++); - } else { - ++i; - } + for (Ordering::iterator i = messages.begin(); i != messages.end(); ++i) { + if (i->second.getState() == AVAILABLE) f(i->second); } } @@ -180,6 +156,6 @@ void MessageMap::erase(Ordering::iterator i) messages.erase(i); } -MessageMap::MessageMap(const std::string& k) : key(k) {} +MessageMap::MessageMap(const std::string& k) : key(k), version(0) {} }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/MessageMap.h b/qpid/cpp/src/qpid/broker/MessageMap.h index 1f0481cb6b..c30606d0ff 100644 --- a/qpid/cpp/src/qpid/broker/MessageMap.h +++ b/qpid/cpp/src/qpid/broker/MessageMap.h @@ -6,7 +6,7 @@ * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information -o * regarding copyright ownership. The ASF licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at @@ -22,6 +22,7 @@ o * regarding copyright ownership. The ASF licenses this file * */ #include "qpid/broker/Messages.h" +#include "qpid/broker/Message.h" #include "qpid/framing/SequenceNumber.h" #include <map> #include <string> @@ -38,32 +39,31 @@ class MessageMap : public Messages { public: MessageMap(const std::string& key); - virtual ~MessageMap() {} size_t size(); bool empty(); - virtual bool deleted(const QueuedMessage&); - void release(const QueuedMessage&); - virtual bool acquire(const framing::SequenceNumber&, QueuedMessage&); - bool find(const framing::SequenceNumber&, QueuedMessage&); - virtual bool browse(const framing::SequenceNumber&, QueuedMessage&, bool); - bool consume(QueuedMessage&); - virtual bool push(const QueuedMessage& added, QueuedMessage& removed); - void setPosition(const framing::SequenceNumber&); + bool deleted(const QueueCursor&); + void publish(const Message& added);//use update instead to get replaced message + Message* next(QueueCursor&); + Message* release(const QueueCursor& cursor); + Message* find(const QueueCursor&); + Message* find(const framing::SequenceNumber&, QueueCursor*); void foreach(Functor); - virtual void removeIf(Predicate); + + bool update(const Message& added, Message& removed); protected: - typedef std::map<std::string, QueuedMessage> Index; - typedef std::map<framing::SequenceNumber, QueuedMessage> Ordering; + typedef std::map<std::string, Message> Index; + typedef std::map<framing::SequenceNumber, Message> Ordering; const std::string key; Index index; Ordering messages; + int32_t version; - std::string getKey(const QueuedMessage&); - virtual const QueuedMessage& replace(const QueuedMessage&, const QueuedMessage&); + std::string getKey(const Message&); + virtual const Message& replace(const Message&, const Message&); void erase(Ordering::iterator); }; }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/Messages.h b/qpid/cpp/src/qpid/broker/Messages.h index 45f5e6cd81..a94ac7e0bf 100644 --- a/qpid/cpp/src/qpid/broker/Messages.h +++ b/qpid/cpp/src/qpid/broker/Messages.h @@ -29,7 +29,8 @@ namespace framing { class SequenceNumber; } namespace broker { -struct QueuedMessage; +class Message; +class QueueCursor; /** * This interface abstracts out the access to the messages held for @@ -39,8 +40,7 @@ struct QueuedMessage; class Messages { public: - typedef boost::function1<void, QueuedMessage&> Functor; - typedef boost::function1<bool, QueuedMessage&> Predicate; + typedef boost::function1<void, Message&> Functor; virtual ~Messages() {} /** @@ -51,47 +51,44 @@ class Messages /** * Called when a message is deleted from the queue. */ - virtual bool deleted(const QueuedMessage&) = 0; + virtual bool deleted(const QueueCursor&) = 0; /** - * Releases an acquired message, making it available again. + * Makes a message available. */ - virtual void release(const QueuedMessage&) = 0; + virtual void publish(const Message& added) = 0; /** - * Acquire the message at the specified position, returning true - * if found, false otherwise. The acquired message is passed back - * via the second parameter. - */ - virtual bool acquire(const framing::SequenceNumber&, QueuedMessage&) = 0; - /** - * Find the message at the specified position, returning true if - * found, false otherwise. The matched message is passed back via - * the second parameter. + * Retrieve the next message for the given cursor. A reference to + * the message is passed back via the second parameter. + * + * @return a pointer to the message if there is one, in which case + * the cursor that points to it is assigned to cursor; null + * otherwise. */ - virtual bool find(const framing::SequenceNumber&, QueuedMessage&) = 0; + virtual Message* next(QueueCursor& cursor) = 0; + /** - * Retrieve the next message to be given to a browsing - * subscription that has reached the specified position. The next - * message is passed back via the second parameter. + * Release the message i.e. return it to the available state + * unless it has already been deleted. * - * @param unacquired, if true, will only browse unacquired messages - * - * @return true if there is another message, false otherwise. + * @return a pointer to the Message if it is still in acquired state and + * hence can be released; null if it has already been deleted */ - virtual bool browse(const framing::SequenceNumber&, QueuedMessage&, bool unacquired) = 0; + virtual Message* release(const QueueCursor& cursor) = 0; /** - * Retrieve the next message available for a consuming - * subscription. - * - * @return true if there is such a message, false otherwise. + * Find the message with the specified sequence number, returning + * a pointer if found, null otherwise. A cursor to the matched + * message can be passed back via the second parameter, regardless + * of whether the message is found, using this cursor to call + * next() will give the next message greater than position if one + * exists. */ - virtual bool consume(QueuedMessage&) = 0; + virtual Message* find(const framing::SequenceNumber&, QueueCursor*) = 0; + /** - * Pushes a message to the back of the 'queue'. For some types of - * queue this may cause another message to be removed; if that is - * the case the method will return true and the removed message - * will be passed out via the second parameter. + * Find the message at the specified position, returning a pointer if + * found, null otherwise. */ - virtual bool push(const QueuedMessage& added, QueuedMessage& removed) = 0; + virtual Message* find(const QueueCursor&) = 0; /** * Add an already acquired message to the queue. @@ -99,25 +96,11 @@ class Messages * Only need be implemented by subclasses that keep track of * acquired messages. */ - virtual void updateAcquired(const QueuedMessage&) { } - - /** - * Set the position of the back of the queue. Next message enqueued will be n+1. - *@pre Any messages with seq > n must already be dequeued. - */ - virtual void setPosition(const framing::SequenceNumber& /*n*/) = 0; - + //virtual void updateAcquired(const QueuedMessage&) { } /** * Apply, the functor to each message held */ - virtual void foreach(Functor) = 0; - /** - * Remove every message held that for which the specified - * predicate returns true - */ - virtual void removeIf(Predicate) = 0; - private: }; }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/Persistable.h b/qpid/cpp/src/qpid/broker/Persistable.h index 36499c7a1a..444aca3295 100644 --- a/qpid/cpp/src/qpid/broker/Persistable.h +++ b/qpid/cpp/src/qpid/broker/Persistable.h @@ -32,7 +32,7 @@ namespace broker { /** * Base class for all persistable objects */ -class Persistable : public RefCounted +class Persistable : public virtual RefCounted { public: /** diff --git a/qpid/cpp/src/qpid/broker/PersistableMessage.cpp b/qpid/cpp/src/qpid/broker/PersistableMessage.cpp index 7ba28eb293..8866675c5c 100644 --- a/qpid/cpp/src/qpid/broker/PersistableMessage.cpp +++ b/qpid/cpp/src/qpid/broker/PersistableMessage.cpp @@ -29,132 +29,23 @@ using namespace qpid::broker; namespace qpid { namespace broker { -class MessageStore; - PersistableMessage::~PersistableMessage() {} - -PersistableMessage::PersistableMessage() : - asyncDequeueCounter(0), - store(0) -{} +PersistableMessage::PersistableMessage() : persistenceId(0) {} void PersistableMessage::flush() { - syncList copy; - { - sys::ScopedLock<sys::Mutex> l(storeLock); - if (store) { - copy = synclist; - } else { - return;//early exit as nothing to do - } - } - for (syncList::iterator i = copy.begin(); i != copy.end(); ++i) { - PersistableQueue::shared_ptr q(i->lock()); - if (q) { - q->flush(); - } - } -} - -void PersistableMessage::setContentReleased() -{ - contentReleaseState.released = true; -} - -bool PersistableMessage::isContentReleased() const -{ - return contentReleaseState.released; -} - - -bool PersistableMessage::isStoredOnQueue(PersistableQueue::shared_ptr queue){ - if (store && (queue->getPersistenceId()!=0)) { - for (syncList::iterator i = synclist.begin(); i != synclist.end(); ++i) { - PersistableQueue::shared_ptr q(i->lock()); - if (q && q->getPersistenceId() == queue->getPersistenceId()) return true; - } - } - return false; + //TODO: is this really the right place for this? } -void PersistableMessage::addToSyncList(PersistableQueue::shared_ptr queue, MessageStore* _store) { - if (_store){ - sys::ScopedLock<sys::Mutex> l(storeLock); - store = _store; - boost::weak_ptr<PersistableQueue> q(queue); - synclist.push_back(q); - } -} - -void PersistableMessage::enqueueAsync(PersistableQueue::shared_ptr queue, MessageStore* _store) { - addToSyncList(queue, _store); - enqueueStart(); -} - -bool PersistableMessage::isDequeueComplete() { - sys::ScopedLock<sys::Mutex> l(asyncDequeueLock); - return asyncDequeueCounter == 0; -} - -void PersistableMessage::dequeueComplete() { - bool notify = false; - { - sys::ScopedLock<sys::Mutex> l(asyncDequeueLock); - if (asyncDequeueCounter > 0) { - if (--asyncDequeueCounter == 0) { - notify = true; - } - } - } - if (notify) allDequeuesComplete(); -} - -void PersistableMessage::dequeueAsync(PersistableQueue::shared_ptr queue, MessageStore* _store) { - if (_store){ - sys::ScopedLock<sys::Mutex> l(storeLock); - store = _store; - boost::weak_ptr<PersistableQueue> q(queue); - synclist.push_back(q); - } - dequeueAsync(); -} - -void PersistableMessage::dequeueAsync() { - sys::ScopedLock<sys::Mutex> l(asyncDequeueLock); - asyncDequeueCounter++; -} - -PersistableMessage::ContentReleaseState::ContentReleaseState() : blocked(false), requested(false), released(false) {} - -void PersistableMessage::setStore(MessageStore* s) -{ - store = s; -} - -void PersistableMessage::requestContentRelease() +void PersistableMessage::enqueueAsync(PersistableQueue::shared_ptr, MessageStore*) { - contentReleaseState.requested = true; -} -void PersistableMessage::blockContentRelease() -{ - contentReleaseState.blocked = true; -} -bool PersistableMessage::checkContentReleasable() -{ - return contentReleaseState.requested && !contentReleaseState.blocked; -} - -bool PersistableMessage::isContentReleaseBlocked() -{ - return contentReleaseState.blocked; + enqueueStart(); } -bool PersistableMessage::isContentReleaseRequested() -{ - return contentReleaseState.requested; -} +bool PersistableMessage::isDequeueComplete() { return false; } +void PersistableMessage::dequeueComplete() {} +void PersistableMessage::dequeueAsync(PersistableQueue::shared_ptr, MessageStore*) {} }} diff --git a/qpid/cpp/src/qpid/broker/PersistableMessage.h b/qpid/cpp/src/qpid/broker/PersistableMessage.h index d29c2c45b4..eb6b444e4a 100644 --- a/qpid/cpp/src/qpid/broker/PersistableMessage.h +++ b/qpid/cpp/src/qpid/broker/PersistableMessage.h @@ -24,29 +24,30 @@ #include <string> #include <list> -#include <boost/shared_ptr.hpp> -#include <boost/weak_ptr.hpp> +#include <map> +#include <boost/intrusive_ptr.hpp> #include "qpid/broker/BrokerImportExport.h" #include "qpid/broker/Persistable.h" #include "qpid/framing/amqp_types.h" +#include "qpid/framing/amqp_framing.h" #include "qpid/sys/Mutex.h" #include "qpid/broker/PersistableQueue.h" #include "qpid/broker/AsyncCompletion.h" namespace qpid { +namespace types { +class Variant; +} namespace broker { class MessageStore; +class Queue; /** * Base class for persistable messages. */ class PersistableMessage : public Persistable { - typedef std::list< boost::weak_ptr<PersistableQueue> > syncList; - sys::Mutex asyncDequeueLock; - sys::Mutex storeLock; - /** * "Ingress" messages == messages sent _to_ the broker. * Tracks the number of outstanding asynchronous operations that must @@ -56,85 +57,44 @@ class PersistableMessage : public Persistable * operations have completed, the transfer of this message from the client * may be considered complete. */ - AsyncCompletion ingressCompletion; - - /** - * Tracks the number of outstanding asynchronous dequeue - * operations. When the message is dequeued asynchronously the - * count is incremented; when that dequeue completes it is - * decremented. Thus when it is 0, there are no outstanding - * dequeues. - */ - int asyncDequeueCounter; - - void dequeueAsync(); - - syncList synclist; - struct ContentReleaseState - { - bool blocked; - bool requested; - bool released; - - ContentReleaseState(); - }; - ContentReleaseState contentReleaseState; - - protected: - /** Called when all dequeues are complete for this message. */ - virtual void allDequeuesComplete() = 0; - - void setContentReleased(); - - MessageStore* store; - + boost::intrusive_ptr<AsyncCompletion> ingressCompletion; + mutable uint64_t persistenceId; public: - typedef boost::shared_ptr<PersistableMessage> shared_ptr; - - /** - * @returns the size of the headers when encoded - */ - virtual uint32_t encodedHeaderSize() const = 0; - virtual ~PersistableMessage(); - PersistableMessage(); void flush(); - - QPID_BROKER_EXTERN bool isContentReleased() const; QPID_BROKER_EXTERN void setStore(MessageStore*); - void requestContentRelease(); - void blockContentRelease(); - bool checkContentReleasable(); - bool isContentReleaseBlocked(); - bool isContentReleaseRequested(); virtual QPID_BROKER_EXTERN bool isPersistent() const = 0; /** track the progress of a message received by the broker - see ingressCompletion above */ - QPID_BROKER_INLINE_EXTERN bool isIngressComplete() { return ingressCompletion.isDone(); } - QPID_BROKER_INLINE_EXTERN AsyncCompletion& getIngressCompletion() { return ingressCompletion; } + QPID_BROKER_INLINE_EXTERN bool isIngressComplete() { return ingressCompletion->isDone(); } + QPID_BROKER_INLINE_EXTERN AsyncCompletion& getIngressCompletion() { return *ingressCompletion; } + QPID_BROKER_INLINE_EXTERN void setIngressCompletion(boost::intrusive_ptr<AsyncCompletion> i) { ingressCompletion = i; } - QPID_BROKER_INLINE_EXTERN void enqueueStart() { ingressCompletion.startCompleter(); } - QPID_BROKER_INLINE_EXTERN void enqueueComplete() { ingressCompletion.finishCompleter(); } + QPID_BROKER_INLINE_EXTERN void enqueueStart() { ingressCompletion->startCompleter(); } + QPID_BROKER_INLINE_EXTERN void enqueueComplete() { ingressCompletion->finishCompleter(); } QPID_BROKER_EXTERN void enqueueAsync(PersistableQueue::shared_ptr queue, MessageStore* _store); QPID_BROKER_EXTERN bool isDequeueComplete(); - QPID_BROKER_EXTERN void dequeueComplete(); - QPID_BROKER_EXTERN void dequeueAsync(PersistableQueue::shared_ptr queue, MessageStore* _store); - bool isStoredOnQueue(PersistableQueue::shared_ptr queue); - - void addToSyncList(PersistableQueue::shared_ptr queue, MessageStore* _store); + uint64_t getPersistenceId() const { return persistenceId; } + void setPersistenceId(uint64_t _persistenceId) const { persistenceId = _persistenceId; } + + + virtual void decodeHeader(framing::Buffer& buffer) = 0; + virtual void decodeContent(framing::Buffer& buffer) = 0; + virtual uint32_t encodedHeaderSize() const = 0; + virtual boost::intrusive_ptr<PersistableMessage> merge(const std::map<std::string, qpid::types::Variant>& annotations) const = 0; }; }} diff --git a/qpid/cpp/src/qpid/broker/PriorityQueue.cpp b/qpid/cpp/src/qpid/broker/PriorityQueue.cpp index 9a0fead744..99488ded13 100644 --- a/qpid/cpp/src/qpid/broker/PriorityQueue.cpp +++ b/qpid/cpp/src/qpid/broker/PriorityQueue.cpp @@ -19,24 +19,53 @@ * */ #include "qpid/broker/PriorityQueue.h" +#include "qpid/broker/Message.h" #include "qpid/broker/Queue.h" -#include "qpid/broker/QueuedMessage.h" +#include "qpid/broker/QueueCursor.h" #include "qpid/framing/reply_exceptions.h" #include "qpid/log/Statement.h" +#include <algorithm> #include <cmath> +#include <boost/bind.hpp> namespace qpid { namespace broker { +namespace { +class PriorityContext : public CursorContext { + public: + std::vector<QueueCursor> position; + PriorityContext(size_t levels, SubscriptionType type) : position(levels, QueueCursor(type)) {} +}; +} + PriorityQueue::PriorityQueue(int l) : levels(l), - messages(levels, Deque()), - frontLevel(0), haveFront(false), cached(false) {} + messages(levels, Deque(boost::bind(&PriorityQueue::priorityPadding, this, _1))), + counters(levels, framing::SequenceNumber()), + fifo(boost::bind(&PriorityQueue::fifoPadding, this, _1)), + frontLevel(0), haveFront(false), cached(false) +{ +} -bool PriorityQueue::deleted(const QueuedMessage& qm) { - bool deleted = fifo.deleted(qm); - if (deleted) erase(qm); - return deleted; +bool PriorityQueue::deleted(const QueueCursor& c) +{ + MessagePointer* ptr = fifo.find(c); + if (ptr && ptr->holder) { + //mark the message as deleted + ptr->holder->message.setState(DELETED); + //clean the deque for the relevant priority level + boost::shared_ptr<PriorityContext> ctxt = boost::dynamic_pointer_cast<PriorityContext>(c.context); + messages[ptr->holder->priority].clean(); + //stop referencing that message holder (it may now have been + //deleted) + ptr->holder = 0; + //clean fifo index + fifo.clean(); + return true; + } else { + return false; + } } size_t PriorityQueue::size() @@ -44,85 +73,69 @@ size_t PriorityQueue::size() return fifo.size(); } -namespace { -bool before(QueuedMessage* a, QueuedMessage* b) { return *a < *b; } -} - -void PriorityQueue::release(const QueuedMessage& message) -{ - QueuedMessage* qm = fifo.releasePtr(message); - if (qm) { - uint p = getPriorityLevel(message); - messages[p].insert( - lower_bound(messages[p].begin(), messages[p].end(), qm, before), qm); - clearCache(); +Message* PriorityQueue::next(QueueCursor& cursor) +{ + boost::shared_ptr<PriorityContext> ctxt = boost::dynamic_pointer_cast<PriorityContext>(cursor.context); + if (!ctxt) { + ctxt = boost::shared_ptr<PriorityContext>(new PriorityContext(levels, CONSUMER)); + cursor.context = ctxt; } -} - - -void PriorityQueue::erase(const QueuedMessage& qm) { - size_t i = getPriorityLevel(qm); - if (!messages[i].empty()) { - long diff = qm.position.getValue() - messages[i].front()->position.getValue(); - if (diff < 0) return; - long maxEnd = std::min(size_t(diff), messages[i].size()); - QueuedMessage mutableQm = qm; // need non-const qm for lower_bound - Deque::iterator l = - lower_bound(messages[i].begin(),messages[i].begin()+maxEnd, &mutableQm, before); - if (l != messages[i].end() && (*l)->position == qm.position) { - messages[i].erase(l); - clearCache(); - return; + if (cursor.type == REPLICATOR) { + //browse in fifo order + MessagePointer* ptr = fifo.next(cursor); + return ptr ? &(ptr->holder->message) : 0; + } else if (cursor.type == PURGE) { + //iterate over message in reverse priority order (i.e. purge lowest priority message first) + //ignore any fairshare configuration here as well + for (int p = 0; p < levels; ++p) { + MessageHolder* holder = messages[p].next(ctxt->position[p]); + if (holder) { + cursor.setPosition(holder->message.getSequence(), 0); + return &(holder->message); + } } + return 0; + } else { + //check each level in turn, in priority order, for any more messages + Priority p = firstLevel(); + do { + MessageHolder* holder = messages[p.current].next(ctxt->position[p.current]); + if (holder) { + cursor.setPosition(holder->message.getSequence(), 0); + return &(holder->message); + } + } while (nextLevel(p)); + return 0; } } -bool PriorityQueue::acquire(const framing::SequenceNumber& position, QueuedMessage& message) +Message* PriorityQueue::find(const QueueCursor& cursor) { - bool acquired = fifo.acquire(position, message); - if (acquired) erase(message); // No longer available - return acquired; + return find(cursor.position, 0); } -bool PriorityQueue::find(const framing::SequenceNumber& position, QueuedMessage& message) +Message* PriorityQueue::find(const framing::SequenceNumber& position, QueueCursor* cursor) { - return fifo.find(position, message); + MessagePointer* ptr = fifo.find(position, cursor); + return ptr ? &(ptr->holder->message) : 0; } -bool PriorityQueue::browse( - const framing::SequenceNumber& position, QueuedMessage& message, bool unacquired) +void PriorityQueue::publish(const Message& published) { - return fifo.browse(position, message, unacquired); + MessageHolder holder; + holder.message = published; + holder.priority = getPriorityLevel(published); + holder.id = ++(counters[holder.priority]); + MessagePointer pointer; + pointer.holder = &(messages[holder.priority].publish(holder)); + pointer.id = published.getSequence(); + fifo.publish(pointer); } -bool PriorityQueue::consume(QueuedMessage& message) -{ - if (checkFront()) { - QueuedMessage* pm = messages[frontLevel].front(); - messages[frontLevel].pop_front(); - clearCache(); - pm->status = QueuedMessage::ACQUIRED; // Updates FIFO index - message = *pm; - return true; - } else { - return false; - } -} - -bool PriorityQueue::push(const QueuedMessage& added, QueuedMessage& /*not needed*/) +Message* PriorityQueue::release(const QueueCursor& cursor) { - QueuedMessage* qmp = fifo.pushPtr(added); - messages[getPriorityLevel(added)].push_back(qmp); - clearCache(); - return false; // Adding a message never causes one to be removed for deque -} - -void PriorityQueue::updateAcquired(const QueuedMessage& acquired) { - fifo.updateAcquired(acquired); -} - -void PriorityQueue::setPosition(const framing::SequenceNumber& n) { - fifo.setPosition(n); + MessagePointer* ptr = fifo.release(cursor); + return ptr ? &(ptr->holder->message) : 0; } void PriorityQueue::foreach(Functor f) @@ -130,62 +143,87 @@ void PriorityQueue::foreach(Functor f) fifo.foreach(f); } -void PriorityQueue::removeIf(Predicate p) -{ - for (int priority = 0; priority < levels; ++priority) { - for (Deque::iterator i = messages[priority].begin(); i != messages[priority].end();) { - if (p(**i)) { - (*i)->status = QueuedMessage::DELETED; // Updates fifo index - i = messages[priority].erase(i); - clearCache(); - } else { - ++i; - } - } - } - fifo.clean(); -} - -uint PriorityQueue::getPriorityLevel(const QueuedMessage& m) const +uint PriorityQueue::getPriorityLevel(const Message& m) const { - uint priority = m.payload->getPriority(); + uint priority = m.getPriority(); //Use AMQP 0-10 approach to mapping priorities to a fixed level //(see rule priority-level-implementation) const uint firstLevel = 5 - uint(std::min(5.0, std::ceil((double) levels/2.0))); if (priority <= firstLevel) return 0; return std::min(priority - firstLevel, (uint)levels-1); } +PriorityQueue::MessagePointer PriorityQueue::fifoPadding(qpid::framing::SequenceNumber id) +{ + PriorityQueue::MessagePointer pointer; + pointer.holder = 0; + pointer.id = id; + return pointer; +} -void PriorityQueue::clearCache() +PriorityQueue::MessageHolder PriorityQueue::priorityPadding(qpid::framing::SequenceNumber id) { - cached = false; + PriorityQueue::MessageHolder holder; + holder.id = id; + holder.message.setState(DELETED); + return holder; } -bool PriorityQueue::findFrontLevel(uint& l, PriorityLevels& m) +PriorityQueue::Priority PriorityQueue::firstLevel() { - for (int p = levels-1; p >= 0; --p) { - if (!m[p].empty()) { - l = p; - return true; - } + return Priority(levels - 1); +} +bool PriorityQueue::nextLevel(Priority& p) +{ + if (p.current > 0) { + --(p.current); + return true; + } else { + return false; } - return false; } -bool PriorityQueue::checkFront() +framing::SequenceNumber PriorityQueue::MessageHolder::getSequence() const +{ + return id; +} +void PriorityQueue::MessageHolder::setState(MessageState s) +{ + message.setState(s); +} +MessageState PriorityQueue::MessageHolder::getState() const +{ + return message.getState(); +} +PriorityQueue::MessageHolder::operator Message&() +{ + return message; +} +framing::SequenceNumber PriorityQueue::MessagePointer::getSequence() const { - if (!cached) { - haveFront = findFrontLevel(frontLevel, messages); - cached = true; + if (holder) { + return holder->message.getSequence(); + } else { + //this is used when the instance is merely acting as padding + return id; } - return haveFront; } - -uint PriorityQueue::getPriority(const QueuedMessage& message) +void PriorityQueue::MessagePointer::setState(MessageState s) { - const PriorityQueue* queue = dynamic_cast<const PriorityQueue*>(&(message.queue->getMessages())); - if (queue) return queue->getPriorityLevel(message); - else return 0; + if (holder) { + holder->message.setState(s); + } +} +MessageState PriorityQueue::MessagePointer::getState() const +{ + if (holder) { + return holder->message.getState(); + } else { + return DELETED; + } +} +PriorityQueue::MessagePointer::operator Message&() +{ + assert(holder); + return holder->message; } - }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/PriorityQueue.h b/qpid/cpp/src/qpid/broker/PriorityQueue.h index 301367358b..16432bfb54 100644 --- a/qpid/cpp/src/qpid/broker/PriorityQueue.h +++ b/qpid/cpp/src/qpid/broker/PriorityQueue.h @@ -22,6 +22,7 @@ * */ #include "qpid/broker/MessageDeque.h" +#include "qpid/broker/IndexedDeque.h" #include "qpid/sys/IntegerTypes.h" #include <deque> #include <vector> @@ -44,42 +45,63 @@ class PriorityQueue : public Messages virtual ~PriorityQueue() {} size_t size(); - bool deleted(const QueuedMessage&); - void release(const QueuedMessage&); - bool acquire(const framing::SequenceNumber&, QueuedMessage&); - bool find(const framing::SequenceNumber&, QueuedMessage&); - bool browse(const framing::SequenceNumber&, QueuedMessage&, bool); - bool consume(QueuedMessage&); - bool push(const QueuedMessage& added, QueuedMessage& removed); - void updateAcquired(const QueuedMessage& acquired); - void setPosition(const framing::SequenceNumber&); - void foreach(Functor); - void removeIf(Predicate); - - static uint getPriority(const QueuedMessage&); + bool deleted(const QueueCursor&); + void publish(const Message& added); + Message* next(QueueCursor&); + Message* release(const QueueCursor& cursor); + Message* find(const QueueCursor&); + Message* find(const framing::SequenceNumber&, QueueCursor*); + void foreach(Functor); + static uint getPriority(const Message&); protected: - typedef std::deque<QueuedMessage*> Deque; - typedef std::vector<Deque> PriorityLevels; - virtual bool findFrontLevel(uint& p, PriorityLevels&); - const int levels; + struct Priority + { + const int start; + int current; + Priority(int s) : start(s), current(start) {} + }; + virtual Priority firstLevel(); + virtual bool nextLevel(Priority& ); private: - /** Available messages separated by priority and sorted in priority order. - * Holds pointers to the QueuedMessages in fifo + struct MessageHolder + { + Message message; + int priority; + framing::SequenceNumber id; + framing::SequenceNumber getSequence() const; + void setState(MessageState); + MessageState getState() const; + operator Message&(); + }; + struct MessagePointer + { + MessageHolder* holder; + framing::SequenceNumber id;//used only for padding + framing::SequenceNumber getSequence() const; + void setState(MessageState); + MessageState getState() const; + operator Message&(); + }; + typedef IndexedDeque<MessageHolder> Deque; + typedef std::vector<Deque> PriorityLevels; + typedef std::vector<framing::SequenceNumber> Counters; + + /** Holds pointers to messages (stored in the fifo index) separated by priority. */ PriorityLevels messages; - /** FIFO index of all messsagse (including acquired messages) for fast browsing and indexing */ - MessageDeque fifo; + Counters counters; + /** FIFO index of messages for fast browsing and indexing */ + IndexedDeque<MessagePointer> fifo; uint frontLevel; bool haveFront; bool cached; - void erase(const QueuedMessage&); - uint getPriorityLevel(const QueuedMessage&) const; - void clearCache(); - bool checkFront(); + uint getPriorityLevel(const Message&) const; + MessageHolder priorityPadding(qpid::framing::SequenceNumber); + MessagePointer fifoPadding(qpid::framing::SequenceNumber); }; }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/Queue.cpp b/qpid/cpp/src/qpid/broker/Queue.cpp index d5267c78dc..0dd4cb7b10 100644 --- a/qpid/cpp/src/qpid/broker/Queue.cpp +++ b/qpid/cpp/src/qpid/broker/Queue.cpp @@ -20,23 +20,23 @@ */ #include "qpid/broker/Queue.h" - #include "qpid/broker/Broker.h" -#include "qpid/broker/QueueEvents.h" +#include "qpid/broker/QueueCursor.h" +#include "qpid/broker/QueueDepth.h" +#include "qpid/broker/QueueSettings.h" #include "qpid/broker/Exchange.h" -#include "qpid/broker/Fairshare.h" #include "qpid/broker/DeliverableMessage.h" -#include "qpid/broker/LegacyLVQ.h" -#include "qpid/broker/MessageDeque.h" -#include "qpid/broker/MessageMap.h" #include "qpid/broker/MessageStore.h" +#include "qpid/broker/MessageDeque.h" +#include "qpid/broker/MessageDistributor.h" +#include "qpid/broker/FifoDistributor.h" #include "qpid/broker/NullMessageStore.h" #include "qpid/broker/QueueRegistry.h" -#include "qpid/broker/QueueFlowLimit.h" -#include "qpid/broker/ThresholdAlerts.h" -#include "qpid/broker/FifoDistributor.h" -#include "qpid/broker/MessageGroupManager.h" +//TODO: get rid of this +#include "qpid/broker/amqp_0_10/MessageTransfer.h" + +#include "qpid/amqp_0_10/Codecs.h" #include "qpid/StringUtils.h" #include "qpid/log/Statement.h" #include "qpid/management/ManagementAgent.h" @@ -76,26 +76,8 @@ namespace _qmf = qmf::org::apache::qpid::broker; namespace { -const std::string qpidMaxSize("qpid.max_size"); -const std::string qpidMaxCount("qpid.max_count"); -const std::string qpidNoLocal("no-local"); -const std::string qpidTraceIdentity("qpid.trace.id"); -const std::string qpidTraceExclude("qpid.trace.exclude"); -const std::string qpidLastValueQueueKey("qpid.last_value_queue_key"); -const std::string qpidLastValueQueue("qpid.last_value_queue"); -const std::string qpidLastValueQueueNoBrowse("qpid.last_value_queue_no_browse"); -const std::string qpidPersistLastNode("qpid.persist_last_node"); -const std::string qpidVQMatchProperty("qpid.LVQ_key"); -const std::string qpidQueueEventGeneration("qpid.queue_event_generation"); -const std::string qpidAutoDeleteTimeout("qpid.auto_delete_timeout"); -//following feature is not ready for general use as it doesn't handle -//the case where a message is enqueued on more than one queue well enough: -const std::string qpidInsertSequenceNumbers("qpid.insert_sequence_numbers"); - -const int ENQUEUE_ONLY=1; -const int ENQUEUE_AND_DEQUEUE=2; - -inline void mgntEnqStats(const boost::intrusive_ptr<Message>& msg, + +inline void mgntEnqStats(const Message& msg, _qmf::Queue* mgmtObject, _qmf::Broker* brokerMgmtObject) { @@ -103,12 +85,12 @@ inline void mgntEnqStats(const boost::intrusive_ptr<Message>& msg, _qmf::Queue::PerThreadStats *qStats = mgmtObject->getStatistics(); _qmf::Broker::PerThreadStats *bStats = brokerMgmtObject->getStatistics(); - uint64_t contentSize = msg->contentSize(); + uint64_t contentSize = msg.getContentSize(); qStats->msgTotalEnqueues +=1; bStats->msgTotalEnqueues += 1; qStats->byteTotalEnqueues += contentSize; bStats->byteTotalEnqueues += contentSize; - if (msg->isPersistent ()) { + if (msg.isPersistent ()) { qStats->msgPersistEnqueues += 1; bStats->msgPersistEnqueues += 1; qStats->bytePersistEnqueues += contentSize; @@ -119,20 +101,20 @@ inline void mgntEnqStats(const boost::intrusive_ptr<Message>& msg, } } -inline void mgntDeqStats(const boost::intrusive_ptr<Message>& msg, +inline void mgntDeqStats(const Message& msg, _qmf::Queue* mgmtObject, _qmf::Broker* brokerMgmtObject) { if (mgmtObject != 0){ _qmf::Queue::PerThreadStats *qStats = mgmtObject->getStatistics(); _qmf::Broker::PerThreadStats *bStats = brokerMgmtObject->getStatistics(); - uint64_t contentSize = msg->contentSize(); + uint64_t contentSize = msg.getContentSize(); qStats->msgTotalDequeues += 1; bStats->msgTotalDequeues += 1; qStats->byteTotalDequeues += contentSize; bStats->byteTotalDequeues += contentSize; - if (msg->isPersistent ()){ + if (msg.isPersistent ()){ qStats->msgPersistDequeues += 1; bStats->msgPersistDequeues += 1; qStats->bytePersistDequeues += contentSize; @@ -143,43 +125,81 @@ inline void mgntDeqStats(const boost::intrusive_ptr<Message>& msg, } } -} // namespace +QueueSettings merge(const QueueSettings& inputs, const Broker::Options& globalOptions) +{ + QueueSettings settings(inputs); + if (!settings.maxDepth.hasSize() && globalOptions.queueLimit) { + settings.maxDepth.setSize(globalOptions.queueLimit); + } + return settings; +} + +} -Queue::Queue(const string& _name, bool _autodelete, +Queue::TxPublish::TxPublish(const Message& m, boost::shared_ptr<Queue> q) : message(m), queue(q), prepared(false) {} +bool Queue::TxPublish::prepare(TransactionContext* ctxt) throw() +{ + try { + prepared = queue->enqueue(ctxt, message); + return true; + } catch (const std::exception& e) { + QPID_LOG(error, "Failed to prepare: " << e.what()); + return false; + } +} +void Queue::TxPublish::commit() throw() +{ + try { + if (prepared) queue->process(message); + } catch (const std::exception& e) { + QPID_LOG(error, "Failed to commit: " << e.what()); + } +} +void Queue::TxPublish::rollback() throw() +{ + try { + if (prepared) queue->enqueueAborted(message); + } catch (const std::exception& e) { + QPID_LOG(error, "Failed to rollback: " << e.what()); + } +} + +Queue::Queue(const string& _name, const QueueSettings& _settings, MessageStore* const _store, - const OwnershipToken* const _owner, Manageable* parent, Broker* b) : name(_name), - autodelete(_autodelete), store(_store), - owner(_owner), + owner(0), consumerCount(0), browserCount(0), exclusive(0), - noLocal(false), persistLastNode(false), inLastNodeFailure(false), messages(new MessageDeque()), persistenceId(0), - policyExceeded(false), + settings(b ? merge(_settings, b->getOptions()) : _settings), mgmtObject(0), brokerMgmtObject(0), eventMode(0), - insertSeqNo(0), broker(b), deleted(false), barrier(*this), - autoDeleteTimeout(0), allocator(new FifoDistributor( *messages )) { + if (settings.maxDepth.hasCount()) current.setCount(0); + if (settings.maxDepth.hasSize()) current.setSize(0); + if (settings.traceExcludes.size()) { + split(traceExclude, settings.traceExcludes, ", "); + } + qpid::amqp_0_10::translate(settings.asMap(), encodableSettings); if (parent != 0 && broker != 0) { ManagementAgent* agent = broker->getManagementAgent(); if (agent != 0) { - mgmtObject = new _qmf::Queue(agent, this, parent, _name, _store != 0, _autodelete); - mgmtObject->set_exclusive(_owner != 0); + mgmtObject = new _qmf::Queue(agent, this, parent, _name, _store != 0, settings.autodelete); + mgmtObject->set_arguments(settings.asMap()); agent->addObject(mgmtObject, 0, store != 0); brokerMgmtObject = (qmf::org::apache::qpid::broker::Broker*) broker->GetManagementObject(); if (brokerMgmtObject) @@ -197,32 +217,36 @@ Queue::~Queue() } } -bool isLocalTo(const OwnershipToken* token, boost::intrusive_ptr<Message>& msg) +bool isLocalTo(const OwnershipToken* token, const Message& msg) { - return token && token->isLocal(msg->getPublisher()); + return token && token->isLocal(msg.getPublisher()); } -bool Queue::isLocal(boost::intrusive_ptr<Message>& msg) +bool Queue::isLocal(const Message& msg) { //message is considered local if it was published on the same //connection as that of the session which declared this queue //exclusive (owner) or which has an exclusive subscription //(exclusive) - return noLocal && (isLocalTo(owner, msg) || isLocalTo(exclusive, msg)); + return settings.noLocal && (isLocalTo(owner, msg) || isLocalTo(exclusive, msg)); } -bool Queue::isExcluded(boost::intrusive_ptr<Message>& msg) +bool Queue::isExcluded(const Message& msg) { - return traceExclude.size() && msg->isExcluded(traceExclude); + return traceExclude.size() && msg.isExcluded(traceExclude); } -void Queue::deliver(boost::intrusive_ptr<Message> msg){ +void Queue::deliver(Message msg, TxBuffer* txn){ + //TODO: move some of this out of the queue and into the publishing + //'link' for whatever protocol is used; that would let protocol + //specific stuff be kept out the queue + // Check for deferred delivery in a cluster. if (broker && broker->deferDelivery(name, msg)) return; - if (msg->isImmediate() && getConsumerCount() == 0) { + if (broker::amqp_0_10::MessageTransfer::isImmediateDeliveryRequired(msg) && getConsumerCount() == 0) { if (alternateExchange) { - DeliverableMessage deliverable(msg); + DeliverableMessage deliverable(msg, 0); alternateExchange->route(deliverable); } } else if (isLocal(msg)) { @@ -232,47 +256,38 @@ void Queue::deliver(boost::intrusive_ptr<Message> msg){ //drop message QPID_LOG(info, "Dropping excluded message from " << getName()); } else { - enqueue(0, msg); - push(msg); - QPID_LOG(debug, "Message " << msg << " enqueued on " << name); + if (txn) { + TxOp::shared_ptr op(new TxPublish(msg, shared_from_this())); + txn->enlist(op); + } else { + if (enqueue(0, msg)) { + push(msg); + QPID_LOG(debug, "Message " << msg << " enqueued on " << name); + } else { + QPID_LOG(debug, "Message " << msg << " dropped from " << name); + } + } } } -void Queue::recoverPrepared(boost::intrusive_ptr<Message>& msg) +void Queue::recoverPrepared(const Message& msg) { Mutex::ScopedLock locker(messageLock); - if (policy.get()) policy->recoverEnqueued(msg); + current += QueueDepth(1, msg.getContentSize()); } -void Queue::recover(boost::intrusive_ptr<Message>& msg) +void Queue::recover(Message& msg) { - { - Mutex::ScopedLock locker(messageLock); - if (policy.get()) policy->recoverEnqueued(msg); - } - + recoverPrepared(msg); push(msg, true); - if (store){ - // setup synclist for recovered messages, so they don't get re-stored on lastNodeFailure - msg->addToSyncList(shared_from_this(), store); - } - - if (store && (!msg->isContentLoaded() || msg->checkContentReleasable())) { - //content has not been loaded, need to ensure that lazy loading mode is set: - //TODO: find a nicer way to do this - msg->releaseContent(store); - // NOTE: The log message in this section are used for flow-to-disk testing (which checks the log for the - // presence of this message). Do not change this without also checking these tests. - QPID_LOG(debug, "Message id=\"" << msg->getProperties<MessageProperties>()->getMessageId() << "\"; pid=0x" << - std::hex << msg->getPersistenceId() << std::dec << ": Content released after recovery"); - } } -void Queue::process(boost::intrusive_ptr<Message>& msg){ +void Queue::process(Message& msg) +{ push(msg); if (mgmtObject != 0){ _qmf::Queue::PerThreadStats *qStats = mgmtObject->getStatistics(); - const uint64_t contentSize = msg->contentSize(); + const uint64_t contentSize = msg.getContentSize(); qStats->msgTxnEnqueues += 1; qStats->byteTxnEnqueues += contentSize; mgmtObject->statisticsUpdated(); @@ -285,46 +300,22 @@ void Queue::process(boost::intrusive_ptr<Message>& msg){ } } -void Queue::requeue(const QueuedMessage& msg){ +void Queue::release(const QueueCursor& position, bool markRedelivered) +{ assertClusterSafe(); QueueListeners::NotificationSet copy; { - if (!isEnqueued(msg)) return; - if (deleted) { - // - // If the queue has been deleted, requeued messages must be sent to the alternate exchange - // if one is configured. - // - if (alternateExchange.get()) { - DeliverableMessage dmsg(msg.payload); - alternateExchange->routeWithAlternate(dmsg); - if (brokerMgmtObject) - brokerMgmtObject->inc_abandonedViaAlt(); - } else { - if (brokerMgmtObject) - brokerMgmtObject->inc_abandoned(); - } - mgntDeqStats(msg.payload, mgmtObject, brokerMgmtObject); - } else { - { - Mutex::ScopedLock locker(messageLock); - messages->release(msg); - observeRequeue(msg, locker); + Mutex::ScopedLock locker(messageLock); + if (!deleted) { + Message* message = messages->release(position); + if (message) { + if (!markRedelivered) message->undeliver(); listeners.populate(copy); - } - - if (mgmtObject) { - mgmtObject->inc_releases(); - if (brokerMgmtObject) - brokerMgmtObject->inc_releases(); - } - - // for persistLastNode - don't force a message twice to disk, but force it if no force before - if(inLastNodeFailure && persistLastNode && !msg.payload->isStoredOnQueue(shared_from_this())) { - msg.payload->forcePersistent(); - if (msg.payload->isForcedPersistent() ){ - boost::intrusive_ptr<Message> payload = msg.payload; - enqueue(0, payload); + observeRequeue(*message, locker); + if (mgmtObject) { + mgmtObject->inc_releases(); + if (brokerMgmtObject) + brokerMgmtObject->inc_releases(); } } } @@ -332,163 +323,118 @@ void Queue::requeue(const QueuedMessage& msg){ copy.notify(); } -bool Queue::acquireMessageAt(const SequenceNumber& position, QueuedMessage& message) +bool Queue::dequeueMessageAt(const SequenceNumber& position) { - assertClusterSafe(); - QPID_LOG(debug, "Attempting to acquire message at " << position); - if (acquire(position, message)) { - QPID_LOG(debug, "Acquired message at " << position << " from " << name); - return true; - } else { - QPID_LOG(debug, "Could not acquire message at " << position << " from " << name << "; no message at that position"); - return false; - } -} - -bool Queue::acquire(const QueuedMessage& msg, const std::string& consumer) -{ - assertClusterSafe(); - QPID_LOG(debug, consumer << " attempting to acquire message at " << msg.position); - bool ok; + boost::intrusive_ptr<PersistableMessage> pmsg; { Mutex::ScopedLock locker(messageLock); - ok = allocator->allocate( consumer, msg ); - } - if (!ok) { - QPID_LOG(debug, "Not permitted to acquire msg at " << msg.position << " from '" << name); - return false; - } - - QueuedMessage copy(msg); - if (acquire( msg.position, copy)) { - QPID_LOG(debug, "Acquired message at " << msg.position << " from " << name); - return true; + assertClusterSafe(); + QPID_LOG(debug, "Attempting to dequeue message at " << position); + QueueCursor cursor; + Message* msg = messages->find(position, &cursor); + if (msg) { + if (msg->isPersistent()) pmsg = msg->getPersistentContext(); + observeDequeue(*msg, locker); + messages->deleted(cursor); + } else { + QPID_LOG(debug, "Could not dequeue message at " << position << "; no such message"); + return false; + } } - QPID_LOG(debug, "Could not acquire message at " << msg.position << " from " << name << "; no message at that position"); - return false; + dequeueFromStore(pmsg); + return true; } -void Queue::notifyListener() +bool Queue::acquire(const QueueCursor& position, const std::string& consumer) { + Mutex::ScopedLock locker(messageLock); assertClusterSafe(); - QueueListeners::NotificationSet set; - { - Mutex::ScopedLock locker(messageLock); - if (messages->size()) { - listeners.populate(set); - } - } - set.notify(); -} + Message* msg; -bool Queue::getNextMessage(QueuedMessage& m, Consumer::shared_ptr& c) -{ - checkNotDeleted(c); - if (c->preAcquires()) { - switch (consumeNextMessage(m, c)) { - case CONSUMED: - return true; - case CANT_CONSUME: - notifyListener();//let someone else try - case NO_MESSAGES: - default: + msg = messages->find(position); + if (msg) { + QPID_LOG(debug, consumer << " attempting to acquire message at " << msg->getSequence()); + if (!allocator->acquire(consumer, *msg)) { + QPID_LOG(debug, "Not permitted to acquire msg at " << msg->getSequence() << " from '" << name); return false; + } else { + observeAcquire(*msg, locker); + QPID_LOG(debug, "Acquired message at " << msg->getSequence() << " from " << name); + return true; } } else { - return browseNextMessage(m, c); + QPID_LOG(debug, "Failed to acquire message which no longer exists on " << name); + return false; } } -Queue::ConsumeCode Queue::consumeNextMessage(QueuedMessage& m, Consumer::shared_ptr& c) +bool Queue::getNextMessage(Message& m, Consumer::shared_ptr& c) { + checkNotDeleted(c); + QueueListeners::NotificationSet set; while (true) { - QueuedMessage msg; - bool found; - { - Mutex::ScopedLock locker(messageLock); - found = allocator->nextConsumableMessage(c, msg); - if (!found) listeners.addListener(c); - } - if (!found) { - QPID_LOG(debug, "No messages to dispatch on queue '" << name << "'"); - return NO_MESSAGES; - } - - if (msg.payload->hasExpired()) { - QPID_LOG(debug, "Message expired from queue '" << name << "'"); - c->setPosition(msg.position); - dequeue(0, msg); - if (mgmtObject) { - mgmtObject->inc_discardsTtl(); - if (brokerMgmtObject) - brokerMgmtObject->inc_discardsTtl(); - } - continue; - } - - if (c->filter(msg.payload)) { - if (c->accept(msg.payload)) { - { - Mutex::ScopedLock locker(messageLock); - bool ok = allocator->allocate( c->getName(), msg ); // inform allocator - (void) ok; assert(ok); - observeAcquire(msg, locker); - } + //TODO: reduce lock scope + Mutex::ScopedLock locker(messageLock); + Message* msg = messages->next(*c); + if (msg) { + if (msg->hasExpired()) { + QPID_LOG(debug, "Message expired from queue '" << name << "'"); + observeDequeue(*msg, locker); + //ERROR: don't hold lock across call to store!! + if (msg->isPersistent()) dequeueFromStore(msg->getPersistentContext()); if (mgmtObject) { - mgmtObject->inc_acquires(); + mgmtObject->inc_discardsTtl(); if (brokerMgmtObject) - brokerMgmtObject->inc_acquires(); + brokerMgmtObject->inc_discardsTtl(); } - m = msg; - return CONSUMED; - } else { - //message(s) are available but consumer hasn't got enough credit - QPID_LOG(debug, "Consumer can't currently accept message from '" << name << "'"); + messages->deleted(*c); + continue; } - } else { - //consumer will never want this message - QPID_LOG(debug, "Consumer doesn't want message from '" << name << "'"); - } - - Mutex::ScopedLock locker(messageLock); - messages->release(msg); - return CANT_CONSUME; - } -} -bool Queue::browseNextMessage(QueuedMessage& m, Consumer::shared_ptr& c) -{ - while (true) { - QueuedMessage msg; - bool found; - { - Mutex::ScopedLock locker(messageLock); - found = allocator->nextBrowsableMessage(c, msg); - if (!found) listeners.addListener(c); - } - if (!found) { // no next available - QPID_LOG(debug, "No browsable messages available for consumer " << - c->getName() << " on queue '" << name << "'"); - return false; - } - - if (c->filter(msg.payload) && !msg.payload->hasExpired()) { - if (c->accept(msg.payload)) { - //consumer wants the message - c->setPosition(msg.position); - m = msg; - return true; + if (c->filter(*msg)) { + if (c->accept(*msg)) { + if (c->preAcquires()) { + QPID_LOG(debug, "Attempting to acquire message " << msg << " from '" << name << "' with state " << msg->getState()); + if (allocator->acquire(c->getName(), *msg)) { + if (mgmtObject) { + mgmtObject->inc_acquires(); + if (brokerMgmtObject) + brokerMgmtObject->inc_acquires(); + } + observeAcquire(*msg, locker); + msg->deliver(); + } else { + QPID_LOG(debug, "Could not acquire message from '" << name << "'"); + continue; //try another message + } + } + QPID_LOG(debug, "Message retrieved from '" << name << "'"); + m = *msg; + return true; + } else { + //message(s) are available but consumer hasn't got enough credit + QPID_LOG(debug, "Consumer can't currently accept message from '" << name << "'"); + if (c->preAcquires()) { + //let someone else try + listeners.populate(set); + } + break; + } } else { - //browser hasn't got enough credit for the message - QPID_LOG(debug, "Browser can't currently accept message from '" << name << "'"); - return false; + //consumer will never want this message, try another one + QPID_LOG(debug, "Consumer doesn't want message from '" << name << "'"); + if (c->preAcquires()) { + //let someone else try to take this one + listeners.populate(set); + } } } else { - //consumer will never want this message, continue seeking - QPID_LOG(debug, "Browser skipping message from '" << name << "'"); - c->setPosition(msg.position); + QPID_LOG(debug, "No messages to dispatch on queue '" << name << "'"); + listeners.addListener(c); + return false; } } + set.notify(); return false; } @@ -507,23 +453,28 @@ void Queue::removeListener(Consumer::shared_ptr c) bool Queue::dispatch(Consumer::shared_ptr c) { - QueuedMessage msg(this); + Message msg; if (getNextMessage(msg, c)) { - c->deliver(msg); + c->deliver(*c, msg); return true; } else { return false; } } -bool Queue::find(SequenceNumber pos, QueuedMessage& msg) const { +bool Queue::find(SequenceNumber pos, Message& msg) const +{ Mutex::ScopedLock locker(messageLock); - if (messages->find(pos, msg)) + Message* ptr = messages->find(pos, 0); + if (ptr) { + msg = *ptr; return true; + } return false; } -void Queue::consume(Consumer::shared_ptr c, bool requestExclusive){ +void Queue::consume(Consumer::shared_ptr c, bool requestExclusive) +{ assertClusterSafe(); { Mutex::ScopedLock locker(messageLock); @@ -550,7 +501,7 @@ void Queue::consume(Consumer::shared_ptr c, bool requestExclusive){ browserCount++; consumerCount++; //reset auto deletion timer if necessary - if (autoDeleteTimeout && autoDeleteTask) { + if (settings.autoDeleteDelay && autoDeleteTask) { autoDeleteTask->cancel(); } observeConsumerAdd(*c, locker); @@ -559,7 +510,8 @@ void Queue::consume(Consumer::shared_ptr c, bool requestExclusive){ mgmtObject->inc_consumerCount (); } -void Queue::cancel(Consumer::shared_ptr c){ +void Queue::cancel(Consumer::shared_ptr c) +{ removeListener(c); { Mutex::ScopedLock locker(messageLock); @@ -572,65 +524,6 @@ void Queue::cancel(Consumer::shared_ptr c){ mgmtObject->dec_consumerCount (); } -QueuedMessage Queue::get(){ - QueuedMessage msg(this); - bool ok; - { - Mutex::ScopedLock locker(messageLock); - ok = messages->consume(msg); - if (ok) observeAcquire(msg, locker); - } - - if (ok && mgmtObject) { - mgmtObject->inc_acquires(); - if (brokerMgmtObject) - brokerMgmtObject->inc_acquires(); - } - - return msg; -} - -namespace { -bool collectIf(QueuedMessage& qm, Messages::Predicate predicate, - std::deque<QueuedMessage>& collection) -{ - if (predicate(qm)) { - collection.push_back(qm); - return true; - } else { - return false; - } -} - -bool isExpired(const QueuedMessage& qm) { return qm.payload->hasExpired(); } -} // namespace - -void Queue::dequeueIf(Messages::Predicate predicate, - std::deque<QueuedMessage>& dequeued) -{ - { - Mutex::ScopedLock locker(messageLock); - messages->removeIf(boost::bind(&collectIf, _1, predicate, boost::ref(dequeued))); - } - if (!dequeued.empty()) { - if (mgmtObject) { - mgmtObject->inc_acquires(dequeued.size()); - if (brokerMgmtObject) - brokerMgmtObject->inc_acquires(dequeued.size()); - } - for (std::deque<QueuedMessage>::const_iterator i = dequeued.begin(); - i != dequeued.end(); ++i) { - { - // KAG: should be safe to retake lock after the removeIf, since - // no other thread can touch these messages after the removeIf() call - Mutex::ScopedLock locker(messageLock); - observeAcquire(*i, locker); - } - dequeue( 0, *i ); - } - } -} - /** *@param lapse: time since the last purgeExpired */ @@ -642,13 +535,17 @@ void Queue::purgeExpired(sys::Duration lapse) { dequeueSincePurge -= count; int seconds = int64_t(lapse)/qpid::sys::TIME_SEC; if (seconds == 0 || count / seconds < 1) { - std::deque<QueuedMessage> dequeued; - dequeueIf(boost::bind(&isExpired, _1), dequeued); - if (dequeued.size()) { - if (mgmtObject) { - mgmtObject->inc_discardsTtl(dequeued.size()); - if (brokerMgmtObject) - brokerMgmtObject->inc_discardsTtl(dequeued.size()); + uint32_t count = remove(0, boost::bind(&Message::hasExpired, _1), 0, CONSUMER); + QPID_LOG(debug, "Purged " << count << " expired messages from " << getName()); + // + // Report the count of discarded-by-ttl messages + // + if (mgmtObject && count) { + mgmtObject->inc_acquires(count); + mgmtObject->inc_discardsTtl(count); + if (brokerMgmtObject) { + brokerMgmtObject->inc_acquires(count); + brokerMgmtObject->inc_discardsTtl(count); } } } @@ -663,7 +560,7 @@ namespace { static const std::string typeKey; static const std::string paramsKey; static MessageFilter *create( const ::qpid::types::Variant::Map *filter ); - virtual bool match( const QueuedMessage& ) const { return true; } + virtual bool match( const Message& ) const { return true; } virtual ~MessageFilter() {} protected: MessageFilter() {}; @@ -687,13 +584,9 @@ namespace { static const std::string valueKey; HeaderMatchFilter( const std::string& _header, const std::string& _value ) : MessageFilter (), header(_header), value(_value) {} - bool match( const QueuedMessage& msg ) const + bool match( const Message& msg ) const { - const qpid::framing::FieldTable* headers = msg.payload->getApplicationHeaders(); - if (!headers) return false; - FieldTable::ValuePtr h = headers->get(header); - if (!h || !h->convertsTo<std::string>()) return false; - return h->get<std::string>() == value; + return msg.getPropertyAsString(header) == value; } private: const std::string header; @@ -730,36 +623,68 @@ namespace { return new MessageFilter(); } - // used by removeIf() to collect all messages matching a filter, maximum match count is - // optional. - struct Collector { - const uint32_t maxMatches; - MessageFilter& filter; - std::deque<QueuedMessage> matches; - Collector(MessageFilter& filter, uint32_t max) - : maxMatches(max), filter(filter) {} - bool operator() (QueuedMessage& qm) - { - if (maxMatches == 0 || matches.size() < maxMatches) { - if (filter.match( qm )) { - matches.push_back(qm); - return true; - } - } + bool reroute(boost::shared_ptr<Exchange> e, const Message& m) + { + if (e) { + DeliverableMessage d(m, 0); + d.getMessage().clearTrace(); + e->routeWithAlternate(d); + return true; + } else { return false; } - }; - + } + void moveTo(boost::shared_ptr<Queue> q, Message& m) + { + if (q) { + q->deliver(m); + } + } } // end namespace +uint32_t Queue::remove(const uint32_t maxCount, MessagePredicate p, MessageFunctor f, SubscriptionType type) +{ + std::deque<Message> removed; + { + QueueCursor c(type); + uint32_t count(0); + Mutex::ScopedLock locker(messageLock); + Message* m = messages->next(c); + while (m){ + if (!p || p(*m)) { + if (!maxCount || count++ < maxCount) { + if (m->getState() == AVAILABLE) { + //don't actually acquire, just act as if we did + observeAcquire(*m, locker); + } + observeDequeue(*m, locker); + removed.push_back(*m);//takes a copy of the message + if (!messages->deleted(c)) { + QPID_LOG(warning, "Failed to correctly remove message from " << name << "; state is not consistent!"); + assert(false); + } + } else { + break; + } + } + m = messages->next(c); + } + } + for (std::deque<Message>::iterator i = removed.begin(); i != removed.end(); ++i) { + if (f) f(*i);//ERROR? need to clear old persistent context? + if (i->isPersistent()) dequeueFromStore(i->getPersistentContext());//do this outside of lock and after any re-routing + } + return removed.size(); +} + /** * purge - for purging all or some messages on a queue * depending on the purge_request * - * purge_request == 0 then purge all messages - * == N then purge N messages from queue - * Sometimes purge_request == 1 to unblock the top of queue + * qty == 0 then purge all messages + * == N then purge N messages from queue + * Sometimes qty == 1 to unblock the top of queue * * The dest exchange may be supplied to re-route messages through the exchange. * It is safe to re-route messages such that they arrive back on the same queue, @@ -768,172 +693,53 @@ namespace { * An optional filter can be supplied that will be applied against each message. The * message is purged only if the filter matches. See MessageDistributor for more detail. */ -uint32_t Queue::purge(const uint32_t purge_request, boost::shared_ptr<Exchange> dest, +uint32_t Queue::purge(const uint32_t qty, boost::shared_ptr<Exchange> dest, const qpid::types::Variant::Map *filter) { std::auto_ptr<MessageFilter> mf(MessageFilter::create(filter)); - Collector c(*mf.get(), purge_request); - - { - Mutex::ScopedLock locker(messageLock); - messages->removeIf( boost::bind<bool>(boost::ref(c), _1) ); - } + uint32_t count = remove(qty, boost::bind(&MessageFilter::match, mf.get(), _1), boost::bind(&reroute, dest, _1), CONSUMER/*?*/); - if (!c.matches.empty()) { - if (mgmtObject) { - mgmtObject->inc_acquires(c.matches.size()); - if (dest.get()) { - mgmtObject->inc_reroutes(c.matches.size()); - if (brokerMgmtObject) { - brokerMgmtObject->inc_acquires(c.matches.size()); - brokerMgmtObject->inc_reroutes(c.matches.size()); - } - } else { - mgmtObject->inc_discardsPurge(c.matches.size()); - if (brokerMgmtObject) { - brokerMgmtObject->inc_acquires(c.matches.size()); - brokerMgmtObject->inc_discardsPurge(c.matches.size()); - } - } - } - - for (std::deque<QueuedMessage>::iterator qmsg = c.matches.begin(); - qmsg != c.matches.end(); ++qmsg) { - - { - // KAG: should be safe to retake lock after the removeIf, since - // no other thread can touch these messages after the removeIf call - Mutex::ScopedLock locker(messageLock); - observeAcquire(*qmsg, locker); + if (mgmtObject && count) { + mgmtObject->inc_acquires(count); + if (dest.get()) { + mgmtObject->inc_reroutes(count); + if (brokerMgmtObject) { + brokerMgmtObject->inc_acquires(count); + brokerMgmtObject->inc_reroutes(count); } - dequeue(0, *qmsg); - QPID_LOG(debug, "Purged message at " << qmsg->position << " from " << getName()); - // now reroute if necessary - if (dest.get()) { - assert(qmsg->payload); - qmsg->payload->clearTrace(); - DeliverableMessage dmsg(qmsg->payload); - dest->routeWithAlternate(dmsg); + } else { + mgmtObject->inc_discardsPurge(count); + if (brokerMgmtObject) { + brokerMgmtObject->inc_acquires(count); + brokerMgmtObject->inc_discardsPurge(count); } } } - return c.matches.size(); + + return count; } uint32_t Queue::move(const Queue::shared_ptr destq, uint32_t qty, const qpid::types::Variant::Map *filter) { std::auto_ptr<MessageFilter> mf(MessageFilter::create(filter)); - Collector c(*mf.get(), qty); - - { - Mutex::ScopedLock locker(messageLock); - messages->removeIf( boost::bind<bool>(boost::ref(c), _1) ); - } - - - if (!c.matches.empty()) { - // Update observers and message state: - - if (mgmtObject) { - mgmtObject->inc_acquires(c.matches.size()); - if (brokerMgmtObject) - brokerMgmtObject->inc_acquires(c.matches.size()); - } - - for (std::deque<QueuedMessage>::iterator qmsg = c.matches.begin(); - qmsg != c.matches.end(); ++qmsg) { - { - Mutex::ScopedLock locker(messageLock); - observeAcquire(*qmsg, locker); - } - dequeue(0, *qmsg); - // and move to destination Queue. - assert(qmsg->payload); - destq->deliver(qmsg->payload); - } - } - return c.matches.size(); + return remove(qty, boost::bind(&MessageFilter::match, mf.get(), _1), boost::bind(&moveTo, destq, _1), CONSUMER/*?*/); } -/** Acquire the message at the given position, return true and msg if acquire succeeds */ -bool Queue::acquire(const qpid::framing::SequenceNumber& position, QueuedMessage& msg) +void Queue::push(Message& message, bool /*isRecovery*/) { - bool ok; - { - Mutex::ScopedLock locker(messageLock); - ok = messages->acquire(position, msg); - if (ok) observeAcquire(msg, locker); - } - if (ok) { - if (mgmtObject) { - mgmtObject->inc_acquires(); - if (brokerMgmtObject) - brokerMgmtObject->inc_acquires(); - } - ++dequeueSincePurge; - return true; - } - return false; -} - -void Queue::push(boost::intrusive_ptr<Message>& msg, bool isRecovery){ assertClusterSafe(); QueueListeners::NotificationSet copy; - QueuedMessage removed, qm(this, msg); - bool dequeueRequired = false; { Mutex::ScopedLock locker(messageLock); - qm.position = ++sequence; - if (messages->push(qm, removed)) { - dequeueRequired = true; - observeAcquire(removed, locker); - } - observeEnqueue(qm, locker); - if (policy.get()) { - policy->enqueued(qm); - } + message.setSequence(++sequence); + messages->publish(message); listeners.populate(copy); - } - if (insertSeqNo) msg->insertCustomProperty(seqNoKey, qm.position); - - mgntEnqStats(msg, mgmtObject, brokerMgmtObject); - - if (dequeueRequired) { - if (mgmtObject) { - mgmtObject->inc_acquires(); - mgmtObject->inc_discardsLvq(); - if (brokerMgmtObject) { - brokerMgmtObject->inc_acquires(); - brokerMgmtObject->inc_discardsLvq(); - } - } - if (isRecovery) { - //can't issue new requests for the store until - //recovery is complete - Mutex::ScopedLock locker(messageLock); - pendingDequeues.push_back(removed); - } else { - dequeue(0, removed); - } + observeEnqueue(message, locker); } copy.notify(); } -void isEnqueueComplete(uint32_t* result, const QueuedMessage& message) -{ - if (message.payload->isIngressComplete()) (*result)++; -} - -/** function only provided for unit tests, or code not in critical message path */ -uint32_t Queue::getEnqueueCompleteMessageCount() const -{ - uint32_t count = 0; - Mutex::ScopedLock locker(messageLock); - messages->foreach(boost::bind(&isEnqueueComplete, &count, _1)); - return count; -} - uint32_t Queue::getMessageCount() const { Mutex::ScopedLock locker(messageLock); @@ -949,7 +755,7 @@ uint32_t Queue::getConsumerCount() const bool Queue::canAutoDelete() const { Mutex::ScopedLock locker(messageLock); - return autodelete && !consumerCount && !owner; + return settings.autodelete && !consumerCount && !owner; } void Queue::clearLastNodeFailure() @@ -957,14 +763,9 @@ void Queue::clearLastNodeFailure() inLastNodeFailure = false; } -void Queue::forcePersistent(QueuedMessage& message) +void Queue::forcePersistent(const Message& /*message*/) { - if(!message.payload->isStoredOnQueue(shared_from_this())) { - message.payload->forcePersistent(); - if (message.payload->isForcedPersistent() ){ - enqueue(0, message.payload); - } - } + //TODO } void Queue::setLastNodeFailure() @@ -982,153 +783,129 @@ void Queue::setLastNodeFailure() } -// return true if store exists, -bool Queue::enqueue(TransactionContext* ctxt, boost::intrusive_ptr<Message>& msg, bool suppressPolicyCheck) +/* + * return true if enqueue succeeded and message should be made + * available; returning false will result in the message being dropped + */ +bool Queue::enqueue(TransactionContext* ctxt, Message& msg) { ScopedUse u(barrier); if (!u.acquired) return false; - if (policy.get() && !suppressPolicyCheck) { - std::deque<QueuedMessage> dequeues; - { - Mutex::ScopedLock locker(messageLock); - try { - policy->tryEnqueue(msg); - } catch(ResourceLimitExceededException&) { - if (mgmtObject) { - mgmtObject->inc_discardsOverflow(); - if (brokerMgmtObject) - brokerMgmtObject->inc_discardsOverflow(); - } - throw; - } - policy->getPendingDequeues(dequeues); - } - //depending on policy, may have some dequeues that need to performed without holding the lock - - // - // Count the dequeues as ring-discards. We know that these aren't rejects because - // policy->tryEnqueue would have thrown an exception. - // - if (mgmtObject && !dequeues.empty()) { - mgmtObject->inc_discardsRing(dequeues.size()); - if (brokerMgmtObject) - brokerMgmtObject->inc_discardsRing(dequeues.size()); + { + Mutex::ScopedLock locker(messageLock); + if (!checkDepth(QueueDepth(1, msg.getContentSize()), msg)) { + return false; } - - for_each(dequeues.begin(), dequeues.end(), boost::bind(&Queue::dequeue, this, (TransactionContext*) 0, _1)); } if (inLastNodeFailure && persistLastNode){ - msg->forcePersistent(); + forcePersistent(msg); } - if (traceId.size()) { - msg->addTraceId(traceId); + if (settings.traceId.size()) { + msg.addTraceId(settings.traceId); } - if ((msg->isPersistent() || msg->checkContentReleasable()) && store) { + if (msg.isPersistent() && store) { // mark the message as being enqueued - the store MUST CALL msg->enqueueComplete() // when it considers the message stored. - msg->enqueueAsync(shared_from_this(), store); - boost::intrusive_ptr<PersistableMessage> pmsg = boost::static_pointer_cast<PersistableMessage>(msg); + boost::intrusive_ptr<PersistableMessage> pmsg = msg.getPersistentContext(); + assert(pmsg); + pmsg->enqueueAsync(shared_from_this(), store); store->enqueue(ctxt, pmsg, *this); - return true; } - if (!store) { - //Messages enqueued on a transient queue should be prevented - //from having their content released as it may not be - //recoverable by these queue for delivery - msg->blockContentRelease(); - } - return false; + return true; } -void Queue::enqueueAborted(boost::intrusive_ptr<Message> msg) +void Queue::enqueueAborted(const Message& msg) { + //Called when any transactional enqueue is aborted (including but + //not limited to a recovered dtx transaction) Mutex::ScopedLock locker(messageLock); - if (policy.get()) policy->enqueueAborted(msg); + current -= QueueDepth(1, msg.getContentSize()); } -// return true if store exists, -bool Queue::dequeue(TransactionContext* ctxt, const QueuedMessage& msg) +void Queue::enqueueCommited(Message& msg) { - ScopedUse u(barrier); - if (!u.acquired) return false; - { - Mutex::ScopedLock locker(messageLock); - if (!isEnqueued(msg)) return false; - if (!ctxt) { - if (policy.get()) policy->dequeued(msg); - messages->deleted(msg); - observeDequeue(msg, locker); - } + //called when a recovered dtx enqueue operation is committed; the + //message is already on disk and space has been reserved in policy + //but it should now be made available + process(msg); +} +void Queue::dequeueAborted(Message& msg) +{ + //called when a recovered dtx dequeue operation is aborted; the + //message should be added back to the queue + push(msg); +} +void Queue::dequeueCommited(const Message& msg) +{ + //called when a recovered dtx dequeue operation is committed; the + //message will at this point have already been removed from the + //store and will not be available for delivery. The only action + //required is to ensure the observers are notified and the + //management stats are correctly decremented + Mutex::ScopedLock locker(messageLock); + observeDequeue(msg, locker); + if (mgmtObject != 0) { + mgmtObject->inc_msgTxnDequeues(); + mgmtObject->inc_byteTxnDequeues(msg.getContentSize()); } +} - if (!ctxt) { - mgntDeqStats(msg.payload, mgmtObject, brokerMgmtObject); - } - // This check prevents messages which have been forced persistent on one queue from dequeuing - // from another on which no forcing has taken place and thus causing a store error. - bool fp = msg.payload->isForcedPersistent(); - if (!fp || (fp && msg.payload->isStoredOnQueue(shared_from_this()))) { - if ((msg.payload->isPersistent() || msg.payload->checkContentReleasable()) && store) { - msg.payload->dequeueAsync(shared_from_this(), store); //increment to async counter -- for message sent to more than one queue - boost::intrusive_ptr<PersistableMessage> pmsg = boost::static_pointer_cast<PersistableMessage>(msg.payload); - store->dequeue(ctxt, pmsg, *this); - return true; - } +void Queue::dequeueFromStore(boost::intrusive_ptr<PersistableMessage> msg) +{ + ScopedUse u(barrier); + if (u.acquired && msg && store) { + store->dequeue(0, msg, *this); } - return false; } -void Queue::dequeueCommitted(const QueuedMessage& msg) +void Queue::dequeue(TransactionContext* ctxt, const QueueCursor& cursor) { + ScopedUse u(barrier); + if (!u.acquired) return; + boost::intrusive_ptr<PersistableMessage> pmsg; { Mutex::ScopedLock locker(messageLock); - if (policy.get()) policy->dequeued(msg); - messages->deleted(msg); - observeDequeue(msg, locker); + Message* msg = messages->find(cursor); + if (msg) { + if (msg->isPersistent()) pmsg = msg->getPersistentContext(); + if (!ctxt) { + observeDequeue(*msg, locker); + messages->deleted(cursor);//message pointer not valid after this + } + } else { + return; + } } - mgntDeqStats(msg.payload, mgmtObject, brokerMgmtObject); - if (mgmtObject != 0) { - _qmf::Queue::PerThreadStats *qStats = mgmtObject->getStatistics(); - const uint64_t contentSize = msg.payload->contentSize(); - qStats->msgTxnDequeues += 1; - qStats->byteTxnDequeues += contentSize; - mgmtObject->statisticsUpdated(); + if (store && pmsg) { + store->dequeue(ctxt, pmsg, *this); + } +} + +void Queue::dequeueCommitted(const QueueCursor& cursor) +{ + Mutex::ScopedLock locker(messageLock); + Message* msg = messages->find(cursor); + if (msg) { + const uint64_t contentSize = msg->getContentSize(); + observeDequeue(*msg, locker); + if (mgmtObject != 0) { + mgmtObject->inc_msgTxnDequeues(); + mgmtObject->inc_byteTxnDequeues(contentSize); + } if (brokerMgmtObject) { _qmf::Broker::PerThreadStats *bStats = brokerMgmtObject->getStatistics(); bStats->msgTxnDequeues += 1; bStats->byteTxnDequeues += contentSize; brokerMgmtObject->statisticsUpdated(); } - } -} - -/** - * Removes the first (oldest) message from the in-memory delivery queue as well dequeing - * it from the logical (and persistent if applicable) queue - */ -bool Queue::popAndDequeue(QueuedMessage& msg) -{ - bool popped; - { - Mutex::ScopedLock locker(messageLock); - popped = messages->consume(msg); - if (popped) observeAcquire(msg, locker); - } - if (popped) { - if (mgmtObject) { - mgmtObject->inc_acquires(); - if (brokerMgmtObject) - brokerMgmtObject->inc_acquires(); - } - dequeue(0, msg); - return true; + messages->deleted(cursor); } else { - return false; + QPID_LOG(error, "Could not find dequeued message on commit"); } } @@ -1136,8 +913,10 @@ bool Queue::popAndDequeue(QueuedMessage& msg) * Updates policy and management when a message has been dequeued, * Requires messageLock be held by caller. */ -void Queue::observeDequeue(const QueuedMessage& msg, const qpid::sys::Mutex::ScopedLock&) +void Queue::observeDequeue(const Message& msg, const Mutex::ScopedLock&) { + current -= QueueDepth(1, msg.getContentSize()); + mgntDeqStats(msg, mgmtObject, brokerMgmtObject); for (Observers::const_iterator i = observers.begin(); i != observers.end(); ++i) { try{ (*i)->dequeued(msg); @@ -1150,7 +929,7 @@ void Queue::observeDequeue(const QueuedMessage& msg, const qpid::sys::Mutex::Sco /** updates queue observers when a message has become unavailable for transfer. * Requires messageLock be held by caller. */ -void Queue::observeAcquire(const QueuedMessage& msg, const qpid::sys::Mutex::ScopedLock&) +void Queue::observeAcquire(const Message& msg, const Mutex::ScopedLock&) { for (Observers::const_iterator i = observers.begin(); i != observers.end(); ++i) { try{ @@ -1164,7 +943,7 @@ void Queue::observeAcquire(const QueuedMessage& msg, const qpid::sys::Mutex::Sco /** updates queue observers when a message has become re-available for transfer * Requires messageLock be held by caller. */ -void Queue::observeRequeue(const QueuedMessage& msg, const qpid::sys::Mutex::ScopedLock&) +void Queue::observeRequeue(const Message& msg, const Mutex::ScopedLock&) { for (Observers::const_iterator i = observers.begin(); i != observers.end(); ++i) { try{ @@ -1202,13 +981,11 @@ void Queue::observeConsumerRemove( const Consumer& c, const qpid::sys::Mutex::Sc } -void Queue::create(const FieldTable& _settings) +void Queue::create() { - settings = _settings; if (store) { - store->create(*this, _settings); + store->create(*this, settings.storeSettings); } - configureImpl(_settings); } @@ -1258,112 +1035,21 @@ bool getBoolSetting(const qpid::framing::FieldTable& settings, const std::string } } -void Queue::configure(const FieldTable& _settings) +void Queue::abandoned(const Message& message) { - settings = _settings; - configureImpl(settings); -} - -void Queue::configureImpl(const FieldTable& _settings) -{ - eventMode = _settings.getAsInt(qpidQueueEventGeneration); - if (eventMode && broker) { - broker->getQueueEvents().observe(*this, eventMode == ENQUEUE_ONLY); - } - - if (QueuePolicy::getType(_settings) == QueuePolicy::FLOW_TO_DISK && - (!store || NullMessageStore::isNullStore(store) || (broker && !(broker->getQueueEvents().isSync())) )) { - if ( NullMessageStore::isNullStore(store)) { - QPID_LOG(warning, "Flow to disk not valid for non-persisted queue:" << getName()); - } else if (broker && !(broker->getQueueEvents().isSync()) ) { - QPID_LOG(warning, "Flow to disk not valid with async Queue Events:" << getName()); - } - FieldTable copy(_settings); - copy.erase(QueuePolicy::typeKey); - setPolicy(QueuePolicy::createQueuePolicy(getName(), copy)); - } else { - setPolicy(QueuePolicy::createQueuePolicy(getName(), _settings)); - } - if (broker && broker->getManagementAgent()) { - ThresholdAlerts::observe(*this, *(broker->getManagementAgent()), _settings, broker->getOptions().queueThresholdEventRatio); - } - - //set this regardless of owner to allow use of no-local with exclusive consumers also - noLocal = getBoolSetting(_settings, qpidNoLocal); - QPID_LOG(debug, "Configured queue " << getName() << " with no-local=" << noLocal); - - std::string lvqKey = _settings.getAsString(qpidLastValueQueueKey); - if (lvqKey.size()) { - QPID_LOG(debug, "Configured queue " << getName() << " as Last Value Queue with key " << lvqKey); - messages = std::auto_ptr<Messages>(new MessageMap(lvqKey)); - allocator = boost::shared_ptr<MessageDistributor>(new FifoDistributor( *messages )); - } else if (getBoolSetting(_settings, qpidLastValueQueueNoBrowse)) { - QPID_LOG(debug, "Configured queue " << getName() << " as Legacy Last Value Queue with 'no-browse' on"); - messages = LegacyLVQ::updateOrReplace(messages, qpidVQMatchProperty, true, broker); - allocator = boost::shared_ptr<MessageDistributor>(new FifoDistributor( *messages )); - } else if (getBoolSetting(_settings, qpidLastValueQueue)) { - QPID_LOG(debug, "Configured queue " << getName() << " as Legacy Last Value Queue"); - messages = LegacyLVQ::updateOrReplace(messages, qpidVQMatchProperty, false, broker); - allocator = boost::shared_ptr<MessageDistributor>(new FifoDistributor( *messages )); - } else { - std::auto_ptr<Messages> m = Fairshare::create(_settings); - if (m.get()) { - messages = m; - allocator = boost::shared_ptr<MessageDistributor>(new FifoDistributor( *messages )); - QPID_LOG(debug, "Configured queue " << getName() << " as priority queue."); - } else { // default (FIFO) queue type - // override default message allocator if message groups configured. - boost::shared_ptr<MessageGroupManager> mgm(MessageGroupManager::create( getName(), *messages, _settings)); - if (mgm) { - allocator = mgm; - addObserver(mgm); - } - } - } - - persistLastNode = getBoolSetting(_settings, qpidPersistLastNode); - if (persistLastNode) QPID_LOG(debug, "Configured queue to Persist data if cluster fails to one node for: " << getName()); - - traceId = _settings.getAsString(qpidTraceIdentity); - std::string excludeList = _settings.getAsString(qpidTraceExclude); - if (excludeList.size()) { - split(traceExclude, excludeList, ", "); - } - QPID_LOG(debug, "Configured queue " << getName() << " with qpid.trace.id='" << traceId - << "' and qpid.trace.exclude='"<< excludeList << "' i.e. " << traceExclude.size() << " elements"); - - FieldTable::ValuePtr p =_settings.get(qpidInsertSequenceNumbers); - if (p && p->convertsTo<std::string>()) insertSequenceNumbers(p->get<std::string>()); - - autoDeleteTimeout = getIntegerSetting(_settings, qpidAutoDeleteTimeout); - if (autoDeleteTimeout) - QPID_LOG(debug, "Configured queue " << getName() << " with qpid.auto_delete_timeout=" << autoDeleteTimeout); - - if (mgmtObject != 0) { - mgmtObject->set_arguments(ManagementAgent::toMap(_settings)); - } - - QueueFlowLimit::observe(*this, _settings); + if (reroute(alternateExchange, message) && brokerMgmtObject) + brokerMgmtObject->inc_abandonedViaAlt(); + else if (brokerMgmtObject) + brokerMgmtObject->inc_abandoned(); } void Queue::destroyed() { unbind(broker->getExchanges()); - - QueuedMessage m; - while(popAndDequeue(m)) { - DeliverableMessage msg(m.payload); - if (alternateExchange.get()) { - if (brokerMgmtObject) - brokerMgmtObject->inc_abandonedViaAlt(); - alternateExchange->routeWithAlternate(msg); - } else { - if (brokerMgmtObject) - brokerMgmtObject->inc_abandoned(); - } - } - if (alternateExchange.get()) + remove(0, 0, boost::bind(&Queue::abandoned, this, _1), REPLICATOR/*even acquired message are treated as abandoned*/); + if (alternateExchange.get()) { alternateExchange->decAlternateUsers(); + } if (store) { barrier.destroy(); @@ -1401,20 +1087,6 @@ void Queue::unbind(ExchangeRegistry& exchanges) bindings.unbind(exchanges, shared_from_this()); } -void Queue::setPolicy(std::auto_ptr<QueuePolicy> _policy) -{ - Mutex::ScopedLock locker(messageLock); - policy = _policy; - if (policy.get()) - policy->setQueue(this); -} - -const QueuePolicy* Queue::getPolicy() -{ - Mutex::ScopedLock locker(messageLock); - return policy.get(); -} - uint64_t Queue::getPersistenceId() const { return persistenceId; @@ -1434,10 +1106,7 @@ void Queue::setPersistenceId(uint64_t _persistenceId) const void Queue::encode(Buffer& buffer) const { buffer.putShortString(name); - buffer.put(settings); - if (policy.get()) { - buffer.put(*policy); - } + buffer.put(encodableSettings); buffer.putShortString(alternateExchange.get() ? alternateExchange->getName() : std::string("")); } @@ -1445,21 +1114,19 @@ uint32_t Queue::encodedSize() const { return name.size() + 1/*short string size octet*/ + (alternateExchange.get() ? alternateExchange->getName().size() : 0) + 1 /* short string */ - + settings.encodedSize() - + (policy.get() ? (*policy).encodedSize() : 0); + + encodableSettings.encodedSize(); } Queue::shared_ptr Queue::restore( QueueRegistry& queues, Buffer& buffer ) { string name; buffer.getShortString(name); - FieldTable settings; - buffer.get(settings); + FieldTable ft; + buffer.get(ft); boost::shared_ptr<Exchange> alternate; - std::pair<Queue::shared_ptr, bool> result = queues.declare(name, true, false, 0, alternate, settings, true); - if (result.first->policy.get() && buffer.available() >= result.first->policy->encodedSize()) { - buffer.get ( *(result.first->policy) ); - } + QueueSettings settings(true, false); + settings.populate(ft, settings.storeSettings); + std::pair<Queue::shared_ptr, bool> result = queues.declare(name, settings, alternate, true); if (buffer.available()) { string altExch; buffer.getShortString(altExch); @@ -1523,8 +1190,8 @@ struct AutoDeleteTask : qpid::sys::TimerTask void Queue::tryAutoDelete(Broker& broker, Queue::shared_ptr queue, const std::string& connectionId, const std::string& userId) { - if (queue->autoDeleteTimeout && queue->canAutoDelete()) { - AbsTime time(now(), Duration(queue->autoDeleteTimeout * TIME_SEC)); + if (queue->settings.autoDeleteDelay && queue->canAutoDelete()) { + AbsTime time(now(), Duration(queue->settings.autoDeleteDelay * TIME_SEC)); queue->autoDeleteTask = boost::intrusive_ptr<qpid::sys::TimerTask>(new AutoDeleteTask(broker, queue, connectionId, userId, time)); broker.getClusterTimer().add(queue->autoDeleteTask); QPID_LOG(debug, "Timed auto-delete for " << queue->getName() << " initiated"); @@ -1543,12 +1210,15 @@ void Queue::releaseExclusiveOwnership() { Mutex::ScopedLock locker(ownershipLock); owner = 0; + if (mgmtObject) { + mgmtObject->set_exclusive(false); + } } bool Queue::setExclusiveOwner(const OwnershipToken* const o) { //reset auto deletion timer if necessary - if (autoDeleteTimeout && autoDeleteTask) { + if (settings.autoDeleteDelay && autoDeleteTask) { autoDeleteTask->cancel(); } Mutex::ScopedLock locker(ownershipLock); @@ -1556,6 +1226,9 @@ bool Queue::setExclusiveOwner(const OwnershipToken* const o) return false; } else { owner = o; + if (mgmtObject) { + mgmtObject->set_exclusive(true); + } return true; } } @@ -1687,7 +1360,7 @@ namespace { struct After { framing::SequenceNumber seq; After(framing::SequenceNumber s) : seq(s) {} - bool operator()(const QueuedMessage& qm) { return qm.position > seq; } + bool operator()(const Message& m) { return m.getSequence() > seq; } }; } // namespace @@ -1695,12 +1368,10 @@ struct After { void Queue::setPosition(SequenceNumber n) { Mutex::ScopedLock locker(messageLock); if (n < sequence) { - std::deque<QueuedMessage> dequeued; - dequeueIf(After(n), dequeued); - messages->setPosition(n); + remove(0, After(n), MessagePredicate(), BROWSER); } sequence = n; - QPID_LOG(trace, "Set position to " << sequence << " on " << getName()); + QPID_LOG(debug, "Set position to " << sequence << " on " << getName()); } SequenceNumber Queue::getPosition() { @@ -1721,25 +1392,16 @@ void Queue::recoveryComplete(ExchangeRegistry& exchanges) << "\": exchange does not exist."); } //process any pending dequeues - std::deque<QueuedMessage> pd; - { - Mutex::ScopedLock locker(messageLock); - pendingDequeues.swap(pd); + for (std::vector<Message>::iterator i = pendingDequeues.begin(); i != pendingDequeues.end(); ++i) { + dequeueFromStore(i->getPersistentContext()); } - for_each(pd.begin(), pd.end(), boost::bind(&Queue::dequeue, this, (TransactionContext*) 0, _1)); -} - -void Queue::insertSequenceNumbers(const std::string& key) -{ - seqNoKey = key; - insertSeqNo = !seqNoKey.empty(); - QPID_LOG(debug, "Inserting sequence numbers as " << key); + pendingDequeues.clear(); } /** updates queue observers and state when a message has become available for transfer * Requires messageLock be held by caller. */ -void Queue::observeEnqueue(const QueuedMessage& m, const qpid::sys::Mutex::ScopedLock&) +void Queue::observeEnqueue(const Message& m, const Mutex::ScopedLock&) { for (Observers::iterator i = observers.begin(); i != observers.end(); ++i) { try { @@ -1748,32 +1410,7 @@ void Queue::observeEnqueue(const QueuedMessage& m, const qpid::sys::Mutex::Scope QPID_LOG(warning, "Exception on notification of enqueue for queue " << getName() << ": " << e.what()); } } -} - -void Queue::updateEnqueued(const QueuedMessage& m) -{ - if (m.payload) { - boost::intrusive_ptr<Message> payload = m.payload; - enqueue(0, payload, true); - { - Mutex::ScopedLock locker(messageLock); - messages->updateAcquired(m); - observeEnqueue(m, locker); - if (policy.get()) { - policy->recoverEnqueued(payload); - policy->enqueued(m); - } - } - mgntEnqStats(m.payload, mgmtObject, brokerMgmtObject); - } else { - QPID_LOG(warning, "Queue informed of enqueued message that has no payload"); - } -} - -bool Queue::isEnqueued(const QueuedMessage& msg) -{ - Mutex::ScopedLock locker(messageLock); - return !policy.get() || policy->isEnqueued(msg); + mgntEnqStats(m, mgmtObject, brokerMgmtObject); } // Note: accessing listeners outside of lock is dangerous. Caller must ensure the queue's @@ -1835,28 +1472,82 @@ void Queue::setDequeueSincePurge(uint32_t value) { dequeueSincePurge = value; } -namespace{ -class FindLowest +void Queue::reject(const QueueCursor& cursor) { - public: - FindLowest() : init(false) {} - void process(const QueuedMessage& message) { - QPID_LOG(debug, "FindLowest processing: " << message.position); - if (!init || message.position < lowest) lowest = message.position; - init = true; - } - bool getLowest(qpid::framing::SequenceNumber& result) { - if (init) { - result = lowest; - return true; + Exchange::shared_ptr alternate = getAlternateExchange(); + Message copy; + boost::intrusive_ptr<PersistableMessage> pmsg; + { + Mutex::ScopedLock locker(messageLock); + Message* message = messages->find(cursor); + if (message) { + if (alternate) copy = *message; + if (message->isPersistent()) pmsg = message->getPersistentContext(); + countRejected(); + observeDequeue(*message, locker); + messages->deleted(cursor); } else { - return false; + return; } } - private: - bool init; - qpid::framing::SequenceNumber lowest; -}; + if (alternate) { + copy.resetDeliveryCount(); + DeliverableMessage delivery(copy, 0); + alternate->routeWithAlternate(delivery); + QPID_LOG(info, "Routed rejected message from " << getName() << " to " + << alternate->getName()); + } else { + //just drop it + QPID_LOG(info, "Dropping rejected message from " << getName()); + } + dequeueFromStore(pmsg); +} + +bool Queue::checkDepth(const QueueDepth& increment, const Message&) +{ + if (current && (settings.maxDepth - current < increment)) { + if (mgmtObject) { + mgmtObject->inc_discardsOverflow(); + if (brokerMgmtObject) + brokerMgmtObject->inc_discardsOverflow(); + } + throw ResourceLimitExceededException(QPID_MSG("Maximum depth exceeded on " << name << ": current=[" << current << "], max=[" << settings.maxDepth << "]")); + } else { + current += increment; + return true; + } +} + +bool Queue::seek(QueueCursor& cursor, MessagePredicate predicate) +{ + Mutex::ScopedLock locker(messageLock); + //hold lock across calls to predicate, or take copy of message? + //currently hold lock, may want to revise depending on any new use + //cases + Message* message = messages->next(cursor); + while (message && (predicate && !predicate(*message))) { + message = messages->next(cursor); + } + return message != 0; +} + +bool Queue::seek(QueueCursor& cursor, MessagePredicate predicate, qpid::framing::SequenceNumber start) +{ + Mutex::ScopedLock locker(messageLock); + //hold lock across calls to predicate, or take copy of message? + //currently hold lock, may want to revise depending on any new use + //cases + Message* message; + message = messages->find(start, &cursor); + if (message && (!predicate || predicate(*message))) return true; + + return seek(cursor, predicate); +} + +bool Queue::seek(QueueCursor& cursor, qpid::framing::SequenceNumber start) +{ + Mutex::ScopedLock locker(messageLock); + return messages->find(start, &cursor); } Queue::UsageBarrier::UsageBarrier(Queue& q) : parent(q), count(0) {} diff --git a/qpid/cpp/src/qpid/broker/Queue.h b/qpid/cpp/src/qpid/broker/Queue.h index a31e0002ea..671a24d53e 100644 --- a/qpid/cpp/src/qpid/broker/Queue.h +++ b/qpid/cpp/src/qpid/broker/Queue.h @@ -28,12 +28,14 @@ #include "qpid/broker/Message.h" #include "qpid/broker/Messages.h" #include "qpid/broker/PersistableQueue.h" -#include "qpid/broker/QueuePolicy.h" #include "qpid/broker/QueueBindings.h" #include "qpid/broker/QueueListeners.h" #include "qpid/broker/QueueObserver.h" +#include "qpid/broker/QueueSettings.h" +#include "qpid/broker/TxOp.h" #include "qpid/framing/FieldTable.h" +#include "qpid/framing/SequenceNumber.h" #include "qpid/sys/AtomicValue.h" #include "qpid/sys/Monitor.h" #include "qpid/sys/Timer.h" @@ -56,10 +58,14 @@ namespace qpid { namespace broker { class Broker; +class Exchange; class MessageStore; +class QueueDepth; class QueueEvents; class QueueRegistry; +class QueueFactory; class TransactionContext; +class TxBuffer; class MessageDistributor; /** @@ -70,7 +76,9 @@ class MessageDistributor; */ class Queue : public boost::enable_shared_from_this<Queue>, public PersistableQueue, public management::Manageable { - + public: + typedef boost::function1<bool, const Message&> MessagePredicate; + protected: struct UsageBarrier { Queue& parent; @@ -90,31 +98,40 @@ class Queue : public boost::enable_shared_from_this<Queue>, ~ScopedUse() { if (acquired) barrier.release(); } }; + class TxPublish : public TxOp + { + Message message; + boost::shared_ptr<Queue> queue; + bool prepared; + public: + TxPublish(const Message&,boost::shared_ptr<Queue>); + bool prepare(TransactionContext* ctxt) throw(); + void commit() throw(); + void rollback() throw(); + }; + typedef std::set< boost::shared_ptr<QueueObserver> > Observers; enum ConsumeCode {NO_MESSAGES=0, CANT_CONSUME=1, CONSUMED=2}; + typedef boost::function1<void, Message&> MessageFunctor; const std::string name; - const bool autodelete; MessageStore* store; const OwnershipToken* owner; uint32_t consumerCount; // Actually a count of all subscriptions, acquiring or not. uint32_t browserCount; // Count of non-acquiring subscriptions. OwnershipToken* exclusive; - bool noLocal; bool persistLastNode; bool inLastNodeFailure; - std::string traceId; std::vector<std::string> traceExclude; QueueListeners listeners; std::auto_ptr<Messages> messages; - std::deque<QueuedMessage> pendingDequeues;//used to avoid dequeuing during recovery + std::vector<Message> pendingDequeues; /** messageLock is used to keep the Queue's state consistent while processing message * events, such as message dispatch, enqueue, acquire, and dequeue. It must be held * while updating certain members in order to keep these members consistent with * each other: * o messages * o sequence - * o policy * o listeners * o allocator * o observeXXX() methods @@ -127,9 +144,9 @@ class Queue : public boost::enable_shared_from_this<Queue>, mutable qpid::sys::Monitor messageLock; mutable qpid::sys::Mutex ownershipLock; mutable uint64_t persistenceId; - framing::FieldTable settings; - std::auto_ptr<QueuePolicy> policy; - bool policyExceeded; + const QueueSettings settings; + qpid::framing::FieldTable encodableSettings; + QueueDepth current; QueueBindings bindings; std::string alternateExchangeName; boost::shared_ptr<Exchange> alternateExchange; @@ -139,43 +156,42 @@ class Queue : public boost::enable_shared_from_this<Queue>, sys::AtomicValue<uint32_t> dequeueSincePurge; // Count dequeues since last purge. int eventMode; Observers observers; - bool insertSeqNo; std::string seqNoKey; Broker* broker; bool deleted; UsageBarrier barrier; - int autoDeleteTimeout; boost::intrusive_ptr<qpid::sys::TimerTask> autoDeleteTask; boost::shared_ptr<MessageDistributor> allocator; - void push(boost::intrusive_ptr<Message>& msg, bool isRecovery=false); - void setPolicy(std::auto_ptr<QueuePolicy> policy); - bool getNextMessage(QueuedMessage& msg, Consumer::shared_ptr& c); - ConsumeCode consumeNextMessage(QueuedMessage& msg, Consumer::shared_ptr& c); - bool browseNextMessage(QueuedMessage& msg, Consumer::shared_ptr& c); - void notifyListener(); + virtual void push(Message& msg, bool isRecovery=false); + void process(Message& msg); + bool enqueue(TransactionContext* ctxt, Message& msg); + bool getNextMessage(Message& msg, Consumer::shared_ptr& c); void removeListener(Consumer::shared_ptr); - bool isExcluded(boost::intrusive_ptr<Message>& msg); + bool isExcluded(const Message& msg); - /** update queue observers, stats, policy, etc when the messages' state changes. - * messageLock is held by caller */ - void observeEnqueue(const QueuedMessage& msg, const sys::Mutex::ScopedLock& lock); - void observeAcquire(const QueuedMessage& msg, const sys::Mutex::ScopedLock& lock); - void observeRequeue(const QueuedMessage& msg, const sys::Mutex::ScopedLock& lock); - void observeDequeue(const QueuedMessage& msg, const sys::Mutex::ScopedLock& lock); + /** update queue observers, stats, policy, etc when the messages' state changes. Lock + * must be held by caller */ + void observeEnqueue(const Message& msg, const sys::Mutex::ScopedLock& lock); + void observeAcquire(const Message& msg, const sys::Mutex::ScopedLock& lock); + void observeRequeue(const Message& msg, const sys::Mutex::ScopedLock& lock); + void observeDequeue(const Message& msg, const sys::Mutex::ScopedLock& lock); void observeConsumerAdd( const Consumer&, const sys::Mutex::ScopedLock& lock); void observeConsumerRemove( const Consumer&, const sys::Mutex::ScopedLock& lock); - bool popAndDequeue(QueuedMessage&); - bool acquire(const qpid::framing::SequenceNumber& position, QueuedMessage& msg); - void forcePersistent(QueuedMessage& msg); + bool acquire(const qpid::framing::SequenceNumber& position, Message& msg, + const qpid::sys::Mutex::ScopedLock& locker); + + void forcePersistent(const Message& msg); int getEventMode(); - void configureImpl(const qpid::framing::FieldTable& settings); - void checkNotDeleted(const Consumer::shared_ptr& c); + void dequeueFromStore(boost::intrusive_ptr<PersistableMessage>); + void abandoned(const Message& message); + void checkNotDeleted(const Consumer::shared_ptr&); void notifyDeleted(); - void dequeueIf(Messages::Predicate predicate, std::deque<QueuedMessage>& dequeued); + uint32_t remove(uint32_t maxCount, MessagePredicate, MessageFunctor, SubscriptionType); + virtual bool checkDepth(const QueueDepth& increment, const Message&); public: @@ -184,12 +200,11 @@ class Queue : public boost::enable_shared_from_this<Queue>, typedef std::vector<shared_ptr> vector; QPID_BROKER_EXTERN Queue(const std::string& name, - bool autodelete = false, + const QueueSettings& settings = QueueSettings(), MessageStore* const store = 0, - const OwnershipToken* const owner = 0, management::Manageable* parent = 0, Broker* broker = 0); - QPID_BROKER_EXTERN ~Queue(); + QPID_BROKER_EXTERN virtual ~Queue(); /** allow the Consumer to consume or browse the next available message */ QPID_BROKER_EXTERN bool dispatch(Consumer::shared_ptr); @@ -198,19 +213,13 @@ class Queue : public boost::enable_shared_from_this<Queue>, * @param msg - message to be acquired. * @return false if message is no longer available for acquire. */ - QPID_BROKER_EXTERN bool acquire(const QueuedMessage& msg, const std::string& consumer); + QPID_BROKER_EXTERN bool acquire(const QueueCursor& msg, const std::string& consumer); /** - * Used to configure a new queue and create a persistent record - * for it in store if required. + * Used to create a persistent record for the queue in store if required. */ - QPID_BROKER_EXTERN void create(const qpid::framing::FieldTable& settings); + QPID_BROKER_EXTERN void create(); - /** - * Used to reconfigure a recovered queue (does not create - * persistent record in store). - */ - QPID_BROKER_EXTERN void configure(const qpid::framing::FieldTable& settings); void destroyed(); QPID_BROKER_EXTERN void bound(const std::string& exchange, const std::string& key, @@ -224,34 +233,36 @@ class Queue : public boost::enable_shared_from_this<Queue>, boost::shared_ptr<Exchange> exchange, const std::string& key, const qpid::framing::FieldTable& arguments=qpid::framing::FieldTable()); - /** Acquire the message at the given position if it is available for acquire. Not to - * be used by clients, but used by the broker for queue management. - * @param message - set to the acquired message if true returned. - * @return true if the message has been acquired. + /** + * Removes (and dequeues) a message by its sequence number (used + * for some broker features, e.g. queue replication) + * + * @param position the sequence number of the message to be dequeued. + * @return true if the message is dequeued. */ - QPID_BROKER_EXTERN bool acquireMessageAt(const qpid::framing::SequenceNumber& position, QueuedMessage& message); + QPID_BROKER_EXTERN bool dequeueMessageAt(const qpid::framing::SequenceNumber& position); /** * Delivers a message to the queue. Will record it as * enqueued if persistent then process it. */ - QPID_BROKER_EXTERN void deliver(boost::intrusive_ptr<Message> msg); - /** - * Dispatches the messages immediately to a consumer if - * one is available or stores it for later if not. - */ - QPID_BROKER_EXTERN void process(boost::intrusive_ptr<Message>& msg); + QPID_BROKER_EXTERN void deliver(Message, TxBuffer* = 0); /** * Returns a message to the in-memory queue (due to lack * of acknowledegement from a receiver). If a consumer is * available it will be dispatched immediately, else it * will be returned to the front of the queue. */ - QPID_BROKER_EXTERN void requeue(const QueuedMessage& msg); + QPID_BROKER_EXTERN void release(const QueueCursor& msg, bool markRedelivered=true); + QPID_BROKER_EXTERN void reject(const QueueCursor& msg); + + QPID_BROKER_EXTERN bool seek(QueueCursor&, MessagePredicate); + QPID_BROKER_EXTERN bool seek(QueueCursor&, MessagePredicate, qpid::framing::SequenceNumber start); + QPID_BROKER_EXTERN bool seek(QueueCursor&, qpid::framing::SequenceNumber start); /** * Used during recovery to add stored messages back to the queue */ - QPID_BROKER_EXTERN void recover(boost::intrusive_ptr<Message>& msg); + QPID_BROKER_EXTERN void recover(Message& msg); QPID_BROKER_EXTERN void consume(Consumer::shared_ptr c, bool exclusive = false); @@ -268,7 +279,6 @@ class Queue : public boost::enable_shared_from_this<Queue>, const qpid::types::Variant::Map *filter=0); QPID_BROKER_EXTERN uint32_t getMessageCount() const; - QPID_BROKER_EXTERN uint32_t getEnqueueCompleteMessageCount() const; QPID_BROKER_EXTERN uint32_t getConsumerCount() const; inline const std::string& getName() const { return name; } QPID_BROKER_EXTERN bool isExclusiveOwner(const OwnershipToken* const o) const; @@ -277,8 +287,9 @@ class Queue : public boost::enable_shared_from_this<Queue>, QPID_BROKER_EXTERN bool hasExclusiveConsumer() const; QPID_BROKER_EXTERN bool hasExclusiveOwner() const; inline bool isDurable() const { return store != 0; } - inline const framing::FieldTable& getSettings() const { return settings; } - inline bool isAutoDelete() const { return autodelete; } + inline const QueueSettings& getSettings() const { return settings; } + inline const qpid::framing::FieldTable& getEncodableSettings() const { return encodableSettings; } + inline bool isAutoDelete() const { return settings.autodelete; } QPID_BROKER_EXTERN bool canAutoDelete() const; const QueueBindings& getBindings() const { return bindings; } @@ -288,48 +299,22 @@ class Queue : public boost::enable_shared_from_this<Queue>, QPID_BROKER_EXTERN void setLastNodeFailure(); QPID_BROKER_EXTERN void clearLastNodeFailure(); - QPID_BROKER_EXTERN bool enqueue(TransactionContext* ctxt, boost::intrusive_ptr<Message>& msg, bool suppressPolicyCheck = false); - QPID_BROKER_EXTERN void enqueueAborted(boost::intrusive_ptr<Message> msg); /** * dequeue from store (only done once messages is acknowledged) */ - QPID_BROKER_EXTERN bool dequeue(TransactionContext* ctxt, const QueuedMessage &msg); + QPID_BROKER_EXTERN void dequeue(TransactionContext* ctxt, const QueueCursor&); /** * Inform the queue that a previous transactional dequeue * committed. */ - QPID_BROKER_EXTERN void dequeueCommitted(const QueuedMessage& msg); - - /** - * Inform queue of messages that were enqueued, have since - * been acquired but not yet accepted or released (and - * thus are still logically on the queue) - used in - * clustered broker. - */ - QPID_BROKER_EXTERN void updateEnqueued(const QueuedMessage& msg); - - /** - * Test whether the specified message (identified by its - * sequence/position), is still enqueued (note this - * doesn't mean it is available for delivery as it may - * have been delievered to a subscriber who has not yet - * accepted it). - */ - QPID_BROKER_EXTERN bool isEnqueued(const QueuedMessage& msg); - - /** - * Acquires the next available (oldest) message - */ - QPID_BROKER_EXTERN QueuedMessage get(); + void dequeueCommitted(const QueueCursor& msg); /** Get the message at position pos, returns true if found and sets msg */ - QPID_BROKER_EXTERN bool find(framing::SequenceNumber pos, QueuedMessage& msg ) const; - - QPID_BROKER_EXTERN const QueuePolicy* getPolicy(); + QPID_BROKER_EXTERN bool find(framing::SequenceNumber pos, Message& msg ) const; QPID_BROKER_EXTERN void setAlternateExchange(boost::shared_ptr<Exchange> exchange); QPID_BROKER_EXTERN boost::shared_ptr<Exchange> getAlternateExchange(); - QPID_BROKER_EXTERN bool isLocal(boost::intrusive_ptr<Message>& msg); + QPID_BROKER_EXTERN bool isLocal(const Message& msg); //PersistableQueue support: QPID_BROKER_EXTERN uint64_t getPersistenceId() const; @@ -410,7 +395,11 @@ class Queue : public boost::enable_shared_from_this<Queue>, * Reserve space in policy for an enqueued message that * has been recovered in the prepared state (dtx only) */ - QPID_BROKER_EXTERN void recoverPrepared(boost::intrusive_ptr<Message>& msg); + QPID_BROKER_EXTERN void recoverPrepared(const Message& msg); + void enqueueAborted(const Message& msg); + void enqueueCommited(Message& msg); + void dequeueAborted(Message& msg); + void dequeueCommited(const Message& msg); QPID_BROKER_EXTERN void flush(); @@ -418,6 +407,7 @@ class Queue : public boost::enable_shared_from_this<Queue>, uint32_t getDequeueSincePurge() { return dequeueSincePurge.get(); } QPID_BROKER_EXTERN void setDequeueSincePurge(uint32_t value); + friend class QueueFactory; }; } } diff --git a/qpid/cpp/src/qpid/broker/QueueCursor.cpp b/qpid/cpp/src/qpid/broker/QueueCursor.cpp new file mode 100644 index 0000000000..e48b18b748 --- /dev/null +++ b/qpid/cpp/src/qpid/broker/QueueCursor.cpp @@ -0,0 +1,44 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "QueueCursor.h" +#include "qpid/broker/Message.h" + +namespace qpid { +namespace broker { +QueueCursor::QueueCursor(SubscriptionType t) : type(t), position(0), version(0), valid(false) {} + +void QueueCursor::setPosition(int32_t p, int32_t v) +{ + position = p; + version = v; + valid = true; +} + +bool QueueCursor::check(const Message& m) +{ + return (m.getState() == AVAILABLE || ((type == REPLICATOR || type == PURGE) && m.getState() == ACQUIRED)); +} + +bool QueueCursor::isValid(int32_t v) +{ + return valid && (valid = (v == version)); +} +}} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/QueueCursor.h b/qpid/cpp/src/qpid/broker/QueueCursor.h new file mode 100644 index 0000000000..2551b64a48 --- /dev/null +++ b/qpid/cpp/src/qpid/broker/QueueCursor.h @@ -0,0 +1,71 @@ +#ifndef QPID_BROKER_QUEUECURSOR_H +#define QPID_BROKER_QUEUECURSOR_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/broker/BrokerImportExport.h" +#include "qpid/sys/IntegerTypes.h" +#include <boost/shared_ptr.hpp> + +namespace qpid { +namespace broker { + +class Message; + +enum SubscriptionType +{ + CONSUMER, + BROWSER, + PURGE, + REPLICATOR +}; + +class CursorContext { + public: + virtual ~CursorContext() {} +}; +/** + * + */ +class QueueCursor +{ + public: + QPID_BROKER_EXTERN QueueCursor(SubscriptionType type = CONSUMER); + + private: + SubscriptionType type; + int32_t position; + int32_t version; + bool valid; + boost::shared_ptr<CursorContext> context; + + void setPosition(int32_t p, int32_t v); + bool check(const Message& m); + bool isValid(int32_t v); + + friend class MessageDeque; + friend class MessageMap; + friend class PriorityQueue; + template <typename T> friend class IndexedDeque; +}; +}} // namespace qpid::broker + +#endif /*!QPID_BROKER_QUEUECURSOR_H*/ diff --git a/qpid/cpp/src/qpid/broker/QueueDepth.cpp b/qpid/cpp/src/qpid/broker/QueueDepth.cpp new file mode 100644 index 0000000000..69ec0ab4ac --- /dev/null +++ b/qpid/cpp/src/qpid/broker/QueueDepth.cpp @@ -0,0 +1,127 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "QueueDepth.h" + +namespace qpid { +namespace broker { + +QueueDepth::QueueDepth() {} +QueueDepth::QueueDepth(uint32_t c, uint64_t s) : count(c), size(s) {} +QueueDepth& QueueDepth::operator+=(const QueueDepth& other) +{ + if (count.valid) count.value += other.count.value; + if (size.valid) size.value += other.size.value; + return *this; +} +QueueDepth& QueueDepth::operator-=(const QueueDepth& other) +{ + if (count.valid) count.value -= other.count.value; + if (size.valid) size.value -= other.size.value; + return *this; +} +bool QueueDepth::operator==(const QueueDepth& other) const +{ + //only compare values, not validity an invalid value is always 0; + //this means that an invalid value will match an empty queue + //depth, which is fine + return (count.value == other.count.value) + && (size.value == other.size.value); +} +bool QueueDepth::operator!=(const QueueDepth& other) const +{ + return !(*this == other); +} +bool QueueDepth::operator<(const QueueDepth& other) const +{ + if (count.valid && size.valid) + return count.value < other.count.value || size.value < other.size.value; + else if (count.valid) + return count.value < other.count.value; + else + return size.value < other.size.value; +} +bool QueueDepth::operator>(const QueueDepth& other) const +{ + if (count.valid && size.valid) + return count.value > other.count.value || size.value > other.size.value; + else if (count.valid) + return count.value > other.count.value; + else + return size.value > other.size.value; +} +bool QueueDepth::hasCount() const { return count.valid; } +uint32_t QueueDepth::getCount() const { return count.value; } +void QueueDepth::setCount(uint32_t c) { count.value = c; count.valid = true; } +bool QueueDepth::hasSize() const { return size.valid; } +uint64_t QueueDepth::getSize() const { return size.value; } +void QueueDepth::setSize(uint64_t c) { size.value = c; size.valid = true; } + +namespace{ + template <typename T> QueueDepth::Optional<T> add(const QueueDepth::Optional<T>& a, const QueueDepth::Optional<T>& b) + { + QueueDepth::Optional<T> result; + if (a.valid && b.valid) { + result.valid = true; + result.value = a.value + b.value; + } + return result; + } + template <typename T> QueueDepth::Optional<T> subtract(const QueueDepth::Optional<T>& a, const QueueDepth::Optional<T>& b) + { + QueueDepth::Optional<T> result; + if (a.valid && b.valid) { + result.valid = true; + result.value = a.value - b.value; + } + return result; + } +} +QueueDepth operator-(const QueueDepth& a, const QueueDepth& b) +{ + QueueDepth result; + result.count = subtract(a.count, b.count); + result.size = subtract(a.size, b.size); + return result; +} + +QueueDepth operator+(const QueueDepth& a, const QueueDepth& b) +{ + QueueDepth result; + result.count = add(a.count, b.count); + result.size = add(a.size, b.size); + return result; + +} + +std::ostream& operator<<(std::ostream& o, const QueueDepth& d) +{ + if (d.hasCount()) o << "count: " << d.getCount(); + if (d.hasSize()) { + if (d.hasCount()) o << ", "; + o << "size: " << d.getSize(); + } + return o; +} + +QueueDepth::operator bool() const { return hasCount() || hasSize(); } + + +}} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/QueueDepth.h b/qpid/cpp/src/qpid/broker/QueueDepth.h new file mode 100644 index 0000000000..d93acb2a7a --- /dev/null +++ b/qpid/cpp/src/qpid/broker/QueueDepth.h @@ -0,0 +1,74 @@ +#ifndef QPID_BROKER_QUEUEDEPTH_H +#define QPID_BROKER_QUEUEDEPTH_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/broker/BrokerImportExport.h" +#include "qpid/sys/IntegerTypes.h" +#include <ostream> + +namespace qpid { +namespace broker { + +/** + * Represents a queue depth in message count and/or aggregate message + * size. + */ +class QueueDepth +{ + public: + QPID_BROKER_EXTERN QueueDepth(); + QPID_BROKER_EXTERN QueueDepth(uint32_t count, uint64_t size); + QPID_BROKER_EXTERN QueueDepth& operator+=(const QueueDepth&); + QPID_BROKER_EXTERN QueueDepth& operator-=(const QueueDepth&); + QPID_BROKER_EXTERN bool operator==(const QueueDepth&) const; + QPID_BROKER_EXTERN bool operator!=(const QueueDepth&) const; + QPID_BROKER_EXTERN bool operator<(const QueueDepth& other) const; + QPID_BROKER_EXTERN bool operator>(const QueueDepth& other) const; + QPID_BROKER_EXTERN operator bool() const; + QPID_BROKER_EXTERN bool hasCount() const; + QPID_BROKER_EXTERN uint32_t getCount() const; + QPID_BROKER_EXTERN void setCount(uint32_t); + QPID_BROKER_EXTERN bool hasSize() const; + QPID_BROKER_EXTERN uint64_t getSize() const; + QPID_BROKER_EXTERN void setSize(uint64_t); + friend QPID_BROKER_EXTERN QueueDepth operator-(const QueueDepth&, const QueueDepth&); + friend QPID_BROKER_EXTERN QueueDepth operator+(const QueueDepth&, const QueueDepth&); + template <typename T> struct Optional + { + T value; + bool valid; + + Optional(T v) : value(v), valid(true) {} + Optional() : value(0), valid(false) {} + }; + private: + Optional<uint32_t> count; + Optional<uint64_t> size; +}; + +QPID_BROKER_EXTERN QueueDepth operator-(const QueueDepth&, const QueueDepth&); +QPID_BROKER_EXTERN QueueDepth operator+(const QueueDepth&, const QueueDepth&); +std::ostream& operator<<(std::ostream&, const QueueDepth&); + +}} // namespace qpid::broker + +#endif /*!QPID_BROKER_QUEUEDEPTH_H*/ diff --git a/qpid/cpp/src/qpid/broker/QueueEvents.cpp b/qpid/cpp/src/qpid/broker/QueueEvents.cpp deleted file mode 100644 index c66bdabf0f..0000000000 --- a/qpid/cpp/src/qpid/broker/QueueEvents.cpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/broker/QueueEvents.h" -#include "qpid/broker/Queue.h" -#include "qpid/broker/QueueObserver.h" -#include "qpid/Exception.h" -#include "qpid/log/Statement.h" - -namespace qpid { -namespace broker { - -QueueEvents::QueueEvents(const boost::shared_ptr<sys::Poller>& poller, bool isSync) : - eventQueue(boost::bind(&QueueEvents::handle, this, _1), poller), enabled(true), sync(isSync) -{ - if (!sync) eventQueue.start(); -} - -QueueEvents::~QueueEvents() -{ - if (!sync) eventQueue.stop(); -} - -void QueueEvents::enqueued(const QueuedMessage& m) -{ - if (enabled) { - Event enq(ENQUEUE, m); - if (sync) { - for (Listeners::iterator j = listeners.begin(); j != listeners.end(); j++) - j->second(enq); - } else { - eventQueue.push(enq); - } - } -} - -void QueueEvents::dequeued(const QueuedMessage& m) -{ - if (enabled) { - Event deq(DEQUEUE, m); - if (sync) { - for (Listeners::iterator j = listeners.begin(); j != listeners.end(); j++) - j->second(deq); - } else { - eventQueue.push(Event(DEQUEUE, m)); - } - } -} - -void QueueEvents::registerListener(const std::string& id, const EventListener& listener) -{ - qpid::sys::Mutex::ScopedLock l(lock); - if (listeners.find(id) == listeners.end()) { - listeners[id] = listener; - } else { - throw Exception(QPID_MSG("Event listener already registered for '" << id << "'")); - } -} - -void QueueEvents::unregisterListener(const std::string& id) -{ - qpid::sys::Mutex::ScopedLock l(lock); - if (listeners.find(id) == listeners.end()) { - throw Exception(QPID_MSG("No event listener registered for '" << id << "'")); - } else { - listeners.erase(id); - } -} - -QueueEvents::EventQueue::Batch::const_iterator -QueueEvents::handle(const EventQueue::Batch& events) { - qpid::sys::Mutex::ScopedLock l(lock); - for (EventQueue::Batch::const_iterator i = events.begin(); i != events.end(); ++i) { - for (Listeners::iterator j = listeners.begin(); j != listeners.end(); j++) { - j->second(*i); - } - } - return events.end(); -} - -void QueueEvents::shutdown() -{ - if (!sync && !eventQueue.empty() && !listeners.empty()) eventQueue.shutdown(); -} - -void QueueEvents::enable() -{ - enabled = true; - QPID_LOG(debug, "Queue events enabled"); -} - -void QueueEvents::disable() -{ - enabled = false; - QPID_LOG(debug, "Queue events disabled"); -} - -bool QueueEvents::isSync() -{ - return sync; -} - -class EventGenerator : public QueueObserver -{ - public: - EventGenerator(QueueEvents& mgr, bool enqOnly) : manager(mgr), enqueueOnly(enqOnly) {} - void enqueued(const QueuedMessage& m) - { - manager.enqueued(m); - } - void dequeued(const QueuedMessage& m) - { - if (!enqueueOnly) manager.dequeued(m); - } - - void acquired(const QueuedMessage&) {}; - void requeued(const QueuedMessage&) {}; - - private: - QueueEvents& manager; - const bool enqueueOnly; -}; - -void QueueEvents::observe(Queue& queue, bool enqueueOnly) -{ - boost::shared_ptr<QueueObserver> observer(new EventGenerator(*this, enqueueOnly)); - queue.addObserver(observer); -} - - -QueueEvents::Event::Event(EventType t, const QueuedMessage& m) : type(t), msg(m) {} - - -}} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/QueueEvents.h b/qpid/cpp/src/qpid/broker/QueueEvents.h deleted file mode 100644 index fcddfe9092..0000000000 --- a/qpid/cpp/src/qpid/broker/QueueEvents.h +++ /dev/null @@ -1,85 +0,0 @@ -#ifndef QPID_BROKER_QUEUEEVENTS_H -#define QPID_BROKER_QUEUEEVENTS_H - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "qpid/broker/BrokerImportExport.h" -#include "qpid/broker/QueuedMessage.h" -#include "qpid/sys/Mutex.h" -#include "qpid/sys/PollableQueue.h" -#include <map> -#include <string> -#include <boost/function.hpp> - -namespace qpid { -namespace broker { - -/** - * Event manager for queue events. Allows queues to indicate when - * events have occured; allows listeners to register for notification - * of this. The notification happens asynchronously, in a separate - * thread. - */ -class QueueEvents -{ - public: - enum EventType {ENQUEUE, DEQUEUE}; - - struct Event - { - EventType type; - QueuedMessage msg; - - QPID_BROKER_EXTERN Event(EventType, const QueuedMessage&); - }; - - typedef boost::function<void (Event)> EventListener; - - QPID_BROKER_EXTERN QueueEvents(const boost::shared_ptr<sys::Poller>& poller, bool isSync = false); - QPID_BROKER_EXTERN ~QueueEvents(); - QPID_BROKER_EXTERN void enqueued(const QueuedMessage&); - QPID_BROKER_EXTERN void dequeued(const QueuedMessage&); - QPID_BROKER_EXTERN void registerListener(const std::string& id, - const EventListener&); - QPID_BROKER_EXTERN void unregisterListener(const std::string& id); - void enable(); - void disable(); - void observe(Queue&, bool enqueueOnly); - //process all outstanding events - QPID_BROKER_EXTERN void shutdown(); - QPID_BROKER_EXTERN bool isSync(); - private: - typedef qpid::sys::PollableQueue<Event> EventQueue; - typedef std::map<std::string, EventListener> Listeners; - - EventQueue eventQueue; - Listeners listeners; - volatile bool enabled; - qpid::sys::Mutex lock;//protect listeners from concurrent access - bool sync; - - EventQueue::Batch::const_iterator handle(const EventQueue::Batch& e); - -}; -}} // namespace qpid::broker - -#endif /*!QPID_BROKER_QUEUEEVENTS_H*/ diff --git a/qpid/cpp/src/qpid/broker/QueueFactory.cpp b/qpid/cpp/src/qpid/broker/QueueFactory.cpp new file mode 100644 index 0000000000..efeb9ae53b --- /dev/null +++ b/qpid/cpp/src/qpid/broker/QueueFactory.cpp @@ -0,0 +1,114 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/broker/QueueFactory.h" +#include "qpid/broker/Broker.h" +#include "qpid/broker/QueueSettings.h" +#include "qpid/broker/Queue.h" +#include "qpid/broker/LossyQueue.h" +#include "qpid/broker/Lvq.h" +#include "qpid/broker/Messages.h" +#include "qpid/broker/MessageDistributor.h" +#include "qpid/broker/MessageGroupManager.h" +#include "qpid/broker/Fairshare.h" +#include "qpid/broker/MessageDeque.h" +#include "qpid/broker/MessageMap.h" +#include "qpid/broker/PriorityQueue.h" +#include "qpid/broker/QueueFlowLimit.h" +#include "qpid/broker/ThresholdAlerts.h" +#include "qpid/broker/FifoDistributor.h" +#include <map> +#include <memory> + +namespace qpid { +namespace broker { + + +QueueFactory::QueueFactory() : broker(0), store(0), parent(0) {} + +boost::shared_ptr<Queue> QueueFactory::create(const std::string& name, const QueueSettings& settings) +{ + settings.validate(); + + //1. determine Queue type (i.e. whether we are subclassing Queue) + // -> if 'ring' policy is in use then subclass + boost::shared_ptr<Queue> queue; + if (settings.dropMessagesAtLimit) { + queue = boost::shared_ptr<Queue>(new LossyQueue(name, settings, settings.durable ? store : 0, parent, broker)); + } else if (settings.lvqKey.size()) { + std::auto_ptr<MessageMap> map(new MessageMap(settings.lvqKey)); + queue = boost::shared_ptr<Queue>(new Lvq(name, map, settings, settings.durable ? store : 0, parent, broker)); + } else { + queue = boost::shared_ptr<Queue>(new Queue(name, settings, settings.durable ? store : 0, parent, broker)); + } + + //2. determine Messages type (i.e. structure) + if (settings.priorities) { + if (settings.defaultFairshare || settings.fairshare.size()) { + queue->messages = Fairshare::create(settings); + } else { + queue->messages = std::auto_ptr<Messages>(new PriorityQueue(settings.priorities)); + } + } else if (settings.lvqKey.empty()) {//LVQ already handled above + queue->messages = std::auto_ptr<Messages>(new MessageDeque()); + } + + //3. determine MessageDistributor type + if (settings.groupKey.size()) { + boost::shared_ptr<MessageGroupManager> mgm(MessageGroupManager::create( name, *(queue->messages), settings)); + queue->allocator = mgm; + queue->addObserver(mgm); + } else { + queue->allocator = boost::shared_ptr<MessageDistributor>(new FifoDistributor( *(queue->messages) )); + } + + + //4. threshold event config + if (broker && broker->getManagementAgent()) { + ThresholdAlerts::observe(*queue, *(broker->getManagementAgent()), settings, broker->getOptions().queueThresholdEventRatio); + } + //5. flow control config + QueueFlowLimit::observe(*queue, settings); + + return queue; +} + +void QueueFactory::setBroker(Broker* b) +{ + broker = b; +} +Broker* QueueFactory::getBroker() +{ + return broker; +} +void QueueFactory::setStore (MessageStore* s) +{ + store = s; +} +MessageStore* QueueFactory::getStore() const +{ + return store; +} +void QueueFactory::setParent(management::Manageable* p) +{ + parent = p; +} + +}} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/QueueFactory.h b/qpid/cpp/src/qpid/broker/QueueFactory.h new file mode 100644 index 0000000000..b6a79f1f1a --- /dev/null +++ b/qpid/cpp/src/qpid/broker/QueueFactory.h @@ -0,0 +1,73 @@ +#ifndef QPID_BROKER_QUEUEFACTORY_H +#define QPID_BROKER_QUEUEFACTORY_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/broker/BrokerImportExport.h" +#include "qpid/types/Variant.h" +#include <boost/shared_ptr.hpp> + +namespace qpid { +namespace management { +class Manageable; +} +namespace broker { +class Broker; +class MessageStore; +class Queue; +struct QueueSettings; + +/** + * Handles the creation and configuration of a Queue instance in order + * to meet the required settings + */ +class QueueFactory +{ + public: + QPID_BROKER_EXTERN QueueFactory(); + + QPID_BROKER_EXTERN boost::shared_ptr<Queue> create(const std::string& name, const QueueSettings& settings); + + void setBroker(Broker*); + Broker* getBroker(); + + /** + * Set the store to use. May only be called once. + */ + void setStore (MessageStore*); + + /** + * Return the message store used. + */ + MessageStore* getStore() const; + + /** + * Register the manageable parent for declared queues + */ + void setParent(management::Manageable*); + private: + Broker* broker; + MessageStore* store; + management::Manageable* parent; +}; +}} // namespace qpid::broker + +#endif /*!QPID_BROKER_QUEUEFACTORY_H*/ diff --git a/qpid/cpp/src/qpid/broker/QueueFlowLimit.cpp b/qpid/cpp/src/qpid/broker/QueueFlowLimit.cpp index 14fe5f4022..11b9cbae63 100644 --- a/qpid/cpp/src/qpid/broker/QueueFlowLimit.cpp +++ b/qpid/cpp/src/qpid/broker/QueueFlowLimit.cpp @@ -20,7 +20,9 @@ */ #include "qpid/broker/QueueFlowLimit.h" #include "qpid/broker/Broker.h" +#include "qpid/broker/Message.h" #include "qpid/broker/Queue.h" +#include "qpid/broker/QueueSettings.h" #include "qpid/Exception.h" #include "qpid/framing/FieldValue.h" #include "qpid/framing/reply_exceptions.h" @@ -57,34 +59,6 @@ namespace { << "=" << max)); } } - - /** extract a capacity value as passed in an argument map - */ - uint64_t getCapacity(const FieldTable& settings, const std::string& key, uint64_t defaultValue) - { - FieldTable::ValuePtr v = settings.get(key); - - int64_t result = 0; - - if (!v) return defaultValue; - if (v->getType() == 0x23) { - QPID_LOG(debug, "Value for " << key << " specified as float: " << v->get<float>()); - } else if (v->getType() == 0x33) { - QPID_LOG(debug, "Value for " << key << " specified as double: " << v->get<double>()); - } else if (v->convertsTo<int64_t>()) { - result = v->get<int64_t>(); - QPID_LOG(debug, "Got integer value for " << key << ": " << result); - if (result >= 0) return result; - } else if (v->convertsTo<std::string>()) { - std::string s(v->get<std::string>()); - QPID_LOG(debug, "Got string value for " << key << ": " << s); - std::istringstream convert(s); - if (convert >> result && result >= 0) return result; - } - - QPID_LOG(warning, "Cannot convert " << key << " to unsigned integer, using default (" << defaultValue << ")"); - return defaultValue; - } } @@ -102,10 +76,8 @@ QueueFlowLimit::QueueFlowLimit(Queue *_queue, if (queue) { queueName = _queue->getName(); - if (queue->getPolicy()) { - maxSize = _queue->getPolicy()->getMaxSize(); - maxCount = _queue->getPolicy()->getMaxCount(); - } + if (queue->getSettings().maxDepth.hasCount()) maxCount = queue->getSettings().maxDepth.getCount(); + if (queue->getSettings().maxDepth.hasCount()) maxSize = queue->getSettings().maxDepth.getSize(); broker = queue->getBroker(); queueMgmtObj = dynamic_cast<_qmfBroker::Queue*> (queue->GetManagementObject()); if (queueMgmtObj) { @@ -125,23 +97,23 @@ QueueFlowLimit::~QueueFlowLimit() sys::Mutex::ScopedLock l(indexLock); if (!index.empty()) { // we're gone - release all pending msgs - for (std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> >::iterator itr = index.begin(); + for (std::map<framing::SequenceNumber, Message >::iterator itr = index.begin(); itr != index.end(); ++itr) if (itr->second) try { - itr->second->getIngressCompletion().finishCompleter(); + itr->second.getPersistentContext()->getIngressCompletion().finishCompleter(); } catch (...) {} // ignore - not safe for a destructor to throw. index.clear(); } } -void QueueFlowLimit::enqueued(const QueuedMessage& msg) +void QueueFlowLimit::enqueued(const Message& msg) { sys::Mutex::ScopedLock l(indexLock); ++count; - size += msg.payload->contentSize(); + size += msg.getContentSize(); if (!flowStopped) { if (flowStopCount && count > flowStopCount) { @@ -160,13 +132,13 @@ void QueueFlowLimit::enqueued(const QueuedMessage& msg) if (flowStopped || !index.empty()) { // ignore flow control if we are populating the queue due to cluster replication: if (broker && broker->isClusterUpdatee()) { - QPID_LOG(trace, "Queue \"" << queueName << "\": ignoring flow control for msg pos=" << msg.position); + QPID_LOG(trace, "Queue \"" << queueName << "\": ignoring flow control for msg pos=" << msg.getSequence()); return; } - QPID_LOG(trace, "Queue \"" << queueName << "\": setting flow control for msg pos=" << msg.position); - msg.payload->getIngressCompletion().startCompleter(); // don't complete until flow resumes + QPID_LOG(trace, "Queue \"" << queueName << "\": setting flow control for msg pos=" << msg.getSequence()); + msg.getPersistentContext()->getIngressCompletion().startCompleter(); // don't complete until flow resumes bool unique; - unique = index.insert(std::pair<framing::SequenceNumber, boost::intrusive_ptr<Message> >(msg.position, msg.payload)).second; + unique = index.insert(std::pair<framing::SequenceNumber, Message >(msg.getSequence(), msg)).second; // Like this to avoid tripping up unused variable warning when NDEBUG set if (!unique) assert(unique); } @@ -174,7 +146,7 @@ void QueueFlowLimit::enqueued(const QueuedMessage& msg) -void QueueFlowLimit::dequeued(const QueuedMessage& msg) +void QueueFlowLimit::dequeued(const Message& msg) { sys::Mutex::ScopedLock l(indexLock); @@ -184,7 +156,7 @@ void QueueFlowLimit::dequeued(const QueuedMessage& msg) throw Exception(QPID_MSG("Flow limit count underflow on dequeue. Queue=" << queueName)); } - uint64_t _size = msg.payload->contentSize(); + uint64_t _size = msg.getContentSize(); if (_size <= size) { size -= _size; } else { @@ -203,16 +175,16 @@ void QueueFlowLimit::dequeued(const QueuedMessage& msg) if (!index.empty()) { if (!flowStopped) { // flow enabled - release all pending msgs - for (std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> >::iterator itr = index.begin(); + for (std::map<framing::SequenceNumber, Message >::iterator itr = index.begin(); itr != index.end(); ++itr) if (itr->second) - itr->second->getIngressCompletion().finishCompleter(); + itr->second.getPersistentContext()->getIngressCompletion().finishCompleter(); index.clear(); } else { // even if flow controlled, we must release this msg as it is being dequeued - std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> >::iterator itr = index.find(msg.position); + std::map<framing::SequenceNumber, Message >::iterator itr = index.find(msg.getSequence()); if (itr != index.end()) { // this msg is flow controlled, release it: - msg.payload->getIngressCompletion().finishCompleter(); + msg.getPersistentContext()->getIngressCompletion().finishCompleter(); index.erase(itr); } } @@ -279,7 +251,7 @@ void QueueFlowLimit::setDefaults(uint64_t maxQueueSize, uint flowStopRatio, uint } -void QueueFlowLimit::observe(Queue& queue, const qpid::framing::FieldTable& settings) +void QueueFlowLimit::observe(Queue& queue, const QueueSettings& settings) { QueueFlowLimit *ptr = createLimit( &queue, settings ); if (ptr) { @@ -289,36 +261,37 @@ void QueueFlowLimit::observe(Queue& queue, const qpid::framing::FieldTable& sett } /** returns ptr to a QueueFlowLimit, else 0 if no limit */ -QueueFlowLimit *QueueFlowLimit::createLimit(Queue *queue, const qpid::framing::FieldTable& settings) +QueueFlowLimit *QueueFlowLimit::createLimit(Queue *queue, const QueueSettings& settings) { - std::string type(QueuePolicy::getType(settings)); - - if (type == QueuePolicy::RING || type == QueuePolicy::RING_STRICT) { + if (settings.dropMessagesAtLimit) { // The size of a RING queue is limited by design - no need for flow control. return 0; } - if (settings.get(flowStopCountKey) || settings.get(flowStopSizeKey) || - settings.get(flowResumeCountKey) || settings.get(flowResumeSizeKey)) { + if (settings.flowStop.hasCount() || settings.flowStop.hasSize()) { // user provided (some) flow settings manually... - uint32_t flowStopCount = getCapacity(settings, flowStopCountKey, 0); - uint32_t flowResumeCount = getCapacity(settings, flowResumeCountKey, 0); - uint64_t flowStopSize = getCapacity(settings, flowStopSizeKey, 0); - uint64_t flowResumeSize = getCapacity(settings, flowResumeSizeKey, 0); - if (flowStopCount == 0 && flowStopSize == 0) { // disable flow control + if (settings.flowStop.getCount() || settings.flowStop.getSize()) { + return new QueueFlowLimit(queue, + settings.flowStop.getCount(), + settings.flowResume.getCount(), + settings.flowStop.getSize(), + settings.flowResume.getSize()); + } else { + //don't have a non-zero value for either the count or the + //size to stop at, yet at least one of these settings was + //provided, i.e it was set to 0 explicitly which we treat + //as turning it off return 0; } - return new QueueFlowLimit(queue, flowStopCount, flowResumeCount, flowStopSize, flowResumeSize); } if (defaultFlowStopRatio) { // broker has a default ratio setup... - uint64_t maxByteCount = getCapacity(settings, QueuePolicy::maxSizeKey, defaultMaxSize); + uint64_t maxByteCount = settings.maxDepth.hasSize() ? settings.maxDepth.getSize() : defaultMaxSize; uint64_t flowStopSize = (uint64_t)(maxByteCount * (defaultFlowStopRatio/100.0) + 0.5); uint64_t flowResumeSize = (uint64_t)(maxByteCount * (defaultFlowResumeRatio/100.0)); - uint32_t maxMsgCount = getCapacity(settings, QueuePolicy::maxCountKey, 0); // no size by default + uint32_t maxMsgCount = settings.maxDepth.hasCount() ? settings.maxDepth.getCount() : 0; uint32_t flowStopCount = (uint32_t)(maxMsgCount * (defaultFlowStopRatio/100.0) + 0.5); uint32_t flowResumeCount = (uint32_t)(maxMsgCount * (defaultFlowResumeRatio/100.0)); - return new QueueFlowLimit(queue, flowStopCount, flowResumeCount, flowStopSize, flowResumeSize); } return 0; @@ -346,7 +319,7 @@ void QueueFlowLimit::getState(qpid::framing::FieldTable& state ) const framing::SequenceSet ss; if (!index.empty()) { /* replicate the set of messages pending flow control */ - for (std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> >::const_iterator itr = index.begin(); + for (std::map<framing::SequenceNumber, Message >::const_iterator itr = index.begin(); itr != index.end(); ++itr) { ss.add(itr->first); } @@ -377,10 +350,10 @@ void QueueFlowLimit::setState(const qpid::framing::FieldTable& state) ++i; fcmsg.add(first, last); for (SequenceNumber seq = first; seq <= last; ++seq) { - QueuedMessage msg; + Message msg; queue->find(seq, msg); // fyi: may not be found if msg is acquired & unacked bool unique; - unique = index.insert(std::pair<framing::SequenceNumber, boost::intrusive_ptr<Message> >(seq, msg.payload)).second; + unique = index.insert(std::pair<framing::SequenceNumber, Message >(seq, msg)).second; // Like this to avoid tripping up unused variable warning when NDEBUG set if (!unique) assert(unique); } diff --git a/qpid/cpp/src/qpid/broker/QueueFlowLimit.h b/qpid/cpp/src/qpid/broker/QueueFlowLimit.h index ad8a2720ef..1bcc388ceb 100644 --- a/qpid/cpp/src/qpid/broker/QueueFlowLimit.h +++ b/qpid/cpp/src/qpid/broker/QueueFlowLimit.h @@ -26,9 +26,9 @@ #include <iostream> #include <memory> #include "qpid/broker/BrokerImportExport.h" -#include "qpid/broker/QueuedMessage.h" #include "qpid/broker/StatefulQueueObserver.h" #include "qpid/framing/FieldTable.h" +#include "qpid/framing/SequenceNumber.h" #include "qpid/sys/AtomicValue.h" #include "qpid/sys/Mutex.h" @@ -45,6 +45,8 @@ namespace qpid { namespace broker { class Broker; +class Queue; +struct QueueSettings; /** * Producer flow control: when level is > flowStop*, flow control is ON. @@ -80,13 +82,13 @@ class Broker; QPID_BROKER_EXTERN virtual ~QueueFlowLimit(); - /** the queue has added QueuedMessage. Returns true if flow state changes */ - QPID_BROKER_EXTERN void enqueued(const QueuedMessage&); - /** the queue has removed QueuedMessage. Returns true if flow state changes */ - QPID_BROKER_EXTERN void dequeued(const QueuedMessage&); + /** the queue has added QueuedMessage */ + QPID_BROKER_EXTERN void enqueued(const Message&); + /** the queue has removed QueuedMessage */ + QPID_BROKER_EXTERN void dequeued(const Message&); /** ignored */ - QPID_BROKER_EXTERN void acquired(const QueuedMessage&) {}; - QPID_BROKER_EXTERN void requeued(const QueuedMessage&) {}; + QPID_BROKER_EXTERN void acquired(const Message&) {}; + QPID_BROKER_EXTERN void requeued(const Message&) {}; /** for clustering: */ QPID_BROKER_EXTERN void getState(qpid::framing::FieldTable&) const; @@ -106,14 +108,14 @@ class Broker; void decode(framing::Buffer& buffer); uint32_t encodedSize() const; - static QPID_BROKER_EXTERN void observe(Queue& queue, const qpid::framing::FieldTable& settings); + static QPID_BROKER_EXTERN void observe(Queue& queue, const QueueSettings& settings); static QPID_BROKER_EXTERN void setDefaults(uint64_t defaultMaxSize, uint defaultFlowStopRatio, uint defaultFlowResumeRatio); friend QPID_BROKER_EXTERN std::ostream& operator<<(std::ostream&, const QueueFlowLimit&); protected: // msgs waiting for flow to become available. - std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> > index; + std::map<framing::SequenceNumber, Message > index; mutable qpid::sys::Mutex indexLock; _qmfBroker::Queue *queueMgmtObj; @@ -123,7 +125,7 @@ class Broker; QPID_BROKER_EXTERN QueueFlowLimit(Queue *queue, uint32_t flowStopCount, uint32_t flowResumeCount, uint64_t flowStopSize, uint64_t flowResumeSize); - static QPID_BROKER_EXTERN QueueFlowLimit *createLimit(Queue *queue, const qpid::framing::FieldTable& settings); + static QPID_BROKER_EXTERN QueueFlowLimit *createLimit(Queue *queue, const QueueSettings& settings); }; }} diff --git a/qpid/cpp/src/qpid/broker/QueueObserver.h b/qpid/cpp/src/qpid/broker/QueueObserver.h index b58becd2ae..29e867253e 100644 --- a/qpid/cpp/src/qpid/broker/QueueObserver.h +++ b/qpid/cpp/src/qpid/broker/QueueObserver.h @@ -24,8 +24,8 @@ namespace qpid { namespace broker { -struct QueuedMessage; class Consumer; +class Message; /** * Interface for notifying classes who want to act as 'observers' of a queue of particular @@ -63,10 +63,10 @@ class QueueObserver virtual ~QueueObserver() {} // note: the Queue will hold the messageLock while calling these methods! - virtual void enqueued(const QueuedMessage&) = 0; - virtual void dequeued(const QueuedMessage&) = 0; - virtual void acquired(const QueuedMessage&) = 0; - virtual void requeued(const QueuedMessage&) = 0; + virtual void enqueued(const Message&) = 0; + virtual void dequeued(const Message&) = 0; + virtual void acquired(const Message&) = 0; + virtual void requeued(const Message&) = 0; virtual void consumerAdded( const Consumer& ) {}; virtual void consumerRemoved( const Consumer& ) {}; private: diff --git a/qpid/cpp/src/qpid/broker/QueuePolicy.cpp b/qpid/cpp/src/qpid/broker/QueuePolicy.cpp deleted file mode 100644 index 3978420f4e..0000000000 --- a/qpid/cpp/src/qpid/broker/QueuePolicy.cpp +++ /dev/null @@ -1,364 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/broker/QueuePolicy.h" -#include "qpid/broker/Queue.h" -#include "qpid/broker/PriorityQueue.h" -#include "qpid/Exception.h" -#include "qpid/framing/FieldValue.h" -#include "qpid/framing/reply_exceptions.h" -#include "qpid/log/Statement.h" -#include <sstream> - -using namespace qpid::broker; -using namespace qpid::framing; - -QueuePolicy::QueuePolicy(const std::string& _name, uint32_t _maxCount, uint64_t _maxSize, const std::string& _type) : - maxCount(_maxCount), maxSize(_maxSize), type(_type), count(0), size(0), policyExceeded(false), queue(0), name(_name) { - QPID_LOG(info, "Queue \"" << name << "\": Policy created: type=" << type << "; maxCount=" << maxCount << "; maxSize=" << maxSize); -} - -void QueuePolicy::enqueued(uint64_t _size) -{ - if (maxCount) ++count; - if (maxSize) size += _size; -} - -void QueuePolicy::dequeued(uint64_t _size) -{ - if (maxCount) { - if (count > 0) { - --count; - } else { - throw Exception(QPID_MSG("Attempted count underflow on dequeue(" << _size << "): " << *this)); - } - } - if (maxSize) { - if (_size > size) { - throw Exception(QPID_MSG("Attempted size underflow on dequeue(" << _size << "): " << *this)); - } else { - size -= _size; - } - } -} - -bool QueuePolicy::checkLimit(boost::intrusive_ptr<Message> m) -{ - bool sizeExceeded = maxSize && (size + m->contentSize()) > maxSize; - bool countExceeded = maxCount && (count + 1) > maxCount; - bool exceeded = sizeExceeded || countExceeded; - if (exceeded) { - if (!policyExceeded) { - policyExceeded = true; - if (sizeExceeded) QPID_LOG(info, "Queue cumulative message size exceeded policy for " << name); - if (countExceeded) QPID_LOG(info, "Queue message count exceeded policy for " << name); - } - } else { - if (policyExceeded) { - policyExceeded = false; - QPID_LOG(info, "Queue cumulative message size and message count within policy for " << name); - } - } - return !exceeded; -} - -void QueuePolicy::tryEnqueue(boost::intrusive_ptr<Message> m) -{ - if (checkLimit(m)) { - enqueued(m->contentSize()); - } else { - throw ResourceLimitExceededException(QPID_MSG("Policy exceeded on " << name << ", policy: " << *this)); - } -} - -void QueuePolicy::recoverEnqueued(boost::intrusive_ptr<Message> m) -{ - tryEnqueue(m); -} - -void QueuePolicy::enqueueAborted(boost::intrusive_ptr<Message> m) -{ - dequeued(m->contentSize()); -} - -void QueuePolicy::enqueued(const QueuedMessage&) {} - -void QueuePolicy::dequeued(const QueuedMessage& m) -{ - dequeued(m.payload->contentSize()); -} - -bool QueuePolicy::isEnqueued(const QueuedMessage&) -{ - return true; -} - -void QueuePolicy::update(FieldTable& settings) -{ - if (maxCount) settings.setInt(maxCountKey, maxCount); - if (maxSize) settings.setInt(maxSizeKey, maxSize); - settings.setString(typeKey, type); -} - -template <typename T> -T getCapacity(const FieldTable& settings, const std::string& key, T defaultValue) -{ - FieldTable::ValuePtr v = settings.get(key); - - T result = 0; - - if (!v) return defaultValue; - if (v->getType() == 0x23) { - QPID_LOG(debug, "Value for " << key << " specified as float: " << v->get<float>()); - } else if (v->getType() == 0x33) { - QPID_LOG(debug, "Value for " << key << " specified as double: " << v->get<double>()); - } else if (v->convertsTo<T>()) { - result = v->get<T>(); - QPID_LOG(debug, "Got integer value for " << key << ": " << result); - if (result >= 0) return result; - } else if (v->convertsTo<std::string>()) { - std::string s(v->get<std::string>()); - QPID_LOG(debug, "Got string value for " << key << ": " << s); - std::istringstream convert(s); - if (convert >> result && result >= 0 && convert.eof()) return result; - } - - throw IllegalArgumentException(QPID_MSG("Cannot convert " << key << " to unsigned integer: " << *v)); -} - -std::string QueuePolicy::getType(const FieldTable& settings) -{ - FieldTable::ValuePtr v = settings.get(typeKey); - if (v && v->convertsTo<std::string>()) { - std::string t = v->get<std::string>(); - std::transform(t.begin(), t.end(), t.begin(), tolower); - if (t == REJECT || t == FLOW_TO_DISK || t == RING || t == RING_STRICT) return t; - } - return REJECT; -} - -void QueuePolicy::setDefaultMaxSize(uint64_t s) -{ - defaultMaxSize = s; -} - -void QueuePolicy::getPendingDequeues(Messages&) {} - - - - -void QueuePolicy::encode(Buffer& buffer) const -{ - buffer.putLong(maxCount); - buffer.putLongLong(maxSize); - buffer.putLong(count); - buffer.putLongLong(size); -} - -void QueuePolicy::decode ( Buffer& buffer ) -{ - maxCount = buffer.getLong(); - maxSize = buffer.getLongLong(); - count = buffer.getLong(); - size = buffer.getLongLong(); -} - - -uint32_t QueuePolicy::encodedSize() const { - return sizeof(uint32_t) + // maxCount - sizeof(uint64_t) + // maxSize - sizeof(uint32_t) + // count - sizeof(uint64_t); // size -} - - - -const std::string QueuePolicy::maxCountKey("qpid.max_count"); -const std::string QueuePolicy::maxSizeKey("qpid.max_size"); -const std::string QueuePolicy::typeKey("qpid.policy_type"); -const std::string QueuePolicy::REJECT("reject"); -const std::string QueuePolicy::FLOW_TO_DISK("flow_to_disk"); -const std::string QueuePolicy::RING("ring"); -const std::string QueuePolicy::RING_STRICT("ring_strict"); -uint64_t QueuePolicy::defaultMaxSize(0); - -FlowToDiskPolicy::FlowToDiskPolicy(const std::string& _name, uint32_t _maxCount, uint64_t _maxSize) : - QueuePolicy(_name, _maxCount, _maxSize, FLOW_TO_DISK) {} - -bool FlowToDiskPolicy::checkLimit(boost::intrusive_ptr<Message> m) -{ - if (!QueuePolicy::checkLimit(m)) { - m->requestContentRelease(); - if (queue) - queue->countFlowedToDisk(m->contentSize()); - } - return true; -} - -RingQueuePolicy::RingQueuePolicy(const std::string& _name, - uint32_t _maxCount, uint64_t _maxSize, const std::string& _type) : - QueuePolicy(_name, _maxCount, _maxSize, _type), strict(_type == RING_STRICT) {} - -bool before(const QueuedMessage& a, const QueuedMessage& b) -{ - int priorityA = PriorityQueue::getPriority(a); - int priorityB = PriorityQueue::getPriority(b); - if (priorityA == priorityB) return a.position < b.position; - else return priorityA < priorityB; -} - -void RingQueuePolicy::enqueued(const QueuedMessage& m) -{ - //need to insert in correct location based on position - queue.insert(lower_bound(queue.begin(), queue.end(), m, before), m); -} - -void RingQueuePolicy::dequeued(const QueuedMessage& m) -{ - //find and remove m from queue - if (find(m, pendingDequeues, true) || find(m, queue, true)) { - //now update count and size - QueuePolicy::dequeued(m); - } -} - -bool RingQueuePolicy::isEnqueued(const QueuedMessage& m) -{ - //for non-strict ring policy, a message can be replaced (and - //therefore dequeued) before it is accepted or released by - //subscriber; need to detect this - return find(m, pendingDequeues, false) || find(m, queue, false); -} - -bool RingQueuePolicy::checkLimit(boost::intrusive_ptr<Message> m) -{ - - // If the message is bigger than the queue size, give up - if (getMaxSize() && m->contentSize() > getMaxSize()) { - QPID_LOG(debug, "Message too large for ring queue " << name - << " [" << *this << "] " - << ": message size = " << m->contentSize() << " bytes" - << ": max queue size = " << getMaxSize() << " bytes"); - return false; - } - - // if within limits, ok to accept - if (QueuePolicy::checkLimit(m)) return true; - - // At this point, we've exceeded maxSize, maxCount, or both. - // - // If we've exceeded maxCount, we've exceeded it by 1, so - // replacing the first message is sufficient. If we've exceeded - // maxSize, we need to pop enough messages to get the space we - // need. - - unsigned int haveSpace = getMaxSize() - getCurrentQueueSize(); - - do { - QueuedMessage oldest = queue.front(); - if (oldest.queue->acquireMessageAt(oldest.position, oldest) || !strict) { - queue.pop_front(); - pendingDequeues.push_back(oldest); - QPID_LOG(debug, "Ring policy triggered in " << name - << ": removed message " << oldest.position << " to make way for new message"); - - haveSpace += oldest.payload->contentSize(); - - } else { - //in strict mode, if oldest message has been delivered (hence - //cannot be acquired) but not yet acked, it should not be - //removed and the attempted enqueue should fail - QPID_LOG(debug, "Ring policy could not be triggered in " << name - << ": oldest message (seq-no=" << oldest.position << ") has been delivered but not yet acknowledged or requeued"); - return false; - } - } while (getMaxSize() && haveSpace < m->contentSize()); - - - return true; -} - -void RingQueuePolicy::getPendingDequeues(Messages& result) -{ - result = pendingDequeues; -} - -bool RingQueuePolicy::find(const QueuedMessage& m, Messages& q, bool remove) -{ - for (Messages::iterator i = q.begin(); i != q.end(); i++) { - if (i->payload == m.payload) { - if (remove) q.erase(i); - return true; - } - } - return false; -} - -std::auto_ptr<QueuePolicy> QueuePolicy::createQueuePolicy(uint32_t maxCount, uint64_t maxSize, const std::string& type) -{ - return createQueuePolicy("<unspecified>", maxCount, maxSize, type); -} - -std::auto_ptr<QueuePolicy> QueuePolicy::createQueuePolicy(const qpid::framing::FieldTable& settings) -{ - return createQueuePolicy("<unspecified>", settings); -} - -std::auto_ptr<QueuePolicy> QueuePolicy::createQueuePolicy(const std::string& name, const qpid::framing::FieldTable& settings) -{ - uint32_t maxCount = getCapacity<int32_t>(settings, maxCountKey, 0); - uint64_t maxSize = getCapacity<int64_t>(settings, maxSizeKey, defaultMaxSize); - if (maxCount || maxSize) { - return createQueuePolicy(name, maxCount, maxSize, getType(settings)); - } else { - return std::auto_ptr<QueuePolicy>(); - } -} - -std::auto_ptr<QueuePolicy> QueuePolicy::createQueuePolicy(const std::string& name, - uint32_t maxCount, uint64_t maxSize, const std::string& type) -{ - if (type == RING || type == RING_STRICT) { - return std::auto_ptr<QueuePolicy>(new RingQueuePolicy(name, maxCount, maxSize, type)); - } else if (type == FLOW_TO_DISK) { - return std::auto_ptr<QueuePolicy>(new FlowToDiskPolicy(name, maxCount, maxSize)); - } else { - return std::auto_ptr<QueuePolicy>(new QueuePolicy(name, maxCount, maxSize, type)); - } - -} - -namespace qpid { - namespace broker { - -std::ostream& operator<<(std::ostream& out, const QueuePolicy& p) -{ - if (p.maxSize) out << "size: max=" << p.maxSize << ", current=" << p.size; - else out << "size: unlimited"; - out << "; "; - if (p.maxCount) out << "count: max=" << p.maxCount << ", current=" << p.count; - else out << "count: unlimited"; - out << "; type=" << p.type; - return out; -} - - } -} - diff --git a/qpid/cpp/src/qpid/broker/QueuePolicy.h b/qpid/cpp/src/qpid/broker/QueuePolicy.h deleted file mode 100644 index f23b709f18..0000000000 --- a/qpid/cpp/src/qpid/broker/QueuePolicy.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#ifndef _QueuePolicy_ -#define _QueuePolicy_ - -#include <deque> -#include <iostream> -#include <memory> -#include "qpid/broker/BrokerImportExport.h" -#include "qpid/broker/QueuedMessage.h" -#include "qpid/framing/FieldTable.h" -#include "qpid/sys/AtomicValue.h" -#include "qpid/sys/Mutex.h" - -namespace qpid { -namespace broker { - -class Queue; - -class QueuePolicy -{ - static uint64_t defaultMaxSize; - - uint32_t maxCount; - uint64_t maxSize; - const std::string type; - uint32_t count; - uint64_t size; - bool policyExceeded; - - protected: - Queue* queue; - uint64_t getCurrentQueueSize() const { return size; } - - public: - typedef std::deque<QueuedMessage> Messages; - static QPID_BROKER_EXTERN const std::string maxCountKey; - static QPID_BROKER_EXTERN const std::string maxSizeKey; - static QPID_BROKER_EXTERN const std::string typeKey; - static QPID_BROKER_EXTERN const std::string REJECT; - static QPID_BROKER_EXTERN const std::string FLOW_TO_DISK; - static QPID_BROKER_EXTERN const std::string RING; - static QPID_BROKER_EXTERN const std::string RING_STRICT; - - virtual ~QueuePolicy() {} - QPID_BROKER_EXTERN void tryEnqueue(boost::intrusive_ptr<Message> msg); - QPID_BROKER_EXTERN void recoverEnqueued(boost::intrusive_ptr<Message> msg); - QPID_BROKER_EXTERN void enqueueAborted(boost::intrusive_ptr<Message> msg); - virtual void enqueued(const QueuedMessage&); - virtual void dequeued(const QueuedMessage&); - virtual bool isEnqueued(const QueuedMessage&); - QPID_BROKER_EXTERN void update(qpid::framing::FieldTable& settings); - uint32_t getMaxCount() const { return maxCount; } - uint64_t getMaxSize() const { return maxSize; } - void encode(framing::Buffer& buffer) const; - void decode ( framing::Buffer& buffer ); - uint32_t encodedSize() const; - virtual void getPendingDequeues(Messages& result); - std::string getType() const { return type; } - void setQueue(Queue* q) { queue = q; } - - static QPID_BROKER_EXTERN std::auto_ptr<QueuePolicy> createQueuePolicy(const std::string& name, const qpid::framing::FieldTable& settings); - static QPID_BROKER_EXTERN std::auto_ptr<QueuePolicy> createQueuePolicy(const std::string& name, uint32_t maxCount, uint64_t maxSize, const std::string& type = REJECT); - static QPID_BROKER_EXTERN std::auto_ptr<QueuePolicy> createQueuePolicy(const qpid::framing::FieldTable& settings); - static QPID_BROKER_EXTERN std::auto_ptr<QueuePolicy> createQueuePolicy(uint32_t maxCount, uint64_t maxSize, const std::string& type = REJECT); - static std::string getType(const qpid::framing::FieldTable& settings); - static void setDefaultMaxSize(uint64_t); - friend QPID_BROKER_EXTERN std::ostream& operator<<(std::ostream&, - const QueuePolicy&); - protected: - const std::string name; - - QueuePolicy(const std::string& name, uint32_t maxCount, uint64_t maxSize, const std::string& type = REJECT); - - virtual bool checkLimit(boost::intrusive_ptr<Message> msg); - void enqueued(uint64_t size); - void dequeued(uint64_t size); -}; - - -class FlowToDiskPolicy : public QueuePolicy -{ - public: - FlowToDiskPolicy(const std::string& name, uint32_t maxCount, uint64_t maxSize); - bool checkLimit(boost::intrusive_ptr<Message> msg); -}; - -class RingQueuePolicy : public QueuePolicy -{ - public: - RingQueuePolicy(const std::string& name, uint32_t maxCount, uint64_t maxSize, const std::string& type = RING); - void enqueued(const QueuedMessage&); - void dequeued(const QueuedMessage&); - bool isEnqueued(const QueuedMessage&); - bool checkLimit(boost::intrusive_ptr<Message> msg); - void getPendingDequeues(Messages& result); - private: - Messages pendingDequeues; - Messages queue; - const bool strict; - - bool find(const QueuedMessage&, Messages&, bool remove); -}; - -}} - - -#endif diff --git a/qpid/cpp/src/qpid/broker/QueueRegistry.cpp b/qpid/cpp/src/qpid/broker/QueueRegistry.cpp index 1401356444..5a30cd2e34 100644 --- a/qpid/cpp/src/qpid/broker/QueueRegistry.cpp +++ b/qpid/cpp/src/qpid/broker/QueueRegistry.cpp @@ -21,7 +21,6 @@ #include "qpid/broker/Broker.h" #include "qpid/broker/Queue.h" #include "qpid/broker/QueueRegistry.h" -#include "qpid/broker/QueueEvents.h" #include "qpid/broker/Exchange.h" #include "qpid/log/Statement.h" #include "qpid/framing/reply_exceptions.h" @@ -32,50 +31,43 @@ using namespace qpid::broker; using namespace qpid::sys; using std::string; -QueueRegistry::QueueRegistry(Broker* b) : - counter(1), store(0), events(0), parent(0), lastNode(false), broker(b) {} +QueueRegistry::QueueRegistry(Broker* b) +{ + setBroker(b); +} QueueRegistry::~QueueRegistry(){} std::pair<Queue::shared_ptr, bool> -QueueRegistry::declare(const string& declareName, bool durable, - bool autoDelete, const OwnershipToken* owner, +QueueRegistry::declare(const string& name, const QueueSettings& settings, boost::shared_ptr<Exchange> alternate, - const qpid::framing::FieldTable& arguments, bool recovering/*true if this declare is a result of recovering queue - definition from persistente + definition from persistent record*/) { - Queue::shared_ptr queue; std::pair<Queue::shared_ptr, bool> result; { RWlock::ScopedWlock locker(lock); - string name = declareName.empty() ? generateName() : declareName; - assert(!name.empty()); QueueMap::iterator i = queues.find(name); - if (i == queues.end()) { - queue.reset(new Queue(name, autoDelete, durable ? store : 0, owner, parent, broker)); + Queue::shared_ptr queue = create(name, settings); + //Move this to factory also? if (alternate) { queue->setAlternateExchange(alternate);//need to do this *before* create alternate->incAlternateUsers(); } if (!recovering) { - //apply settings & create persistent record if required - queue->create(arguments); - } else { - //i.e. recovering a queue for which we already have a persistent record - queue->configure(arguments); + //create persistent record if required + queue->create(); } queues[name] = queue; - if (lastNode) queue->setLastNodeFailure(); result = std::pair<Queue::shared_ptr, bool>(queue, true); } else { result = std::pair<Queue::shared_ptr, bool>(i->second, false); } } - if (broker && queue) broker->getConfigurationObservers().queueCreate(queue); + if (getBroker() && result.second) getBroker()->getConfigurationObservers().queueCreate(result.first); return result; } @@ -89,7 +81,7 @@ void QueueRegistry::destroy(const string& name) { queues.erase(i); } } - if (broker && q) broker->getConfigurationObservers().queueDestroy(q); + if (getBroker() && q) getBroker()->getConfigurationObservers().queueDestroy(q); } Queue::shared_ptr QueueRegistry::find(const string& name){ @@ -108,36 +100,17 @@ Queue::shared_ptr QueueRegistry::get(const string& name) { return q; } -string QueueRegistry::generateName(){ - string name; - do { - std::stringstream ss; - ss << "tmp_" << counter++; - name = ss.str(); - // Thread safety: Private function, only called with lock held - // so this is OK. - } while(queues.find(name) != queues.end()); - return name; -} - void QueueRegistry::setStore (MessageStore* _store) { - store = _store; + QueueFactory::setStore(_store); } -MessageStore* QueueRegistry::getStore() const { - return store; +MessageStore* QueueRegistry::getStore() const +{ + return QueueFactory::getStore(); } -void QueueRegistry::updateQueueClusterState(bool _lastNode) +void QueueRegistry::setParent(qpid::management::Manageable* _parent) { - RWlock::ScopedRlock locker(lock); - for (QueueMap::iterator i = queues.begin(); i != queues.end(); i++) { - if (_lastNode){ - i->second->setLastNodeFailure(); - } else { - i->second->clearLastNodeFailure(); - } - } - lastNode = _lastNode; + QueueFactory::setParent(_parent); } diff --git a/qpid/cpp/src/qpid/broker/QueueRegistry.h b/qpid/cpp/src/qpid/broker/QueueRegistry.h index a354513c5f..7fce90c679 100644 --- a/qpid/cpp/src/qpid/broker/QueueRegistry.h +++ b/qpid/cpp/src/qpid/broker/QueueRegistry.h @@ -22,8 +22,8 @@ #define _QueueRegistry_ #include "qpid/broker/BrokerImportExport.h" +#include "qpid/broker/QueueFactory.h" #include "qpid/sys/Mutex.h" -#include "qpid/management/Manageable.h" #include "qpid/framing/FieldTable.h" #include <boost/bind.hpp> #include <boost/shared_ptr.hpp> @@ -34,11 +34,8 @@ namespace qpid { namespace broker { class Queue; -class QueueEvents; class Exchange; class OwnershipToken; -class Broker; -class MessageStore; /** * A registry of queues indexed by queue name. @@ -47,7 +44,7 @@ class MessageStore; * are deleted when and only when they are no longer in use. * */ -class QueueRegistry { +class QueueRegistry : QueueFactory { public: QPID_BROKER_EXTERN QueueRegistry(Broker* b = 0); QPID_BROKER_EXTERN ~QueueRegistry(); @@ -60,11 +57,8 @@ class QueueRegistry { */ QPID_BROKER_EXTERN std::pair<boost::shared_ptr<Queue>, bool> declare( const std::string& name, - bool durable = false, - bool autodelete = false, - const OwnershipToken* owner = 0, + const QueueSettings& settings, boost::shared_ptr<Exchange> alternateExchange = boost::shared_ptr<Exchange>(), - const qpid::framing::FieldTable& args = framing::FieldTable(), bool recovering = false); /** @@ -101,11 +95,6 @@ class QueueRegistry { QPID_BROKER_EXTERN boost::shared_ptr<Queue> get(const std::string& name); /** - * Generate unique queue name. - */ - std::string generateName(); - - /** * Set the store to use. May only be called once. */ void setStore (MessageStore*); @@ -118,7 +107,7 @@ class QueueRegistry { /** * Register the manageable parent for declared queues */ - void setParent (management::Manageable* _parent) { parent = _parent; } + void setParent (management::Manageable*); /** Call f for each queue in the registry. */ template <class F> void eachQueue(F f) const { @@ -127,22 +116,10 @@ class QueueRegistry { f(i->second); } - /** - * Change queue mode when cluster size drops to 1 node, expands again - * in practice allows flow queue to disk when last name to be exectuted - */ - void updateQueueClusterState(bool lastNode); - private: typedef std::map<std::string, boost::shared_ptr<Queue> > QueueMap; QueueMap queues; mutable qpid::sys::RWlock lock; - int counter; - MessageStore* store; - QueueEvents* events; - management::Manageable* parent; - bool lastNode; //used to set mode on queue declare - Broker* broker; }; diff --git a/qpid/cpp/src/qpid/broker/QueueSettings.cpp b/qpid/cpp/src/qpid/broker/QueueSettings.cpp new file mode 100644 index 0000000000..e96cec1701 --- /dev/null +++ b/qpid/cpp/src/qpid/broker/QueueSettings.cpp @@ -0,0 +1,228 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "QueueSettings.h" +#include "QueueFlowLimit.h" +#include "MessageGroupManager.h" +#include "qpid/types/Variant.h" +#include "qpid/framing/reply_exceptions.h" +#include "qpid/log/Statement.h" +#include "qpid/amqp_0_10/Codecs.h" + + +namespace qpid { +namespace broker { + +namespace { +const std::string MAX_COUNT("qpid.max_count"); +const std::string MAX_SIZE("qpid.max_size"); +const std::string POLICY_TYPE("qpid.policy_type"); +const std::string POLICY_TYPE_REJECT("reject"); +const std::string POLICY_TYPE_RING("ring"); +const std::string NO_LOCAL("no-local"); +const std::string TRACE_ID("qpid.trace.id"); +const std::string TRACE_EXCLUDES("qpid.trace.exclude"); +const std::string LVQ_KEY("qpid.last_value_queue_key"); +const std::string AUTO_DELETE_TIMEOUT("qpid.auto_delete_timeout"); +const std::string ALERT_REPEAT_GAP("qpid.alert_repeat_gap"); +const std::string ALERT_COUNT("qpid.alert_count"); +const std::string ALERT_SIZE("qpid.alert_size"); +const std::string PRIORITIES("qpid.priorities"); +const std::string FAIRSHARE("qpid.fairshare"); +const std::string FAIRSHARE_ALIAS("x-qpid-fairshare"); + +const std::string LVQ_LEGACY("qpid.last_value_queue"); +const std::string LVQ_LEGACY_KEY("qpid.LVQ_key"); +const std::string LVQ_LEGACY_NOBROWSE("qpid.last_value_queue_no_browse"); + + +bool handleFairshareSetting(const std::string& basename, const std::string& key, const qpid::types::Variant& value, QueueSettings& settings) +{ + if (key.find(basename) == 0) { + qpid::types::Variant index(key.substr(basename.size()+1)); + settings.fairshare[index] = value; + return true; + } else { + return false; + } +} +bool isFairshareSetting(const std::string& key, const qpid::types::Variant& value, QueueSettings& settings) +{ + return handleFairshareSetting(FAIRSHARE, key, value, settings) || handleFairshareSetting(FAIRSHARE_ALIAS, key, value, settings); +} +} + +const QueueSettings::Aliases QueueSettings::aliases; + +QueueSettings::QueueSettings(bool d, bool a) : + durable(d), + autodelete(a), + priorities(0), + defaultFairshare(0), + shareGroups(false), + addTimestamp(false), + dropMessagesAtLimit(false), + noLocal(false), + autoDeleteDelay(0), + alertRepeatInterval(60) +{} + +bool QueueSettings::handle(const std::string& key, const qpid::types::Variant& value) +{ + if (key == MAX_COUNT && value.asUint32() > 0) { + maxDepth.setCount(value); + return true; + } else if (key == MAX_SIZE && value.asUint64() > 0) { + maxDepth.setSize(value); + return true; + } else if (key == POLICY_TYPE) { + if (value.getString() == POLICY_TYPE_RING) { + dropMessagesAtLimit = true; + return true; + } else if (value.getString() == POLICY_TYPE_REJECT) { + //do nothing, thats the default + return true; + } else { + QPID_LOG(warning, "Unrecognised policy option: " << value); + return false; + } + } else if (key == NO_LOCAL) { + noLocal = true; + return true; + } else if (key == TRACE_ID) { + traceId = value.asString(); + return true; + } else if (key == TRACE_EXCLUDES) { + traceExcludes = value.asString(); + return true; + } else if (key == PRIORITIES) { + priorities = value; + return true; + } else if (key == FAIRSHARE) { + defaultFairshare = value; + return true; + } else if (isFairshareSetting(key, value, *this)) { + return true; + } else if (key == MessageGroupManager::qpidMessageGroupKey) { + groupKey = value.asString(); + return true; + } else if (key == MessageGroupManager::qpidSharedGroup) { + shareGroups = value; + return true; + } else if (key == MessageGroupManager::qpidMessageGroupTimestamp) { + addTimestamp = value; + return true; + } else if (key == LVQ_KEY) { + lvqKey = value.asString(); + return true; + } else if (key == LVQ_LEGACY) { + if (lvqKey.empty()) lvqKey = LVQ_LEGACY_KEY; + return true; + } else if (key == LVQ_LEGACY_NOBROWSE) { + QPID_LOG(warning, "Ignoring 'no-browse' directive for LVQ; it is no longer necessary"); + if (lvqKey.empty()) lvqKey = LVQ_LEGACY_KEY; + return true; + } else if (key == AUTO_DELETE_TIMEOUT) { + autoDeleteDelay = value; + return true; + } else if (key == QueueFlowLimit::flowStopCountKey) { + flowStop.setCount(value); + return true; + } else if (key == QueueFlowLimit::flowResumeCountKey) { + flowResume.setCount(value); + return true; + } else if (key == QueueFlowLimit::flowStopSizeKey) { + flowStop.setSize(value); + return true; + } else if (key == QueueFlowLimit::flowResumeSizeKey) { + flowResume.setSize(value); + return true; + } else if (key == ALERT_REPEAT_GAP) { + alertRepeatInterval = value; + return true; + } else if (key == ALERT_COUNT) { + alertThreshold.setCount(value); + return true; + } else if (key == ALERT_SIZE) { + alertThreshold.setSize(value); + return true; + } else { + return false; + } +} + +void QueueSettings::validate() const +{ + if (lvqKey.size() && priorities > 0) + throw qpid::framing::InvalidArgumentException(QPID_MSG("Cannot specify " << LVQ_KEY << " and " << PRIORITIES << " for the same queue")); + if ((fairshare.size() || defaultFairshare) && priorities == 0) + throw qpid::framing::InvalidArgumentException(QPID_MSG("Cannot specify fairshare settings when queue is not enabled for priorities")); + if (fairshare.size() > priorities) + throw qpid::framing::InvalidArgumentException(QPID_MSG("Cannot have fairshare set for priority levels greater than " << priorities)); + if (groupKey.size() && lvqKey.size()) + throw qpid::framing::InvalidArgumentException(QPID_MSG("Cannot specify " << LVQ_KEY << " and " << MessageGroupManager::qpidMessageGroupKey << " for the same queue")); + if (groupKey.size() && priorities) + throw qpid::framing::InvalidArgumentException(QPID_MSG("Cannot specify " << PRIORITIES << " and " << MessageGroupManager::qpidMessageGroupKey << " for the same queue")); + if (shareGroups && groupKey.empty()) { + throw qpid::framing::InvalidArgumentException(QPID_MSG("Can only specify " << MessageGroupManager::qpidSharedGroup + << " if " << MessageGroupManager::qpidMessageGroupKey << " is set")); + } + if (addTimestamp && groupKey.empty()) { + throw qpid::framing::InvalidArgumentException(QPID_MSG("Can only specify " << MessageGroupManager::qpidMessageGroupTimestamp + << " if " << MessageGroupManager::qpidMessageGroupKey << " is set")); + } + + // @todo: remove once "sticky" consumers are supported - see QPID-3347 + if (!shareGroups && groupKey.size()) { + throw qpid::framing::InvalidArgumentException(QPID_MSG("Only shared groups are supported at present; " << MessageGroupManager::qpidSharedGroup + << " is required if " << MessageGroupManager::qpidMessageGroupKey << " is set")); + } +} + +void QueueSettings::populate(const std::map<std::string, qpid::types::Variant>& inputs, std::map<std::string, qpid::types::Variant>& unused) +{ + original = inputs; + for (qpid::types::Variant::Map::const_iterator i = inputs.begin(); i != inputs.end(); ++i) { + Aliases::const_iterator a = aliases.find(i->first); + if (!handle((a != aliases.end() ? a->second : i->first), i->second)) unused.insert(*i); + } +} +void QueueSettings::populate(const qpid::framing::FieldTable& inputs, qpid::framing::FieldTable& unused) +{ + qpid::types::Variant::Map o; + qpid::amqp_0_10::translate(inputs, original); + populate(original, o); + qpid::amqp_0_10::translate(o, unused); +} +std::map<std::string, qpid::types::Variant> QueueSettings::asMap() const +{ + return original; +} + +QueueSettings::Aliases::Aliases() +{ + insert(value_type("x-qpid-priorities", "qpid.priorities")); + insert(value_type("x-qpid-fairshare", "qpid.fairshare")); + insert(value_type("x-qpid-minimum-alert-repeat-gap", "qpid.alert_repeat_gap")); + insert(value_type("x-qpid-maximum-message-count", "qpid.alert_count")); + insert(value_type("x-qpid-maximum-message-size", "qpid.alert_size")); +} + +}} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/QueueSettings.h b/qpid/cpp/src/qpid/broker/QueueSettings.h new file mode 100644 index 0000000000..2443624615 --- /dev/null +++ b/qpid/cpp/src/qpid/broker/QueueSettings.h @@ -0,0 +1,92 @@ +#ifndef QPID_BROKER_QUEUESETTINGS_H +#define QPID_BROKER_QUEUESETTINGS_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/broker/BrokerImportExport.h" +#include "qpid/broker/QueueDepth.h" +#include "qpid/sys/IntegerTypes.h" +#include "qpid/framing/FieldTable.h" +#include <string> +#include <map> + +namespace qpid { +namespace types { +class Variant; +} +namespace broker { + +/** + * Defines the various queue configuration settings that can be specified + */ +struct QueueSettings +{ + QPID_BROKER_EXTERN QueueSettings(bool durable=false, bool autodelete=false); + + bool durable; + bool autodelete; + + //basic queue types: + std::string lvqKey; + uint32_t priorities; + uint32_t defaultFairshare; + std::map<uint32_t,uint32_t> fairshare; + + //message groups: + std::string groupKey; + bool shareGroups; + bool addTimestamp;//not actually used; always on at present? + + QueueDepth maxDepth; + bool dropMessagesAtLimit;//aka ring queue policy + + bool noLocal; + std::string traceId; + std::string traceExcludes; + uint64_t autoDeleteDelay;//queueTtl? + + //flow control: + QueueDepth flowStop; + QueueDepth flowResume; + + //threshold events: + QueueDepth alertThreshold; + int64_t alertRepeatInterval; + + //yuck, yuck + qpid::framing::FieldTable storeSettings; + std::map<std::string, qpid::types::Variant> original; + + bool handle(const std::string& key, const qpid::types::Variant& value); + void validate() const; + QPID_BROKER_EXTERN void populate(const std::map<std::string, qpid::types::Variant>& inputs, std::map<std::string, qpid::types::Variant>& unused); + QPID_BROKER_EXTERN void populate(const qpid::framing::FieldTable& inputs, qpid::framing::FieldTable& unused); + std::map<std::string, qpid::types::Variant> asMap() const; + + struct Aliases : std::map<std::string, std::string> + { + Aliases(); + }; + static const Aliases aliases; +}; +}} // namespace qpid::broker + +#endif /*!QPID_BROKER_QUEUESETTINGS_H*/ diff --git a/qpid/cpp/src/qpid/broker/QueuedMessage.h b/qpid/cpp/src/qpid/broker/QueuedMessage.h index 9d008193a0..c80fff900a 100644 --- a/qpid/cpp/src/qpid/broker/QueuedMessage.h +++ b/qpid/cpp/src/qpid/broker/QueuedMessage.h @@ -22,8 +22,8 @@ #define _QueuedMessage_ #include "qpid/broker/Message.h" -#include "BrokerImportExport.h" -#include <iosfwd> +#include "qpid/framing/SequenceNumber.h" +#include "qpid/broker/BrokerImportExport.h" namespace qpid { namespace broker { @@ -32,20 +32,19 @@ class Queue; struct QueuedMessage { - boost::intrusive_ptr<Message> payload; + Message message; framing::SequenceNumber position; - typedef enum { AVAILABLE, ACQUIRED, DELETED, REMOVED } Status; - Status status; + enum {AVAILABLE, ACQUIRED, DELETED, REMOVED} status; Queue* queue; - QueuedMessage(Queue* q=0, - boost::intrusive_ptr<Message> msg=0, - framing::SequenceNumber sn=0, - Status st=AVAILABLE - ) : payload(msg), position(sn), status(st), queue(q) {} + QueuedMessage() : queue(0) {} + QueuedMessage(Queue* q, Message msg, framing::SequenceNumber sn) : + message(msg), position(sn), queue(q) {} + QueuedMessage(Queue* q) : queue(q) {} }; -inline bool operator<(const QueuedMessage& a, const QueuedMessage& b) { +inline bool operator<(const QueuedMessage& a, const QueuedMessage& b) +{ return a.position < b.position; } diff --git a/qpid/cpp/src/qpid/broker/RecoveredDequeue.cpp b/qpid/cpp/src/qpid/broker/RecoveredDequeue.cpp index cd6735328f..6e21a5bc21 100644 --- a/qpid/cpp/src/qpid/broker/RecoveredDequeue.cpp +++ b/qpid/cpp/src/qpid/broker/RecoveredDequeue.cpp @@ -22,10 +22,9 @@ #include "qpid/broker/Queue.h" #include "qpid/broker/RecoveredDequeue.h" -using boost::intrusive_ptr; using namespace qpid::broker; -RecoveredDequeue::RecoveredDequeue(Queue::shared_ptr _queue, intrusive_ptr<Message> _msg) : queue(_queue), msg(_msg) +RecoveredDequeue::RecoveredDequeue(Queue::shared_ptr _queue, Message _msg) : queue(_queue), msg(_msg) { queue->recoverPrepared(msg); } @@ -38,11 +37,11 @@ bool RecoveredDequeue::prepare(TransactionContext*) throw() void RecoveredDequeue::commit() throw() { - queue->enqueueAborted(msg); + queue->dequeueCommited(msg); } void RecoveredDequeue::rollback() throw() { - queue->process(msg); + queue->dequeueAborted(msg); } diff --git a/qpid/cpp/src/qpid/broker/RecoveredDequeue.h b/qpid/cpp/src/qpid/broker/RecoveredDequeue.h index 66e66f1d5f..87f768eefd 100644 --- a/qpid/cpp/src/qpid/broker/RecoveredDequeue.h +++ b/qpid/cpp/src/qpid/broker/RecoveredDequeue.h @@ -26,8 +26,6 @@ #include "qpid/broker/MessageStore.h" #include "qpid/broker/TxOp.h" -#include <boost/intrusive_ptr.hpp> - #include <algorithm> #include <functional> #include <list> @@ -36,18 +34,17 @@ namespace qpid { namespace broker { class RecoveredDequeue : public TxOp{ boost::shared_ptr<Queue> queue; - boost::intrusive_ptr<Message> msg; + Message msg; public: - RecoveredDequeue(boost::shared_ptr<Queue> queue, boost::intrusive_ptr<Message> msg); + RecoveredDequeue(boost::shared_ptr<Queue> queue, Message msg); virtual bool prepare(TransactionContext* ctxt) throw(); virtual void commit() throw(); virtual void rollback() throw(); virtual ~RecoveredDequeue(){} - virtual void accept(TxOpConstVisitor& visitor) const { visitor(*this); } boost::shared_ptr<Queue> getQueue() const { return queue; } - boost::intrusive_ptr<Message> getMessage() const { return msg; } + Message getMessage() const { return msg; } }; } } diff --git a/qpid/cpp/src/qpid/broker/RecoveredEnqueue.cpp b/qpid/cpp/src/qpid/broker/RecoveredEnqueue.cpp index 6d2eaee6c4..296d5194c0 100644 --- a/qpid/cpp/src/qpid/broker/RecoveredEnqueue.cpp +++ b/qpid/cpp/src/qpid/broker/RecoveredEnqueue.cpp @@ -22,10 +22,9 @@ #include "qpid/broker/Queue.h" #include "qpid/broker/RecoveredEnqueue.h" -using boost::intrusive_ptr; using namespace qpid::broker; -RecoveredEnqueue::RecoveredEnqueue(Queue::shared_ptr _queue, intrusive_ptr<Message> _msg) : queue(_queue), msg(_msg) +RecoveredEnqueue::RecoveredEnqueue(Queue::shared_ptr _queue, Message _msg) : queue(_queue), msg(_msg) { queue->recoverPrepared(msg); } @@ -36,7 +35,7 @@ bool RecoveredEnqueue::prepare(TransactionContext*) throw(){ } void RecoveredEnqueue::commit() throw(){ - queue->process(msg); + queue->enqueueCommited(msg); } void RecoveredEnqueue::rollback() throw(){ diff --git a/qpid/cpp/src/qpid/broker/RecoveredEnqueue.h b/qpid/cpp/src/qpid/broker/RecoveredEnqueue.h index 5f718001d5..d1f8e1106c 100644 --- a/qpid/cpp/src/qpid/broker/RecoveredEnqueue.h +++ b/qpid/cpp/src/qpid/broker/RecoveredEnqueue.h @@ -26,8 +26,6 @@ #include "qpid/broker/MessageStore.h" #include "qpid/broker/TxOp.h" -#include <boost/intrusive_ptr.hpp> - #include <algorithm> #include <functional> #include <list> @@ -36,19 +34,17 @@ namespace qpid { namespace broker { class RecoveredEnqueue : public TxOp{ boost::shared_ptr<Queue> queue; - boost::intrusive_ptr<Message> msg; + Message msg; public: - RecoveredEnqueue(boost::shared_ptr<Queue> queue, boost::intrusive_ptr<Message> msg); + RecoveredEnqueue(boost::shared_ptr<Queue> queue, Message msg); virtual bool prepare(TransactionContext* ctxt) throw(); virtual void commit() throw(); virtual void rollback() throw(); virtual ~RecoveredEnqueue(){} - virtual void accept(TxOpConstVisitor& visitor) const { visitor(*this); } boost::shared_ptr<Queue> getQueue() const { return queue; } - boost::intrusive_ptr<Message> getMessage() const { return msg; } - + Message getMessage() const { return msg; } }; } } diff --git a/qpid/cpp/src/qpid/broker/RecoveryManagerImpl.cpp b/qpid/cpp/src/qpid/broker/RecoveryManagerImpl.cpp index 858535637a..7deeba5e65 100644 --- a/qpid/cpp/src/qpid/broker/RecoveryManagerImpl.cpp +++ b/qpid/cpp/src/qpid/broker/RecoveryManagerImpl.cpp @@ -21,11 +21,13 @@ #include "qpid/broker/RecoveryManagerImpl.h" #include "qpid/broker/Message.h" +#include "qpid/broker/PersistableMessage.h" #include "qpid/broker/Queue.h" #include "qpid/broker/Link.h" #include "qpid/broker/Bridge.h" #include "qpid/broker/RecoveredEnqueue.h" #include "qpid/broker/RecoveredDequeue.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/framing/reply_exceptions.h" using boost::dynamic_pointer_cast; @@ -43,9 +45,9 @@ RecoveryManagerImpl::~RecoveryManagerImpl() {} class RecoverableMessageImpl : public RecoverableMessage { - intrusive_ptr<Message> msg; + Message msg; public: - RecoverableMessageImpl(const intrusive_ptr<Message>& _msg); + RecoverableMessageImpl(const Message& _msg); ~RecoverableMessageImpl() {}; void setPersistenceId(uint64_t id); void setRedelivered(); @@ -128,9 +130,10 @@ RecoverableQueue::shared_ptr RecoveryManagerImpl::recoverQueue(framing::Buffer& RecoverableMessage::shared_ptr RecoveryManagerImpl::recoverMessage(framing::Buffer& buffer) { - boost::intrusive_ptr<Message> message(new Message()); - message->decodeHeader(buffer); - return RecoverableMessage::shared_ptr(new RecoverableMessageImpl(message)); + //TODO: determine encoding/version actually used + boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> transfer(new qpid::broker::amqp_0_10::MessageTransfer()); + transfer->decodeHeader(buffer); + return RecoverableMessage::shared_ptr(new RecoverableMessageImpl(Message(transfer, transfer))); } RecoverableTransaction::shared_ptr RecoveryManagerImpl::recoverTransaction(const std::string& xid, @@ -163,12 +166,7 @@ void RecoveryManagerImpl::recoveryComplete() exchanges.eachExchange(boost::bind(&Exchange::recoveryComplete, _1, boost::ref(exchanges))); } -RecoverableMessageImpl:: RecoverableMessageImpl(const intrusive_ptr<Message>& _msg) : msg(_msg) -{ - if (!msg->isPersistent()) { - msg->forcePersistent(); // set so that message will get dequeued from store. - } -} +RecoverableMessageImpl:: RecoverableMessageImpl(const Message& _msg) : msg(_msg) {} bool RecoverableMessageImpl::loadContent(uint64_t /*available*/) { @@ -177,7 +175,7 @@ bool RecoverableMessageImpl::loadContent(uint64_t /*available*/) void RecoverableMessageImpl::decodeContent(framing::Buffer& buffer) { - msg->decodeContent(buffer); + msg.getPersistentContext()->decodeContent(buffer); } void RecoverableMessageImpl::recover(Queue::shared_ptr queue) @@ -187,12 +185,12 @@ void RecoverableMessageImpl::recover(Queue::shared_ptr queue) void RecoverableMessageImpl::setPersistenceId(uint64_t id) { - msg->setPersistenceId(id); + msg.getPersistentContext()->setPersistenceId(id); } void RecoverableMessageImpl::setRedelivered() { - msg->redeliver(); + msg.deliver();//increment delivery count (but at present that isn't recorded durably) } void RecoverableQueueImpl::recover(RecoverableMessage::shared_ptr msg) @@ -204,7 +202,7 @@ void RecoverableQueueImpl::setPersistenceId(uint64_t id) { queue->setPersistenceId(id); } - + uint64_t RecoverableQueueImpl::getPersistenceId() const { return queue->getPersistenceId(); diff --git a/qpid/cpp/src/qpid/broker/SemanticState.cpp b/qpid/cpp/src/qpid/broker/SemanticState.cpp index 9a84db547c..5d96467bbf 100644 --- a/qpid/cpp/src/qpid/broker/SemanticState.cpp +++ b/qpid/cpp/src/qpid/broker/SemanticState.cpp @@ -29,7 +29,7 @@ #include "qpid/broker/SessionContext.h" #include "qpid/broker/SessionOutputException.h" #include "qpid/broker/TxAccept.h" -#include "qpid/broker/TxPublish.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/framing/reply_exceptions.h" #include "qpid/framing/MessageTransferBody.h" #include "qpid/framing/SequenceSet.h" @@ -65,9 +65,8 @@ using qpid::management::Manageable; using qpid::management::Args; namespace _qmf = qmf::org::apache::qpid::broker; -SemanticState::SemanticState(DeliveryAdapter& da, SessionContext& ss) +SemanticState::SemanticState(SessionState& ss) : session(ss), - deliveryAdapter(da), tagGenerator("sgen"), dtxSelected(false), authMsg(getSession().getBroker().getOptions().auth && !getSession().getConnection().isUserProxyAuth()), @@ -89,7 +88,7 @@ void SemanticState::closed() { if (dtxBuffer.get()) { dtxBuffer->fail(); } - recover(true); + requeue(); //now unsubscribe, which may trigger queue deletion and thus //needs to occur after the requeueing of unacked messages @@ -124,7 +123,7 @@ void SemanticState::consume(const string& tag, resumeId, resumeTtl, arguments); if (!c) // Create plain consumer c = ConsumerImpl::shared_ptr( - new ConsumerImpl(this, name, queue, ackRequired, acquire, exclusive, tag, + new ConsumerImpl(this, name, queue, ackRequired, acquire ? CONSUMER : BROWSER, exclusive, tag, resumeId, resumeTtl, arguments)); queue->consume(c, exclusive);//may throw exception consumers[tag] = c; @@ -281,7 +280,7 @@ SemanticState::ConsumerImpl::ConsumerImpl(SemanticState* _parent, const string& _name, Queue::shared_ptr _queue, bool ack, - bool _acquire, + SubscriptionType type, bool _exclusive, const string& _tag, const string& _resumeId, @@ -289,11 +288,11 @@ SemanticState::ConsumerImpl::ConsumerImpl(SemanticState* _parent, const framing::FieldTable& _arguments ) : - Consumer(_name, _acquire), +Consumer(_name, type), parent(_parent), queue(_queue), ackExpected(ack), - acquire(_acquire), + acquire(type == CONSUMER), blocked(true), exclusive(_exclusive), resumeId(_resumeId), @@ -340,32 +339,42 @@ OwnershipToken* SemanticState::ConsumerImpl::getSession() return &(parent->session); } -bool SemanticState::ConsumerImpl::deliver(QueuedMessage& msg) +bool SemanticState::ConsumerImpl::deliver(const QueueCursor& cursor, const Message& msg) +{ + return deliver(cursor, msg, shared_from_this()); +} +bool SemanticState::ConsumerImpl::deliver(const QueueCursor& cursor, const Message& msg, boost::shared_ptr<Consumer> consumer) { assertClusterSafe(); - allocateCredit(msg.payload); - DeliveryRecord record(msg, msg.queue->shared_from_this(), getTag(), - shared_from_this(), acquire, !ackExpected, credit.isWindowMode(), 0); + allocateCredit(msg); + DeliveryRecord record(cursor, msg.getSequence(), queue, getTag(), + consumer, acquire, !ackExpected, credit.isWindowMode(), amqp_0_10::MessageTransfer::getRequiredCredit(msg)); bool sync = syncFrequency && ++deliveryCount >= syncFrequency; if (sync) deliveryCount = 0;//reset - parent->deliver(record, sync); + const amqp_0_10::MessageTransfer* transfer = dynamic_cast<const amqp_0_10::MessageTransfer*>(&msg.getEncoding()); + + record.setId(parent->session.deliver(*transfer, getTag(), msg.isRedelivered(), msg.getTtl(), msg.getTimestamp(), + ackExpected ? message::ACCEPT_MODE_EXPLICIT : message::ACCEPT_MODE_NONE, + acquire ? message::ACQUIRE_MODE_PRE_ACQUIRED : message::ACQUIRE_MODE_NOT_ACQUIRED, + msg.getAnnotations(), + sync)); if (credit.isWindowMode() || ackExpected || !acquire) { parent->record(record); } if (acquire && !ackExpected) { // auto acquire && auto accept - msg.queue->dequeue(0, msg); + queue->dequeue(0 /*ctxt*/, cursor); record.setEnded(); } if (mgmtObject) { mgmtObject->inc_delivered(); } return true; } -bool SemanticState::ConsumerImpl::filter(intrusive_ptr<Message>) +bool SemanticState::ConsumerImpl::filter(const Message&) { return true; } -bool SemanticState::ConsumerImpl::accept(intrusive_ptr<Message> msg) +bool SemanticState::ConsumerImpl::accept(const Message& msg) { assertClusterSafe(); // TODO aconway 2009-06-08: if we have byte & message credit but @@ -389,21 +398,21 @@ ostream& operator<<(ostream& o, const ConsumerName& pc) { } } -void SemanticState::ConsumerImpl::allocateCredit(intrusive_ptr<Message>& msg) +void SemanticState::ConsumerImpl::allocateCredit(const Message& msg) { assertClusterSafe(); Credit original = credit; - credit.consume(1, msg->getRequiredCredit()); + credit.consume(1, qpid::broker::amqp_0_10::MessageTransfer::getRequiredCredit(msg)); QPID_LOG(debug, "Credit allocated for " << ConsumerName(*this) << ", was " << original << " now " << credit); } -bool SemanticState::ConsumerImpl::checkCredit(intrusive_ptr<Message>& msg) +bool SemanticState::ConsumerImpl::checkCredit(const Message& msg) { - bool enoughCredit = credit.check(1, msg->getRequiredCredit()); + bool enoughCredit = credit.check(1, qpid::broker::amqp_0_10::MessageTransfer::getRequiredCredit(msg)); QPID_LOG(debug, "Subscription " << ConsumerName(*this) << " has " << (enoughCredit ? "sufficient " : "insufficient") - << " credit for message of " << msg->getRequiredCredit() << " bytes: " + << " credit for message of " << qpid::broker::amqp_0_10::MessageTransfer::getRequiredCredit(msg) << " bytes: " << credit); return enoughCredit; } @@ -421,7 +430,6 @@ void SemanticState::disable(ConsumerImpl::shared_ptr c) session.getConnection().outputTasks.removeOutputTask(c.get()); } - void SemanticState::cancel(ConsumerImpl::shared_ptr c) { disable(c); @@ -435,49 +443,20 @@ void SemanticState::cancel(ConsumerImpl::shared_ptr c) c->cancel(); } -void SemanticState::handle(intrusive_ptr<Message> msg) { - if (txBuffer.get()) { - TxPublish* deliverable(new TxPublish(msg)); - TxOp::shared_ptr op(deliverable); - route(msg, *deliverable); - txBuffer->enlist(op); - } else { - DeliverableMessage deliverable(msg); - route(msg, deliverable); - if (msg->isContentReleaseRequested()) { - // NOTE: The log messages in this section are used for flow-to-disk testing (which checks the log for the - // presence of these messages). Do not change these without also checking these tests. - if (msg->isContentReleaseBlocked()) { - QPID_LOG(debug, "Message id=\"" << msg->getProperties<MessageProperties>()->getMessageId() << "\"; pid=0x" << - std::hex << msg->getPersistenceId() << std::dec << ": Content release blocked"); - } else { - msg->releaseContent(); - QPID_LOG(debug, "Message id=\"" << msg->getProperties<MessageProperties>()->getMessageId() << "\"; pid=0x" << - std::hex << msg->getPersistenceId() << std::dec << ": Content released"); - } - } - } -} - -namespace +TxBuffer* SemanticState::getTxBuffer() { -const std::string nullstring; + return txBuffer.get(); } -void SemanticState::route(intrusive_ptr<Message> msg, Deliverable& strategy) { - msg->computeExpiration(getSession().getBroker().getExpiryPolicy()); +void SemanticState::route(Message& msg, Deliverable& strategy) { + msg.computeExpiration(getSession().getBroker().getExpiryPolicy()); - std::string exchangeName = msg->getExchangeName(); - if (!cacheExchange || cacheExchange->getName() != exchangeName - || cacheExchange->isDestroyed()) - { + std::string exchangeName = qpid::broker::amqp_0_10::MessageTransfer::get(msg).getExchangeName(); + if (!cacheExchange || cacheExchange->getName() != exchangeName || cacheExchange->isDestroyed()) cacheExchange = session.getBroker().getExchanges().get(exchangeName); - } - cacheExchange->setProperties(msg); /* verify the userid if specified: */ - std::string id = - msg->hasProperties<MessageProperties>() ? msg->getProperties<MessageProperties>()->getUserId() : nullstring; + std::string id = msg.getUserId(); if (authMsg && !id.empty() && !session.getConnection().isAuthenticatedUser(id)) { QPID_LOG(debug, "authorised user id : " << userID << " but user id in message declared as " << id); @@ -487,9 +466,9 @@ void SemanticState::route(intrusive_ptr<Message> msg, Deliverable& strategy) { AclModule* acl = getSession().getBroker().getAcl(); if (acl && acl->doTransferAcl()) { - if (!acl->authorise(getSession().getConnection().getUserId(),acl::ACT_PUBLISH,acl::OBJ_EXCHANGE,exchangeName, msg->getRoutingKey() )) + if (!acl->authorise(getSession().getConnection().getUserId(),acl::ACT_PUBLISH,acl::OBJ_EXCHANGE,exchangeName, msg.getRoutingKey() )) throw UnauthorizedAccessException(QPID_MSG(userID << " cannot publish to " << - exchangeName << " with routing-key " << msg->getRoutingKey())); + exchangeName << " with routing-key " << msg.getRoutingKey())); } cacheExchange->route(strategy); @@ -501,9 +480,6 @@ void SemanticState::route(intrusive_ptr<Message> msg, Deliverable& strategy) { if (cacheExchange->getAlternate()) { cacheExchange->getAlternate()->route(strategy); } - if (!strategy.delivered) { - msg->destroy(); - } } } @@ -543,28 +519,20 @@ void SemanticState::ConsumerImpl::complete(DeliveryRecord& delivery) } } -void SemanticState::recover(bool requeue) +void SemanticState::requeue() { - if(requeue){ - //take copy and clear unacked as requeue may result in redelivery to this session - //which will in turn result in additions to unacked - DeliveryRecords copy = unacked; - unacked.clear(); - for_each(copy.rbegin(), copy.rend(), mem_fun_ref(&DeliveryRecord::requeue)); - }else{ - for_each(unacked.begin(), unacked.end(), boost::bind(&DeliveryRecord::redeliver, _1, this)); - //unconfirmed messages re redelivered and therefore have their - //id adjusted, confirmed messages are not and so the ordering - //w.r.t id is lost - sort(unacked.begin(), unacked.end()); - } + //take copy and clear unacked as requeue may result in redelivery to this session + //which will in turn result in additions to unacked + DeliveryRecords copy = unacked; + unacked.clear(); + for_each(copy.rbegin(), copy.rend(), mem_fun_ref(&DeliveryRecord::requeue)); getSession().setUnackedCount(unacked.size()); } -void SemanticState::deliver(DeliveryRecord& msg, bool sync) -{ - return deliveryAdapter.deliver(msg, sync); -} + +SessionContext& SemanticState::getSession() { return session; } +const SessionContext& SemanticState::getSession() const { return session; } + const SemanticState::ConsumerImpl::shared_ptr SemanticState::find(const std::string& destination) const { diff --git a/qpid/cpp/src/qpid/broker/SemanticState.h b/qpid/cpp/src/qpid/broker/SemanticState.h index 15928ce599..67cfe808d0 100644 --- a/qpid/cpp/src/qpid/broker/SemanticState.h +++ b/qpid/cpp/src/qpid/broker/SemanticState.h @@ -26,7 +26,6 @@ #include "qpid/broker/Consumer.h" #include "qpid/broker/Credit.h" #include "qpid/broker/Deliverable.h" -#include "qpid/broker/DeliveryAdapter.h" #include "qpid/broker/DeliveryRecord.h" #include "qpid/broker/DtxBuffer.h" #include "qpid/broker/DtxManager.h" @@ -34,12 +33,15 @@ #include "qpid/broker/QueueObserver.h" #include "qpid/broker/TxBuffer.h" +#include "qpid/framing/FieldTable.h" #include "qpid/framing/FrameHandler.h" #include "qpid/framing/SequenceSet.h" #include "qpid/framing/Uuid.h" #include "qpid/sys/AggregateOutput.h" #include "qpid/sys/Mutex.h" #include "qpid/sys/AtomicValue.h" +#include "qpid/broker/AclModule.h" +#include "qpid/management/Manageable.h" #include "qmf/org/apache/qpid/broker/Subscription.h" #include <list> @@ -47,13 +49,15 @@ #include <vector> #include <boost/enable_shared_from_this.hpp> -#include <boost/intrusive_ptr.hpp> #include <boost/cast.hpp> namespace qpid { namespace broker { +class Exchange; +class MessageStore; class SessionContext; +class SessionState; /** * @@ -94,28 +98,28 @@ class SemanticState : private boost::noncopyable { int deliveryCount; qmf::org::apache::qpid::broker::Subscription* mgmtObject; - bool checkCredit(boost::intrusive_ptr<Message>& msg); - void allocateCredit(boost::intrusive_ptr<Message>& msg); + bool checkCredit(const Message& msg); + void allocateCredit(const Message& msg); bool haveCredit(); protected: QPID_BROKER_EXTERN virtual bool doDispatch(); size_t unacked() { return parent->unacked.size(); } + QPID_BROKER_EXTERN bool deliver(const QueueCursor&, const Message&, boost::shared_ptr<Consumer>); public: typedef boost::shared_ptr<ConsumerImpl> shared_ptr; - QPID_BROKER_EXTERN ConsumerImpl( - SemanticState* parent, - const std::string& name, boost::shared_ptr<Queue> queue, - bool ack, bool acquire, bool exclusive, - const std::string& tag, const std::string& resumeId, uint64_t resumeTtl, - const framing::FieldTable& arguments); - QPID_BROKER_EXTERN virtual ~ConsumerImpl(); + QPID_BROKER_EXTERN ConsumerImpl(SemanticState* parent, + const std::string& name, boost::shared_ptr<Queue> queue, + bool ack, SubscriptionType type, bool exclusive, + const std::string& tag, const std::string& resumeId, + uint64_t resumeTtl, const framing::FieldTable& arguments); + QPID_BROKER_EXTERN ~ConsumerImpl(); QPID_BROKER_EXTERN OwnershipToken* getSession(); - QPID_BROKER_EXTERN virtual bool deliver(QueuedMessage& msg); - QPID_BROKER_EXTERN bool filter(boost::intrusive_ptr<Message> msg); - QPID_BROKER_EXTERN bool accept(boost::intrusive_ptr<Message> msg); + QPID_BROKER_EXTERN bool deliver(const QueueCursor&, const Message&); + QPID_BROKER_EXTERN bool filter(const Message&); + QPID_BROKER_EXTERN bool accept(const Message&); QPID_BROKER_EXTERN void cancel() {} QPID_BROKER_EXTERN void disableNotify(); @@ -153,7 +157,7 @@ class SemanticState : private boost::noncopyable { SemanticState& getParent() { return *parent; } const SemanticState& getParent() const { return *parent; } - void acknowledged(const broker::QueuedMessage&) {} + void acknowledged(const DeliveryRecord&) {} // manageable entry points QPID_BROKER_EXTERN management::ManagementObject* @@ -168,8 +172,7 @@ class SemanticState : private boost::noncopyable { private: typedef std::map<std::string, ConsumerImpl::shared_ptr> ConsumerImplMap; - SessionContext& session; - DeliveryAdapter& deliveryAdapter; + SessionState& session; ConsumerImplMap consumers; NameGenerator tagGenerator; DeliveryRecords unacked; @@ -185,7 +188,6 @@ class SemanticState : private boost::noncopyable { //needed for queue delete events in auto-delete: const std::string connectionId; - void route(boost::intrusive_ptr<Message> msg, Deliverable& strategy); void checkDtxTimeout(); bool complete(DeliveryRecord&); @@ -196,11 +198,11 @@ class SemanticState : private boost::noncopyable { public: - SemanticState(DeliveryAdapter&, SessionContext&); + SemanticState(SessionState&); ~SemanticState(); - SessionContext& getSession() { return session; } - const SessionContext& getSession() const { return session; } + SessionContext& getSession(); + const SessionContext& getSession() const; const ConsumerImpl::shared_ptr find(const std::string& destination) const; bool find(const std::string& destination, ConsumerImpl::shared_ptr&) const; @@ -239,12 +241,12 @@ class SemanticState : private boost::noncopyable { void endDtx(const std::string& xid, bool fail); void suspendDtx(const std::string& xid); void resumeDtx(const std::string& xid); - void recover(bool requeue); - void deliver(DeliveryRecord& message, bool sync); + TxBuffer* getTxBuffer(); + void requeue(); void acquire(DeliveryId first, DeliveryId last, DeliveryIds& acquired); void release(DeliveryId first, DeliveryId last, bool setRedelivered); void reject(DeliveryId first, DeliveryId last); - void handle(boost::intrusive_ptr<Message> msg); + void route(Message& msg, Deliverable& strategy); void completed(const framing::SequenceSet& commands); void accepted(const framing::SequenceSet& commands); diff --git a/qpid/cpp/src/qpid/broker/SessionAdapter.cpp b/qpid/cpp/src/qpid/broker/SessionAdapter.cpp index ae994a6bd5..c973098020 100644 --- a/qpid/cpp/src/qpid/broker/SessionAdapter.cpp +++ b/qpid/cpp/src/qpid/broker/SessionAdapter.cpp @@ -264,7 +264,7 @@ QueueQueryResult SessionAdapter::QueueHandlerImpl::query(const string& name) queue->isDurable(), queue->hasExclusiveOwner(), queue->isAutoDelete(), - queue->getSettings(), + queue->getEncodableSettings(), queue->getMessageCount(), queue->getConsumerCount()); } else { @@ -294,19 +294,24 @@ void SessionAdapter::QueueHandlerImpl::declare(const string& name, const string& queue = getQueue(name); //TODO: check alternate-exchange is as expected } else { + QueueSettings settings(durable, autoDelete); + try { + settings.populate(arguments, settings.storeSettings); + } catch (const qpid::types::Exception& e) { + throw InvalidArgumentException(e.what()); + } + std::pair<Queue::shared_ptr, bool> queue_created = - getBroker().createQueue(name, durable, - autoDelete, + getBroker().createQueue(name, settings, exclusive ? &session : 0, alternateExchange, - arguments, getConnection().getUserId(), getConnection().getUrl()); queue = queue_created.first; assert(queue); if (queue_created.second) { // This is a new queue //handle automatic cleanup: - if (exclusive) { + if (exclusive && queue->setExclusiveOwner(&session)) { exclusiveQueues.push_back(queue); } } else { diff --git a/qpid/cpp/src/qpid/broker/SessionHandler.cpp b/qpid/cpp/src/qpid/broker/SessionHandler.cpp index 23fa2ee0ca..9888d12be2 100644 --- a/qpid/cpp/src/qpid/broker/SessionHandler.cpp +++ b/qpid/cpp/src/qpid/broker/SessionHandler.cpp @@ -32,7 +32,7 @@ using namespace std; using namespace qpid::sys; SessionHandler::SessionHandler(Connection& c, ChannelId ch) - : amqp_0_10::SessionHandler(&c.getOutput(), ch), + : qpid::amqp_0_10::SessionHandler(&c.getOutput(), ch), connection(c), proxy(out), clusterOrderProxy(c.getClusterOrderOutput() ? @@ -75,7 +75,7 @@ ConnectionState& SessionHandler::getConnection() { return connection; } const ConnectionState& SessionHandler::getConnection() const { return connection; } void SessionHandler::handleDetach() { - amqp_0_10::SessionHandler::handleDetach(); + qpid::amqp_0_10::SessionHandler::handleDetach(); assert(&connection.getChannel(channel.get()) == this); if (session.get()) connection.getBroker().getSessionManager().detach(session); @@ -125,7 +125,7 @@ void SessionHandler::attached(const std::string& name) { if (session.get()) { session->addManagementObject(); // Delayed from attachAs() - amqp_0_10::SessionHandler::attached(name); + qpid::amqp_0_10::SessionHandler::attached(name); } else { SessionId id(connection.getUserId(), name); SessionState::Configuration config = connection.broker.getSessionManager().getSessionConfig(); diff --git a/qpid/cpp/src/qpid/broker/SessionHandler.h b/qpid/cpp/src/qpid/broker/SessionHandler.h index ab87cf41a4..21c736fa37 100644 --- a/qpid/cpp/src/qpid/broker/SessionHandler.h +++ b/qpid/cpp/src/qpid/broker/SessionHandler.h @@ -41,7 +41,7 @@ class SessionState; * receives incoming frames, handles session controls and manages the * association between the channel and a session. */ -class SessionHandler : public amqp_0_10::SessionHandler { +class SessionHandler : public qpid::amqp_0_10::SessionHandler { public: class ErrorListener { public: diff --git a/qpid/cpp/src/qpid/broker/SessionState.cpp b/qpid/cpp/src/qpid/broker/SessionState.cpp index cc02d9ec94..88cdf7e03a 100644 --- a/qpid/cpp/src/qpid/broker/SessionState.cpp +++ b/qpid/cpp/src/qpid/broker/SessionState.cpp @@ -21,6 +21,7 @@ #include "qpid/broker/SessionState.h" #include "qpid/broker/Broker.h" #include "qpid/broker/ConnectionState.h" +#include "qpid/broker/DeliverableMessage.h" #include "qpid/broker/DeliveryRecord.h" #include "qpid/broker/SessionManager.h" #include "qpid/broker/SessionHandler.h" @@ -28,6 +29,7 @@ #include "qpid/framing/AMQContentBody.h" #include "qpid/framing/AMQHeaderBody.h" #include "qpid/framing/AMQMethodBody.h" +#include "qpid/framing/MessageTransferBody.h" #include "qpid/framing/reply_exceptions.h" #include "qpid/framing/ServerInvoker.h" #include "qpid/log/Statement.h" @@ -55,9 +57,8 @@ SessionState::SessionState( const SessionState::Configuration& config, bool delayManagement) : qpid::SessionState(id, config), broker(b), handler(&h), - semanticState(*this, *this), + semanticState(*this), adapter(semanticState), - msgBuilder(&broker.getStore()), mgmtObject(0), asyncCommandCompleter(new AsyncCommandCompleter(this)) { @@ -208,7 +209,7 @@ void SessionState::handleContent(AMQFrame& frame, const SequenceNumber& id) { if (frame.getBof() && frame.getBos()) //start of frameset msgBuilder.start(id); - intrusive_ptr<Message> msg(msgBuilder.getMessage()); + intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> msg(msgBuilder.getMessage()); msgBuilder.handle(frame); if (frame.getEof() && frame.getEos()) {//end of frameset if (frame.getBof()) { @@ -218,13 +219,16 @@ void SessionState::handleContent(AMQFrame& frame, const SequenceNumber& id) header.setEof(false); msg->getFrames().append(header); } + DeliverableMessage deliverable(Message(msg, msg), semanticState.getTxBuffer()); if (broker.isTimestamping()) - msg->setTimestamp(); - msg->setPublisher(&getConnection()); + deliverable.getMessage().setTimestamp(); + deliverable.getMessage().setPublisher(&getConnection()); + + + IncompleteIngressMsgXfer xfer(this, msg); msg->getIngressCompletion().begin(); - semanticState.handle(msg); + semanticState.route(deliverable.getMessage(), deliverable); msgBuilder.end(); - IncompleteIngressMsgXfer xfer(this, msg); msg->getIngressCompletion().end(xfer); // allows msg to complete xfer } } @@ -294,18 +298,28 @@ void SessionState::handleOut(AMQFrame& frame) { handler->out(frame); } -void SessionState::deliver(DeliveryRecord& msg, bool sync) +DeliveryId SessionState::deliver(const qpid::broker::amqp_0_10::MessageTransfer& message, + const std::string& destination, bool isRedelivered, uint64_t ttl, uint64_t timestamp, + qpid::framing::message::AcceptMode acceptMode, qpid::framing::message::AcquireMode acquireMode, + const qpid::types::Variant::Map& annotations, bool sync) { uint32_t maxFrameSize = getConnection().getFrameMax(); assert(senderGetCommandPoint().offset == 0); SequenceNumber commandId = senderGetCommandPoint().command; - msg.deliver(getProxy().getHandler(), commandId, maxFrameSize); + + framing::AMQFrame method((framing::MessageTransferBody(framing::ProtocolVersion(), destination, acceptMode, acquireMode))); + method.setEof(false); + getProxy().getHandler().handle(method); + message.sendHeader(getProxy().getHandler(), maxFrameSize, isRedelivered, ttl, timestamp, annotations); + message.sendContent(getProxy().getHandler(), maxFrameSize); + assert(senderGetCommandPoint() == SessionPoint(commandId+1, 0)); // Delivery has moved sendPoint. if (sync) { AMQP_ClientProxy::Execution& p(getProxy().getExecution()); Proxy::ScopedSync s(p); p.sync(); } + return commandId; } void SessionState::sendCompletion() { @@ -349,7 +363,6 @@ void SessionState::addPendingExecutionSync() } } - /** factory for creating a reference-counted IncompleteIngressMsgXfer object * which will be attached to a message that will be completed asynchronously. */ @@ -408,10 +421,10 @@ void SessionState::AsyncCommandCompleter::schedule(boost::intrusive_ptr<AsyncCom /** Track an ingress message that is pending completion */ -void SessionState::AsyncCommandCompleter::addPendingMessage(boost::intrusive_ptr<Message> msg) +void SessionState::AsyncCommandCompleter::addPendingMessage(boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> msg) { qpid::sys::ScopedLock<qpid::sys::Mutex> l(completerLock); - std::pair<SequenceNumber, boost::intrusive_ptr<Message> > item(msg->getCommandId(), msg); + std::pair<SequenceNumber, boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> > item(msg->getCommandId(), msg); bool unique = pendingMsgs.insert(item).second; if (!unique) { assert(false); @@ -430,13 +443,13 @@ void SessionState::AsyncCommandCompleter::deletePendingMessage(SequenceNumber id /** done when an execution.sync arrives */ void SessionState::AsyncCommandCompleter::flushPendingMessages() { - std::map<SequenceNumber, boost::intrusive_ptr<Message> > copy; + std::map<SequenceNumber, boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> > copy; { qpid::sys::ScopedLock<qpid::sys::Mutex> l(completerLock); pendingMsgs.swap(copy); // we've only tracked these in case a flush is needed, so nuke 'em now. } // drop lock, so it is safe to call "flush()" - for (std::map<SequenceNumber, boost::intrusive_ptr<Message> >::iterator i = copy.begin(); + for (std::map<SequenceNumber, boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> >::iterator i = copy.begin(); i != copy.end(); ++i) { i->second->flush(); } diff --git a/qpid/cpp/src/qpid/broker/SessionState.h b/qpid/cpp/src/qpid/broker/SessionState.h index a8ff7feff9..5e3a77d7ed 100644 --- a/qpid/cpp/src/qpid/broker/SessionState.h +++ b/qpid/cpp/src/qpid/broker/SessionState.h @@ -23,17 +23,18 @@ */ #include "qpid/SessionState.h" +#include "qpid/framing/enum.h" #include "qpid/framing/FrameHandler.h" #include "qpid/framing/SequenceSet.h" #include "qpid/sys/Time.h" #include "qpid/management/Manageable.h" #include "qmf/org/apache/qpid/broker/Session.h" #include "qpid/broker/SessionAdapter.h" -#include "qpid/broker/DeliveryAdapter.h" #include "qpid/broker/AsyncCompletion.h" #include "qpid/broker/MessageBuilder.h" #include "qpid/broker/SessionContext.h" #include "qpid/broker/SemanticState.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/sys/Monitor.h" #include <boost/noncopyable.hpp> @@ -58,7 +59,6 @@ namespace broker { class Broker; class ConnectionState; -class Message; class SessionHandler; class SessionManager; @@ -68,7 +68,6 @@ class SessionManager; */ class SessionState : public qpid::SessionState, public SessionContext, - public DeliveryAdapter, public management::Manageable, public framing::FrameHandler::InOutHandler { @@ -105,8 +104,10 @@ class SessionState : public qpid::SessionState, void sendCompletion(); - //delivery adapter methods: - void deliver(DeliveryRecord&, bool sync); + DeliveryId deliver(const qpid::broker::amqp_0_10::MessageTransfer& message, + const std::string& destination, bool isRedelivered, uint64_t ttl, uint64_t timestamp, + qpid::framing::message::AcceptMode, qpid::framing::message::AcquireMode, + const qpid::types::Variant::Map& annotations, bool sync); // Manageable entry points management::ManagementObject* GetManagementObject (void) const; @@ -117,7 +118,7 @@ class SessionState : public qpid::SessionState, // Used by cluster to create replica sessions. SemanticState& getSemanticState() { return semanticState; } - boost::intrusive_ptr<Message> getMessageInProgress() { return msgBuilder.getMessage(); } + boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> getMessageInProgress() { return msgBuilder.getMessage(); } SessionAdapter& getSessionAdapter() { return adapter; } const SessionId& getSessionId() const { return getId(); } @@ -199,7 +200,7 @@ class SessionState : public qpid::SessionState, // If an ingress message does not require a Sync, we need to // hold a reference to it in case an Execution.Sync command is received and we // have to manually flush the message. - std::map<SequenceNumber, boost::intrusive_ptr<Message> > pendingMsgs; + std::map<SequenceNumber, boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> > pendingMsgs; /** complete all pending commands, runs in IO thread */ void completeCommands(); @@ -212,7 +213,7 @@ class SessionState : public qpid::SessionState, ~AsyncCommandCompleter() {}; /** track a message pending ingress completion */ - void addPendingMessage(boost::intrusive_ptr<Message> m); + void addPendingMessage(boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> m); void deletePendingMessage(SequenceNumber id); void flushPendingMessages(); /** schedule the processing of a completed ingress message.transfer command */ @@ -246,29 +247,29 @@ class SessionState : public qpid::SessionState, { public: IncompleteIngressMsgXfer( SessionState *ss, - boost::intrusive_ptr<Message> m ) + boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> m) : AsyncCommandContext(ss, m->getCommandId()), - session(ss), - msg(m), - requiresAccept(m->requiresAccept()), - requiresSync(m->getFrames().getMethod()->isSync()), - pending(false) {} + session(ss), + msg(m), + requiresAccept(m->requiresAccept()), + requiresSync(m->getFrames().getMethod()->isSync()), + pending(false) {} IncompleteIngressMsgXfer( const IncompleteIngressMsgXfer& x ) : AsyncCommandContext(x.session, x.msg->getCommandId()), - session(x.session), - msg(x.msg), - requiresAccept(x.requiresAccept), - requiresSync(x.requiresSync), - pending(x.pending) {} + session(x.session), + msg(x.msg), + requiresAccept(x.requiresAccept), + requiresSync(x.requiresSync), + pending(x.pending) {} - virtual ~IncompleteIngressMsgXfer() {}; + virtual ~IncompleteIngressMsgXfer() {}; virtual void completed(bool); virtual boost::intrusive_ptr<AsyncCompletion::Callback> clone(); private: SessionState *session; // only valid if sync flag in callback is true - boost::intrusive_ptr<Message> msg; + boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> msg; bool requiresAccept; bool requiresSync; bool pending; // true if msg saved on pending list... diff --git a/qpid/cpp/src/qpid/broker/ThresholdAlerts.cpp b/qpid/cpp/src/qpid/broker/ThresholdAlerts.cpp index 3c9e210d4d..9b4e948e4f 100644 --- a/qpid/cpp/src/qpid/broker/ThresholdAlerts.cpp +++ b/qpid/cpp/src/qpid/broker/ThresholdAlerts.cpp @@ -20,7 +20,8 @@ */ #include "qpid/broker/ThresholdAlerts.h" #include "qpid/broker/Queue.h" -#include "qpid/broker/QueuedMessage.h" +#include "qpid/broker/Message.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/amqp_0_10/Codecs.h" #include "qpid/log/Statement.h" #include "qpid/management/ManagementAgent.h" @@ -30,21 +31,20 @@ namespace qpid { namespace broker { namespace { const qmf::org::apache::qpid::broker::EventQueueThresholdExceeded EVENT("dummy", 0, 0); -bool isQMFv2(const boost::intrusive_ptr<Message> message) +bool isQMFv2(const Message& message) { - const qpid::framing::MessageProperties* props = message->getProperties<qpid::framing::MessageProperties>(); + const qpid::framing::MessageProperties* props = qpid::broker::amqp_0_10::MessageTransfer::get(message).getProperties<qpid::framing::MessageProperties>(); return props && props->getAppId() == "qmf2"; } -bool isThresholdEvent(const boost::intrusive_ptr<Message> message) +bool isThresholdEvent(const Message& message) { - if (message->getIsManagementMessage()) { + if (message.getIsManagementMessage()) { //is this a qmf event? if so is it a threshold event? if (isQMFv2(message)) { - const qpid::framing::FieldTable* headers = message->getApplicationHeaders(); - if (headers && headers->getAsString("qmf.content") == "_event") { + if (message.getPropertyAsString("qmf.content") == "_event") { //decode as list - std::string content = message->getFrames().getContent(); + std::string content = qpid::broker::amqp_0_10::MessageTransfer::get(message).getFrames().getContent(); qpid::types::Variant::List list; qpid::amqp_0_10::ListCodec::decode(content, list); if (list.empty() || list.front().getType() != qpid::types::VAR_MAP) return false; @@ -57,7 +57,7 @@ bool isThresholdEvent(const boost::intrusive_ptr<Message> message) } } } else { - std::string content = message->getFrames().getContent(); + std::string content = qpid::broker::amqp_0_10::MessageTransfer::get(message).getFrames().getContent(); qpid::framing::Buffer buffer(const_cast<char*>(content.data()), content.size()); if (buffer.getOctet() == 'A' && buffer.getOctet() == 'M' && buffer.getOctet() == '2' && buffer.getOctet() == 'e') { buffer.getLong();//sequence @@ -83,9 +83,9 @@ ThresholdAlerts::ThresholdAlerts(const std::string& n, repeatInterval(repeat ? repeat*qpid::sys::TIME_SEC : 0), count(0), size(0), lastAlert(qpid::sys::EPOCH) {} -void ThresholdAlerts::enqueued(const QueuedMessage& m) +void ThresholdAlerts::enqueued(const Message& m) { - size += m.payload->contentSize(); + size += m.getContentSize(); ++count; if ((countThreshold && count >= countThreshold) || (sizeThreshold && size >= sizeThreshold)) { if ((repeatInterval == 0 && lastAlert == qpid::sys::EPOCH) @@ -94,7 +94,7 @@ void ThresholdAlerts::enqueued(const QueuedMessage& m) //enqueued on queues; it may even be that this event //causes a message to be enqueued on the queue we are //tracking, and so we need to avoid recursing - if (isThresholdEvent(m.payload)) return; + if (isThresholdEvent(m)) return; lastAlert = qpid::sys::now(); agent.raiseEvent(qmf::org::apache::qpid::broker::EventQueueThresholdExceeded(name, count, size)); QPID_LOG(info, "Threshold event triggered for " << name << ", count=" << count << ", size=" << size); @@ -102,9 +102,9 @@ void ThresholdAlerts::enqueued(const QueuedMessage& m) } } -void ThresholdAlerts::dequeued(const QueuedMessage& m) +void ThresholdAlerts::dequeued(const Message& m) { - size -= m.payload->contentSize(); + size -= m.getContentSize(); --count; if ((countThreshold && count < countThreshold) || (sizeThreshold && size < sizeThreshold)) { lastAlert = qpid::sys::EPOCH; @@ -127,65 +127,14 @@ void ThresholdAlerts::observe(Queue& queue, qpid::management::ManagementAgent& a } void ThresholdAlerts::observe(Queue& queue, qpid::management::ManagementAgent& agent, - const qpid::framing::FieldTable& settings, uint16_t limitRatio) - -{ - qpid::types::Variant::Map map; - qpid::amqp_0_10::translate(settings, map); - observe(queue, agent, map, limitRatio); -} - -template <class T> -class Option -{ - public: - Option(const std::string& name, T d) : defaultValue(d) { names.push_back(name); } - void addAlias(const std::string& name) { names.push_back(name); } - T get(const qpid::types::Variant::Map& settings) const - { - T value(defaultValue); - for (std::vector<std::string>::const_iterator i = names.begin(); i != names.end(); ++i) { - if (get(settings, *i, value)) break; - } - return value; - } - private: - std::vector<std::string> names; - T defaultValue; - - bool get(const qpid::types::Variant::Map& settings, const std::string& name, T& value) const - { - qpid::types::Variant::Map::const_iterator i = settings.find(name); - if (i != settings.end()) { - try { - value = (T) i->second; - } catch (const qpid::types::InvalidConversion&) { - QPID_LOG(warning, "Bad value for" << name << ": " << i->second); - } - return true; - } else { - return false; - } - } -}; - -void ThresholdAlerts::observe(Queue& queue, qpid::management::ManagementAgent& agent, - const qpid::types::Variant::Map& settings, uint16_t limitRatio) - + const QueueSettings& settings, uint16_t limitRatio) { - //Note: aliases are keys defined by java broker - Option<int64_t> repeatInterval("qpid.alert_repeat_gap", 60); - repeatInterval.addAlias("x-qpid-minimum-alert-repeat-gap"); - //If no explicit threshold settings were given use specified //percentage of any limit from the policy. - const QueuePolicy* policy = queue.getPolicy(); - Option<uint32_t> countThreshold("qpid.alert_count", (uint32_t) (policy && limitRatio ? (policy->getMaxCount()*limitRatio/100) : 0)); - countThreshold.addAlias("x-qpid-maximum-message-count"); - Option<uint64_t> sizeThreshold("qpid.alert_size", (uint64_t) (policy && limitRatio ? (policy->getMaxSize()*limitRatio/100) : 0)); - sizeThreshold.addAlias("x-qpid-maximum-message-size"); + uint32_t countThreshold = settings.alertThreshold.hasCount() ? settings.alertThreshold.getCount() : (settings.maxDepth.getCount()*limitRatio/100); + uint32_t sizeThreshold = settings.alertThreshold.hasSize() ? settings.alertThreshold.getSize() : (settings.maxDepth.getSize()*limitRatio/100); - observe(queue, agent, countThreshold.get(settings), sizeThreshold.get(settings), repeatInterval.get(settings)); + observe(queue, agent, countThreshold, sizeThreshold, settings.alertRepeatInterval); } }} diff --git a/qpid/cpp/src/qpid/broker/ThresholdAlerts.h b/qpid/cpp/src/qpid/broker/ThresholdAlerts.h index 2b4a46b736..4f985522e2 100644 --- a/qpid/cpp/src/qpid/broker/ThresholdAlerts.h +++ b/qpid/cpp/src/qpid/broker/ThresholdAlerts.h @@ -27,15 +27,13 @@ #include <string> namespace qpid { -namespace framing { -class FieldTable; -} namespace management { class ManagementAgent; } namespace broker { class Queue; +struct QueueSettings; /** * Class to manage generation of QMF alerts when particular thresholds * are breached on a queue. @@ -48,19 +46,17 @@ class ThresholdAlerts : public QueueObserver const uint32_t countThreshold, const uint64_t sizeThreshold, const long repeatInterval); - void enqueued(const QueuedMessage&); - void dequeued(const QueuedMessage&); - void acquired(const QueuedMessage&) {}; - void requeued(const QueuedMessage&) {}; + void enqueued(const Message&); + void dequeued(const Message&); + void acquired(const Message&) {}; + void requeued(const Message&) {}; static void observe(Queue& queue, qpid::management::ManagementAgent& agent, const uint64_t countThreshold, const uint64_t sizeThreshold, const long repeatInterval); static void observe(Queue& queue, qpid::management::ManagementAgent& agent, - const qpid::framing::FieldTable& settings, uint16_t limitRatio); - static void observe(Queue& queue, qpid::management::ManagementAgent& agent, - const qpid::types::Variant::Map& settings, uint16_t limitRatio); + const QueueSettings& settings, uint16_t limitRatio); private: const std::string name; qpid::management::ManagementAgent& agent; diff --git a/qpid/cpp/src/qpid/broker/TxAccept.h b/qpid/cpp/src/qpid/broker/TxAccept.h index 314a150176..a59e69a85f 100644 --- a/qpid/cpp/src/qpid/broker/TxAccept.h +++ b/qpid/cpp/src/qpid/broker/TxAccept.h @@ -71,7 +71,6 @@ namespace qpid { virtual void commit() throw(); virtual void rollback() throw(); virtual ~TxAccept(){} - virtual void accept(TxOpConstVisitor& visitor) const { visitor(*this); } // Used by cluster replication. const framing::SequenceSet& getAcked() const { return acked; } diff --git a/qpid/cpp/src/qpid/broker/TxBuffer.cpp b/qpid/cpp/src/qpid/broker/TxBuffer.cpp index d92e6ace48..7663cc525f 100644 --- a/qpid/cpp/src/qpid/broker/TxBuffer.cpp +++ b/qpid/cpp/src/qpid/broker/TxBuffer.cpp @@ -28,7 +28,7 @@ using namespace qpid::broker; bool TxBuffer::prepare(TransactionContext* const ctxt) { - for(op_iterator i = ops.begin(); i < ops.end(); i++){ + for(op_iterator i = ops.begin(); i != ops.end(); i++){ if(!(*i)->prepare(ctxt)){ return false; } @@ -74,7 +74,3 @@ bool TxBuffer::commitLocal(TransactionalStore* const store) } return false; } - -void TxBuffer::accept(TxOpConstVisitor& v) const { - std::for_each(ops.begin(), ops.end(), boost::bind(&TxOp::accept, _1, boost::ref(v))); -} diff --git a/qpid/cpp/src/qpid/broker/TxBuffer.h b/qpid/cpp/src/qpid/broker/TxBuffer.h index d49c8ba16a..22e2f06be1 100644 --- a/qpid/cpp/src/qpid/broker/TxBuffer.h +++ b/qpid/cpp/src/qpid/broker/TxBuffer.h @@ -108,9 +108,6 @@ namespace qpid { * commit */ QPID_BROKER_EXTERN bool commitLocal(TransactionalStore* const store); - - // Used by cluster to replicate transaction status. - void accept(TxOpConstVisitor& v) const; }; } } diff --git a/qpid/cpp/src/qpid/broker/TxOp.h b/qpid/cpp/src/qpid/broker/TxOp.h index a8fa1c2621..775efc92f7 100644 --- a/qpid/cpp/src/qpid/broker/TxOp.h +++ b/qpid/cpp/src/qpid/broker/TxOp.h @@ -21,7 +21,6 @@ #ifndef _TxOp_ #define _TxOp_ -#include "qpid/broker/TxOpVisitor.h" #include "qpid/broker/TransactionalStore.h" #include <boost/shared_ptr.hpp> @@ -36,8 +35,6 @@ namespace qpid { virtual void commit() throw() = 0; virtual void rollback() throw() = 0; virtual ~TxOp(){} - - virtual void accept(TxOpConstVisitor&) const = 0; }; }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/TxOpVisitor.h b/qpid/cpp/src/qpid/broker/TxOpVisitor.h index ceb894896e..e69de29bb2 100644 --- a/qpid/cpp/src/qpid/broker/TxOpVisitor.h +++ b/qpid/cpp/src/qpid/broker/TxOpVisitor.h @@ -1,97 +0,0 @@ -#ifndef QPID_BROKER_TXOPVISITOR_H -#define QPID_BROKER_TXOPVISITOR_H - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -namespace qpid { -namespace broker { - -class DtxAck; -class RecoveredDequeue; -class RecoveredEnqueue; -class TxAccept; -class TxPublish; - -/** - * Visitor for TxOp familly of classes. - */ -struct TxOpConstVisitor -{ - virtual ~TxOpConstVisitor() {} - virtual void operator()(const DtxAck&) = 0; - virtual void operator()(const RecoveredDequeue&) = 0; - virtual void operator()(const RecoveredEnqueue&) = 0; - virtual void operator()(const TxAccept&) = 0; - virtual void operator()(const TxPublish&) = 0; -}; - -}} // namespace qpid::broker - -#endif /*!QPID_BROKER_TXOPVISITOR_H*/ -#ifndef QPID_BROKER_TXOPVISITOR_H -#define QPID_BROKER_TXOPVISITOR_H - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -namespace qpid { -namespace broker { - -class DtxAck; -class RecoveredDequeue; -class RecoveredEnqueue; -class TxAccept; -class TxPublish; - -/** - * Visitor for TxOp familly of classes. - */ -struct TxOpConstVisitor -{ - virtual ~TxOpConstVisitor() {} - virtual void operator()(const DtxAck&) = 0; - virtual void operator()(const RecoveredDequeue&) = 0; - virtual void operator()(const RecoveredEnqueue&) = 0; - virtual void operator()(const TxAccept&) = 0; - virtual void operator()(const TxPublish&) = 0; -}; - -}} // namespace qpid::broker - -#endif /*!QPID_BROKER_TXOPVISITOR_H*/ diff --git a/qpid/cpp/src/qpid/broker/TxPublish.cpp b/qpid/cpp/src/qpid/broker/TxPublish.cpp index 9c2cf4a467..e69de29bb2 100644 --- a/qpid/cpp/src/qpid/broker/TxPublish.cpp +++ b/qpid/cpp/src/qpid/broker/TxPublish.cpp @@ -1,111 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/log/Statement.h" -#include "qpid/broker/TxPublish.h" -#include "qpid/broker/Queue.h" - -using boost::intrusive_ptr; -using namespace qpid::broker; - -TxPublish::TxPublish(intrusive_ptr<Message> _msg) : msg(_msg) {} - -bool TxPublish::prepare(TransactionContext* ctxt) throw() -{ - try{ - while (!queues.empty()) { - prepare(ctxt, queues.front()); - prepared.push_back(queues.front()); - queues.pop_front(); - } - return true; - }catch(const std::exception& e){ - QPID_LOG(error, "Failed to prepare: " << e.what()); - }catch(...){ - QPID_LOG(error, "Failed to prepare (unknown error)"); - } - return false; -} - -void TxPublish::commit() throw() -{ - try { - for_each(prepared.begin(), prepared.end(), Commit(msg)); - if (msg->isContentReleaseRequested()) { - // NOTE: The log messages in this section are used for flow-to-disk testing (which checks the log for the - // presence of these messages). Do not change these without also checking these tests. - if (msg->isContentReleaseBlocked()) { - QPID_LOG(debug, "Message id=\"" << msg->getProperties<qpid::framing::MessageProperties>()->getMessageId() << "\"; pid=0x" << - std::hex << msg->getPersistenceId() << std::dec << ": Content release blocked on commit"); - } else { - msg->releaseContent(); - QPID_LOG(debug, "Message id=\"" << msg->getProperties<qpid::framing::MessageProperties>()->getMessageId() << "\"; pid=0x" << - std::hex << msg->getPersistenceId() << std::dec << ": Content released on commit"); - } - } - } catch (const std::exception& e) { - QPID_LOG(error, "Failed to commit: " << e.what()); - } catch(...) { - QPID_LOG(error, "Failed to commit (unknown error)"); - } -} - -void TxPublish::rollback() throw() -{ - try { - for_each(prepared.begin(), prepared.end(), Rollback(msg)); - } catch (const std::exception& e) { - QPID_LOG(error, "Failed to complete rollback: " << e.what()); - } catch(...) { - QPID_LOG(error, "Failed to complete rollback (unknown error)"); - } - -} - -void TxPublish::deliverTo(const boost::shared_ptr<Queue>& queue){ - if (!queue->isLocal(msg)) { - queues.push_back(queue); - delivered = true; - } else { - QPID_LOG(debug, "Won't enqueue local message for " << queue->getName()); - } -} - -void TxPublish::prepare(TransactionContext* ctxt, const boost::shared_ptr<Queue> queue) -{ - queue->enqueue(ctxt, msg); -} - -TxPublish::Commit::Commit(intrusive_ptr<Message>& _msg) : msg(_msg){} - -void TxPublish::Commit::operator()(const boost::shared_ptr<Queue>& queue){ - queue->process(msg); -} - -TxPublish::Rollback::Rollback(intrusive_ptr<Message>& _msg) : msg(_msg){} - -void TxPublish::Rollback::operator()(const boost::shared_ptr<Queue>& queue){ - queue->enqueueAborted(msg); -} - -uint64_t TxPublish::contentSize () -{ - return msg->contentSize (); -} diff --git a/qpid/cpp/src/qpid/broker/TxPublish.h b/qpid/cpp/src/qpid/broker/TxPublish.h index dba7878af2..e69de29bb2 100644 --- a/qpid/cpp/src/qpid/broker/TxPublish.h +++ b/qpid/cpp/src/qpid/broker/TxPublish.h @@ -1,92 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#ifndef _TxPublish_ -#define _TxPublish_ - -#include "qpid/broker/BrokerImportExport.h" -#include "qpid/broker/Deliverable.h" -#include "qpid/broker/Message.h" -#include "qpid/broker/MessageStore.h" -#include "qpid/broker/TxOp.h" - -#include <algorithm> -#include <functional> -#include <list> - -#include <boost/intrusive_ptr.hpp> - -namespace qpid { -namespace broker { -/** - * Defines the behaviour for publish operations on a - * transactional channel. Messages are routed through - * exchanges when received but are not at that stage delivered - * to the matching queues, rather the queues are held in an - * instance of this class. On prepare() the message is marked - * enqueued to the relevant queues in the MessagesStore. On - * commit() the messages will be passed to the queue for - * dispatch or to be added to the in-memory queue. - */ -class QPID_BROKER_CLASS_EXTERN TxPublish : public TxOp, public Deliverable{ - - class Commit{ - boost::intrusive_ptr<Message>& msg; - public: - Commit(boost::intrusive_ptr<Message>& msg); - void operator()(const boost::shared_ptr<Queue>& queue); - }; - class Rollback{ - boost::intrusive_ptr<Message>& msg; - public: - Rollback(boost::intrusive_ptr<Message>& msg); - void operator()(const boost::shared_ptr<Queue>& queue); - }; - - boost::intrusive_ptr<Message> msg; - std::list<boost::shared_ptr<Queue> > queues; - std::list<boost::shared_ptr<Queue> > prepared; - - void prepare(TransactionContext* ctxt, boost::shared_ptr<Queue>); - - public: - QPID_BROKER_EXTERN TxPublish(boost::intrusive_ptr<Message> msg); - QPID_BROKER_EXTERN virtual bool prepare(TransactionContext* ctxt) throw(); - QPID_BROKER_EXTERN virtual void commit() throw(); - QPID_BROKER_EXTERN virtual void rollback() throw(); - - virtual Message& getMessage() { return *msg; }; - - QPID_BROKER_EXTERN virtual void deliverTo(const boost::shared_ptr<Queue>& queue); - - virtual ~TxPublish(){} - virtual void accept(TxOpConstVisitor& visitor) const { visitor(*this); } - - QPID_BROKER_EXTERN uint64_t contentSize(); - - boost::intrusive_ptr<Message> getMessage() const { return msg; } - const std::list<boost::shared_ptr<Queue> >& getQueues() const { return queues; } - const std::list<boost::shared_ptr<Queue> >& getPrepared() const { return prepared; } -}; -} -} - - -#endif diff --git a/qpid/cpp/src/qpid/broker/amqp_0_10/MessageTransfer.cpp b/qpid/cpp/src/qpid/broker/amqp_0_10/MessageTransfer.cpp new file mode 100644 index 0000000000..e343abde8a --- /dev/null +++ b/qpid/cpp/src/qpid/broker/amqp_0_10/MessageTransfer.cpp @@ -0,0 +1,366 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "MessageTransfer.h" +#include "qpid/broker/MapHandler.h" +#include "qpid/broker/Message.h" +#include "qpid/framing/MessageTransferBody.h" +#include "qpid/framing/MessageProperties.h" +#include "qpid/framing/DeliveryProperties.h" +#include "qpid/framing/FieldValue.h" +#include "qpid/framing/frame_functors.h" +#include "qpid/framing/TypeFilter.h" +#include "qpid/framing/SendContent.h" +#include "qpid/log/Statement.h" + +using namespace qpid::framing; + +namespace qpid { +namespace broker { +namespace amqp_0_10 { +namespace { +const std::string QMF2("qmf2"); +const std::string PARTIAL("partial"); +} +MessageTransfer::MessageTransfer() : frames(framing::SequenceNumber()) {} +MessageTransfer::MessageTransfer(const framing::SequenceNumber& id) : frames(id) {} + +uint64_t MessageTransfer::getContentSize() const +{ + return frames.getContentSize(); +} + +std::string MessageTransfer::getAnnotationAsString(const std::string& key) const +{ + const qpid::framing::MessageProperties* mp = getProperties<qpid::framing::MessageProperties>(); + if (mp && mp->hasApplicationHeaders()) { + return mp->getApplicationHeaders().getAsString(key); + } else { + return std::string(); + } +} +std::string MessageTransfer::getPropertyAsString(const std::string& key) const { return getAnnotationAsString(key); } + +bool MessageTransfer::getTtl(uint64_t& result) const +{ + const qpid::framing::DeliveryProperties* dp = getProperties<qpid::framing::DeliveryProperties>(); + if (dp && dp->hasTtl()) { + result = dp->getTtl(); + return true; + } else { + return false; + } +} +bool MessageTransfer::hasExpiration() const +{ + const qpid::framing::DeliveryProperties* dp = getProperties<qpid::framing::DeliveryProperties>(); + if (dp && dp->hasExpiration()) { + return true; + } else { + return false; + } +} + +uint8_t MessageTransfer::getPriority() const +{ + const qpid::framing::DeliveryProperties* dp = getProperties<qpid::framing::DeliveryProperties>(); + if (dp && dp->hasPriority()) { + return dp->getPriority(); + } else { + return 0; + } +} + +std::string MessageTransfer::getExchangeName() const +{ + return getFrames().as<framing::MessageTransferBody>()->getDestination(); +} + +bool MessageTransfer::requiresAccept() const +{ + const framing::MessageTransferBody* b = getFrames().as<framing::MessageTransferBody>(); + return b && b->getAcceptMode() == 0/*EXPLICIT == 0*/; +} +uint32_t MessageTransfer::getRequiredCredit() const +{ + return requiredCredit; +} +void MessageTransfer::computeRequiredCredit() +{ + //add up payload for all header and content frames in the frameset + qpid::framing::SumBodySize sum; + frames.map_if(sum, qpid::framing::TypeFilter2<qpid::framing::HEADER_BODY, qpid::framing::CONTENT_BODY>()); + requiredCredit = sum.getSize(); +} +uint32_t MessageTransfer::getRequiredCredit(const qpid::broker::Message& msg) +{ + //TODO: may need to reflect annotations and other modifications in this also + return get(msg).getRequiredCredit(); +} + +qpid::framing::FrameSet& MessageTransfer::getFrames() +{ + return frames; +} +const qpid::framing::FrameSet& MessageTransfer::getFrames() const +{ + return frames; +} +void MessageTransfer::sendContent(framing::FrameHandler& out, uint16_t maxFrameSize) const +{ + qpid::framing::Count c; + frames.map_if(c, qpid::framing::TypeFilter<qpid::framing::CONTENT_BODY>()); + + qpid::framing::SendContent f(out, maxFrameSize, c.getCount()); + frames.map_if(f, qpid::framing::TypeFilter<qpid::framing::CONTENT_BODY>()); +} + +class SendHeader +{ + public: + SendHeader(FrameHandler& h, bool r, uint64_t t, uint64_t ts, const qpid::types::Variant::Map& a) : handler(h), redelivered(r), ttl(t), timestamp(ts), annotations(a) {} + void operator()(const AMQFrame& f) + { + AMQFrame copy = f; + if (redelivered || ttl || timestamp || annotations.size()) { + copy.cloneBody(); + if (annotations.size()) { + MessageProperties* props = + copy.castBody<AMQHeaderBody>()->get<MessageProperties>(true); + for (qpid::types::Variant::Map::const_iterator i = annotations.begin(); + i != annotations.end(); ++i) { + props->getApplicationHeaders().setString(i->first, i->second.asString()); + } + } + if (redelivered || ttl || timestamp) { + DeliveryProperties* dp = + copy.castBody<AMQHeaderBody>()->get<DeliveryProperties>(true); + if (ttl) dp->setTtl(ttl); + if (redelivered) dp->setRedelivered(redelivered); + if (timestamp) dp->setTimestamp(timestamp); + } + } + handler.handle(copy); + } + private: + FrameHandler& handler; + bool redelivered; + uint64_t ttl; + uint64_t timestamp; + const qpid::types::Variant::Map& annotations; +}; + +void MessageTransfer::sendHeader(framing::FrameHandler& out, uint16_t /*maxFrameSize*/, + bool redelivered, uint64_t ttl, uint64_t timestamp, + const qpid::types::Variant::Map& annotations) const +{ + SendHeader f(out, redelivered, ttl, timestamp, annotations); + frames.map_if(f, TypeFilter<HEADER_BODY>()); +} +bool MessageTransfer::isImmediateDeliveryRequired(const qpid::broker::Message& /*message*/) +{ + return false;//TODO +} + +const framing::SequenceNumber& MessageTransfer::getCommandId() const { return frames.getId(); } + +std::string MessageTransfer::getRoutingKey() const +{ + const qpid::framing::DeliveryProperties* dp = getProperties<qpid::framing::DeliveryProperties>(); + if (dp && dp->hasRoutingKey()) { + return dp->getRoutingKey(); + } else { + return std::string(); + } +} +bool MessageTransfer::isPersistent() const +{ + const qpid::framing::DeliveryProperties* dp = getProperties<qpid::framing::DeliveryProperties>(); + if (dp && dp->hasDeliveryMode()) { + return dp->getDeliveryMode() == 2; + } else { + return false; + } +} + +std::string MessageTransfer::getContent() const +{ + return frames.getContent(); +} + +void MessageTransfer::decodeHeader(framing::Buffer& buffer) +{ + AMQFrame method; + method.decode(buffer); + frames.append(method); + + AMQFrame header; + header.decode(buffer); + frames.append(header); +} +void MessageTransfer::decodeContent(framing::Buffer& buffer) +{ + if (buffer.available()) { + //get the data as a string and set that as the content + //body on a frame then add that frame to the frameset + AMQFrame frame((AMQContentBody())); + frame.castBody<AMQContentBody>()->decode(buffer, buffer.available()); + frame.setFirstSegment(false); + frames.append(frame); + } else { + //adjust header flags + MarkLastSegment f; + frames.map_if(f, TypeFilter<HEADER_BODY>()); + } +} + +void MessageTransfer::encode(framing::Buffer& buffer) const +{ + //encode method and header frames + EncodeFrame f1(buffer); + frames.map_if(f1, TypeFilter2<METHOD_BODY, HEADER_BODY>()); + + //then encode the payload of each content frame + framing::EncodeBody f2(buffer); + frames.map_if(f2, TypeFilter<CONTENT_BODY>()); +} + +void MessageTransfer::encodeContent(framing::Buffer& buffer) const +{ + //encode the payload of each content frame + EncodeBody f2(buffer); + frames.map_if(f2, TypeFilter<CONTENT_BODY>()); +} + +uint32_t MessageTransfer::encodedSize() const +{ + return encodedHeaderSize() + encodedContentSize(); +} + +uint32_t MessageTransfer::encodedContentSize() const +{ + return frames.getContentSize(); +} + +uint32_t MessageTransfer::encodedHeaderSize() const +{ + //add up the size for all method and header frames in the frameset + SumFrameSize sum; + frames.map_if(sum, TypeFilter2<METHOD_BODY, HEADER_BODY>()); + return sum.getSize(); +} + +bool MessageTransfer::isQMFv2() const +{ + const framing::MessageProperties* props = getProperties<framing::MessageProperties>(); + return props && props->getAppId() == QMF2 && props->hasApplicationHeaders(); +} + +bool MessageTransfer::isQMFv2(const qpid::broker::Message& message) +{ + const MessageTransfer* transfer = dynamic_cast<const MessageTransfer*>(&message.getEncoding()); + return transfer && transfer->isQMFv2(); +} + +bool MessageTransfer::isLastQMFResponse(const std::string correlation) const +{ + const framing::MessageProperties* props = getProperties<framing::MessageProperties>(); + return props && props->getCorrelationId() == correlation + && props->hasApplicationHeaders() && !props->getApplicationHeaders().isSet(PARTIAL); +} + +bool MessageTransfer::isLastQMFResponse(const qpid::broker::Message& message, const std::string correlation) +{ + const MessageTransfer* transfer = dynamic_cast<const MessageTransfer*>(&message.getEncoding()); + return transfer && transfer->isLastQMFResponse(correlation); +} + + +void MessageTransfer::processProperties(qpid::broker::MapHandler& handler) const +{ + const qpid::framing::MessageProperties* mp = getProperties<qpid::framing::MessageProperties>(); + if (mp && mp->hasApplicationHeaders()) { + const FieldTable ft = mp->getApplicationHeaders(); + for (FieldTable::const_iterator i = ft.begin(); i != ft.end(); ++i) { + qpid::broker::MapHandler::CharSequence key; + key.data = i->first.data(); + key.size = i->first.size(); + FieldTable::ValuePtr v = i->second; + //TODO: something more sophisticated... + if (v->empty()) { + handler.handleVoid(key); + } else if (v->convertsTo<uint64_t>()) { + handler.handleUint64(key, v->get<uint64_t>()); + } else if (v->convertsTo<int64_t>()) { + handler.handleInt64(key, v->get<int64_t>()); + } else if (v->convertsTo<std::string>()) { + std::string s = v->get<std::string>(); + qpid::broker::MapHandler::CharSequence value; + value.data = s.data(); + value.size = s.size(); + qpid::broker::MapHandler::CharSequence encoding; encoding.size = 0; encoding.data = 0; + handler.handleString(key, value, encoding); + } else { + QPID_LOG(debug, "Unhandled key!" << *v); + } + } + } +} + +std::string MessageTransfer::getUserId() const +{ + const qpid::framing::MessageProperties* mp = getProperties<qpid::framing::MessageProperties>(); + if (mp && mp->hasUserId()) return mp->getUserId(); + else return std::string(); + +} +MessageTransfer::MessageTransfer(const qpid::framing::FrameSet& f) : frames(f), requiredCredit(0) {} + +boost::intrusive_ptr<PersistableMessage> MessageTransfer::merge(const std::map<std::string, qpid::types::Variant>& annotations) const +{ + boost::intrusive_ptr<MessageTransfer> clone(new MessageTransfer(this->frames)); + qpid::framing::MessageProperties* mp = clone->frames.getHeaders()->get<qpid::framing::MessageProperties>(true); + for (qpid::types::Variant::Map::const_iterator i = annotations.begin(); i != annotations.end(); ++i) { + mp->getApplicationHeaders().setString(i->first, i->second); + } + return clone; +} +} +// qpid::broker namespace, TODO: move these elsewhere! +void encode(const Message& in, std::string& out) +{ + const amqp_0_10::MessageTransfer& transfer = amqp_0_10::MessageTransfer::get(in); + uint32_t size = transfer.encodedSize(); + std::vector<char> data(size); + qpid::framing::Buffer buffer(&(data[0]), size); + transfer.encode(buffer); + buffer.reset(); + buffer.getRawData(out, size); +} +void decode(const std::string& in, Message& out) +{ + boost::intrusive_ptr<amqp_0_10::MessageTransfer> transfer(new amqp_0_10::MessageTransfer); + qpid::framing::Buffer buffer(const_cast<char*>(in.data()), in.size()); + transfer->decodeHeader(buffer); + transfer->decodeContent(buffer); + out = Message(transfer, transfer); +} + +}} // namespace qpid::broker::amqp_0_10 diff --git a/qpid/cpp/src/qpid/broker/amqp_0_10/MessageTransfer.h b/qpid/cpp/src/qpid/broker/amqp_0_10/MessageTransfer.h new file mode 100644 index 0000000000..0d7ecb3956 --- /dev/null +++ b/qpid/cpp/src/qpid/broker/amqp_0_10/MessageTransfer.h @@ -0,0 +1,132 @@ +#ifndef QPID_BROKER_AMQP_0_10_MESSAGETRANSFER_H +#define QPID_BROKER_AMQP_0_10_MESSAGETRANSFER_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/broker/BrokerImportExport.h" +#include "qpid/framing/FrameSet.h" +#include "qpid/broker/Message.h" +#include "qpid/broker/PersistableMessage.h" +#include "qpid/types/Variant.h" + +namespace qpid { +namespace broker { +class Queue; +namespace amqp_0_10 { + +/** + * + */ +class MessageTransfer : public qpid::broker::Message::Encoding, public qpid::broker::PersistableMessage +{ + public: + QPID_BROKER_EXTERN MessageTransfer(); + QPID_BROKER_EXTERN MessageTransfer(const qpid::framing::SequenceNumber&); + + std::string getRoutingKey() const; + bool isPersistent() const; + uint8_t getPriority() const; + uint64_t getContentSize() const; + std::string getPropertyAsString(const std::string& key) const; + std::string getAnnotationAsString(const std::string& key) const; + bool getTtl(uint64_t&) const; + bool hasExpiration() const; + std::string getExchangeName() const; + void processProperties(MapHandler&) const; + std::string getUserId() const; + + bool requiresAccept() const; + const qpid::framing::SequenceNumber& getCommandId() const; + QPID_BROKER_EXTERN qpid::framing::FrameSet& getFrames(); + QPID_BROKER_EXTERN const qpid::framing::FrameSet& getFrames() const; + + template <class T> const T* getProperties() const { + const qpid::framing::AMQHeaderBody* p = frames.getHeaders(); + return p->get<T>(); + } + + template <class T> const T* hasProperties() const { + const qpid::framing::AMQHeaderBody* p = frames.getHeaders(); + return p->get<T>(); + } + template <class T> const T* getMethod() const { + return frames.as<T>(); + } + + template <class T> T* getMethod() { + return frames.as<T>(); + } + + template <class T> bool isA() const { + return frames.isA<T>(); + } + + template <class T> void eraseProperties() { + qpid::framing::AMQHeaderBody* p = frames.getHeaders(); + p->erase<T>(); + } + std::string getContent() const; + uint32_t getRequiredCredit() const; + void computeRequiredCredit(); + + void clearApplicationHeadersFlag(); + void sendContent(framing::FrameHandler& out, uint16_t maxFrameSize) const; + void sendHeader(framing::FrameHandler& out, uint16_t maxFrameSize, bool redelivered, uint64_t ttl, uint64_t timestamp, const qpid::types::Variant::Map& annotations) const; + + void decodeHeader(framing::Buffer& buffer); + void decodeContent(framing::Buffer& buffer); + + void encode(framing::Buffer& buffer) const; + uint32_t encodedSize() const; + + /** + * @returns the size of the buffer needed to encode the + * 'header' of this message (not just the header frame, + * but other meta data e.g.routing key and exchange) + */ + uint32_t encodedHeaderSize() const; + boost::intrusive_ptr<PersistableMessage> merge(const std::map<std::string, qpid::types::Variant>& annotations) const; + + QPID_BROKER_EXTERN bool isQMFv2() const; + QPID_BROKER_EXTERN bool isLastQMFResponse(const std::string correlation) const; + + static bool isImmediateDeliveryRequired(const qpid::broker::Message& message); + static uint32_t getRequiredCredit(const qpid::broker::Message&); + static MessageTransfer& get(qpid::broker::Message& message) { + return *dynamic_cast<MessageTransfer*>(&message.getEncoding()); + } + static const MessageTransfer& get(const qpid::broker::Message& message) { + return *dynamic_cast<const MessageTransfer*>(&message.getEncoding()); + } + QPID_BROKER_EXTERN static bool isQMFv2(const qpid::broker::Message& message); + QPID_BROKER_EXTERN static bool isLastQMFResponse(const qpid::broker::Message& message, const std::string correlation); + private: + qpid::framing::FrameSet frames; + uint32_t requiredCredit; + + MessageTransfer(const qpid::framing::FrameSet&); + void encodeHeader(framing::Buffer& buffer) const; + uint32_t encodedContentSize() const; + void encodeContent(framing::Buffer& buffer) const; +}; +}}} // namespace qpid::broker::amqp_0_10 + +#endif /*!QPID_BROKER_AMQP_0_10_MESSAGETRANSFER_H*/ diff --git a/qpid/cpp/src/qpid/ha/Backup.cpp b/qpid/cpp/src/qpid/ha/Backup.cpp index e099554df6..6852a58b0c 100644 --- a/qpid/cpp/src/qpid/ha/Backup.cpp +++ b/qpid/cpp/src/qpid/ha/Backup.cpp @@ -35,6 +35,7 @@ #include "qpid/framing/MessageTransferBody.h" #include "qpid/sys/SystemInfo.h" #include "qpid/types/Variant.h" +#include "qpid/log/Statement.h" namespace qpid { namespace ha { diff --git a/qpid/cpp/src/qpid/ha/BrokerReplicator.cpp b/qpid/cpp/src/qpid/ha/BrokerReplicator.cpp index 4a12745d7b..fe32753b4e 100644 --- a/qpid/cpp/src/qpid/ha/BrokerReplicator.cpp +++ b/qpid/cpp/src/qpid/ha/BrokerReplicator.cpp @@ -24,7 +24,9 @@ #include "qpid/broker/Broker.h" #include "qpid/broker/Connection.h" #include "qpid/broker/Queue.h" +#include "qpid/broker/QueueSettings.h" #include "qpid/broker/Link.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/framing/FieldTable.h" #include "qpid/log/Statement.h" #include "qpid/amqp_0_10/Codecs.h" @@ -117,11 +119,6 @@ const string _QUERY_REQUEST("_query_request"); const string BROKER("broker"); const string MEMBERS("members"); -bool isQMFv2(const Message& message) { - const framing::MessageProperties* props = message.getProperties<framing::MessageProperties>(); - return props && props->getAppId() == QMF2; -} - template <class T> bool match(Variant::Map& schema) { return T::match(schema[CLASS_NAME], schema[PACKAGE_NAME]); } @@ -253,18 +250,15 @@ void BrokerReplicator::route(Deliverable& msg) { haBroker.setStatus(CATCHUP); QPID_LOG(notice, logPrefix << "Connected to primary " << primary); } - - const framing::FieldTable* headers = msg.getMessage().getApplicationHeaders(); - const MessageProperties* messageProperties = msg.getMessage().getProperties<MessageProperties>(); Variant::List list; try { - if (!isQMFv2(msg.getMessage()) || !headers || !messageProperties) + if (!qpid::broker::amqp_0_10::MessageTransfer::isQMFv2(msg.getMessage())) throw Exception("Unexpected message, not QMF2 event or query response."); // decode as list - string content = msg.getMessage().getFrames().getContent(); - amqp_0_10::ListCodec::decode(content, list); - QPID_LOG(trace, "Broker replicator received: " << *messageProperties); - if (headers->getAsString(QMF_CONTENT) == EVENT) { + string content = msg.getMessage().getContent(); + qpid::amqp_0_10::ListCodec::decode(content, list); + + if (msg.getMessage().getPropertyAsString(QMF_CONTENT) == EVENT) { for (Variant::List::iterator i = list.begin(); i != list.end(); ++i) { Variant::Map& map = i->asMap(); QPID_LOG(trace, "Broker replicator event: " << map); @@ -278,20 +272,20 @@ void BrokerReplicator::route(Deliverable& msg) { else if (match<EventUnbind>(schema)) doEventUnbind(values); else if (match<EventMembersUpdate>(schema)) doEventMembersUpdate(values); } - } else if (headers->getAsString(QMF_OPCODE) == QUERY_RESPONSE) { + } else if (msg.getMessage().getPropertyAsString(QMF_OPCODE) == QUERY_RESPONSE) { for (Variant::List::iterator i = list.begin(); i != list.end(); ++i) { Variant::Map& map = i->asMap(); QPID_LOG(trace, "Broker replicator response: " << map); string type = map[SCHEMA_ID].asMap()[CLASS_NAME].asString(); Variant::Map& values = map[VALUES].asMap(); framing::FieldTable args; - amqp_0_10::translate(asMapVoid(values[ARGUMENTS]), args); + qpid::amqp_0_10::translate(asMapVoid(values[ARGUMENTS]), args); if (type == QUEUE) doResponseQueue(values); else if (type == EXCHANGE) doResponseExchange(values); else if (type == BINDING) doResponseBind(values); else if (type == HA_BROKER) doResponseHaBroker(values); } - if (messageProperties->getCorrelationId() == EXCHANGE && !headers->isSet(PARTIAL)) { + if (qpid::broker::amqp_0_10::MessageTransfer::isLastQMFResponse(msg.getMessage(), EXCHANGE)) { // We have received all of the exchange response. alternates.clear(); } @@ -309,13 +303,11 @@ void BrokerReplicator::doEventQueueDeclare(Variant::Map& values) { Variant::Map argsMap = asMapVoid(values[ARGS]); bool autoDel = values[AUTODEL].asBool(); bool excl = values[EXCL].asBool(); - if (values[DISP] == CREATED && - replicationTest.isReplicated(CONFIGURATION, argsMap, autoDel, excl)) - { + if (values[DISP] == CREATED && replicationTest.isReplicated(CONFIGURATION, argsMap, autoDel, excl)) { string name = values[QNAME].asString(); - QPID_LOG(debug, logPrefix << "Queue declare event: " << name); + QueueSettings settings(values[DURABLE].asBool(), values[AUTODEL].asBool()); framing::FieldTable args; - amqp_0_10::translate(argsMap, args); + qpid::amqp_0_10::translate(argsMap, args); // If we already have a queue with this name, replace it. // The queue was definitely created on the primary. if (broker.getQueues().find(name)) { @@ -323,10 +315,17 @@ void BrokerReplicator::doEventQueueDeclare(Variant::Map& values) { broker.getQueues().destroy(name); stopQueueReplicator(name); } - boost::shared_ptr<Queue> queue = createQueue( - name, values[DURABLE].asBool(), autoDel, args, values[ALTEX].asString()); - assert(queue); // Should be created since we destroed the previous queue above. - if (queue) startQueueReplicator(queue); + settings.populate(args, settings.storeSettings); + std::pair<boost::shared_ptr<Queue>, bool> result = + broker.createQueue( + name, + settings, + 0 /*i.e. no owner regardless of exclusivity on master*/, + values[ALTEX].asString(), + userId, + remoteHost); + assert(result.second); // Should be true since we destroyed existing queue above + startQueueReplicator(result.first); } } @@ -343,7 +342,7 @@ void BrokerReplicator::doEventQueueDelete(Variant::Map& values) { // sessions may be closed by a "queue deleted" exception. string name = values[QNAME].asString(); boost::shared_ptr<Queue> queue = broker.getQueues().find(name); - if (queue && replicationTest.replicateLevel(queue->getSettings())) { + if (queue && replicationTest.replicateLevel(queue->getSettings().storeSettings)) { QPID_LOG(debug, logPrefix << "Queue delete event: " << name); stopQueueReplicator(name); broker.deleteQueue(name, userId, remoteHost); @@ -357,7 +356,7 @@ void BrokerReplicator::doEventExchangeDeclare(Variant::Map& values) { string name = values[EXNAME].asString(); QPID_LOG(debug, logPrefix << "Exchange declare event: " << name); framing::FieldTable args; - amqp_0_10::translate(argsMap, args); + qpid::amqp_0_10::translate(argsMap, args); // If we already have a exchange with this name, replace it. // The exchange was definitely created on the primary. if (broker.getExchanges().find(name)) { @@ -391,10 +390,10 @@ void BrokerReplicator::doEventBind(Variant::Map& values) { // We only replicate binds for a replicated queue to replicated // exchange that both exist locally. if (exchange && replicationTest.replicateLevel(exchange->getArgs()) && - queue && replicationTest.replicateLevel(queue->getSettings())) + queue && replicationTest.replicateLevel(queue->getSettings().storeSettings)) { framing::FieldTable args; - amqp_0_10::translate(asMapVoid(values[ARGS]), args); + qpid::amqp_0_10::translate(asMapVoid(values[ARGS]), args); string key = values[KEY].asString(); QPID_LOG(debug, logPrefix << "Bind event: exchange=" << exchange->getName() << " queue=" << queue->getName() @@ -411,10 +410,10 @@ void BrokerReplicator::doEventUnbind(Variant::Map& values) { // We only replicate unbinds for a replicated queue to replicated // exchange that both exist locally. if (exchange && replicationTest.replicateLevel(exchange->getArgs()) && - queue && replicationTest.replicateLevel(queue->getSettings())) + queue && replicationTest.replicateLevel(queue->getSettings().storeSettings)) { framing::FieldTable args; - amqp_0_10::translate(asMapVoid(values[ARGS]), args); + qpid::amqp_0_10::translate(asMapVoid(values[ARGS]), args); string key = values[KEY].asString(); QPID_LOG(debug, logPrefix << "Unbind event: exchange=" << exchange->getName() << " queue=" << queue->getName() @@ -455,7 +454,7 @@ void BrokerReplicator::doResponseQueue(Variant::Map& values) { string name(values[NAME].asString()); QPID_LOG(debug, logPrefix << "Queue response: " << name); framing::FieldTable args; - amqp_0_10::translate(argsMap, args); + qpid::amqp_0_10::translate(argsMap, args); boost::shared_ptr<Queue> queue = createQueue(name, values[DURABLE].asBool(), values[AUTODELETE].asBool(), args, getAltExchange(values[ALTEXCHANGE])); @@ -470,7 +469,7 @@ void BrokerReplicator::doResponseExchange(Variant::Map& values) { string name = values[NAME].asString(); QPID_LOG(debug, logPrefix << "Exchange response: " << name); framing::FieldTable args; - amqp_0_10::translate(argsMap, args); + qpid::amqp_0_10::translate(argsMap, args); boost::shared_ptr<Exchange> exchange = createExchange( name, values[TYPE].asString(), values[DURABLE].asBool(), args, getAltExchange(values[ALTEXCHANGE])); @@ -507,14 +506,14 @@ void BrokerReplicator::doResponseBind(Variant::Map& values) { // Automatically replicate binding if queue and exchange exist and are replicated if (exchange && replicationTest.replicateLevel(exchange->getArgs()) && - queue && replicationTest.replicateLevel(queue->getSettings())) + queue && replicationTest.replicateLevel(queue->getSettings().storeSettings)) { string key = values[KEY].asString(); QPID_LOG(debug, logPrefix << "Bind response: exchange:" << exName << " queue:" << qName << " key:" << key); framing::FieldTable args; - amqp_0_10::translate(asMapVoid(values[ARGUMENTS]), args); + qpid::amqp_0_10::translate(asMapVoid(values[ARGUMENTS]), args); exchange->bind(queue, key, &args); } } @@ -544,7 +543,7 @@ void BrokerReplicator::doResponseHaBroker(Variant::Map& values) { void BrokerReplicator::startQueueReplicator(const boost::shared_ptr<Queue>& queue) { - if (replicationTest.replicateLevel(queue->getSettings()) == ALL) { + if (replicationTest.replicateLevel(queue->getSettings().storeSettings) == ALL) { boost::shared_ptr<QueueReplicator> qr( new QueueReplicator(haBroker, queue, link)); if (!broker.getExchanges().registerExchange(qr)) @@ -570,14 +569,14 @@ boost::shared_ptr<Queue> BrokerReplicator::createQueue( const qpid::framing::FieldTable& arguments, const std::string& alternateExchange) { + QueueSettings settings(durable, autodelete); + settings.populate(arguments, settings.storeSettings); std::pair<boost::shared_ptr<Queue>, bool> result = broker.createQueue( name, - durable, - autodelete, - 0, // no owner regardless of exclusivity on primary + settings, + 0,// no owner regardless of exclusivity on primary string(), // Set alternate exchange below - arguments, userId, remoteHost); if (result.second) { diff --git a/qpid/cpp/src/qpid/ha/Primary.cpp b/qpid/cpp/src/qpid/ha/Primary.cpp index 4462d91062..e4bf9671b8 100644 --- a/qpid/cpp/src/qpid/ha/Primary.cpp +++ b/qpid/cpp/src/qpid/ha/Primary.cpp @@ -180,7 +180,7 @@ void Primary::readyReplica(const ReplicatingSubscription& rs) { void Primary::queueCreate(const QueuePtr& q) { // Throw if there is an invalid replication level in the queue settings. - haBroker.getReplicationTest().replicateLevel(q->getSettings()); + haBroker.getReplicationTest().replicateLevel(q->getSettings().storeSettings); Mutex::ScopedLock l(lock); for (BackupMap::iterator i = backups.begin(); i != backups.end(); ++i) { i->second->queueCreate(q); diff --git a/qpid/cpp/src/qpid/ha/QueueGuard.cpp b/qpid/cpp/src/qpid/ha/QueueGuard.cpp index a30ab1f73c..77e1f81a38 100644 --- a/qpid/cpp/src/qpid/ha/QueueGuard.cpp +++ b/qpid/cpp/src/qpid/ha/QueueGuard.cpp @@ -39,10 +39,10 @@ class QueueGuard::QueueObserver : public broker::QueueObserver { public: QueueObserver(QueueGuard& g) : guard(g) {} - void enqueued(const broker::QueuedMessage& qm) { guard.enqueued(qm); } - void dequeued(const broker::QueuedMessage& qm) { guard.dequeued(qm); } - void acquired(const broker::QueuedMessage&) {} - void requeued(const broker::QueuedMessage&) {} + void enqueued(const broker::Message& m) { guard.enqueued(m); } + void dequeued(const broker::Message& m) { guard.dequeued(m); } + void acquired(const broker::Message&) {} + void requeued(const broker::Message&) {} private: QueueGuard& guard; }; @@ -64,39 +64,47 @@ QueueGuard::QueueGuard(broker::Queue& q, const BrokerInfo& info) QueueGuard::~QueueGuard() { cancel(); } // NOTE: Called with message lock held. -void QueueGuard::enqueued(const QueuedMessage& qm) { - assert(qm.queue == &queue); +void QueueGuard::enqueued(const Message& m) { // Delay completion - QPID_LOG(trace, logPrefix << "Delayed completion of " << qm); - qm.payload->getIngressCompletion().startCompleter(); + QPID_LOG(trace, logPrefix << "Delayed completion of " << m); + m.getIngressCompletion()->startCompleter(); { Mutex::ScopedLock l(lock); - assert(!delayed.contains(qm.position)); - delayed += qm.position; + if (!delayed.insert(Delayed::value_type(m.getSequence(), m.getIngressCompletion())).second) { + QPID_LOG(critical, logPrefix << "Second enqueue for message with sequence " << m.getSequence()); + assert(false); + } } } // NOTE: Called with message lock held. -void QueueGuard::dequeued(const QueuedMessage& qm) { - assert(qm.queue == &queue); - QPID_LOG(trace, logPrefix << "Dequeued " << qm); +void QueueGuard::dequeued(const Message& m) { + QPID_LOG(trace, logPrefix << "Dequeued " << m); ReplicatingSubscription* rs=0; { Mutex::ScopedLock l(lock); rs = subscription; } - if (rs) rs->dequeued(qm); - complete(qm); + if (rs) rs->dequeued(m); + complete(m.getSequence()); +} + +void QueueGuard::completeRange(Delayed::iterator begin, Delayed::iterator end) { + for (Delayed::iterator i = begin; i != end; ++i) { + QPID_LOG(trace, logPrefix << "Completed " << i->first); + i->second->finishCompleter(); + } } void QueueGuard::cancel() { queue.removeObserver(observer); + Delayed removed; { Mutex::ScopedLock l(lock); if (delayed.empty()) return; // No need if no delayed messages. + delayed.swap(removed); } - // FIXME aconway 2012-06-15: optimize, only messages in delayed set. - queue.eachMessage(boost::bind(&QueueGuard::complete, this, _1)); + completeRange(removed.begin(), removed.end()); } void QueueGuard::attach(ReplicatingSubscription& rs) { @@ -104,36 +112,39 @@ void QueueGuard::attach(ReplicatingSubscription& rs) { subscription = &rs; } -namespace { -void completeBefore(QueueGuard* guard, SequenceNumber position, const QueuedMessage& qm) { - if (qm.position <= position) guard->complete(qm); -} -} - bool QueueGuard::subscriptionStart(SequenceNumber position) { - // Complete any messages before or at the ReplicatingSubscription start position. - // Those messages are already on the backup. - if (!delayed.empty() && delayed.front() <= position) { - // FIXME aconway 2012-06-15: queue iteration, only messages in delayed - queue.eachMessage(boost::bind(&completeBefore, this, position, _1)); + Delayed removed; + { + Mutex::ScopedLock l(lock); + // Complete any messages before or at the ReplicatingSubscription start position. + // Those messages are already on the backup. + for (Delayed::iterator i = delayed.begin(); i != delayed.end() && i->first <= position;) { + removed.insert(*i); + delayed.erase(i++); + } } + completeRange(removed.begin(), removed.end()); return position >= range.back; } -void QueueGuard::complete(const QueuedMessage& qm) { - assert(qm.queue == &queue); +void QueueGuard::complete(SequenceNumber sequence) { + boost::intrusive_ptr<broker::AsyncCompletion> m; { Mutex::ScopedLock l(lock); // The same message can be completed twice, by // ReplicatingSubscription::acknowledged and dequeued. Remove it - // from the set so we only call finishCompleter() once - if (delayed.contains(qm.position)) - delayed -= qm.position; - else - return; + // from the map so we only call finishCompleter() once + Delayed::iterator i = delayed.find(sequence); + if (i != delayed.end()) { + m = i->second; + delayed.erase(i); + } + + } + if (m) { + QPID_LOG(trace, logPrefix << "Completed " << sequence); + m->finishCompleter(); } - QPID_LOG(trace, logPrefix << "Completed " << qm); - qm.payload->getIngressCompletion().finishCompleter(); } }} // namespaces qpid::ha diff --git a/qpid/cpp/src/qpid/ha/QueueGuard.h b/qpid/cpp/src/qpid/ha/QueueGuard.h index bc8f40b65f..3904b3bd3f 100644 --- a/qpid/cpp/src/qpid/ha/QueueGuard.h +++ b/qpid/cpp/src/qpid/ha/QueueGuard.h @@ -63,15 +63,15 @@ class QueueGuard { /** QueueObserver override. Delay completion of the message. * NOTE: Called under the queues message lock. */ - void enqueued(const broker::QueuedMessage&); + void enqueued(const broker::Message&); /** QueueObserver override: Complete a delayed message. * NOTE: Called under the queues message lock. */ - void dequeued(const broker::QueuedMessage&); + void dequeued(const broker::Message&); /** Complete a delayed message. */ - void complete(const broker::QueuedMessage&); + void complete(framing::SequenceNumber); /** Complete all delayed messages. */ void cancel(); @@ -108,10 +108,13 @@ class QueueGuard { sys::Mutex lock; std::string logPrefix; broker::Queue& queue; - framing::SequenceSet delayed; + typedef std::map<framing::SequenceNumber, boost::intrusive_ptr<broker::AsyncCompletion> > Delayed; + Delayed delayed; ReplicatingSubscription* subscription; boost::shared_ptr<QueueObserver> observer; QueueRange range; + + void completeRange(Delayed::iterator begin, Delayed::iterator end); }; }} // namespace qpid::ha diff --git a/qpid/cpp/src/qpid/ha/QueueReplicator.cpp b/qpid/cpp/src/qpid/ha/QueueReplicator.cpp index be910a087f..ae53f89404 100644 --- a/qpid/cpp/src/qpid/ha/QueueReplicator.cpp +++ b/qpid/cpp/src/qpid/ha/QueueReplicator.cpp @@ -120,8 +120,10 @@ void QueueReplicator::initializeBridge(Bridge& bridge, SessionHandler& sessionHa settings.setTable(ReplicatingSubscription::QPID_BROKER_INFO, brokerInfo.asFieldTable()); SequenceNumber front; - if (ReplicatingSubscription::getFront(*queue, front)) + if (ReplicatingSubscription::getFront(*queue, front)) { settings.setInt(ReplicatingSubscription::QPID_FRONT, front); + QPID_LOG(debug, "QPID_FRONT for " << queue->getName() << " is " << front); + } peer.getMessage().subscribe( args.i_src, args.i_dest, 0/*accept-explicit*/, 1/*not-acquired*/, false/*exclusive*/, "", 0, settings); @@ -137,8 +139,7 @@ void QueueReplicator::initializeBridge(Bridge& bridge, SessionHandler& sessionHa namespace { template <class T> T decodeContent(Message& m) { - std::string content; - m.getFrames().getContent(content); + std::string content = m.getContent(); Buffer buffer(const_cast<char*>(content.c_str()), content.size()); T result; result.decode(buffer); @@ -148,9 +149,7 @@ template <class T> T decodeContent(Message& m) { void QueueReplicator::dequeue(SequenceNumber n, sys::Mutex::ScopedLock&) { // Thread safe: only calls thread safe Queue functions. - QueuedMessage message; - if (queue->acquireMessageAt(n, message)) - queue->dequeue(0, message); + queue->dequeueMessageAt(n); } // Called in connection thread of the queues bridge to primary. diff --git a/qpid/cpp/src/qpid/ha/RemoteBackup.cpp b/qpid/cpp/src/qpid/ha/RemoteBackup.cpp index 9c7bf1e694..3421380940 100644 --- a/qpid/cpp/src/qpid/ha/RemoteBackup.cpp +++ b/qpid/cpp/src/qpid/ha/RemoteBackup.cpp @@ -23,6 +23,7 @@ #include "qpid/broker/Broker.h" #include "qpid/broker/Queue.h" #include "qpid/broker/QueueRegistry.h" +#include "qpid/log/Statement.h" #include <boost/bind.hpp> namespace qpid { diff --git a/qpid/cpp/src/qpid/ha/ReplicatingSubscription.cpp b/qpid/cpp/src/qpid/ha/ReplicatingSubscription.cpp index c960758eaf..6f7519cd1f 100644 --- a/qpid/cpp/src/qpid/ha/ReplicatingSubscription.cpp +++ b/qpid/cpp/src/qpid/ha/ReplicatingSubscription.cpp @@ -27,6 +27,7 @@ #include "qpid/broker/Queue.h" #include "qpid/broker/SessionContext.h" #include "qpid/broker/ConnectionState.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/framing/AMQFrame.h" #include "qpid/framing/MessageTransferBody.h" #include "qpid/log/Statement.h" @@ -66,10 +67,10 @@ class DequeueScanner at = front - 1; } - void operator()(const QueuedMessage& qm) { - if (qm.position >= front && qm.position <= back) { - if (qm.position > at+1) subscription.dequeued(at+1, qm.position-1); - at = qm.position; + void operator()(const Message& m) { + if (m.getSequence() >= front && m.getSequence() <= back) { + if (m.getSequence() > at+1) subscription.dequeued(at+1, m.getSequence()-1); + at = m.getSequence(); } } @@ -90,37 +91,23 @@ string mask(const string& in) return DOLLAR + in + INTERNAL; } - -/** Dummy consumer used to get the front position on the queue */ -class GetPositionConsumer : public Consumer +namespace { +bool getSequence(const Message& message, SequenceNumber& result) { - public: - GetPositionConsumer() : - Consumer("ha.GetPositionConsumer."+types::Uuid(true).str(), false) {} - bool deliver(broker::QueuedMessage& ) { return true; } - void notify() {} - bool filter(boost::intrusive_ptr<broker::Message>) { return true; } - bool accept(boost::intrusive_ptr<broker::Message>) { return true; } - void cancel() {} - void acknowledged(const broker::QueuedMessage&) {} - bool browseAcquired() const { return true; } - broker::OwnershipToken* getSession() { return 0; } -}; - - + result = message.getSequence(); + return true; +} +} bool ReplicatingSubscription::getNext( broker::Queue& q, SequenceNumber from, SequenceNumber& result) { - boost::shared_ptr<Consumer> c(new GetPositionConsumer); - c->setPosition(from); - if (!q.dispatch(c)) return false; - result = c->getPosition(); - return true; + QueueCursor cursor(REPLICATOR); + return q.seek(cursor, boost::bind(&getSequence, _1, boost::ref(result)), from); } bool ReplicatingSubscription::getFront(broker::Queue& q, SequenceNumber& front) { - // FIXME aconway 2012-05-23: won't wrap, assumes 0 is < all messages in queue. - return getNext(q, 0, front); + QueueCursor cursor(REPLICATOR); + return q.seek(cursor, boost::bind(&getSequence, _1, boost::ref(front))); } /* Called by SemanticState::consume to create a consumer */ @@ -152,15 +139,14 @@ ReplicatingSubscription::ReplicatingSubscription( const string& name, Queue::shared_ptr queue, bool ack, - bool acquire, + bool /*acquire*/, bool exclusive, const string& tag, const string& resumeId, uint64_t resumeTtl, const framing::FieldTable& arguments -) : ConsumerImpl(parent, name, queue, ack, acquire, exclusive, tag, +) : ConsumerImpl(parent, name, queue, ack, REPLICATOR, exclusive, tag, resumeId, resumeTtl, arguments), - dummy(new Queue(mask(name))), ready(false) { try { @@ -213,6 +199,8 @@ ReplicatingSubscription::ReplicatingSubscription( queue->eachMessage(boost::ref(scan)); // Remove missing messages in between. scan.finish(); position = backup.back; + //move cursor to position + queue->seek(*this, position); } // NOTE: we are assuming that the messages that are on the backup are // consistent with those on the primary. If the backup is a replica @@ -260,32 +248,31 @@ void ReplicatingSubscription::initialize() { } // Message is delivered in the subscription's connection thread. -bool ReplicatingSubscription::deliver(QueuedMessage& qm) { +bool ReplicatingSubscription::deliver(const qpid::broker::QueueCursor& c, const qpid::broker::Message& m) { + position = m.getSequence(); try { - // Add position events for the subscribed queue, not the internal event queue. - if (qm.queue == getQueue().get()) { - QPID_LOG(trace, logPrefix << "Replicating " << qm); - { - Mutex::ScopedLock l(lock); - assert(position == qm.position); - // qm.position is the position of the newly enqueued qm on local queue. - // backupPosition is latest position on backup queue before enqueueing - if (qm.position <= backupPosition) - throw Exception( - QPID_MSG("Expected position > " << backupPosition - << " but got " << qm.position)); - if (qm.position - backupPosition > 1) { - // Position has advanced because of messages dequeued ahead of us. - // Send the position before qm was enqueued. - sendPositionEvent(qm.position-1, l); - } - // Backup will automatically advance by 1 on delivery of message. - backupPosition = qm.position; + QPID_LOG(trace, logPrefix << "Replicating " << getQueue()->getName() << "[" << m.getSequence() << "]"); + { + Mutex::ScopedLock l(lock); + //FIXME GRS: position is no longer set//assert(position == m.getSequence()); + + // m.getSequence() is the position of the newly enqueued message on local queue. + // backupPosition is latest position on backup queue before enqueueing + if (m.getSequence() <= backupPosition) + throw Exception( + QPID_MSG("Expected position > " << backupPosition + << " but got " << m.getSequence())); + if (m.getSequence() - backupPosition > 1) { + // Position has advanced because of messages dequeued ahead of us. + // Send the position before message was enqueued. + sendPositionEvent(m.getSequence()-1, l); } + // Backup will automatically advance by 1 on delivery of message. + backupPosition = m.getSequence(); } - return ConsumerImpl::deliver(qm); + return ConsumerImpl::deliver(c, m); } catch (const std::exception& e) { - QPID_LOG(critical, logPrefix << "Error replicating " << qm + QPID_LOG(critical, logPrefix << "Error replicating " << getQueue()->getName() << "[" << m.getSequence() << "]" << ": " << e.what()); throw; } @@ -310,15 +297,13 @@ void ReplicatingSubscription::cancel() } // Consumer override, called on primary in the backup's IO thread. -void ReplicatingSubscription::acknowledged(const QueuedMessage& qm) { - if (qm.queue == getQueue().get()) { // Don't complete messages on the internal queue - // Finish completion of message, it has been acknowledged by the backup. - QPID_LOG(trace, logPrefix << "Acknowledged " << qm); - guard->complete(qm); - // If next message is protected by the guard then we are ready - if (qm.position >= guard->getRange().back) setReady(); - } - ConsumerImpl::acknowledged(qm); +void ReplicatingSubscription::acknowledged(const broker::DeliveryRecord& r) { + // Finish completion of message, it has been acknowledged by the backup. + QPID_LOG(trace, logPrefix << "Acknowledged " << getQueue()->getName() << "[" << r.getMessageId() << "]"); + guard->complete(r.getMessageId()); + // If next message is protected by the guard then we are ready + if (r.getMessageId() >= guard->getRange().back) setReady(); + ConsumerImpl::acknowledged(r); } // Called with lock held. Called in subscription's connection thread. @@ -341,13 +326,12 @@ void ReplicatingSubscription::sendDequeueEvent(Mutex::ScopedLock&) // Called after the message has been removed // from the deque and under the messageLock in the queue. Called in // arbitrary connection threads. -void ReplicatingSubscription::dequeued(const QueuedMessage& qm) +void ReplicatingSubscription::dequeued(const Message& m) { - assert (qm.queue == getQueue().get()); - QPID_LOG(trace, logPrefix << "Dequeued " << qm); + QPID_LOG(trace, logPrefix << "Dequeued " << getQueue()->getName() << "[" << m.getSequence() << "]"); { Mutex::ScopedLock l(lock); - dequeues.add(qm.position); + dequeues.add(m.getSequence()); } notify(); // Ensure a call to doDispatch } @@ -379,7 +363,7 @@ void ReplicatingSubscription::sendPositionEvent(SequenceNumber pos, Mutex::Scope void ReplicatingSubscription::sendEvent(const std::string& key, framing::Buffer& buffer) { //generate event message - boost::intrusive_ptr<Message> event = new Message(); + boost::intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> event(new qpid::broker::amqp_0_10::MessageTransfer()); AMQFrame method((MessageTransferBody(ProtocolVersion(), string(), 0, 0))); AMQFrame header((AMQHeaderBody())); AMQFrame content((AMQContentBody())); @@ -400,10 +384,8 @@ void ReplicatingSubscription::sendEvent(const std::string& key, framing::Buffer& event->getFrames().getHeaders()->get<DeliveryProperties>(true); props->setRoutingKey(key); // Send the event directly to the base consumer implementation. - // We don't really need a queue here but we pass a dummy queue - // to conform to the consumer API. - QueuedMessage qm(dummy.get(), event); - ConsumerImpl::deliver(qm); + //dummy consumer prevents acknowledgements being handled, which is what we want for events + ConsumerImpl::deliver(QueueCursor(), Message(event, 0), boost::shared_ptr<Consumer>()); } diff --git a/qpid/cpp/src/qpid/ha/ReplicatingSubscription.h b/qpid/cpp/src/qpid/ha/ReplicatingSubscription.h index a80141a6c2..8a2984846e 100644 --- a/qpid/cpp/src/qpid/ha/ReplicatingSubscription.h +++ b/qpid/cpp/src/qpid/ha/ReplicatingSubscription.h @@ -101,15 +101,15 @@ class ReplicatingSubscription : public broker::SemanticState::ConsumerImpl // Called via QueueGuard::dequeued. //@return true if the message requires completion. - void dequeued(const broker::QueuedMessage& qm); + void dequeued(const broker::Message&); // Called during initial scan for dequeues. void dequeued(framing::SequenceNumber first, framing::SequenceNumber last); // Consumer overrides. - bool deliver(broker::QueuedMessage& msg); + bool deliver(const broker::QueueCursor& cursor, const broker::Message& msg); void cancel(); - void acknowledged(const broker::QueuedMessage&); + void acknowledged(const broker::DeliveryRecord&); bool browseAcquired() const { return true; } // Hide the "queue deleted" error for a ReplicatingSubscription when a // queue is deleted, this is normal and not an error. @@ -127,8 +127,8 @@ class ReplicatingSubscription : public broker::SemanticState::ConsumerImpl private: std::string logPrefix; - boost::shared_ptr<broker::Queue> dummy; // Used to send event messages framing::SequenceSet dequeues; + framing::SequenceNumber position; framing::SequenceNumber backupPosition; bool ready; BrokerInfo info; diff --git a/qpid/cpp/src/qpid/ha/ReplicationTest.cpp b/qpid/cpp/src/qpid/ha/ReplicationTest.cpp index 18e0953930..88a969dbfd 100644 --- a/qpid/cpp/src/qpid/ha/ReplicationTest.cpp +++ b/qpid/cpp/src/qpid/ha/ReplicationTest.cpp @@ -68,7 +68,7 @@ bool ReplicationTest::isReplicated( bool ReplicationTest::isReplicated(ReplicateLevel level, const broker::Queue& q) { - return isReplicated(level, q.getSettings(), q.isAutoDelete(), q.hasExclusiveOwner()); + return isReplicated(level, q.getSettings().storeSettings, q.isAutoDelete(), q.hasExclusiveOwner()); } diff --git a/qpid/cpp/src/qpid/management/ManagementAgent.cpp b/qpid/cpp/src/qpid/management/ManagementAgent.cpp index 7d90ed99d0..7aed5a3c14 100644 --- a/qpid/cpp/src/qpid/management/ManagementAgent.cpp +++ b/qpid/cpp/src/qpid/management/ManagementAgent.cpp @@ -31,6 +31,7 @@ #include <qpid/broker/Message.h> #include "qpid/framing/MessageTransferBody.h" #include "qpid/framing/FieldValue.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/sys/Time.h" #include "qpid/sys/Thread.h" #include "qpid/broker/ConnectionState.h" @@ -535,7 +536,7 @@ void ManagementAgent::sendBufferLH(Buffer& buf, } if (exchange.get() == 0) return; - intrusive_ptr<Message> msg(new Message()); + intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> transfer(new qpid::broker::amqp_0_10::MessageTransfer()); AMQFrame method((MessageTransferBody(ProtocolVersion(), exchange->getName (), 0, 0))); AMQFrame header((AMQHeaderBody())); AMQFrame content((AMQContentBody())); @@ -547,24 +548,26 @@ void ManagementAgent::sendBufferLH(Buffer& buf, header.setEof(false); content.setBof(false); - msg->getFrames().append(method); - msg->getFrames().append(header); + transfer->getFrames().append(method); + transfer->getFrames().append(header); MessageProperties* props = - msg->getFrames().getHeaders()->get<MessageProperties>(true); + transfer->getFrames().getHeaders()->get<MessageProperties>(true); props->setContentLength(length); DeliveryProperties* dp = - msg->getFrames().getHeaders()->get<DeliveryProperties>(true); + transfer->getFrames().getHeaders()->get<DeliveryProperties>(true); dp->setRoutingKey(routingKey); - msg->getFrames().append(content); - msg->setIsManagementMessage(true); + transfer->getFrames().append(content); + + Message msg(transfer, transfer); + msg.setIsManagementMessage(true); { sys::Mutex::ScopedUnlock u(userLock); - DeliverableMessage deliverable (msg); + DeliverableMessage deliverable (msg, 0); try { exchange->route(deliverable); } catch(exception&) {} @@ -602,7 +605,7 @@ void ManagementAgent::sendBufferLH(const string& data, } if (exchange.get() == 0) return; - intrusive_ptr<Message> msg(new Message()); + intrusive_ptr<qpid::broker::amqp_0_10::MessageTransfer> transfer(new qpid::broker::amqp_0_10::MessageTransfer()); AMQFrame method((MessageTransferBody(ProtocolVersion(), exchange->getName (), 0, 0))); AMQFrame header((AMQHeaderBody())); AMQFrame content((AMQContentBody(data))); @@ -612,11 +615,11 @@ void ManagementAgent::sendBufferLH(const string& data, header.setEof(false); content.setBof(false); - msg->getFrames().append(method); - msg->getFrames().append(header); + transfer->getFrames().append(method); + transfer->getFrames().append(header); MessageProperties* props = - msg->getFrames().getHeaders()->get<MessageProperties>(true); + transfer->getFrames().getHeaders()->get<MessageProperties>(true); props->setContentLength(data.length()); if (!cid.empty()) { props->setCorrelationId(cid); @@ -625,23 +628,24 @@ void ManagementAgent::sendBufferLH(const string& data, props->setAppId("qmf2"); for (i = headers.begin(); i != headers.end(); ++i) { - msg->insertCustomProperty(i->first, i->second.asString()); + props->getApplicationHeaders().setString(i->first, i->second.asString()); } DeliveryProperties* dp = - msg->getFrames().getHeaders()->get<DeliveryProperties>(true); + transfer->getFrames().getHeaders()->get<DeliveryProperties>(true); dp->setRoutingKey(routingKey); if (ttl_msec) { dp->setTtl(ttl_msec); - msg->computeExpiration(broker->getExpiryPolicy()); } - msg->getFrames().append(content); - msg->setIsManagementMessage(true); + transfer->getFrames().append(content); + Message msg(transfer, transfer); + msg.setIsManagementMessage(true); + msg.computeExpiration(broker->getExpiryPolicy()); { sys::Mutex::ScopedUnlock u(userLock); - DeliverableMessage deliverable (msg); + DeliverableMessage deliverable (msg, 0); try { exchange->route(deliverable); } catch(exception&) {} @@ -2135,19 +2139,20 @@ bool ManagementAgent::authorizeAgentMessageLH(Message& msg) // authorized or not. In this case, return true (authorized) if there is no ACL in place, // otherwise return false; // - if (msg.encodedSize() > MA_BUFFER_SIZE) + if (msg.getContentSize() > MA_BUFFER_SIZE) return broker->getAcl() == 0; - msg.encodeContent(inBuffer); + inBuffer.putRawData(msg.getContent()); uint32_t bufferLen = inBuffer.getPosition(); inBuffer.reset(); + qpid::broker::amqp_0_10::MessageTransfer& transfer(qpid::broker::amqp_0_10::MessageTransfer::get(msg)); const framing::MessageProperties* p = - msg.getFrames().getHeaders()->get<framing::MessageProperties>(); + transfer.getFrames().getHeaders()->get<framing::MessageProperties>(); - const framing::FieldTable *headers = msg.getApplicationHeaders(); + const framing::FieldTable *headers = p ? &p->getApplicationHeaders() : 0; - if (headers && msg.getAppId() == "qmf2") + if (headers && p->getAppId() == "qmf2") { mapMsg = true; @@ -2238,8 +2243,9 @@ bool ManagementAgent::authorizeAgentMessageLH(Message& msg) // authorization failed, send reply if replyTo present + qpid::broker::amqp_0_10::MessageTransfer& transfer(qpid::broker::amqp_0_10::MessageTransfer::get(msg)); const framing::MessageProperties* p = - msg.getFrames().getHeaders()->get<framing::MessageProperties>(); + transfer.getFrames().getHeaders()->get<framing::MessageProperties>(); if (p && p->hasReplyTo()) { const framing::ReplyTo& rt = p->getReplyTo(); string rte = rt.getExchange(); @@ -2277,8 +2283,9 @@ void ManagementAgent::dispatchAgentCommandLH(Message& msg, bool viaLocal) { string rte; string rtk; + qpid::broker::amqp_0_10::MessageTransfer& transfer(qpid::broker::amqp_0_10::MessageTransfer::get(msg)); const framing::MessageProperties* p = - msg.getFrames().getHeaders()->get<framing::MessageProperties>(); + transfer.getFrames().getHeaders()->get<framing::MessageProperties>(); if (p && p->hasReplyTo()) { const framing::ReplyTo& rt = p->getReplyTo(); rte = rt.getExchange(); @@ -2290,19 +2297,19 @@ void ManagementAgent::dispatchAgentCommandLH(Message& msg, bool viaLocal) Buffer inBuffer(inputBuffer, MA_BUFFER_SIZE); uint8_t opcode; - if (msg.encodedSize() > MA_BUFFER_SIZE) { + if (msg.getContentSize() > MA_BUFFER_SIZE) { QPID_LOG(debug, "ManagementAgent::dispatchAgentCommandLH: Message too large: " << - msg.encodedSize()); + msg.getContentSize()); return; } - msg.encodeContent(inBuffer); + inBuffer.putRawData(msg.getContent()); uint32_t bufferLen = inBuffer.getPosition(); inBuffer.reset(); ScopedManagementContext context((const qpid::broker::ConnectionState*) msg.getPublisher()); - const framing::FieldTable *headers = msg.getApplicationHeaders(); - if (headers && msg.getAppId() == "qmf2") + const framing::FieldTable *headers = p ? &p->getApplicationHeaders() : 0; + if (headers && p->getAppId() == "qmf2") { string opcode = headers->getAsString("qmf.opcode"); string contentType = headers->getAsString("qmf.content"); diff --git a/qpid/cpp/src/qpid/management/ManagementDirectExchange.cpp b/qpid/cpp/src/qpid/management/ManagementDirectExchange.cpp index 9432a21b3a..1c1d6ef3db 100644 --- a/qpid/cpp/src/qpid/management/ManagementDirectExchange.cpp +++ b/qpid/cpp/src/qpid/management/ManagementDirectExchange.cpp @@ -43,11 +43,9 @@ ManagementDirectExchange::ManagementDirectExchange(const std::string& _name, void ManagementDirectExchange::route(Deliverable& msg) { bool routeIt = true; - const std::string& routingKey = msg.getMessage().getRoutingKey(); - const FieldTable* args = msg.getMessage().getApplicationHeaders(); if (managementAgent) - routeIt = managementAgent->dispatchCommand(msg, routingKey, args, false, qmfVersion); + routeIt = managementAgent->dispatchCommand(msg, msg.getMessage().getRoutingKey(), 0/*args - TODO*/, false, qmfVersion); if (routeIt) DirectExchange::route(msg); diff --git a/qpid/cpp/src/qpid/management/ManagementTopicExchange.cpp b/qpid/cpp/src/qpid/management/ManagementTopicExchange.cpp index e5b659f217..c8bfef3785 100644 --- a/qpid/cpp/src/qpid/management/ManagementTopicExchange.cpp +++ b/qpid/cpp/src/qpid/management/ManagementTopicExchange.cpp @@ -42,12 +42,10 @@ ManagementTopicExchange::ManagementTopicExchange(const std::string& _name, void ManagementTopicExchange::route(Deliverable& msg) { bool routeIt = true; - const std::string& routingKey = msg.getMessage().getRoutingKey(); - const FieldTable* args = msg.getMessage().getApplicationHeaders(); // Intercept management agent commands if (managementAgent) - routeIt = managementAgent->dispatchCommand(msg, routingKey, args, true, qmfVersion); + routeIt = managementAgent->dispatchCommand(msg, msg.getMessage().getRoutingKey(), 0/*args - TODO*/, true, qmfVersion); if (routeIt) TopicExchange::route(msg); diff --git a/qpid/cpp/src/qpid/replication/ReplicatingEventListener.cpp b/qpid/cpp/src/qpid/replication/ReplicatingEventListener.cpp deleted file mode 100644 index 9284bda388..0000000000 --- a/qpid/cpp/src/qpid/replication/ReplicatingEventListener.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/replication/ReplicatingEventListener.h" -#include "qpid/replication/constants.h" -#include "qpid/broker/Broker.h" -#include "qpid/broker/DeliverableMessage.h" -#include "qpid/broker/QueueEvents.h" -#include "qpid/framing/AMQFrame.h" -#include "qpid/framing/FrameHandler.h" -#include "qpid/framing/MessageTransferBody.h" -#include "qpid/log/Statement.h" - -namespace qpid { -namespace replication { - -using namespace qpid::broker; -using namespace qpid::framing; -using namespace qpid::replication::constants; - -void ReplicatingEventListener::handle(QueueEvents::Event event) -{ - switch (event.type) { - case QueueEvents::ENQUEUE: - deliverEnqueueMessage(event.msg); - QPID_LOG(debug, "Queuing 'enqueue' event on " << event.msg.queue->getName() << " for replication"); - break; - case QueueEvents::DEQUEUE: - deliverDequeueMessage(event.msg); - QPID_LOG(debug, "Queuing 'dequeue' event from " << event.msg.queue->getName() << " for replication, (from position " - << event.msg.position << ")"); - break; - } -} - -namespace { -const std::string EMPTY; -} - -void ReplicatingEventListener::deliverDequeueMessage(const QueuedMessage& dequeued) -{ - FieldTable headers; - headers.setString(REPLICATION_TARGET_QUEUE, dequeued.queue->getName()); - headers.setInt(REPLICATION_EVENT_TYPE, DEQUEUE); - headers.setInt(DEQUEUED_MESSAGE_POSITION, dequeued.position); - boost::intrusive_ptr<Message> msg(createMessage(headers)); - DeliveryProperties* props = msg->getFrames().getHeaders()->get<DeliveryProperties>(true); - props->setRoutingKey(dequeued.queue->getName()); - route(msg); -} - -void ReplicatingEventListener::deliverEnqueueMessage(const QueuedMessage& enqueued) -{ - boost::intrusive_ptr<Message> msg(cloneMessage(*(enqueued.queue), enqueued.payload)); - msg->insertCustomProperty(REPLICATION_TARGET_QUEUE, enqueued.queue->getName()); - msg->insertCustomProperty(REPLICATION_EVENT_TYPE, ENQUEUE); - msg->insertCustomProperty(QUEUE_MESSAGE_POSITION,enqueued.position); - route(msg); -} - -void ReplicatingEventListener::route(boost::intrusive_ptr<qpid::broker::Message> msg) -{ - try { - if (exchange) { - DeliverableMessage deliverable(msg); - exchange->route(deliverable); - } else if (queue) { - queue->deliver(msg); - } else { - QPID_LOG(error, "Cannot route replication event, neither replication queue nor exchange configured"); - } - } catch (const std::exception& e) { - QPID_LOG(error, "Error enqueing replication event: " << e.what()); - } -} - - -boost::intrusive_ptr<Message> ReplicatingEventListener::createMessage(const FieldTable& headers) -{ - boost::intrusive_ptr<Message> msg(new Message()); - AMQFrame method((MessageTransferBody(ProtocolVersion(), EMPTY, 0, 0))); - AMQFrame header((AMQHeaderBody())); - header.setBof(false); - header.setEof(true); - header.setBos(true); - header.setEos(true); - msg->getFrames().append(method); - msg->getFrames().append(header); - MessageProperties* props = msg->getFrames().getHeaders()->get<MessageProperties>(true); - props->setApplicationHeaders(headers); - return msg; -} - -struct AppendingHandler : FrameHandler -{ - boost::intrusive_ptr<Message> msg; - - AppendingHandler(boost::intrusive_ptr<Message> m) : msg(m) {} - - void handle(AMQFrame& f) - { - msg->getFrames().append(f); - } -}; - -boost::intrusive_ptr<Message> ReplicatingEventListener::cloneMessage(Queue& queue, boost::intrusive_ptr<Message> original) -{ - boost::intrusive_ptr<Message> copy(new Message()); - AMQFrame method((MessageTransferBody(ProtocolVersion(), EMPTY, 0, 0))); - AppendingHandler handler(copy); - handler.handle(method); - - //To avoid modifying original headers, create new frame with - //cloned body: - AMQFrame header(*original->getFrames().getHeaders()); - header.setBof(false); - header.setEof(!original->getFrames().hasContent());//if there are any content frames then the header is not the end of the frameset - header.setBos(true); - header.setEos(true); - handler.handle(header); - - original->sendContent(queue, handler, std::numeric_limits<int16_t>::max()); - return copy; -} - -Options* ReplicatingEventListener::getOptions() -{ - return &options; -} - -void ReplicatingEventListener::initialize(Plugin::Target& target) -{ - Broker* broker = dynamic_cast<broker::Broker*>(&target); - if (broker) { - broker->addFinalizer(boost::bind(&ReplicatingEventListener::shutdown, this)); - if (!options.exchange.empty()) { - if (!options.queue.empty()) { - QPID_LOG(warning, "Replication queue option ignored as replication exchange has been specified"); - } - try { - exchange = broker->getExchanges().declare(options.exchange, options.exchangeType).first; - } catch (const UnknownExchangeTypeException&) { - QPID_LOG(error, "Replication disabled due to invalid type: " << options.exchangeType); - } - } else if (!options.queue.empty()) { - if (options.createQueue) { - queue = broker->getQueues().declare(options.queue).first; - } else { - queue = broker->getQueues().find(options.queue); - } - if (queue) { - queue->insertSequenceNumbers(REPLICATION_EVENT_SEQNO); - } else { - QPID_LOG(error, "Replication queue named '" << options.queue << "' does not exist; replication plugin disabled."); - } - } - if (queue || exchange) { - QueueEvents::EventListener callback = boost::bind(&ReplicatingEventListener::handle, this, _1); - broker->getQueueEvents().registerListener(options.name, callback); - QPID_LOG(info, "Registered replicating queue event listener"); - } - } -} - -void ReplicatingEventListener::earlyInitialize(Target&) {} -void ReplicatingEventListener::shutdown() { queue.reset(); exchange.reset(); } - -ReplicatingEventListener::PluginOptions::PluginOptions() : Options("Queue Replication Options"), - exchangeType("direct"), - name("replicator"), - createQueue(false) -{ - addOptions() - ("replication-exchange-name", optValue(exchange, "EXCHANGE"), "Exchange to which events for other queues are routed") - ("replication-exchange-type", optValue(exchangeType, "direct|topic etc"), "Type of exchange to use") - ("replication-queue", optValue(queue, "QUEUE"), "Queue on which events for other queues are recorded") - ("replication-listener-name", optValue(name, "NAME"), "name by which to register the replicating event listener") - ("create-replication-queue", optValue(createQueue), "if set, the replication will be created if it does not exist"); -} - -static ReplicatingEventListener plugin; - -}} // namespace qpid::replication diff --git a/qpid/cpp/src/qpid/replication/ReplicatingEventListener.h b/qpid/cpp/src/qpid/replication/ReplicatingEventListener.h deleted file mode 100644 index 74418d00e6..0000000000 --- a/qpid/cpp/src/qpid/replication/ReplicatingEventListener.h +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef QPID_REPLICATION_REPLICATINGEVENTLISTENER_H -#define QPID_REPLICATION_REPLICATINGEVENTLISTENER_H - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "qpid/Plugin.h" -#include "qpid/Options.h" -#include "qpid/broker/Exchange.h" -#include "qpid/broker/Message.h" -#include "qpid/broker/Queue.h" -#include "qpid/broker/QueueEvents.h" -#include "qpid/framing/FieldTable.h" -#include "qpid/framing/SequenceNumber.h" - -namespace qpid { -namespace replication { - -/** - * An event listener plugin that records queue events as messages on a - * replication queue, from where they can be consumed (e.g. by an - * inter-broker link to the corresponding QueueReplicationExchange - * plugin. - */ -class ReplicatingEventListener : public Plugin -{ - public: - Options* getOptions(); - void earlyInitialize(Plugin::Target& target); - void initialize(Plugin::Target& target); - void handle(qpid::broker::QueueEvents::Event); - private: - struct PluginOptions : public Options - { - std::string queue; - std::string exchange; - std::string exchangeType; - std::string name; - bool createQueue; - - PluginOptions(); - }; - - PluginOptions options; - qpid::broker::Queue::shared_ptr queue; - qpid::broker::Exchange::shared_ptr exchange; - - void deliverDequeueMessage(const qpid::broker::QueuedMessage& enqueued); - void deliverEnqueueMessage(const qpid::broker::QueuedMessage& enqueued); - void route(boost::intrusive_ptr<qpid::broker::Message>); - void shutdown(); - - boost::intrusive_ptr<qpid::broker::Message> createMessage(const qpid::framing::FieldTable& headers); - boost::intrusive_ptr<qpid::broker::Message> cloneMessage(qpid::broker::Queue& queue, - boost::intrusive_ptr<qpid::broker::Message> original); -}; - -}} // namespace qpid::replication - -#endif /*!QPID_REPLICATION_REPLICATINGEVENTLISTENER_H*/ diff --git a/qpid/cpp/src/qpid/replication/ReplicationExchange.cpp b/qpid/cpp/src/qpid/replication/ReplicationExchange.cpp deleted file mode 100644 index bcb7c7f293..0000000000 --- a/qpid/cpp/src/qpid/replication/ReplicationExchange.cpp +++ /dev/null @@ -1,240 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/replication/ReplicationExchange.h" -#include "qpid/replication/constants.h" -#include "qpid/Plugin.h" -#include "qpid/broker/Broker.h" -#include "qpid/broker/Queue.h" -#include "qpid/broker/QueueRegistry.h" -#include "qpid/broker/ExchangeRegistry.h" -#include "qpid/framing/reply_exceptions.h" -#include "qpid/log/Statement.h" -#include <boost/bind.hpp> - -namespace qpid { -namespace replication { - -using namespace qpid::broker; -using namespace qpid::framing; -using namespace qpid::replication::constants; - -const std::string SEQUENCE_VALUE("qpid.replication-event.sequence"); -ReplicationExchange::ReplicationExchange(const std::string& name, bool durable, - const FieldTable& _args, - QueueRegistry& qr, - Manageable* parent, Broker* broker) - : Exchange(name, durable, _args, parent, broker), queues(qr), sequence(args.getAsInt64(SEQUENCE_VALUE)), init(false) -{ - args.setInt64(SEQUENCE_VALUE, sequence); - if (mgmtExchange != 0) - mgmtExchange->set_type(typeName); -} - -std::string ReplicationExchange::getType() const { return typeName; } - -void ReplicationExchange::route(Deliverable& msg) -{ - const FieldTable* args = msg.getMessage().getApplicationHeaders(); - if (mgmtExchange != 0) { - mgmtExchange->inc_msgReceives(); - mgmtExchange->inc_byteReceives(msg.contentSize()); - } - if (args) { - int eventType = args->getAsInt(REPLICATION_EVENT_TYPE); - if (eventType) { - if (isDuplicate(args)) { - if (mgmtExchange != 0) { - mgmtExchange->inc_msgDrops(); - mgmtExchange->inc_byteDrops(msg.contentSize()); - } - return; - } - switch (eventType) { - case ENQUEUE: - handleEnqueueEvent(args, msg); - return; - case DEQUEUE: - handleDequeueEvent(args, msg); - return; - default: - throw IllegalArgumentException(QPID_MSG("Illegal value for " << REPLICATION_EVENT_TYPE << ": " << eventType)); - } - } - } else { - QPID_LOG(warning, "Dropping unexpected message with no headers"); - if (mgmtExchange != 0) { - mgmtExchange->inc_msgDrops(); - mgmtExchange->inc_byteDrops(msg.contentSize()); - } - } -} - -void ReplicationExchange::handleEnqueueEvent(const FieldTable* args, Deliverable& msg) -{ - std::string queueName = args->getAsString(REPLICATION_TARGET_QUEUE); - Queue::shared_ptr queue = queues.find(queueName); - if (queue) { - - SequenceNumber seqno1(args->getAsInt(QUEUE_MESSAGE_POSITION)); - - // note that queue will ++ before enqueue. - if (queue->getPosition() > --seqno1) // test queue.pos < seqnumber - { - QPID_LOG(error, "Cannot enqueue replicated message. Destination Queue " << queueName << " ahead of source queue"); - mgmtExchange->inc_msgDrops(); - mgmtExchange->inc_byteDrops(msg.contentSize()); - } else { - queue->setPosition(seqno1); - - msg.getMessage().removeCustomProperty(REPLICATION_TARGET_QUEUE); - msg.getMessage().removeCustomProperty(REPLICATION_EVENT_SEQNO); - msg.getMessage().removeCustomProperty(REPLICATION_EVENT_TYPE); - msg.getMessage().removeCustomProperty(QUEUE_MESSAGE_POSITION); - msg.deliverTo(queue); - QPID_LOG(debug, "Enqueued replicated message onto " << queueName); - if (mgmtExchange != 0) { - mgmtExchange->inc_msgRoutes(); - mgmtExchange->inc_byteRoutes( msg.contentSize()); - } - } - } else { - QPID_LOG(error, "Cannot enqueue replicated message. Queue " << queueName << " does not exist"); - if (mgmtExchange != 0) { - mgmtExchange->inc_msgDrops(); - mgmtExchange->inc_byteDrops(msg.contentSize()); - } - } -} - -void ReplicationExchange::handleDequeueEvent(const FieldTable* args, Deliverable& msg) -{ - std::string queueName = args->getAsString(REPLICATION_TARGET_QUEUE); - Queue::shared_ptr queue = queues.find(queueName); - if (queue) { - SequenceNumber position(args->getAsInt(DEQUEUED_MESSAGE_POSITION)); - QueuedMessage dequeued; - if (queue->acquireMessageAt(position, dequeued)) { - queue->dequeue(0, dequeued); - QPID_LOG(debug, "Processed replicated 'dequeue' event from " << queueName << " at position " << position); - if (mgmtExchange != 0) { - mgmtExchange->inc_msgRoutes(); - mgmtExchange->inc_byteRoutes(msg.contentSize()); - } - } else { - QPID_LOG(warning, "Could not acquire message " << position << " from " << queueName); - if (mgmtExchange != 0) { - mgmtExchange->inc_msgDrops(); - mgmtExchange->inc_byteDrops(msg.contentSize()); - } - } - } else { - QPID_LOG(error, "Cannot process replicated 'dequeue' event. Queue " << queueName << " does not exist"); - if (mgmtExchange != 0) { - mgmtExchange->inc_msgDrops(); - mgmtExchange->inc_byteDrops(msg.contentSize()); - } - } -} - -bool ReplicationExchange::isDuplicate(const FieldTable* args) -{ - if (!args->get(REPLICATION_EVENT_SEQNO)) return false; - SequenceNumber seqno(args->getAsInt(REPLICATION_EVENT_SEQNO)); - if (!init) { - init = true; - sequence = seqno; - return false; - } else if (seqno > sequence) { - if (seqno - sequence > 1) { - QPID_LOG(error, "Gap in replication event sequence between: " << sequence << " and " << seqno); - } - sequence = seqno; - return false; - } else { - QPID_LOG(info, "Duplicate detected: seqno=" << seqno << " (last seqno=" << sequence << ")"); - return true; - } -} - -bool ReplicationExchange::bind(Queue::shared_ptr /*queue*/, const std::string& /*routingKey*/, const FieldTable* /*args*/) -{ - throw NotImplementedException("Replication exchange does not support bind operation"); -} - -bool ReplicationExchange::unbind(Queue::shared_ptr /*queue*/, const std::string& /*routingKey*/, const FieldTable* /*args*/) -{ - throw NotImplementedException("Replication exchange does not support unbind operation"); -} - -bool ReplicationExchange::isBound(Queue::shared_ptr /*queue*/, const std::string* const /*routingKey*/, const FieldTable* const /*args*/) -{ - return false; -} - -const std::string ReplicationExchange::typeName("replication"); - - -void ReplicationExchange::encode(Buffer& buffer) const -{ - args.setInt64(std::string(SEQUENCE_VALUE), sequence); - Exchange::encode(buffer); -} - - -struct ReplicationExchangePlugin : Plugin -{ - Broker* broker; - - ReplicationExchangePlugin(); - void earlyInitialize(Plugin::Target& target); - void initialize(Plugin::Target& target); - Exchange::shared_ptr create(const std::string& name, bool durable, - const framing::FieldTable& args, - management::Manageable* parent, - qpid::broker::Broker* broker); -}; - -ReplicationExchangePlugin::ReplicationExchangePlugin() : broker(0) {} - -Exchange::shared_ptr ReplicationExchangePlugin::create(const std::string& name, bool durable, - const framing::FieldTable& args, - management::Manageable* parent, qpid::broker::Broker* broker) -{ - Exchange::shared_ptr e(new ReplicationExchange(name, durable, args, broker->getQueues(), parent, broker)); - return e; -} - - -void ReplicationExchangePlugin::earlyInitialize(Plugin::Target& target) -{ - broker = dynamic_cast<broker::Broker*>(&target); - if (broker) { - ExchangeRegistry::FactoryFunction f = boost::bind(&ReplicationExchangePlugin::create, this, _1, _2, _3, _4, _5); - broker->getExchanges().registerType(ReplicationExchange::typeName, f); - QPID_LOG(info, "Registered replication exchange"); - } -} - -void ReplicationExchangePlugin::initialize(Target&) {} - -static ReplicationExchangePlugin exchangePlugin; - -}} // namespace qpid::replication diff --git a/qpid/cpp/src/qpid/replication/ReplicationExchange.h b/qpid/cpp/src/qpid/replication/ReplicationExchange.h deleted file mode 100644 index ff0a98c48e..0000000000 --- a/qpid/cpp/src/qpid/replication/ReplicationExchange.h +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef QPID_REPLICATION_REPLICATIONEXCHANGE_H -#define QPID_REPLICATION_REPLICATIONEXCHANGE_H - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/broker/Exchange.h" -#include "qpid/framing/Buffer.h" -#include "qpid/framing/SequenceNumber.h" - -namespace qpid { - -namespace broker { -class QueueRegistry; -} - -namespace replication { - -/** - * A custom exchange plugin that processes incoming messages - * representing enqueue or dequeue events for particular queues and - * carries out the corresponding action to replicate that on the local - * broker. - */ -class ReplicationExchange : public qpid::broker::Exchange -{ - public: - static const std::string typeName; - - ReplicationExchange(const std::string& name, bool durable, - const qpid::framing::FieldTable& args, - qpid::broker::QueueRegistry& queues, - qpid::management::Manageable* parent = 0, - qpid::broker::Broker* broker = 0); - - std::string getType() const; - - void route(qpid::broker::Deliverable& msg); - - bool bind(boost::shared_ptr<broker::Queue> queue, const std::string& routingKey, const qpid::framing::FieldTable* args); - bool unbind(boost::shared_ptr<broker::Queue> queue, const std::string& routingKey, const qpid::framing::FieldTable* args); - bool isBound(boost::shared_ptr<broker::Queue> queue, const std::string* const routingKey, const qpid::framing::FieldTable* const args); - private: - qpid::broker::QueueRegistry& queues; - qpid::framing::SequenceNumber sequence; - bool init; - - bool isDuplicate(const qpid::framing::FieldTable* args); - void handleEnqueueEvent(const qpid::framing::FieldTable* args, qpid::broker::Deliverable& msg); - void handleDequeueEvent(const qpid::framing::FieldTable* args, qpid::broker::Deliverable& msg); - void encode(framing::Buffer& buffer) const; -}; -}} // namespace qpid::replication - -#endif /*!QPID_REPLICATION_REPLICATIONEXCHANGE_H*/ diff --git a/qpid/cpp/src/qpid/store/MessageStorePlugin.cpp b/qpid/cpp/src/qpid/store/MessageStorePlugin.cpp index 20231bf910..c6b0e1a53a 100644 --- a/qpid/cpp/src/qpid/store/MessageStorePlugin.cpp +++ b/qpid/cpp/src/qpid/store/MessageStorePlugin.cpp @@ -249,7 +249,7 @@ MessageStorePlugin::destroy(const broker::PersistableConfig& config) void MessageStorePlugin::stage(const boost::intrusive_ptr<broker::PersistableMessage>& msg) { - if (msg->getPersistenceId() == 0 && !msg->isContentReleased()) { + if (msg->getPersistenceId() == 0) { provider->second->stage(msg); } } diff --git a/qpid/cpp/src/qpid/xml/XmlExchange.cpp b/qpid/cpp/src/qpid/xml/XmlExchange.cpp index 3fb11394d0..f88acb04ee 100644 --- a/qpid/cpp/src/qpid/xml/XmlExchange.cpp +++ b/qpid/cpp/src/qpid/xml/XmlExchange.cpp @@ -27,6 +27,7 @@ #include "qpid/log/Statement.h" #include "qpid/broker/FedOps.h" +#include "qpid/broker/MapHandler.h" #include "qpid/framing/FieldTable.h" #include "qpid/framing/FieldValue.h" #include "qpid/framing/reply_exceptions.h" @@ -198,7 +199,52 @@ bool XmlExchange::unbind(Queue::shared_ptr queue, const std::string& bindingKey, } } -bool XmlExchange::matches(Query& query, Deliverable& msg, const qpid::framing::FieldTable* args, bool parse_message_content) +namespace { +class DefineExternals : public MapHandler +{ + public: + DefineExternals(DynamicContext* c) : context(c) { assert(context); } + void handleUint8(const MapHandler::CharSequence& key, uint8_t value) { process(std::string(key.data, key.size), (int) value); } + void handleUint16(const MapHandler::CharSequence& key, uint16_t value) { process(std::string(key.data, key.size), (int) value); } + void handleUint32(const MapHandler::CharSequence& key, uint32_t value) { process(std::string(key.data, key.size), (int) value); } + void handleUint64(const MapHandler::CharSequence& key, uint64_t value) { process(std::string(key.data, key.size), (int) value); } + void handleInt8(const MapHandler::CharSequence& key, int8_t value) { process(std::string(key.data, key.size), (int) value); } + void handleInt16(const MapHandler::CharSequence& key, int16_t value) { process(std::string(key.data, key.size), (int) value); } + void handleInt32(const MapHandler::CharSequence& key, int32_t value) { process(std::string(key.data, key.size), (int) value); } + void handleInt64(const MapHandler::CharSequence& key, int64_t value) { process(std::string(key.data, key.size), (int) value); } + void handleFloat(const MapHandler::CharSequence& key, float value) { process(std::string(key.data, key.size), value); } + void handleDouble(const MapHandler::CharSequence& key, double value) { process(std::string(key.data, key.size), value); } + void handleString(const MapHandler::CharSequence& key, const MapHandler::CharSequence& value, const MapHandler::CharSequence& /*encoding*/) + { + process(std::string(key.data, key.size), std::string(value.data, value.size)); + } + void handleVoid(const MapHandler::CharSequence&) {} + private: + void process(const std::string& key, double value) + { + QPID_LOG(trace, "XmlExchange, external variable (double): " << key << " = " << value); + Item::Ptr item = context->getItemFactory()->createDouble(value, context); + context->setExternalVariable(X(key.c_str()), item); + } + void process(const std::string& key, int value) + { + QPID_LOG(trace, "XmlExchange, external variable (int):" << key << " = " << value); + Item::Ptr item = context->getItemFactory()->createInteger(value, context); + context->setExternalVariable(X(key.c_str()), item); + } + void process(const std::string& key, const std::string& value) + { + QPID_LOG(trace, "XmlExchange, external variable (string):" << key << " = " << value); + Item::Ptr item = context->getItemFactory()->createString(X(value.c_str()), context); + context->setExternalVariable(X(key.c_str()), item); + } + + DynamicContext* context; +}; + +} + +bool XmlExchange::matches(Query& query, Deliverable& msg, bool parse_message_content) { std::string msgContent; @@ -212,7 +258,7 @@ bool XmlExchange::matches(Query& query, Deliverable& msg, const qpid::framing::F if (parse_message_content) { - msg.getMessage().getFrames().getContent(msgContent); + msgContent = msg.getMessage().getContent(); QPID_LOG(trace, "matches: message content is [" << msgContent << "]"); @@ -231,28 +277,8 @@ bool XmlExchange::matches(Query& query, Deliverable& msg, const qpid::framing::F } } - if (args) { - FieldTable::ValueMap::const_iterator v = args->begin(); - for(; v != args->end(); ++v) { - - if (v->second->convertsTo<double>()) { - QPID_LOG(trace, "XmlExchange, external variable (double): " << v->first << " = " << v->second->get<double>()); - Item::Ptr value = context->getItemFactory()->createDouble(v->second->get<double>(), context.get()); - context->setExternalVariable(X(v->first.c_str()), value); - } - else if (v->second->convertsTo<int>()) { - QPID_LOG(trace, "XmlExchange, external variable (int):" << v->first << " = " << v->second->getData().getInt()); - Item::Ptr value = context->getItemFactory()->createInteger(v->second->get<int>(), context.get()); - context->setExternalVariable(X(v->first.c_str()), value); - } - else if (v->second->convertsTo<std::string>()) { - QPID_LOG(trace, "XmlExchange, external variable (string):" << v->first << " = " << v->second->getData().getString().c_str()); - Item::Ptr value = context->getItemFactory()->createString(X(v->second->get<std::string>().c_str()), context.get()); - context->setExternalVariable(X(v->first.c_str()), value); - } - - } - } + DefineExternals f(context.get()); + msg.getMessage().processProperties(f); Result result = query->execute(context.get()); #ifdef XQ_EFFECTIVE_BOOLEAN_VALUE_HPP @@ -286,7 +312,6 @@ bool XmlExchange::matches(Query& query, Deliverable& msg, const qpid::framing::F void XmlExchange::route(Deliverable& msg) { const std::string& routingKey = msg.getMessage().getRoutingKey(); - const FieldTable* args = msg.getMessage().getApplicationHeaders(); PreRoute pr(msg, this); try { XmlBinding::vector::ConstPtr p; @@ -298,7 +323,7 @@ void XmlExchange::route(Deliverable& msg) } for (std::vector<XmlBinding::shared_ptr>::const_iterator i = p->begin(); i != p->end(); i++) { - if (matches((*i)->xquery, msg, args, (*i)->parse_message_content)) { + if (matches((*i)->xquery, msg, (*i)->parse_message_content)) { b->push_back(*i); } } diff --git a/qpid/cpp/src/qpid/xml/XmlExchange.h b/qpid/cpp/src/qpid/xml/XmlExchange.h index 1d4723f9c4..7b04781ad5 100644 --- a/qpid/cpp/src/qpid/xml/XmlExchange.h +++ b/qpid/cpp/src/qpid/xml/XmlExchange.h @@ -65,7 +65,7 @@ class XmlExchange : public virtual Exchange { qpid::sys::RWlock lock; - bool matches(Query& query, Deliverable& msg, const qpid::framing::FieldTable* args, bool parse_message_content); + bool matches(Query& query, Deliverable& msg, bool parse_message_content); public: static const std::string typeName; diff --git a/qpid/cpp/src/replication.mk b/qpid/cpp/src/replication.mk deleted file mode 100644 index e5da32f88b..0000000000 --- a/qpid/cpp/src/replication.mk +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Make file for building two plugins for asynchronously replicating -# queues. - -dmoduleexec_LTLIBRARIES += replicating_listener.la replication_exchange.la - -# a queue event listener plugin that creates messages on a replication -# queue corresponding to enqueue and dequeue events: -replicating_listener_la_SOURCES = \ - qpid/replication/constants.h \ - qpid/replication/ReplicatingEventListener.cpp \ - qpid/replication/ReplicatingEventListener.h - -replicating_listener_la_LIBADD = libqpidbroker.la -if SUNOS - replicating_listener_la_LIBADD += libqpidcommon.la -lboost_program_options -luuid $(SUNCC_RUNTIME_LIBS) -endif -replicating_listener_la_LDFLAGS = $(PLUGINLDFLAGS) - -# a custom exchange plugin that allows an exchange to be created that -# can process the messages from a replication queue (populated on the -# source system by the replicating listener plugin above) and take the -# corresponding action on the local queues -replication_exchange_la_SOURCES = \ - qpid/replication/constants.h \ - qpid/replication/ReplicationExchange.cpp \ - qpid/replication/ReplicationExchange.h - -replication_exchange_la_LIBADD = libqpidbroker.la - -if SUNOS - replication_exchange_la_LIBADD += libqpidcommon.la -lboost_program_options $(SUNCC_RUNTIME_LIBS) -luuid -endif -replication_exchange_la_LDFLAGS = $(PLUGINLDFLAGS) - diff --git a/qpid/cpp/src/tests/CMakeLists.txt b/qpid/cpp/src/tests/CMakeLists.txt index 9a77514f5f..f88c0eb58e 100644 --- a/qpid/cpp/src/tests/CMakeLists.txt +++ b/qpid/cpp/src/tests/CMakeLists.txt @@ -126,6 +126,7 @@ set(unit_tests_to_build ExchangeTest HeadersExchangeTest MessageTest + QueueDepth QueueRegistryTest QueuePolicyTest QueueFlowLimitTest @@ -135,16 +136,12 @@ set(unit_tests_to_build TimerTest TopicExchangeTest TxBufferTest - TxPublishTest - MessageBuilderTest ManagementTest MessageReplayTracker ConsoleTest - QueueEvents ProxyTest RetryList FrameDecoder - ReplicationTest ClientMessageTest PollableCondition Variant diff --git a/qpid/cpp/src/tests/ClientSessionTest.cpp b/qpid/cpp/src/tests/ClientSessionTest.cpp index 1905219bf2..1f07d2b83f 100644 --- a/qpid/cpp/src/tests/ClientSessionTest.cpp +++ b/qpid/cpp/src/tests/ClientSessionTest.cpp @@ -621,7 +621,7 @@ QPID_AUTO_TEST_CASE(testQueueDeleted) fix.session.queueDeclare(arg::queue="my-queue"); LocalQueue queue; fix.subs.subscribe(queue, "my-queue"); - + ScopedSuppressLogging sl; fix.session.queueDelete(arg::queue="my-queue"); BOOST_CHECK_THROW(queue.get(1*qpid::sys::TIME_SEC), qpid::framing::ResourceDeletedException); diff --git a/qpid/cpp/src/tests/DeliveryRecordTest.cpp b/qpid/cpp/src/tests/DeliveryRecordTest.cpp index fb7bd2f727..c83bd9a6a4 100644 --- a/qpid/cpp/src/tests/DeliveryRecordTest.cpp +++ b/qpid/cpp/src/tests/DeliveryRecordTest.cpp @@ -49,7 +49,7 @@ QPID_AUTO_TEST_CASE(testSort) list<DeliveryRecord> records; for (list<SequenceNumber>::iterator i = ids.begin(); i != ids.end(); i++) { - DeliveryRecord r(QueuedMessage(0), Queue::shared_ptr(), "tag", Consumer::shared_ptr(), false, false, false); + DeliveryRecord r(QueueCursor(CONSUMER), framing::SequenceNumber(), Queue::shared_ptr(), "tag", Consumer::shared_ptr(), false, false, false); r.setId(*i); records.push_back(r); } diff --git a/qpid/cpp/src/tests/ExchangeTest.cpp b/qpid/cpp/src/tests/ExchangeTest.cpp index 66a16b9178..4f18b91b5a 100644 --- a/qpid/cpp/src/tests/ExchangeTest.cpp +++ b/qpid/cpp/src/tests/ExchangeTest.cpp @@ -35,7 +35,6 @@ using std::string; -using boost::intrusive_ptr; using namespace qpid::broker; using namespace qpid::framing; using namespace qpid::sys; @@ -62,11 +61,9 @@ QPID_AUTO_TEST_CASE(testMe) queue.reset(); queue2.reset(); - intrusive_ptr<Message> msgPtr(MessageUtils::createMessage("exchange", "abc", false, "id")); - DeliverableMessage msg(msgPtr); + DeliverableMessage msg(MessageUtils::createMessage("exchange", "abc"), 0); topic.route(msg); direct.route(msg); - } QPID_AUTO_TEST_CASE(testIsBound) @@ -170,16 +167,6 @@ QPID_AUTO_TEST_CASE(testDeleteGetAndRedeclare) BOOST_CHECK_EQUAL(string("direct"), response.first->getType()); } -intrusive_ptr<Message> cmessage(std::string exchange, std::string routingKey) { - intrusive_ptr<Message> msg(new Message()); - AMQFrame method((MessageTransferBody(ProtocolVersion(), exchange, 0, 0))); - AMQFrame header((AMQHeaderBody())); - msg->getFrames().append(method); - msg->getFrames().append(header); - msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setRoutingKey(routingKey); - return msg; -} - QPID_AUTO_TEST_CASE(testSequenceOptions) { FieldTable args; @@ -189,46 +176,35 @@ QPID_AUTO_TEST_CASE(testSequenceOptions) { DirectExchange direct("direct1", false, args); - intrusive_ptr<Message> msg1 = cmessage("e", "abc"); - intrusive_ptr<Message> msg2 = cmessage("e", "abc"); - intrusive_ptr<Message> msg3 = cmessage("e", "abc"); - - DeliverableMessage dmsg1(msg1); - DeliverableMessage dmsg2(msg2); - DeliverableMessage dmsg3(msg3); + DeliverableMessage msg1(MessageUtils::createMessage("e", "abc"), 0); + DeliverableMessage msg2(MessageUtils::createMessage("e", "abc"), 0); + DeliverableMessage msg3(MessageUtils::createMessage("e", "abc"), 0); - direct.route(dmsg1); - direct.route(dmsg2); - direct.route(dmsg3); + direct.route(msg1); + direct.route(msg2); + direct.route(msg3); - BOOST_CHECK_EQUAL(1, msg1->getApplicationHeaders()->getAsInt64("qpid.msg_sequence")); - BOOST_CHECK_EQUAL(2, msg2->getApplicationHeaders()->getAsInt64("qpid.msg_sequence")); - BOOST_CHECK_EQUAL(3, msg3->getApplicationHeaders()->getAsInt64("qpid.msg_sequence")); + BOOST_CHECK_EQUAL(1, msg1.getMessage().getAnnotation("qpid.msg_sequence").asInt64()); + BOOST_CHECK_EQUAL(2, msg2.getMessage().getAnnotation("qpid.msg_sequence").asInt64()); + BOOST_CHECK_EQUAL(3, msg3.getMessage().getAnnotation("qpid.msg_sequence").asInt64()); FanOutExchange fanout("fanout1", false, args); HeadersExchange header("headers1", false, args); TopicExchange topic ("topic1", false, args); // check other exchanges, that they preroute - intrusive_ptr<Message> msg4 = cmessage("e", "abc"); - intrusive_ptr<Message> msg5 = cmessage("e", "abc"); + DeliverableMessage msg4(MessageUtils::createMessage("e", "abc"), 0); + DeliverableMessage msg5(MessageUtils::createMessage("e", "abc"), 0); + DeliverableMessage msg6(MessageUtils::createMessage("e", "abc"), 0); - // Need at least empty header for the HeadersExchange to route at all - msg5->insertCustomProperty("", ""); - intrusive_ptr<Message> msg6 = cmessage("e", "abc"); + fanout.route(msg4); + BOOST_CHECK_EQUAL(1, msg4.getMessage().getAnnotation("qpid.msg_sequence").asInt64()); - DeliverableMessage dmsg4(msg4); - DeliverableMessage dmsg5(msg5); - DeliverableMessage dmsg6(msg6); + header.route(msg5); + BOOST_CHECK_EQUAL(1, msg5.getMessage().getAnnotation("qpid.msg_sequence").asInt64()); - fanout.route(dmsg4); - BOOST_CHECK_EQUAL(1, msg4->getApplicationHeaders()->getAsInt64("qpid.msg_sequence")); - - header.route(dmsg5); - BOOST_CHECK_EQUAL(1, msg5->getApplicationHeaders()->getAsInt64("qpid.msg_sequence")); - - topic.route(dmsg6); - BOOST_CHECK_EQUAL(1, msg6->getApplicationHeaders()->getAsInt64("qpid.msg_sequence")); + topic.route(msg6); + BOOST_CHECK_EQUAL(1, msg6.getMessage().getAnnotation("qpid.msg_sequence").asInt64()); direct.encode(buffer); } { @@ -237,11 +213,10 @@ QPID_AUTO_TEST_CASE(testSequenceOptions) buffer.reset(); DirectExchange::shared_ptr exch_dec = Exchange::decode(exchanges, buffer); - intrusive_ptr<Message> msg1 = cmessage("e", "abc"); - DeliverableMessage dmsg1(msg1); - exch_dec->route(dmsg1); + DeliverableMessage msg1(MessageUtils::createMessage("e", "abc"), 0); + exch_dec->route(msg1); - BOOST_CHECK_EQUAL(4, msg1->getApplicationHeaders()->getAsInt64("qpid.msg_sequence")); + BOOST_CHECK_EQUAL(4, msg1.getMessage().getAnnotation("qpid.msg_sequence").asInt64()); } delete [] buff; @@ -256,9 +231,11 @@ QPID_AUTO_TEST_CASE(testIVEOption) HeadersExchange header("headers1", false, args); TopicExchange topic ("topic1", false, args); - intrusive_ptr<Message> msg1 = cmessage("direct1", "abc"); - msg1->insertCustomProperty("a", "abc"); - DeliverableMessage dmsg1(msg1); + qpid::types::Variant::Map properties; + properties["routing-key"] = "abc"; + properties["a"] = "abc"; + Message msg1 = MessageUtils::createMessage(properties, "my-message", "direct1"); + DeliverableMessage dmsg1(msg1, 0); FieldTable args2; args2.setString("x-match", "any"); @@ -273,8 +250,6 @@ QPID_AUTO_TEST_CASE(testIVEOption) Queue::shared_ptr queue2(new Queue("queue2", true)); Queue::shared_ptr queue3(new Queue("queue3", true)); - BOOST_CHECK(HeadersExchange::match(args2, msg1->getProperties<MessageProperties>()->getApplicationHeaders())); - BOOST_CHECK(direct.bind(queue, "abc", 0)); BOOST_CHECK(fanout.bind(queue1, "abc", 0)); BOOST_CHECK(header.bind(queue2, "", &args2)); @@ -287,7 +262,6 @@ QPID_AUTO_TEST_CASE(testIVEOption) } - QPID_AUTO_TEST_SUITE_END() }} // namespace qpid::tests diff --git a/qpid/cpp/src/tests/Makefile.am b/qpid/cpp/src/tests/Makefile.am index b04ec6b43e..f9eed9270b 100644 --- a/qpid/cpp/src/tests/Makefile.am +++ b/qpid/cpp/src/tests/Makefile.am @@ -96,6 +96,7 @@ unit_test_SOURCES= unit_test.cpp unit_test.h \ ExchangeTest.cpp \ HeadersExchangeTest.cpp \ MessageTest.cpp \ + QueueDepth.cpp \ QueueRegistryTest.cpp \ QueuePolicyTest.cpp \ QueueFlowLimitTest.cpp \ @@ -105,19 +106,15 @@ unit_test_SOURCES= unit_test.cpp unit_test.h \ TimerTest.cpp \ TopicExchangeTest.cpp \ TxBufferTest.cpp \ - TxPublishTest.cpp \ - MessageBuilderTest.cpp \ ConnectionOptions.h \ ForkedBroker.h \ ForkedBroker.cpp \ ManagementTest.cpp \ MessageReplayTracker.cpp \ ConsoleTest.cpp \ - QueueEvents.cpp \ ProxyTest.cpp \ RetryList.cpp \ FrameDecoder.cpp \ - ReplicationTest.cpp \ ClientMessageTest.cpp \ PollableCondition.cpp \ Variant.cpp \ diff --git a/qpid/cpp/src/tests/MessageBuilderTest.cpp b/qpid/cpp/src/tests/MessageBuilderTest.cpp deleted file mode 100644 index 9adb133d40..0000000000 --- a/qpid/cpp/src/tests/MessageBuilderTest.cpp +++ /dev/null @@ -1,190 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/broker/Message.h" -#include "qpid/broker/MessageBuilder.h" -#include "qpid/broker/NullMessageStore.h" -#include "qpid/framing/frame_functors.h" -#include "qpid/framing/MessageTransferBody.h" -#include "qpid/framing/TypeFilter.h" -#include "unit_test.h" -#include <list> - -using namespace qpid::broker; -using namespace qpid::framing; -using namespace qpid::sys; - -namespace qpid { -namespace tests { - -class MockMessageStore : public NullMessageStore -{ - enum Op {STAGE=1, APPEND=2}; - - uint64_t id; - boost::intrusive_ptr<PersistableMessage> expectedMsg; - std::string expectedData; - std::list<Op> ops; - - void checkExpectation(Op actual) - { - BOOST_CHECK_EQUAL(ops.front(), actual); - ops.pop_front(); - } - - public: - MockMessageStore() : id(0), expectedMsg(0) {} - - void expectStage(PersistableMessage& msg) - { - expectedMsg = &msg; - ops.push_back(STAGE); - } - - void expectAppendContent(PersistableMessage& msg, const std::string& data) - { - expectedMsg = &msg; - expectedData = data; - ops.push_back(APPEND); - } - - void stage(const boost::intrusive_ptr<PersistableMessage>& msg) - { - checkExpectation(STAGE); - BOOST_CHECK_EQUAL(expectedMsg, msg); - msg->setPersistenceId(++id); - } - - void appendContent(const boost::intrusive_ptr<const PersistableMessage>& msg, - const std::string& data) - { - checkExpectation(APPEND); - BOOST_CHECK_EQUAL(boost::static_pointer_cast<const PersistableMessage>(expectedMsg), msg); - BOOST_CHECK_EQUAL(expectedData, data); - } - - bool expectationsMet() - { - return ops.empty(); - } - - //don't treat this store as a null impl - bool isNull() const - { - return false; - } - -}; - -QPID_AUTO_TEST_SUITE(MessageBuilderTestSuite) - -QPID_AUTO_TEST_CASE(testHeaderOnly) -{ - MessageBuilder builder(0); - builder.start(SequenceNumber()); - - std::string exchange("builder-exchange"); - std::string key("builder-exchange"); - - AMQFrame method((MessageTransferBody(ProtocolVersion(), exchange, 0, 0))); - AMQFrame header((AMQHeaderBody())); - - header.castBody<AMQHeaderBody>()->get<MessageProperties>(true)->setContentLength(0); - header.castBody<AMQHeaderBody>()->get<DeliveryProperties>(true)->setRoutingKey(key); - - builder.handle(method); - builder.handle(header); - - BOOST_CHECK(builder.getMessage()); - BOOST_CHECK_EQUAL(exchange, builder.getMessage()->getExchangeName()); - BOOST_CHECK_EQUAL(key, builder.getMessage()->getRoutingKey()); - BOOST_CHECK(builder.getMessage()->getFrames().isComplete()); -} - -QPID_AUTO_TEST_CASE(test1ContentFrame) -{ - MessageBuilder builder(0); - builder.start(SequenceNumber()); - - std::string data("abcdefg"); - std::string exchange("builder-exchange"); - std::string key("builder-exchange"); - - AMQFrame method((MessageTransferBody(ProtocolVersion(), exchange, 0, 0))); - AMQFrame header((AMQHeaderBody())); - AMQFrame content((AMQContentBody(data))); - method.setEof(false); - header.setBof(false); - header.setEof(false); - content.setBof(false); - - header.castBody<AMQHeaderBody>()->get<MessageProperties>(true)->setContentLength(data.size()); - header.castBody<AMQHeaderBody>()->get<DeliveryProperties>(true)->setRoutingKey(key); - - builder.handle(method); - BOOST_CHECK(builder.getMessage()); - BOOST_CHECK(!builder.getMessage()->getFrames().isComplete()); - - builder.handle(header); - BOOST_CHECK(builder.getMessage()); - BOOST_CHECK(!builder.getMessage()->getFrames().isComplete()); - - builder.handle(content); - BOOST_CHECK(builder.getMessage()); - BOOST_CHECK(builder.getMessage()->getFrames().isComplete()); -} - -QPID_AUTO_TEST_CASE(test2ContentFrames) -{ - MessageBuilder builder(0); - builder.start(SequenceNumber()); - - std::string data1("abcdefg"); - std::string data2("hijklmn"); - std::string exchange("builder-exchange"); - std::string key("builder-exchange"); - - AMQFrame method((MessageTransferBody(ProtocolVersion(), exchange, 0, 0))); - AMQFrame header((AMQHeaderBody())); - AMQFrame content1((AMQContentBody(data1))); - AMQFrame content2((AMQContentBody(data2))); - method.setEof(false); - header.setBof(false); - header.setEof(false); - content1.setBof(false); - content1.setEof(false); - content2.setBof(false); - - header.castBody<AMQHeaderBody>()->get<MessageProperties>(true)->setContentLength(data1.size() + data2.size()); - header.castBody<AMQHeaderBody>()->get<DeliveryProperties>(true)->setRoutingKey(key); - - builder.handle(method); - builder.handle(header); - builder.handle(content1); - BOOST_CHECK(builder.getMessage()); - BOOST_CHECK(!builder.getMessage()->getFrames().isComplete()); - - builder.handle(content2); - BOOST_CHECK(builder.getMessage()); - BOOST_CHECK(builder.getMessage()->getFrames().isComplete()); -} -QPID_AUTO_TEST_SUITE_END() - -}} // namespace qpid::tests diff --git a/qpid/cpp/src/tests/MessageTest.cpp b/qpid/cpp/src/tests/MessageTest.cpp index 3a3ed061f9..fe670a274e 100644 --- a/qpid/cpp/src/tests/MessageTest.cpp +++ b/qpid/cpp/src/tests/MessageTest.cpp @@ -24,6 +24,7 @@ #include "qpid/framing/MessageTransferBody.h" #include "qpid/framing/FieldValue.h" #include "qpid/framing/Uuid.h" +#include "MessageUtils.h" #include "unit_test.h" @@ -43,49 +44,29 @@ QPID_AUTO_TEST_CASE(testEncodeDecode) { string exchange = "MyExchange"; string routingKey = "MyRoutingKey"; + uint64_t ttl(60); Uuid messageId(true); - string data1("abcdefg"); - string data2("hijklmn"); + string data("abcdefghijklmn"); - boost::intrusive_ptr<Message> msg(new Message()); + qpid::types::Variant::Map properties; + properties["routing-key"] = routingKey; + properties["ttl"] = ttl; + properties["durable"] = true; + properties["message-id"] = qpid::types::Uuid(messageId.data()); + properties["abc"] = "xyz"; + Message msg = MessageUtils::createMessage(properties, data); - AMQFrame method((MessageTransferBody(ProtocolVersion(), exchange, 0, 0))); - AMQFrame header((AMQHeaderBody())); - AMQFrame content1((AMQContentBody(data1))); - AMQFrame content2((AMQContentBody(data2))); + std::string buffer; + encode(msg, buffer); + msg = Message(); + decode(buffer, msg); - msg->getFrames().append(method); - msg->getFrames().append(header); - msg->getFrames().append(content1); - msg->getFrames().append(content2); - - MessageProperties* mProps = msg->getFrames().getHeaders()->get<MessageProperties>(true); - mProps->setContentLength(data1.size() + data2.size()); - mProps->setMessageId(messageId); - FieldTable applicationHeaders; - applicationHeaders.setString("abc", "xyz"); - mProps->setApplicationHeaders(applicationHeaders); - DeliveryProperties* dProps = msg->getFrames().getHeaders()->get<DeliveryProperties>(true); - dProps->setRoutingKey(routingKey); - dProps->setDeliveryMode(PERSISTENT); - BOOST_CHECK(msg->isPersistent()); - - std::vector<char> buff(msg->encodedSize()); - Buffer wbuffer(&buff[0], msg->encodedSize()); - msg->encode(wbuffer); - - Buffer rbuffer(&buff[0], msg->encodedSize()); - msg = new Message(); - msg->decodeHeader(rbuffer); - msg->decodeContent(rbuffer); - BOOST_CHECK_EQUAL(exchange, msg->getExchangeName()); - BOOST_CHECK_EQUAL(routingKey, msg->getRoutingKey()); - BOOST_CHECK_EQUAL((uint64_t) data1.size() + data2.size(), msg->contentSize()); - BOOST_CHECK_EQUAL((uint64_t) data1.size() + data2.size(), msg->getProperties<MessageProperties>()->getContentLength()); - BOOST_CHECK_EQUAL(messageId, msg->getProperties<MessageProperties>()->getMessageId()); - BOOST_CHECK_EQUAL(string("xyz"), msg->getProperties<MessageProperties>()->getApplicationHeaders().getAsString("abc")); - BOOST_CHECK_EQUAL((uint8_t) PERSISTENT, msg->getProperties<DeliveryProperties>()->getDeliveryMode()); - BOOST_CHECK(msg->isPersistent()); + BOOST_CHECK_EQUAL(routingKey, msg.getRoutingKey()); + BOOST_CHECK_EQUAL((uint64_t) data.size(), msg.getContentSize()); + BOOST_CHECK_EQUAL(data, msg.getContent()); + //BOOST_CHECK_EQUAL(messageId, msg->getProperties<MessageProperties>()->getMessageId()); + BOOST_CHECK_EQUAL(string("xyz"), msg.getPropertyAsString("abc")); + BOOST_CHECK(msg.isPersistent()); } QPID_AUTO_TEST_SUITE_END() diff --git a/qpid/cpp/src/tests/MessageUtils.h b/qpid/cpp/src/tests/MessageUtils.h index 991e2a2714..c2eabd804d 100644 --- a/qpid/cpp/src/tests/MessageUtils.h +++ b/qpid/cpp/src/tests/MessageUtils.h @@ -20,9 +20,11 @@ */ #include "qpid/broker/Message.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/framing/AMQFrame.h" #include "qpid/framing/MessageTransferBody.h" #include "qpid/framing/Uuid.h" +#include "qpid/types/Variant.h" using namespace qpid; using namespace broker; @@ -33,11 +35,46 @@ namespace tests { struct MessageUtils { - static boost::intrusive_ptr<Message> createMessage(const std::string& exchange="", const std::string& routingKey="", - const bool durable = false, const Uuid& messageId=Uuid(true), - uint64_t contentSize = 0) + static Message createMessage(const qpid::types::Variant::Map& properties, const std::string& content="", const std::string& destination = "") { - boost::intrusive_ptr<broker::Message> msg(new broker::Message()); + boost::intrusive_ptr<broker::amqp_0_10::MessageTransfer> msg(new broker::amqp_0_10::MessageTransfer()); + + AMQFrame method(( MessageTransferBody(ProtocolVersion(), destination, 0, 0))); + AMQFrame header((AMQHeaderBody())); + + msg->getFrames().append(method); + msg->getFrames().append(header); + if (content.size()) { + msg->getFrames().getHeaders()->get<MessageProperties>(true)->setContentLength(content.size()); + AMQFrame data((AMQContentBody(content))); + msg->getFrames().append(data); + } + for (qpid::types::Variant::Map::const_iterator i = properties.begin(); i != properties.end(); ++i) { + if (i->first == "routing-key" && !i->second.isVoid()) { + msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setRoutingKey(i->second); + } else if (i->first == "message-id" && !i->second.isVoid()) { + qpid::types::Uuid id = i->second; + qpid::framing::Uuid id2(id.data()); + msg->getFrames().getHeaders()->get<MessageProperties>(true)->setMessageId(id2); + } else if (i->first == "ttl" && !i->second.isVoid()) { + msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setTtl(i->second); + } else if (i->first == "priority" && !i->second.isVoid()) { + msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setPriority(i->second); + } else if (i->first == "durable" && !i->second.isVoid()) { + msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setDeliveryMode(i->second.asBool() ? 2 : 1); + } else { + msg->getFrames().getHeaders()->get<MessageProperties>(true)->getApplicationHeaders().setString(i->first, i->second); + } + } + return Message(msg, msg); + } + + + static Message createMessage(const std::string& exchange="", const std::string& routingKey="", + uint64_t ttl = 0, bool durable = false, const Uuid& messageId=Uuid(true), + const std::string& content="") + { + boost::intrusive_ptr<broker::amqp_0_10::MessageTransfer> msg(new broker::amqp_0_10::MessageTransfer()); AMQFrame method(( MessageTransferBody(ProtocolVersion(), exchange, 0, 0))); AMQFrame header((AMQHeaderBody())); @@ -45,18 +82,18 @@ struct MessageUtils msg->getFrames().append(method); msg->getFrames().append(header); MessageProperties* props = msg->getFrames().getHeaders()->get<MessageProperties>(true); - props->setContentLength(contentSize); + props->setContentLength(content.size()); props->setMessageId(messageId); msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setRoutingKey(routingKey); if (durable) msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setDeliveryMode(2); - return msg; - } - - static void addContent(boost::intrusive_ptr<Message> msg, const std::string& data) - { - AMQFrame content((AMQContentBody(data))); - msg->getFrames().append(content); + if (ttl) + msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setTtl(ttl); + if (content.size()) { + AMQFrame data((AMQContentBody(content))); + msg->getFrames().append(data); + } + return Message(msg, msg); } }; diff --git a/qpid/cpp/src/tests/QueueDepth.cpp b/qpid/cpp/src/tests/QueueDepth.cpp new file mode 100644 index 0000000000..73556141ca --- /dev/null +++ b/qpid/cpp/src/tests/QueueDepth.cpp @@ -0,0 +1,105 @@ +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/broker/QueueDepth.h" + +#include "unit_test.h" + +namespace qpid { +namespace tests { + +QPID_AUTO_TEST_SUITE(QueueDepthTestSuite) + +using namespace qpid::broker; + +QPID_AUTO_TEST_CASE(testCompare) +{ + QueueDepth a(0, 0); + QueueDepth b(1, 1); + QueueDepth c(2, 2); + QueueDepth d(1, 1); + + BOOST_CHECK(a < b); + BOOST_CHECK(b < c); + BOOST_CHECK(a < c); + + BOOST_CHECK(b > a); + BOOST_CHECK(c > b); + BOOST_CHECK(c > a); + + BOOST_CHECK(b == d); + BOOST_CHECK(d == b); + BOOST_CHECK(a != b); + BOOST_CHECK(b != a); + + QueueDepth e; e.setCount(1); + QueueDepth f; f.setCount(2); + BOOST_CHECK(e < f); + BOOST_CHECK(f > e); + + QueueDepth g; g.setSize(1); + QueueDepth h; h.setSize(2); + BOOST_CHECK(g < h); + BOOST_CHECK(h > g); +} + +QPID_AUTO_TEST_CASE(testIncrement) +{ + QueueDepth a(5, 10); + QueueDepth b(3, 6); + QueueDepth c(8, 16); + a += b; + BOOST_CHECK(a == c); + BOOST_CHECK_EQUAL(8, a.getCount()); + BOOST_CHECK_EQUAL(16, a.getSize()); +} + +QPID_AUTO_TEST_CASE(testDecrement) +{ + QueueDepth a(5, 10); + QueueDepth b(3, 6); + QueueDepth c(2, 4); + a -= b; + BOOST_CHECK(a == c); + BOOST_CHECK_EQUAL(2, a.getCount()); + BOOST_CHECK_EQUAL(4, a.getSize()); +} + +QPID_AUTO_TEST_CASE(testAddition) +{ + QueueDepth a(5, 10); + QueueDepth b(3, 6); + + QueueDepth c = a + b; + BOOST_CHECK_EQUAL(8, c.getCount()); + BOOST_CHECK_EQUAL(16, c.getSize()); +} + +QPID_AUTO_TEST_CASE(testSubtraction) +{ + QueueDepth a(5, 10); + QueueDepth b(3, 6); + + QueueDepth c = a - b; + BOOST_CHECK_EQUAL(2, c.getCount()); + BOOST_CHECK_EQUAL(4, c.getSize()); +} + +QPID_AUTO_TEST_SUITE_END() + +}} // namespace qpid::tests diff --git a/qpid/cpp/src/tests/QueueEvents.cpp b/qpid/cpp/src/tests/QueueEvents.cpp deleted file mode 100644 index cea8bbf0db..0000000000 --- a/qpid/cpp/src/tests/QueueEvents.cpp +++ /dev/null @@ -1,238 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "MessageUtils.h" -#include "unit_test.h" -#include "BrokerFixture.h" -#include "qpid/broker/Message.h" -#include "qpid/broker/Queue.h" -#include "qpid/broker/QueueEvents.h" -#include "qpid/client/QueueOptions.h" -#include "qpid/framing/SequenceNumber.h" -#include "qpid/sys/Dispatcher.h" -#include <boost/bind.hpp> -#include <boost/format.hpp> - -namespace qpid { -namespace tests { - -QPID_AUTO_TEST_SUITE(QueueEventsSuite) - -using namespace qpid::client; -using namespace qpid::broker; -using namespace qpid::sys; -using qpid::framing::SequenceNumber; - -struct EventChecker -{ - typedef std::deque<QueueEvents::Event> Events; - - Events events; - boost::shared_ptr<Poller> poller; - - void handle(QueueEvents::Event e) - { - if (events.empty()) { - BOOST_FAIL("Unexpected event received"); - } else { - BOOST_CHECK_EQUAL(events.front().type, e.type); - BOOST_CHECK_EQUAL(events.front().msg.queue, e.msg.queue); - BOOST_CHECK_EQUAL(events.front().msg.payload, e.msg.payload); - BOOST_CHECK_EQUAL(events.front().msg.position, e.msg.position); - events.pop_front(); - } - if (events.empty() && poller) poller->shutdown(); - } - - void expect(QueueEvents::Event e) - { - events.push_back(e); - } -}; - -QPID_AUTO_TEST_CASE(testBasicEventProcessing) -{ - boost::shared_ptr<Poller> poller(new Poller()); - sys::Dispatcher dispatcher(poller); - Thread dispatchThread(dispatcher); - QueueEvents events(poller); - EventChecker listener; - listener.poller = poller; - events.registerListener("dummy", boost::bind(&EventChecker::handle, &listener, _1)); - //signal occurence of some events: - Queue queue("queue1"); - SequenceNumber id; - QueuedMessage event1(&queue, MessageUtils::createMessage(), id); - QueuedMessage event2(&queue, MessageUtils::createMessage(), ++id); - - //define events expected by listener: - listener.expect(QueueEvents::Event(QueueEvents::ENQUEUE, event1)); - listener.expect(QueueEvents::Event(QueueEvents::ENQUEUE, event2)); - listener.expect(QueueEvents::Event(QueueEvents::DEQUEUE, event1)); - - events.enqueued(event1); - events.enqueued(event2); - events.dequeued(event1); - - dispatchThread.join(); - events.shutdown(); - events.unregisterListener("dummy"); -} - - -struct EventRecorder -{ - struct EventRecord - { - QueueEvents::EventType type; - std::string queue; - std::string content; - SequenceNumber position; - }; - - typedef std::deque<EventRecord> Events; - - Events events; - - void handle(QueueEvents::Event event) - { - EventRecord record; - record.type = event.type; - record.queue = event.msg.queue->getName(); - event.msg.payload->getFrames().getContent(record.content); - record.position = event.msg.position; - events.push_back(record); - } - - void check(QueueEvents::EventType type, const std::string& queue, const std::string& content, const SequenceNumber& position) - { - if (events.empty()) { - BOOST_FAIL("Missed event"); - } else { - BOOST_CHECK_EQUAL(events.front().type, type); - BOOST_CHECK_EQUAL(events.front().queue, queue); - BOOST_CHECK_EQUAL(events.front().content, content); - BOOST_CHECK_EQUAL(events.front().position, position); - events.pop_front(); - } - } - void checkEnqueue(const std::string& queue, const std::string& data, const SequenceNumber& position) - { - check(QueueEvents::ENQUEUE, queue, data, position); - } - - void checkDequeue(const std::string& queue, const std::string& data, const SequenceNumber& position) - { - check(QueueEvents::DEQUEUE, queue, data, position); - } -}; - -QPID_AUTO_TEST_CASE(testSystemLevelEventProcessing) -{ - SessionFixture fixture; - //register dummy event listener to broker - EventRecorder listener; - fixture.broker->getQueueEvents().registerListener("recorder", boost::bind(&EventRecorder::handle, &listener, _1)); - - //declare queue with event options specified - QueueOptions options; - options.enableQueueEvents(false); - std::string q("queue-events-test"); - fixture.session.queueDeclare(arg::queue=q, arg::arguments=options); - //send and consume some messages - LocalQueue incoming; - Subscription sub = fixture.subs.subscribe(incoming, q); - for (int i = 0; i < 5; i++) { - fixture.session.messageTransfer(arg::content=client::Message((boost::format("%1%_%2%") % "Message" % (i+1)).str(), q)); - } - for (int i = 0; i < 3; i++) { - BOOST_CHECK_EQUAL(incoming.pop().getData(), (boost::format("%1%_%2%") % "Message" % (i+1)).str()); - } - for (int i = 5; i < 10; i++) { - fixture.session.messageTransfer(arg::content=client::Message((boost::format("%1%_%2%") % "Message" % (i+1)).str(), q)); - } - for (int i = 3; i < 10; i++) { - BOOST_CHECK_EQUAL(incoming.pop().getData(), (boost::format("%1%_%2%") % "Message" % (i+1)).str()); - } - fixture.connection.close(); - fixture.broker->getQueueEvents().shutdown(); - - //check listener was notified of all events, and in correct order - SequenceNumber enqueueId(1); - SequenceNumber dequeueId(1); - for (int i = 0; i < 5; i++) { - listener.checkEnqueue(q, (boost::format("%1%_%2%") % "Message" % (i+1)).str(), enqueueId++); - } - for (int i = 0; i < 3; i++) { - listener.checkDequeue(q, (boost::format("%1%_%2%") % "Message" % (i+1)).str(), dequeueId++); - } - for (int i = 5; i < 10; i++) { - listener.checkEnqueue(q, (boost::format("%1%_%2%") % "Message" % (i+1)).str(), enqueueId++); - } - for (int i = 3; i < 10; i++) { - listener.checkDequeue(q, (boost::format("%1%_%2%") % "Message" % (i+1)).str(), dequeueId++); - } -} - -QPID_AUTO_TEST_CASE(testSystemLevelEventProcessing_enqueuesOnly) -{ - SessionFixture fixture; - //register dummy event listener to broker - EventRecorder listener; - fixture.broker->getQueueEvents().registerListener("recorder", boost::bind(&EventRecorder::handle, &listener, _1)); - - //declare queue with event options specified - QueueOptions options; - options.enableQueueEvents(true); - std::string q("queue-events-test"); - fixture.session.queueDeclare(arg::queue=q, arg::arguments=options); - //send and consume some messages - LocalQueue incoming; - Subscription sub = fixture.subs.subscribe(incoming, q); - for (int i = 0; i < 5; i++) { - fixture.session.messageTransfer(arg::content=client::Message((boost::format("%1%_%2%") % "Message" % (i+1)).str(), q)); - } - for (int i = 0; i < 3; i++) { - BOOST_CHECK_EQUAL(incoming.pop().getData(), (boost::format("%1%_%2%") % "Message" % (i+1)).str()); - } - for (int i = 5; i < 10; i++) { - fixture.session.messageTransfer(arg::content=client::Message((boost::format("%1%_%2%") % "Message" % (i+1)).str(), q)); - } - for (int i = 3; i < 10; i++) { - BOOST_CHECK_EQUAL(incoming.pop().getData(), (boost::format("%1%_%2%") % "Message" % (i+1)).str()); - } - fixture.connection.close(); - fixture.broker->getQueueEvents().shutdown(); - - //check listener was notified of all events, and in correct order - SequenceNumber enqueueId(1); - SequenceNumber dequeueId(1); - for (int i = 0; i < 5; i++) { - listener.checkEnqueue(q, (boost::format("%1%_%2%") % "Message" % (i+1)).str(), enqueueId++); - } - for (int i = 5; i < 10; i++) { - listener.checkEnqueue(q, (boost::format("%1%_%2%") % "Message" % (i+1)).str(), enqueueId++); - } -} - -QPID_AUTO_TEST_SUITE_END() - -}} // namespace qpid::tests diff --git a/qpid/cpp/src/tests/QueueFlowLimitTest.cpp b/qpid/cpp/src/tests/QueueFlowLimitTest.cpp index bd868398f8..d305ca452b 100644 --- a/qpid/cpp/src/tests/QueueFlowLimitTest.cpp +++ b/qpid/cpp/src/tests/QueueFlowLimitTest.cpp @@ -23,8 +23,8 @@ #include "unit_test.h" #include "test_tools.h" -#include "qpid/broker/QueuePolicy.h" #include "qpid/broker/QueueFlowLimit.h" +#include "qpid/broker/QueueSettings.h" #include "qpid/sys/Time.h" #include "qpid/framing/reply_exceptions.h" #include "qpid/framing/FieldValue.h" @@ -66,21 +66,19 @@ public: return new TestFlow(flowStopCount, flowResumeCount, flowStopSize, flowResumeSize); } - static QueueFlowLimit *getQueueFlowLimit(const qpid::framing::FieldTable& settings) + static QueueFlowLimit *getQueueFlowLimit(const qpid::framing::FieldTable& arguments) { + QueueSettings settings; + settings.populate(arguments, settings.storeSettings); return QueueFlowLimit::createLimit(0, settings); } }; - - -QueuedMessage createMessage(uint32_t size) +Message createMessage(uint32_t size) { static uint32_t seqNum; - QueuedMessage msg; - msg.payload = MessageUtils::createMessage(); - msg.position = ++seqNum; - MessageUtils::addContent(msg.payload, std::string (size, 'x')); + Message msg = MessageUtils::createMessage(qpid::types::Variant::Map(), std::string (size, 'x')); + msg.setSequence(++seqNum); return msg; } } @@ -100,7 +98,7 @@ QPID_AUTO_TEST_CASE(testFlowCount) BOOST_CHECK(!flow->isFlowControlActive()); BOOST_CHECK(flow->monitorFlowControl()); - std::deque<QueuedMessage> msgs; + std::deque<Message> msgs; for (size_t i = 0; i < 6; i++) { msgs.push_back(createMessage(10)); flow->enqueued(msgs.back()); @@ -135,7 +133,6 @@ QPID_AUTO_TEST_CASE(testFlowCount) BOOST_CHECK(!flow->isFlowControlActive()); // 4 on queue, OFF } - QPID_AUTO_TEST_CASE(testFlowSize) { FieldTable args; @@ -151,7 +148,7 @@ QPID_AUTO_TEST_CASE(testFlowSize) BOOST_CHECK(!flow->isFlowControlActive()); BOOST_CHECK(flow->monitorFlowControl()); - std::deque<QueuedMessage> msgs; + std::deque<Message> msgs; for (size_t i = 0; i < 6; i++) { msgs.push_back(createMessage(10)); flow->enqueued(msgs.back()); @@ -161,14 +158,14 @@ QPID_AUTO_TEST_CASE(testFlowSize) BOOST_CHECK_EQUAL(6u, flow->getFlowCount()); BOOST_CHECK_EQUAL(60u, flow->getFlowSize()); - QueuedMessage msg_9 = createMessage(9); + Message msg_9 = createMessage(9); flow->enqueued(msg_9); BOOST_CHECK(!flow->isFlowControlActive()); // 69 on queue - QueuedMessage tinyMsg_1 = createMessage(1); + Message tinyMsg_1 = createMessage(1); flow->enqueued(tinyMsg_1); BOOST_CHECK(!flow->isFlowControlActive()); // 70 on queue - QueuedMessage tinyMsg_2 = createMessage(1); + Message tinyMsg_2 = createMessage(1); flow->enqueued(tinyMsg_2); BOOST_CHECK(flow->isFlowControlActive()); // 71 on queue, ON msgs.push_back(createMessage(10)); @@ -233,12 +230,12 @@ QPID_AUTO_TEST_CASE(testFlowCombo) args.setUInt64(QueueFlowLimit::flowStopSizeKey, 200); args.setUInt64(QueueFlowLimit::flowResumeSizeKey, 100); - std::deque<QueuedMessage> msgs_1; - std::deque<QueuedMessage> msgs_10; - std::deque<QueuedMessage> msgs_50; - std::deque<QueuedMessage> msgs_100; + std::deque<Message> msgs_1; + std::deque<Message> msgs_10; + std::deque<Message> msgs_50; + std::deque<Message> msgs_100; - QueuedMessage msg; + Message msg; std::auto_ptr<TestFlow> flow(TestFlow::createTestFlow(args)); BOOST_CHECK(!flow->isFlowControlActive()); // count:0 size:0 @@ -458,7 +455,6 @@ QPID_AUTO_TEST_CASE(testFlowDisable) } } - QPID_AUTO_TEST_SUITE_END() }} // namespace qpid::tests diff --git a/qpid/cpp/src/tests/QueuePolicyTest.cpp b/qpid/cpp/src/tests/QueuePolicyTest.cpp index f735e09449..00e964602a 100644 --- a/qpid/cpp/src/tests/QueuePolicyTest.cpp +++ b/qpid/cpp/src/tests/QueuePolicyTest.cpp @@ -22,12 +22,10 @@ #include "unit_test.h" #include "test_tools.h" -#include "qpid/broker/QueuePolicy.h" #include "qpid/broker/QueueFlowLimit.h" #include "qpid/client/QueueOptions.h" #include "qpid/sys/Time.h" #include "qpid/framing/reply_exceptions.h" -#include "MessageUtils.h" #include "BrokerFixture.h" using namespace qpid::broker; @@ -39,118 +37,10 @@ namespace tests { QPID_AUTO_TEST_SUITE(QueuePolicyTestSuite) -namespace { -QueuedMessage createMessage(uint32_t size) -{ - QueuedMessage msg; - msg.payload = MessageUtils::createMessage(); - MessageUtils::addContent(msg.payload, std::string (size, 'x')); - return msg; -} -} - -QPID_AUTO_TEST_CASE(testCount) -{ - std::auto_ptr<QueuePolicy> policy(QueuePolicy::createQueuePolicy("test", 5, 0)); - BOOST_CHECK_EQUAL((uint64_t) 0, policy->getMaxSize()); - BOOST_CHECK_EQUAL((uint32_t) 5, policy->getMaxCount()); - - QueuedMessage msg = createMessage(10); - for (size_t i = 0; i < 5; i++) { - policy->tryEnqueue(msg.payload); - } - try { - policy->tryEnqueue(msg.payload); - BOOST_FAIL("Policy did not fail on enqueuing sixth message"); - } catch (const ResourceLimitExceededException&) {} - - policy->dequeued(msg); - policy->tryEnqueue(msg.payload); - - try { - policy->tryEnqueue(msg.payload); - BOOST_FAIL("Policy did not fail on enqueuing sixth message (after dequeue)"); - } catch (const ResourceLimitExceededException&) {} -} - -QPID_AUTO_TEST_CASE(testSize) -{ - std::auto_ptr<QueuePolicy> policy(QueuePolicy::createQueuePolicy("test", 0, 50)); - QueuedMessage msg = createMessage(10); - - for (size_t i = 0; i < 5; i++) { - policy->tryEnqueue(msg.payload); - } - try { - policy->tryEnqueue(msg.payload); - BOOST_FAIL("Policy did not fail on aggregate size exceeding 50. " << *policy); - } catch (const ResourceLimitExceededException&) {} - - policy->dequeued(msg); - policy->tryEnqueue(msg.payload); - - try { - policy->tryEnqueue(msg.payload); - BOOST_FAIL("Policy did not fail on aggregate size exceeding 50 (after dequeue). " << *policy); - } catch (const ResourceLimitExceededException&) {} -} - -QPID_AUTO_TEST_CASE(testBoth) -{ - std::auto_ptr<QueuePolicy> policy(QueuePolicy::createQueuePolicy("test", 5, 50)); - try { - QueuedMessage msg = createMessage(51); - policy->tryEnqueue(msg.payload); - BOOST_FAIL("Policy did not fail on single message exceeding 50. " << *policy); - } catch (const ResourceLimitExceededException&) {} - - std::vector<QueuedMessage> messages; - messages.push_back(createMessage(15)); - messages.push_back(createMessage(10)); - messages.push_back(createMessage(11)); - messages.push_back(createMessage(2)); - messages.push_back(createMessage(7)); - for (size_t i = 0; i < messages.size(); i++) { - policy->tryEnqueue(messages[i].payload); - } - //size = 45 at this point, count = 5 - try { - QueuedMessage msg = createMessage(5); - policy->tryEnqueue(msg.payload); - BOOST_FAIL("Policy did not fail on count exceeding 6. " << *policy); - } catch (const ResourceLimitExceededException&) {} - try { - QueuedMessage msg = createMessage(10); - policy->tryEnqueue(msg.payload); - BOOST_FAIL("Policy did not fail on aggregate size exceeding 50. " << *policy); - } catch (const ResourceLimitExceededException&) {} - - - policy->dequeued(messages[0]); - try { - QueuedMessage msg = createMessage(20); - policy->tryEnqueue(msg.payload); - } catch (const ResourceLimitExceededException&) { - BOOST_FAIL("Policy failed incorrectly after dequeue. " << *policy); - } -} - -QPID_AUTO_TEST_CASE(testSettings) -{ - //test reading and writing the policy from/to field table - std::auto_ptr<QueuePolicy> a(QueuePolicy::createQueuePolicy("test", 101, 303)); - FieldTable settings; - a->update(settings); - std::auto_ptr<QueuePolicy> b(QueuePolicy::createQueuePolicy("test", settings)); - BOOST_CHECK_EQUAL(a->getMaxCount(), b->getMaxCount()); - BOOST_CHECK_EQUAL(a->getMaxSize(), b->getMaxSize()); -} - QPID_AUTO_TEST_CASE(testRingPolicyCount) { - FieldTable args; - std::auto_ptr<QueuePolicy> policy = QueuePolicy::createQueuePolicy("test", 5, 0, QueuePolicy::RING); - policy->update(args); + QueueOptions args; + args.setSizePolicy(RING, 0, 5); SessionFixture f; std::string q("my-ring-queue"); @@ -183,9 +73,8 @@ QPID_AUTO_TEST_CASE(testRingPolicySize) // Ring queue, 500 bytes maxSize - FieldTable args; - std::auto_ptr<QueuePolicy> policy = QueuePolicy::createQueuePolicy("test", 0, 500, QueuePolicy::RING); - policy->update(args); + QueueOptions args; + args.setSizePolicy(RING, 500, 0); SessionFixture f; std::string q("my-ring-queue"); @@ -255,9 +144,9 @@ QPID_AUTO_TEST_CASE(testRingPolicySize) QPID_AUTO_TEST_CASE(testStrictRingPolicy) { - FieldTable args; - std::auto_ptr<QueuePolicy> policy = QueuePolicy::createQueuePolicy("test", 5, 0, QueuePolicy::RING_STRICT); - policy->update(args); + QueueOptions args; + args.setSizePolicy(RING_STRICT, 0, 5); + args.setString("qpid.flow_stop_count", "0"); SessionFixture f; std::string q("my-ring-queue"); @@ -281,9 +170,8 @@ QPID_AUTO_TEST_CASE(testStrictRingPolicy) QPID_AUTO_TEST_CASE(testPolicyWithDtx) { - FieldTable args; - std::auto_ptr<QueuePolicy> policy = QueuePolicy::createQueuePolicy("test", 5, 0, QueuePolicy::REJECT); - policy->update(args); + QueueOptions args; + args.setSizePolicy(REJECT, 0, 5); SessionFixture f; std::string q("my-policy-queue"); @@ -367,9 +255,8 @@ QPID_AUTO_TEST_CASE(testFlowToDiskWithNoStore) QPID_AUTO_TEST_CASE(testPolicyFailureOnCommit) { - FieldTable args; - std::auto_ptr<QueuePolicy> policy = QueuePolicy::createQueuePolicy("test", 5, 0, QueuePolicy::REJECT); - policy->update(args); + QueueOptions args; + args.setSizePolicy(REJECT, 0, 5); SessionFixture f; std::string q("q"); diff --git a/qpid/cpp/src/tests/QueueRegistryTest.cpp b/qpid/cpp/src/tests/QueueRegistryTest.cpp index ae555539a4..364d66c525 100644 --- a/qpid/cpp/src/tests/QueueRegistryTest.cpp +++ b/qpid/cpp/src/tests/QueueRegistryTest.cpp @@ -19,6 +19,7 @@ #include "qpid/broker/QueueRegistry.h" #include "qpid/broker/Queue.h" +#include "qpid/broker/QueueSettings.h" #include "unit_test.h" #include <string> @@ -36,33 +37,23 @@ QPID_AUTO_TEST_CASE(testDeclare) QueueRegistry reg; std::pair<Queue::shared_ptr, bool> qc; - qc = reg.declare(foo, false, 0, 0); + qc = reg.declare(foo, QueueSettings()); Queue::shared_ptr q = qc.first; BOOST_CHECK(q); BOOST_CHECK(qc.second); // New queue BOOST_CHECK_EQUAL(foo, q->getName()); - qc = reg.declare(foo, false, 0, 0); + qc = reg.declare(foo, QueueSettings()); BOOST_CHECK_EQUAL(q, qc.first); BOOST_CHECK(!qc.second); - qc = reg.declare(bar, false, 0, 0); + qc = reg.declare(bar, QueueSettings()); q = qc.first; BOOST_CHECK(q); BOOST_CHECK_EQUAL(true, qc.second); BOOST_CHECK_EQUAL(bar, q->getName()); } -QPID_AUTO_TEST_CASE(testDeclareTmp) -{ - QueueRegistry reg; - std::pair<Queue::shared_ptr, bool> qc; - - qc = reg.declare(std::string(), false, 0, 0); - BOOST_CHECK(qc.second); - BOOST_CHECK_EQUAL(std::string("tmp_1"), qc.first->getName()); -} - QPID_AUTO_TEST_CASE(testFind) { std::string foo("foo"); @@ -72,8 +63,8 @@ QPID_AUTO_TEST_CASE(testFind) BOOST_CHECK(reg.find(foo) == 0); - reg.declare(foo, false, 0, 0); - reg.declare(bar, false, 0, 0); + reg.declare(foo, QueueSettings()); + reg.declare(bar, QueueSettings()); Queue::shared_ptr q = reg.find(bar); BOOST_CHECK(q); BOOST_CHECK_EQUAL(bar, q->getName()); @@ -85,7 +76,7 @@ QPID_AUTO_TEST_CASE(testDestroy) QueueRegistry reg; std::pair<Queue::shared_ptr, bool> qc; - qc = reg.declare(foo, false, 0, 0); + qc = reg.declare(foo, QueueSettings()); reg.destroy(foo); // Queue is gone from the registry. BOOST_CHECK(reg.find(foo) == 0); diff --git a/qpid/cpp/src/tests/QueueTest.cpp b/qpid/cpp/src/tests/QueueTest.cpp index aa7132828a..3dfe3863f4 100644 --- a/qpid/cpp/src/tests/QueueTest.cpp +++ b/qpid/cpp/src/tests/QueueTest.cpp @@ -38,8 +38,8 @@ #include "qpid/framing/AMQFrame.h" #include "qpid/framing/MessageTransferBody.h" #include "qpid/framing/reply_exceptions.h" -#include "qpid/broker/QueuePolicy.h" #include "qpid/broker/QueueFlowLimit.h" +#include "qpid/broker/QueueSettings.h" #include <iostream> #include <vector> @@ -56,196 +56,47 @@ using namespace qpid::sys; namespace qpid { namespace tests { - class TestConsumer : public virtual Consumer{ public: typedef boost::shared_ptr<TestConsumer> shared_ptr; - QueuedMessage last; + QueueCursor lastCursor; + Message lastMessage; bool received; - TestConsumer(std::string name="test", bool acquire = true):Consumer(name, acquire), received(false) {}; + TestConsumer(std::string name="test", bool acquire = true) : Consumer(name, acquire ? CONSUMER : BROWSER), received(false) {}; - virtual bool deliver(QueuedMessage& msg){ - last = msg; + virtual bool deliver(const QueueCursor& cursor, const Message& message){ + lastCursor = cursor; + lastMessage = message; received = true; return true; }; void notify() {} void cancel() {} - void acknowledged(const QueuedMessage&) {} + void acknowledged(const DeliveryRecord&) {} OwnershipToken* getSession() { return 0; } }; class FailOnDeliver : public Deliverable { - boost::intrusive_ptr<Message> msg; + Message msg; public: FailOnDeliver() : msg(MessageUtils::createMessage()) {} void deliverTo(const boost::shared_ptr<Queue>& queue) { throw Exception(QPID_MSG("Invalid delivery to " << queue->getName())); } - Message& getMessage() { return *(msg.get()); } + Message& getMessage() { return msg; } }; -intrusive_ptr<Message> createMessage(std::string exchange, std::string routingKey, uint64_t ttl = 0) { - intrusive_ptr<Message> msg(new Message()); - AMQFrame method((MessageTransferBody(ProtocolVersion(), exchange, 0, 0))); - AMQFrame header((AMQHeaderBody())); - msg->getFrames().append(method); - msg->getFrames().append(header); - msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setRoutingKey(routingKey); - if (ttl) msg->getFrames().getHeaders()->get<DeliveryProperties>(true)->setTtl(ttl); - return msg; -} - -intrusive_ptr<Message> contentMessage(string content) { - intrusive_ptr<Message> m(MessageUtils::createMessage()); - MessageUtils::addContent(m, content); - return m; -} - -string getContent(intrusive_ptr<Message> m) { - return m->getFrames().getContent(); -} - QPID_AUTO_TEST_SUITE(QueueTestSuite) -QPID_AUTO_TEST_CASE(testAsyncMessage) { - Queue::shared_ptr queue(new Queue("my_test_queue", true)); - intrusive_ptr<Message> received; - - TestConsumer::shared_ptr c1(new TestConsumer()); - queue->consume(c1); - - - //Test basic delivery: - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - msg1->enqueueAsync(queue, 0);//this is done on enqueue which is not called from process - queue->process(msg1); - sleep(2); - - BOOST_CHECK(!c1->received); - msg1->enqueueComplete(); - - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg1.get(), received.get()); -} - - -QPID_AUTO_TEST_CASE(testAsyncMessageCount){ - Queue::shared_ptr queue(new Queue("my_test_queue", true)); - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - msg1->enqueueAsync(queue, 0);//this is done on enqueue which is not called from process - - queue->process(msg1); - sleep(2); - uint32_t compval=0; - BOOST_CHECK_EQUAL(compval, queue->getEnqueueCompleteMessageCount()); - msg1->enqueueComplete(); - compval=1; - BOOST_CHECK_EQUAL(compval, queue->getEnqueueCompleteMessageCount()); - BOOST_CHECK_EQUAL(compval, queue->getMessageCount()); -} - -QPID_AUTO_TEST_CASE(testConsumers){ - Queue::shared_ptr queue(new Queue("my_queue", true)); - - //Test adding consumers: - TestConsumer::shared_ptr c1(new TestConsumer()); - TestConsumer::shared_ptr c2(new TestConsumer()); - queue->consume(c1); - queue->consume(c2); - - BOOST_CHECK_EQUAL(uint32_t(2), queue->getConsumerCount()); - - //Test basic delivery: - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "B"); - intrusive_ptr<Message> msg3 = createMessage("e", "C"); - - queue->deliver(msg1); - BOOST_CHECK(queue->dispatch(c1)); - BOOST_CHECK_EQUAL(msg1.get(), c1->last.payload.get()); - - queue->deliver(msg2); - BOOST_CHECK(queue->dispatch(c2)); - BOOST_CHECK_EQUAL(msg2.get(), c2->last.payload.get()); - - c1->received = false; - queue->deliver(msg3); - BOOST_CHECK(queue->dispatch(c1)); - BOOST_CHECK_EQUAL(msg3.get(), c1->last.payload.get()); - - //Test cancellation: - queue->cancel(c1); - BOOST_CHECK_EQUAL(uint32_t(1), queue->getConsumerCount()); - queue->cancel(c2); - BOOST_CHECK_EQUAL(uint32_t(0), queue->getConsumerCount()); -} - -QPID_AUTO_TEST_CASE(testRegistry){ - //Test use of queues in registry: - QueueRegistry registry; - registry.declare("queue1", true, true); - registry.declare("queue2", true, true); - registry.declare("queue3", true, true); - - BOOST_CHECK(registry.find("queue1")); - BOOST_CHECK(registry.find("queue2")); - BOOST_CHECK(registry.find("queue3")); - - registry.destroy("queue1"); - registry.destroy("queue2"); - registry.destroy("queue3"); - - BOOST_CHECK(!registry.find("queue1")); - BOOST_CHECK(!registry.find("queue2")); - BOOST_CHECK(!registry.find("queue3")); -} - -QPID_AUTO_TEST_CASE(testDequeue){ - Queue::shared_ptr queue(new Queue("my_queue", true)); - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "B"); - intrusive_ptr<Message> msg3 = createMessage("e", "C"); - intrusive_ptr<Message> received; - - queue->deliver(msg1); - queue->deliver(msg2); - queue->deliver(msg3); - - BOOST_CHECK_EQUAL(uint32_t(3), queue->getMessageCount()); - - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg1.get(), received.get()); - BOOST_CHECK_EQUAL(uint32_t(2), queue->getMessageCount()); - - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg2.get(), received.get()); - BOOST_CHECK_EQUAL(uint32_t(1), queue->getMessageCount()); - - TestConsumer::shared_ptr consumer(new TestConsumer()); - queue->consume(consumer); - queue->dispatch(consumer); - if (!consumer->received) - sleep(2); - - BOOST_CHECK_EQUAL(msg3.get(), consumer->last.payload.get()); - BOOST_CHECK_EQUAL(uint32_t(0), queue->getMessageCount()); - - received = queue->get().payload; - BOOST_CHECK(!received); - BOOST_CHECK_EQUAL(uint32_t(0), queue->getMessageCount()); - -} - QPID_AUTO_TEST_CASE(testBound){ //test the recording of bindings, and use of those to allow a queue to be unbound string key("my-key"); FieldTable args; - Queue::shared_ptr queue(new Queue("my-queue", true)); + Queue::shared_ptr queue(new Queue("my-queue")); ExchangeRegistry exchanges; //establish bindings from exchange->queue and notify the queue as it is bound: Exchange::shared_ptr exchange1 = exchanges.declare("my-exchange-1", "direct").first; @@ -273,423 +124,69 @@ QPID_AUTO_TEST_CASE(testBound){ exchange3->route(deliverable); } -QPID_AUTO_TEST_CASE(testPersistLastNodeStanding){ - client::QueueOptions args; - args.setPersistLastNode(); - - Queue::shared_ptr queue(new Queue("my-queue", true)); - queue->configure(args); - - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "B"); - intrusive_ptr<Message> msg3 = createMessage("e", "C"); - - //enqueue 2 messages - queue->deliver(msg1); - queue->deliver(msg2); - - //change mode - queue->setLastNodeFailure(); - - //enqueue 1 message - queue->deliver(msg3); - - //check all have persistent ids. - BOOST_CHECK(msg1->isPersistent()); - BOOST_CHECK(msg2->isPersistent()); - BOOST_CHECK(msg3->isPersistent()); - -} - - -QPID_AUTO_TEST_CASE(testSeek){ - - Queue::shared_ptr queue(new Queue("my-queue", true)); - - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "B"); - intrusive_ptr<Message> msg3 = createMessage("e", "C"); - - //enqueue 2 messages - queue->deliver(msg1); - queue->deliver(msg2); - queue->deliver(msg3); - - TestConsumer::shared_ptr consumer(new TestConsumer("test", false)); - SequenceNumber seq(2); - consumer->setPosition(seq); - - QueuedMessage qm; - queue->dispatch(consumer); - - BOOST_CHECK_EQUAL(msg3.get(), consumer->last.payload.get()); - queue->dispatch(consumer); - queue->dispatch(consumer); // make sure over-run is safe - -} - -QPID_AUTO_TEST_CASE(testSearch){ - - Queue::shared_ptr queue(new Queue("my-queue", true)); - - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "B"); - intrusive_ptr<Message> msg3 = createMessage("e", "C"); - - //enqueue 2 messages - queue->deliver(msg1); - queue->deliver(msg2); - queue->deliver(msg3); - - SequenceNumber seq(2); - QueuedMessage qm; - TestConsumer::shared_ptr c1(new TestConsumer()); - - BOOST_CHECK(queue->find(seq, qm)); +QPID_AUTO_TEST_CASE(testLVQ){ - BOOST_CHECK_EQUAL(seq.getValue(), qm.position.getValue()); - - queue->acquire(qm, c1->getName()); - BOOST_CHECK_EQUAL(queue->getMessageCount(), 2u); - SequenceNumber seq1(3); - QueuedMessage qm1; - BOOST_CHECK(queue->find(seq1, qm1)); - BOOST_CHECK_EQUAL(seq1.getValue(), qm1.position.getValue()); - -} -const std::string nullxid = ""; - -class SimpleDummyCtxt : public TransactionContext {}; + QueueSettings settings; + string key="key"; + settings.lvqKey = key; + QueueFactory factory; + Queue::shared_ptr q(factory.create("my-queue", settings)); -class DummyCtxt : public TPCTransactionContext -{ - const std::string xid; - public: - DummyCtxt(const std::string& _xid) : xid(_xid) {} - static std::string getXid(TransactionContext& ctxt) - { - DummyCtxt* c(dynamic_cast<DummyCtxt*>(&ctxt)); - return c ? c->xid : nullxid; + const char* values[] = { "a", "b", "c", "a"}; + for (size_t i = 0; i < sizeof(values)/sizeof(values[0]); ++i) { + qpid::types::Variant::Map properties; + properties[key] = values[i]; + q->deliver(MessageUtils::createMessage(properties, boost::lexical_cast<string>(i+1))); } -}; - -class TestMessageStoreOC : public MessageStore -{ - std::set<std::string> prepared; - uint64_t nextPersistenceId; - public: - - uint enqCnt; - uint deqCnt; - bool error; - - TestMessageStoreOC() : MessageStore(),nextPersistenceId(1),enqCnt(0),deqCnt(0),error(false) {} - ~TestMessageStoreOC(){} + BOOST_CHECK_EQUAL(q->getMessageCount(), 3u); - virtual void dequeue(TransactionContext*, - const boost::intrusive_ptr<PersistableMessage>& /*msg*/, - const PersistableQueue& /*queue*/) - { - if (error) throw Exception("Dequeue error test"); - deqCnt++; - } + TestConsumer::shared_ptr c(new TestConsumer("test", true)); + BOOST_CHECK(q->dispatch(c)); + BOOST_CHECK_EQUAL(std::string("2"), c->lastMessage.getContent()); + BOOST_CHECK(q->dispatch(c)); + BOOST_CHECK_EQUAL(std::string("3"), c->lastMessage.getContent()); + BOOST_CHECK(q->dispatch(c)); + BOOST_CHECK_EQUAL(std::string("4"), c->lastMessage.getContent()); - virtual void enqueue(TransactionContext*, - const boost::intrusive_ptr<PersistableMessage>& msg, - const PersistableQueue& /* queue */) - { - if (error) throw Exception("Enqueue error test"); - enqCnt++; - msg->enqueueComplete(); - } - void createError() - { - error=true; + const char* values2[] = { "a", "b", "c"}; + for (size_t i = 0; i < sizeof(values2)/sizeof(values2[0]); ++i) { + qpid::types::Variant::Map properties; + properties[key] = values[i]; + q->deliver(MessageUtils::createMessage(properties, boost::lexical_cast<string>(i+5))); } + BOOST_CHECK_EQUAL(q->getMessageCount(), 3u); - bool init(const Options*) { return true; } - void truncateInit(const bool) {} - void create(PersistableQueue& queue, const framing::FieldTable&) { queue.setPersistenceId(nextPersistenceId++); } - void destroy(PersistableQueue&) {} - void create(const PersistableExchange& exchange, const framing::FieldTable&) { exchange.setPersistenceId(nextPersistenceId++); } - void destroy(const PersistableExchange&) {} - void bind(const PersistableExchange&, const PersistableQueue&, const std::string&, const framing::FieldTable&) {} - void unbind(const PersistableExchange&, const PersistableQueue&, const std::string&, const framing::FieldTable&) {} - void create(const PersistableConfig& config) { config.setPersistenceId(nextPersistenceId++); } - void destroy(const PersistableConfig&) {} - void stage(const boost::intrusive_ptr<PersistableMessage>&) {} - void destroy(PersistableMessage&) {} - void appendContent(const boost::intrusive_ptr<const PersistableMessage>&, const std::string&) {} - void loadContent(const qpid::broker::PersistableQueue&, const boost::intrusive_ptr<const PersistableMessage>&, - std::string&, uint64_t, uint32_t) { throw qpid::framing::InternalErrorException("Can't load content; persistence not enabled"); } - void flush(const qpid::broker::PersistableQueue&) {} - uint32_t outstandingQueueAIO(const PersistableQueue&) { return 0; } - - std::auto_ptr<TransactionContext> begin() { return std::auto_ptr<TransactionContext>(new SimpleDummyCtxt()); } - std::auto_ptr<TPCTransactionContext> begin(const std::string& xid) { return std::auto_ptr<TPCTransactionContext>(new DummyCtxt(xid)); } - void prepare(TPCTransactionContext& ctxt) { prepared.insert(DummyCtxt::getXid(ctxt)); } - void commit(TransactionContext& ctxt) { prepared.erase(DummyCtxt::getXid(ctxt)); } - void abort(TransactionContext& ctxt) { prepared.erase(DummyCtxt::getXid(ctxt)); } - void collectPreparedXids(std::set<std::string>& out) { out.insert(prepared.begin(), prepared.end()); } - - void recover(RecoveryManager&) {} -}; - - -QPID_AUTO_TEST_CASE(testLVQOrdering){ - - client::QueueOptions args; - // set queue mode - args.setOrdering(client::LVQ); - - Queue::shared_ptr queue(new Queue("my-queue", true )); - queue->configure(args); - - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "B"); - intrusive_ptr<Message> msg3 = createMessage("e", "C"); - intrusive_ptr<Message> msg4 = createMessage("e", "D"); - intrusive_ptr<Message> received; - - //set deliever match for LVQ a,b,c,a - - string key; - args.getLVQKey(key); - BOOST_CHECK_EQUAL(key, "qpid.LVQ_key"); - - - msg1->insertCustomProperty(key,"a"); - msg2->insertCustomProperty(key,"b"); - msg3->insertCustomProperty(key,"c"); - msg4->insertCustomProperty(key,"a"); - - //enqueue 4 message - queue->deliver(msg1); - queue->deliver(msg2); - queue->deliver(msg3); - queue->deliver(msg4); - - BOOST_CHECK_EQUAL(queue->getMessageCount(), 3u); - - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg4.get(), received.get()); - - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg2.get(), received.get()); - - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg3.get(), received.get()); - - intrusive_ptr<Message> msg5 = createMessage("e", "A"); - intrusive_ptr<Message> msg6 = createMessage("e", "B"); - intrusive_ptr<Message> msg7 = createMessage("e", "C"); - msg5->insertCustomProperty(key,"a"); - msg6->insertCustomProperty(key,"b"); - msg7->insertCustomProperty(key,"c"); - queue->deliver(msg5); - queue->deliver(msg6); - queue->deliver(msg7); - - BOOST_CHECK_EQUAL(queue->getMessageCount(), 3u); - - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg5.get(), received.get()); - - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg6.get(), received.get()); - - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg7.get(), received.get()); - + BOOST_CHECK(q->dispatch(c)); + BOOST_CHECK_EQUAL(std::string("5"), c->lastMessage.getContent()); + BOOST_CHECK(q->dispatch(c)); + BOOST_CHECK_EQUAL(std::string("6"), c->lastMessage.getContent()); + BOOST_CHECK(q->dispatch(c)); + BOOST_CHECK_EQUAL(std::string("7"), c->lastMessage.getContent()); } QPID_AUTO_TEST_CASE(testLVQEmptyKey){ - client::QueueOptions args; - // set queue mode - args.setOrdering(client::LVQ); - - Queue::shared_ptr queue(new Queue("my-queue", true )); - queue->configure(args); - - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "B"); - - string key; - args.getLVQKey(key); - BOOST_CHECK_EQUAL(key, "qpid.LVQ_key"); - - - msg1->insertCustomProperty(key,"a"); - queue->deliver(msg1); - queue->deliver(msg2); - BOOST_CHECK_EQUAL(queue->getMessageCount(), 2u); - -} - -QPID_AUTO_TEST_CASE(testLVQAcquire){ - - client::QueueOptions args; - // set queue mode - args.setOrdering(client::LVQ); - // disable flow control, as this test violates the enqueue/dequeue sequence. - args.setInt(QueueFlowLimit::flowStopCountKey, 0); - - Queue::shared_ptr queue(new Queue("my-queue", true )); - queue->configure(args); - - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "B"); - intrusive_ptr<Message> msg3 = createMessage("e", "C"); - intrusive_ptr<Message> msg4 = createMessage("e", "D"); - intrusive_ptr<Message> msg5 = createMessage("e", "F"); - intrusive_ptr<Message> msg6 = createMessage("e", "G"); - - //set deliever match for LVQ a,b,c,a - - string key; - args.getLVQKey(key); - BOOST_CHECK_EQUAL(key, "qpid.LVQ_key"); - - - msg1->insertCustomProperty(key,"a"); - msg2->insertCustomProperty(key,"b"); - msg3->insertCustomProperty(key,"c"); - msg4->insertCustomProperty(key,"a"); - msg5->insertCustomProperty(key,"b"); - msg6->insertCustomProperty(key,"c"); - - //enqueue 4 message - queue->deliver(msg1); - queue->deliver(msg2); - queue->deliver(msg3); - queue->deliver(msg4); - - BOOST_CHECK_EQUAL(queue->getMessageCount(), 3u); - - framing::SequenceNumber sequence(1); - QueuedMessage qmsg(queue.get(), msg1, sequence); - QueuedMessage qmsg2(queue.get(), msg2, ++sequence); - framing::SequenceNumber sequence1(10); - QueuedMessage qmsg3(queue.get(), 0, sequence1); - TestConsumer::shared_ptr dummy(new TestConsumer()); - - BOOST_CHECK(!queue->acquire(qmsg, dummy->getName())); - BOOST_CHECK(queue->acquire(qmsg2, dummy->getName())); - // Acquire the massage again to test failure case. - BOOST_CHECK(!queue->acquire(qmsg2, dummy->getName())); - BOOST_CHECK(!queue->acquire(qmsg3, dummy->getName())); - - BOOST_CHECK_EQUAL(queue->getMessageCount(), 2u); - - queue->deliver(msg5); - BOOST_CHECK_EQUAL(queue->getMessageCount(), 3u); - - // set mode to no browse and check - args.setOrdering(client::LVQ_NO_BROWSE); - queue->configure(args); - TestConsumer::shared_ptr c1(new TestConsumer("test", false)); - - queue->dispatch(c1); - queue->dispatch(c1); - queue->dispatch(c1); - - queue->deliver(msg6); - BOOST_CHECK_EQUAL(queue->getMessageCount(), 3u); - - intrusive_ptr<Message> received; - received = queue->get().payload; - BOOST_CHECK_EQUAL(msg4.get(), received.get()); - -} - -QPID_AUTO_TEST_CASE(testLVQMultiQueue){ - - client::QueueOptions args; - // set queue mode - args.setOrdering(client::LVQ); - - Queue::shared_ptr queue1(new Queue("my-queue", true )); - Queue::shared_ptr queue2(new Queue("my-queue", true )); - intrusive_ptr<Message> received; - queue1->configure(args); - queue2->configure(args); - - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "A"); - - string key; - args.getLVQKey(key); - BOOST_CHECK_EQUAL(key, "qpid.LVQ_key"); - - msg1->insertCustomProperty(key,"a"); - msg2->insertCustomProperty(key,"a"); - - queue1->deliver(msg1); - queue2->deliver(msg1); - queue1->deliver(msg2); - - received = queue1->get().payload; - BOOST_CHECK_EQUAL(msg2.get(), received.get()); - - received = queue2->get().payload; - BOOST_CHECK_EQUAL(msg1.get(), received.get()); + QueueSettings settings; + string key="key"; + settings.lvqKey = key; + QueueFactory factory; + Queue::shared_ptr q(factory.create("my-queue", settings)); -} -QPID_AUTO_TEST_CASE(testLVQRecover){ - -/* simulate this - 1. start 2 nodes - 2. create cluster durable lvq - 3. send a transient message to the queue - 4. kill one of the nodes (to trigger force persistent behaviour)... - 5. then restart it (to turn off force persistent behaviour) - 6. send another transient message with same lvq key as in 3 - 7. kill the second node again (retrigger force persistent) - 8. stop and recover the first node -*/ - TestMessageStoreOC testStore; - client::QueueOptions args; - // set queue mode - args.setOrdering(client::LVQ); - args.setPersistLastNode(); - - Queue::shared_ptr queue1(new Queue("my-queue", true, &testStore)); - intrusive_ptr<Message> received; - queue1->create(args); - - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - intrusive_ptr<Message> msg2 = createMessage("e", "A"); - // 2 - string key; - args.getLVQKey(key); - BOOST_CHECK_EQUAL(key, "qpid.LVQ_key"); - - msg1->insertCustomProperty(key,"a"); - msg2->insertCustomProperty(key,"a"); - // 3 - queue1->deliver(msg1); - // 4 - queue1->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 1u); - // 5 - queue1->clearLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 1u); - // 6 - queue1->deliver(msg2); - BOOST_CHECK_EQUAL(testStore.enqCnt, 1u); - queue1->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 2u); - BOOST_CHECK_EQUAL(testStore.deqCnt, 1u); + qpid::types::Variant::Map properties; + properties["key"] = "a"; + q->deliver(MessageUtils::createMessage(properties, "one")); + properties.clear(); + q->deliver(MessageUtils::createMessage(properties, "two")); + BOOST_CHECK_EQUAL(q->getMessageCount(), 2u); } void addMessagesToQueue(uint count, Queue& queue, uint oddTtl = 200, uint evenTtl = 0) { for (uint i = 0; i < count; i++) { - intrusive_ptr<Message> m = createMessage("exchange", "key", i % 2 ? oddTtl : evenTtl); - m->computeExpiration(new broker::ExpiryPolicy); + Message m = MessageUtils::createMessage("exchange", "key", i % 2 ? oddTtl : evenTtl); + m.computeExpiration(new broker::ExpiryPolicy); queue.deliver(m); } } @@ -706,7 +203,7 @@ QPID_AUTO_TEST_CASE(testPurgeExpired) { QPID_AUTO_TEST_CASE(testQueueCleaner) { Timer timer; QueueRegistry queues; - Queue::shared_ptr queue = queues.declare("my-queue").first; + Queue::shared_ptr queue = queues.declare("my-queue", QueueSettings()).first; addMessagesToQueue(10, *queue, 200, 400); BOOST_CHECK_EQUAL(queue->getMessageCount(), 10u); @@ -717,44 +214,57 @@ QPID_AUTO_TEST_CASE(testQueueCleaner) { ::usleep(300*1000); BOOST_CHECK_EQUAL(queue->getMessageCount(), 0u); } - - namespace { - // helper for group tests - void verifyAcquire( Queue::shared_ptr queue, - TestConsumer::shared_ptr c, - std::deque<QueuedMessage>& results, - const std::string& expectedGroup, - const int expectedId ) - { - BOOST_CHECK(queue->dispatch(c)); - results.push_back(c->last); - std::string group = c->last.payload->getProperties<MessageProperties>()->getApplicationHeaders().getAsString("GROUP-ID"); - int id = c->last.payload->getProperties<MessageProperties>()->getApplicationHeaders().getAsInt("MY-ID"); +int getIntProperty(const Message& message, const std::string& key) +{ + qpid::types::Variant v = message.getProperty(key); + int i(0); + if (!v.isVoid()) i = v; + return i; +} +// helper for group tests +void verifyAcquire( Queue::shared_ptr queue, + TestConsumer::shared_ptr c, + std::deque<QueueCursor>& results, + const std::string& expectedGroup, + const int expectedId ) +{ + bool success = queue->dispatch(c); + BOOST_CHECK(success); + if (success) { + results.push_back(c->lastCursor); + std::string group = c->lastMessage.getPropertyAsString("GROUP-ID"); + int id = getIntProperty(c->lastMessage, "MY-ID"); BOOST_CHECK_EQUAL( group, expectedGroup ); BOOST_CHECK_EQUAL( id, expectedId ); } } +Message createGroupMessage(int id, const std::string& group) +{ + qpid::types::Variant::Map properties; + properties["GROUP-ID"] = group; + properties["MY-ID"] = id; + return MessageUtils::createMessage(properties); +} +} + QPID_AUTO_TEST_CASE(testGroupsMultiConsumer) { // // Verify that consumers of grouped messages own the groups once a message is acquired, // and release the groups once all acquired messages have been dequeued or requeued // - FieldTable args; - Queue::shared_ptr queue(new Queue("my_queue", true)); - args.setString("qpid.group_header_key", "GROUP-ID"); - args.setInt("qpid.shared_msg_group", 1); - queue->configure(args); + QueueSettings settings; + settings.shareGroups = 1; + settings.groupKey = "GROUP-ID"; + QueueFactory factory; + Queue::shared_ptr queue(factory.create("my_queue", settings)); std::string groups[] = { std::string("a"), std::string("a"), std::string("a"), std::string("b"), std::string("b"), std::string("b"), std::string("c"), std::string("c"), std::string("c") }; for (int i = 0; i < 9; ++i) { - intrusive_ptr<Message> msg = createMessage("e", "A"); - msg->insertCustomProperty("GROUP-ID", groups[i]); - msg->insertCustomProperty("MY-ID", i); - queue->deliver(msg); + queue->deliver(createGroupMessage(i, groups[i])); } // Queue = a-0, a-1, a-2, b-3, b-4, b-5, c-6, c-7, c-8... @@ -768,8 +278,8 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumer) { queue->consume(c1); queue->consume(c2); - std::deque<QueuedMessage> dequeMeC1; - std::deque<QueuedMessage> dequeMeC2; + std::deque<QueueCursor> dequeMeC1; + std::deque<QueueCursor> dequeMeC2; verifyAcquire(queue, c1, dequeMeC1, "a", 0 ); // c1 now owns group "a" (acquire a-0) @@ -828,9 +338,9 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumer) { // Owners= ^C2, ^C2, ^C1, ^C1, ^C2, ^C2 // what happens if C-2 "requeues" a-1 and a-2? - queue->requeue( dequeMeC2.front() ); + queue->release( dequeMeC2.front() ); dequeMeC2.pop_front(); - queue->requeue( dequeMeC2.front() ); + queue->release( dequeMeC2.front() ); dequeMeC2.pop_front(); // now just has c-7 acquired // Queue = a-1, a-2, b-4, b-5, c-7, c-8... @@ -855,9 +365,9 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumer) { gotOne = queue->dispatch(c2); BOOST_CHECK( !gotOne ); - // requeue all of C1's acquired messages, then cancel C1 + // release all of C1's acquired messages, then cancel C1 while (!dequeMeC1.empty()) { - queue->requeue(dequeMeC1.front()); + queue->release(dequeMeC1.front()); dequeMeC1.pop_front(); } queue->cancel(c1); @@ -877,7 +387,7 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumer) { // Owners= ---, ---, --- TestConsumer::shared_ptr c3(new TestConsumer("C3")); - std::deque<QueuedMessage> dequeMeC3; + std::deque<QueueCursor> dequeMeC3; verifyAcquire(queue, c3, dequeMeC3, "a", 2 ); verifyAcquire(queue, c2, dequeMeC2, "b", 4 ); @@ -897,11 +407,7 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumer) { // Queue = a-2, // Owners= ^C3, - - intrusive_ptr<Message> msg = createMessage("e", "A"); - msg->insertCustomProperty("GROUP-ID", "a"); - msg->insertCustomProperty("MY-ID", 9); - queue->deliver(msg); + queue->deliver(createGroupMessage(9, "a")); // Queue = a-2, a-9 // Owners= ^C3, ^C3 @@ -909,10 +415,7 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumer) { gotOne = queue->dispatch(c2); BOOST_CHECK( !gotOne ); - msg = createMessage("e", "A"); - msg->insertCustomProperty("GROUP-ID", "b"); - msg->insertCustomProperty("MY-ID", 10); - queue->deliver(msg); + queue->deliver(createGroupMessage(10, "b")); // Queue = a-2, a-9, b-10 // Owners= ^C3, ^C3, ---- @@ -933,17 +436,17 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumerDefaults) { // Verify that the same default group name is automatically applied to messages that // do not specify a group name. // - FieldTable args; - Queue::shared_ptr queue(new Queue("my_queue", true)); - args.setString("qpid.group_header_key", "GROUP-ID"); - args.setInt("qpid.shared_msg_group", 1); - queue->configure(args); + QueueSettings settings; + settings.shareGroups = 1; + settings.groupKey = "GROUP-ID"; + QueueFactory factory; + Queue::shared_ptr queue(factory.create("my_queue", settings)); for (int i = 0; i < 3; ++i) { - intrusive_ptr<Message> msg = createMessage("e", "A"); + qpid::types::Variant::Map properties; // no "GROUP-ID" header - msg->insertCustomProperty("MY-ID", i); - queue->deliver(msg); + properties["MY-ID"] = i; + queue->deliver(MessageUtils::createMessage(properties)); } // Queue = 0, 1, 2 @@ -956,20 +459,20 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumerDefaults) { queue->consume(c1); queue->consume(c2); - std::deque<QueuedMessage> dequeMeC1; - std::deque<QueuedMessage> dequeMeC2; + std::deque<QueueCursor> dequeMeC1; + std::deque<QueueCursor> dequeMeC2; queue->dispatch(c1); // c1 now owns default group (acquired 0) - dequeMeC1.push_back(c1->last); - int id = c1->last.payload->getProperties<MessageProperties>()->getApplicationHeaders().getAsInt("MY-ID"); + dequeMeC1.push_back(c1->lastCursor); + int id = getIntProperty(c1->lastMessage, "MY-ID"); BOOST_CHECK_EQUAL( id, 0 ); bool gotOne = queue->dispatch(c2); // c2 should get nothing BOOST_CHECK( !gotOne ); queue->dispatch(c1); // c1 now acquires 1 - dequeMeC1.push_back(c1->last); - id = c1->last.payload->getProperties<MessageProperties>()->getApplicationHeaders().getAsInt("MY-ID"); + dequeMeC1.push_back(c1->lastCursor); + id = getIntProperty(c1->lastMessage, "MY-ID"); BOOST_CHECK_EQUAL( id, 1 ); gotOne = queue->dispatch(c2); // c2 should still get nothing @@ -982,7 +485,7 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumerDefaults) { // now default group should be available... queue->dispatch(c2); // c2 now owns default group (acquired 2) - id = c2->last.payload->getProperties<MessageProperties>()->getApplicationHeaders().getAsInt("MY-ID"); + id = c2->lastMessage.getProperty("MY-ID"); BOOST_CHECK_EQUAL( id, 2 ); gotOne = queue->dispatch(c1); // c1 should get nothing @@ -992,556 +495,128 @@ QPID_AUTO_TEST_CASE(testGroupsMultiConsumerDefaults) { queue->cancel(c2); } -QPID_AUTO_TEST_CASE(testMultiQueueLastNode){ - - TestMessageStoreOC testStore; - client::QueueOptions args; - args.setPersistLastNode(); - - Queue::shared_ptr queue1(new Queue("queue1", true, &testStore )); - queue1->create(args); - Queue::shared_ptr queue2(new Queue("queue2", true, &testStore )); - queue2->create(args); - - intrusive_ptr<Message> msg1 = createMessage("e", "A"); - - queue1->deliver(msg1); - queue2->deliver(msg1); - - //change mode - queue1->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 1u); - queue2->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 2u); - - // check they don't get stored twice - queue1->setLastNodeFailure(); - queue2->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 2u); - - intrusive_ptr<Message> msg2 = createMessage("e", "B"); - queue1->deliver(msg2); - queue2->deliver(msg2); - - queue1->clearLastNodeFailure(); - queue2->clearLastNodeFailure(); - // check only new messages get forced - queue1->setLastNodeFailure(); - queue2->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 4u); - - // check no failure messages are stored - queue1->clearLastNodeFailure(); - queue2->clearLastNodeFailure(); - - intrusive_ptr<Message> msg3 = createMessage("e", "B"); - queue1->deliver(msg3); - queue2->deliver(msg3); - BOOST_CHECK_EQUAL(testStore.enqCnt, 4u); - queue1->setLastNodeFailure(); - queue2->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 6u); - - /** - * TODO: Fix or replace the following test which incorrectly requeues a - * message that was never on the queue in the first place. This relied on - * internal details not part of the queue abstraction. - - // check requeue 1 - intrusive_ptr<Message> msg4 = createMessage("e", "C"); - intrusive_ptr<Message> msg5 = createMessage("e", "D"); - - framing::SequenceNumber sequence(1); - QueuedMessage qmsg1(queue1.get(), msg4, sequence); - QueuedMessage qmsg2(queue2.get(), msg5, ++sequence); - - queue1->requeue(qmsg1); - BOOST_CHECK_EQUAL(testStore.enqCnt, 7u); - - // check requeue 2 - queue2->clearLastNodeFailure(); - queue2->requeue(qmsg2); - BOOST_CHECK_EQUAL(testStore.enqCnt, 7u); - queue2->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 8u); - - queue2->clearLastNodeFailure(); - queue2->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 8u); - */ -} - -QPID_AUTO_TEST_CASE(testLastNodeRecoverAndFail){ -/* -simulate this: - 1. start two nodes - 2. create cluster durable queue and add some messages - 3. kill one node (trigger force-persistent behaviour) - 4. stop and recover remaining node - 5. add another node - 6. kill that new node again -make sure that an attempt to re-enqueue a message does not happen which will -result in the last man standing exiting with an error. - -we need to make sure that recover is safe, i.e. messages are -not requeued to the store. -*/ - TestMessageStoreOC testStore; - client::QueueOptions args; - // set queue mode - args.setPersistLastNode(); - - Queue::shared_ptr queue1(new Queue("my-queue", true, &testStore)); - intrusive_ptr<Message> received; - queue1->create(args); - - // check requeue 1 - intrusive_ptr<Message> msg1 = createMessage("e", "C"); - intrusive_ptr<Message> msg2 = createMessage("e", "D"); - - queue1->recover(msg1); - - queue1->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 0u); - - queue1->clearLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 0u); - - queue1->deliver(msg2); - BOOST_CHECK_EQUAL(testStore.enqCnt, 0u); - queue1->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 1u); - -} - -QPID_AUTO_TEST_CASE(testLastNodeJournalError){ -/* -simulate store exception going into last node standing - -*/ - TestMessageStoreOC testStore; - client::QueueOptions args; - // set queue mode - args.setPersistLastNode(); - - Queue::shared_ptr queue1(new Queue("my-queue", true, &testStore)); - intrusive_ptr<Message> received; - queue1->configure(args); - - // check requeue 1 - intrusive_ptr<Message> msg1 = createMessage("e", "C"); - - queue1->deliver(msg1); - testStore.createError(); - - ScopedSuppressLogging sl; // Suppress messages for expected errors. - queue1->setLastNodeFailure(); - BOOST_CHECK_EQUAL(testStore.enqCnt, 0u); - -} - -intrusive_ptr<Message> mkMsg(MessageStore& store, std::string content = "", bool durable = false) -{ - intrusive_ptr<Message> msg = MessageUtils::createMessage("", "", durable); - if (content.size()) MessageUtils::addContent(msg, content); - msg->setStore(&store); - return msg; -} - -QPID_AUTO_TEST_CASE(testFlowToDiskBlocking){ - - TestMessageStoreOC testStore; - client::QueueOptions args0; // No size policy - client::QueueOptions args1; - args1.setSizePolicy(FLOW_TO_DISK, 0, 1); - client::QueueOptions args2; - args2.setSizePolicy(FLOW_TO_DISK, 0, 2); - - // --- Fanout exchange bound to single transient queue ------------------------------------------------------------- - - FanOutExchange sbtFanout1("sbtFanout1", false, args0); // single binding to transient queue - Queue::shared_ptr tq1(new Queue("tq1", true)); // transient w/ limit - tq1->configure(args1); - sbtFanout1.bind(tq1, "", 0); - - intrusive_ptr<Message> msg01 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content - DeliverableMessage dmsg01(msg01); - sbtFanout1.route(dmsg01); // Brings queue 1 to capacity limit - msg01->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg01->isContentReleased(), false); - BOOST_CHECK_EQUAL(1u, tq1->getMessageCount()); - - intrusive_ptr<Message> msg02 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content - DeliverableMessage dmsg02(msg02); - { - ScopedSuppressLogging sl; // suppress expected error messages. - BOOST_CHECK_THROW(sbtFanout1.route(dmsg02), ResourceLimitExceededException); - } - msg02->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg02->isContentReleased(), false); - BOOST_CHECK_EQUAL(1u, tq1->getMessageCount()); - - intrusive_ptr<Message> msg03 = mkMsg(testStore, std::string(5, 'X'), true); // durable w/ content - DeliverableMessage dmsg03(msg03); - { - ScopedSuppressLogging sl; // suppress expected error messages. - BOOST_CHECK_THROW(sbtFanout1.route(dmsg03), ResourceLimitExceededException); - } - msg03->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg03->isContentReleased(), false); - BOOST_CHECK_EQUAL(1u, tq1->getMessageCount()); - - intrusive_ptr<Message> msg04 = mkMsg(testStore); // transient no content - DeliverableMessage dmsg04(msg04); - { - ScopedSuppressLogging sl; // suppress expected error messages. - BOOST_CHECK_THROW(sbtFanout1.route(dmsg04), ResourceLimitExceededException); - } - msg04->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg04->isContentReleased(), false); - BOOST_CHECK_EQUAL(1u, tq1->getMessageCount()); - - intrusive_ptr<Message> msg05 = mkMsg(testStore, "", true); // durable no content - DeliverableMessage dmsg05(msg05); - { - ScopedSuppressLogging sl; // suppress expected error messages. - BOOST_CHECK_THROW(sbtFanout1.route(dmsg05), ResourceLimitExceededException); - } - msg05->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg05->isContentReleased(), false); - BOOST_CHECK_EQUAL(1u, tq1->getMessageCount()); - - // --- Fanout exchange bound to single durable queue --------------------------------------------------------------- - - FanOutExchange sbdFanout2("sbdFanout2", false, args0); // single binding to durable queue - Queue::shared_ptr dq2(new Queue("dq2", true, &testStore)); // durable w/ limit - dq2->configure(args1); - sbdFanout2.bind(dq2, "", 0); - - intrusive_ptr<Message> msg06 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content - DeliverableMessage dmsg06(msg06); - sbdFanout2.route(dmsg06); // Brings queue 2 to capacity limit - msg06->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg06->isContentReleased(), false); - BOOST_CHECK_EQUAL(1u, dq2->getMessageCount()); - - intrusive_ptr<Message> msg07 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content - DeliverableMessage dmsg07(msg07); - sbdFanout2.route(dmsg07); - msg07->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg07->isContentReleased(), true); - BOOST_CHECK_EQUAL(2u, dq2->getMessageCount()); - - intrusive_ptr<Message> msg08 = mkMsg(testStore, std::string(5, 'X'), true); // durable w/ content - DeliverableMessage dmsg08(msg08); - sbdFanout2.route(dmsg08); - msg08->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg08->isContentReleased(), true); - BOOST_CHECK_EQUAL(3u, dq2->getMessageCount()); - - intrusive_ptr<Message> msg09 = mkMsg(testStore); // transient no content - DeliverableMessage dmsg09(msg09); - sbdFanout2.route(dmsg09); - msg09->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg09->isContentReleased(), true); - BOOST_CHECK_EQUAL(4u, dq2->getMessageCount()); - - intrusive_ptr<Message> msg10 = mkMsg(testStore, "", true); // durable no content - DeliverableMessage dmsg10(msg10); - sbdFanout2.route(dmsg10); - msg10->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg10->isContentReleased(), true); - BOOST_CHECK_EQUAL(5u, dq2->getMessageCount()); - - // --- Fanout exchange bound to multiple durable queues ------------------------------------------------------------ - - FanOutExchange mbdFanout3("mbdFanout3", false, args0); // multiple bindings to durable queues - Queue::shared_ptr dq3(new Queue("dq3", true, &testStore)); // durable w/ limit 2 - dq3->configure(args2); - mbdFanout3.bind(dq3, "", 0); - Queue::shared_ptr dq4(new Queue("dq4", true, &testStore)); // durable w/ limit 1 - dq4->configure(args1); - mbdFanout3.bind(dq4, "", 0); - Queue::shared_ptr dq5(new Queue("dq5", true, &testStore)); // durable no limit - dq5->configure(args0); - mbdFanout3.bind(dq5, "", 0); - - intrusive_ptr<Message> msg11 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content - DeliverableMessage dmsg11(msg11); - mbdFanout3.route(dmsg11); // Brings queues 3 and 4 to capacity limit - msg11->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg11->isContentReleased(), false); - BOOST_CHECK_EQUAL(1u, dq3->getMessageCount()); - BOOST_CHECK_EQUAL(1u, dq4->getMessageCount()); - BOOST_CHECK_EQUAL(1u, dq5->getMessageCount()); - - intrusive_ptr<Message> msg12 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content - DeliverableMessage dmsg12(msg12); - mbdFanout3.route(dmsg12); - msg12->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg12->isContentReleased(), false); // XXXX - consequence of transient msg multi-queue ftd policy-handling limitations, fix in broker at some point! - BOOST_CHECK_EQUAL(2u, dq3->getMessageCount()); - BOOST_CHECK_EQUAL(2u, dq4->getMessageCount()); - BOOST_CHECK_EQUAL(2u, dq5->getMessageCount()); - - intrusive_ptr<Message> msg13 = mkMsg(testStore, std::string(5, 'X'), true); // durable w/ content - DeliverableMessage dmsg13(msg13); - mbdFanout3.route(dmsg13); - msg13->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg13->isContentReleased(), true); - BOOST_CHECK_EQUAL(3u, dq3->getMessageCount()); - BOOST_CHECK_EQUAL(3u, dq4->getMessageCount()); - BOOST_CHECK_EQUAL(3u, dq5->getMessageCount()); - - intrusive_ptr<Message> msg14 = mkMsg(testStore); // transient no content - DeliverableMessage dmsg14(msg14); - mbdFanout3.route(dmsg14); - msg14->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg14->isContentReleased(), false); // XXXX - consequence of transient msg multi-queue ftd policy-handling limitations, fix in broker at some point! - BOOST_CHECK_EQUAL(4u, dq3->getMessageCount()); - BOOST_CHECK_EQUAL(4u, dq4->getMessageCount()); - BOOST_CHECK_EQUAL(4u, dq5->getMessageCount()); - - intrusive_ptr<Message> msg15 = mkMsg(testStore, "", true); // durable no content - DeliverableMessage dmsg15(msg15); - mbdFanout3.route(dmsg15); - msg15->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg15->isContentReleased(), true); - BOOST_CHECK_EQUAL(5u, dq3->getMessageCount()); - BOOST_CHECK_EQUAL(5u, dq4->getMessageCount()); - BOOST_CHECK_EQUAL(5u, dq5->getMessageCount()); - - // Bind a transient queue, this should block the release of any further messages. - // Note: this will result in a violation of the count policy of dq3 and dq4 - but this - // is expected until a better overall multi-queue design is implemented. Similarly - // for the other tests in this section. - - Queue::shared_ptr tq6(new Queue("tq6", true)); // transient no limit - tq6->configure(args0); - mbdFanout3.bind(tq6, "", 0); - - intrusive_ptr<Message> msg16 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content - DeliverableMessage dmsg16(msg16); - mbdFanout3.route(dmsg16); - msg16->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg16->isContentReleased(), false); - BOOST_CHECK_EQUAL(6u, dq3->getMessageCount()); - BOOST_CHECK_EQUAL(6u, dq4->getMessageCount()); - BOOST_CHECK_EQUAL(6u, dq5->getMessageCount()); - - intrusive_ptr<Message> msg17 = mkMsg(testStore, std::string(5, 'X'), true); // durable w/ content - DeliverableMessage dmsg17(msg17); - mbdFanout3.route(dmsg17); - msg17->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg17->isContentReleased(), false); - BOOST_CHECK_EQUAL(7u, dq3->getMessageCount()); - BOOST_CHECK_EQUAL(7u, dq4->getMessageCount()); - BOOST_CHECK_EQUAL(7u, dq5->getMessageCount()); - - intrusive_ptr<Message> msg18 = mkMsg(testStore); // transient no content - DeliverableMessage dmsg18(msg18); - mbdFanout3.route(dmsg18); - msg18->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg18->isContentReleased(), false); - BOOST_CHECK_EQUAL(8u, dq3->getMessageCount()); - BOOST_CHECK_EQUAL(8u, dq4->getMessageCount()); - BOOST_CHECK_EQUAL(8u, dq5->getMessageCount()); - - intrusive_ptr<Message> msg19 = mkMsg(testStore, "", true); // durable no content - DeliverableMessage dmsg19(msg19); - mbdFanout3.route(dmsg19); - msg19->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg19->isContentReleased(), false); - BOOST_CHECK_EQUAL(9u, dq3->getMessageCount()); - BOOST_CHECK_EQUAL(9u, dq4->getMessageCount()); - BOOST_CHECK_EQUAL(9u, dq5->getMessageCount()); - - - // --- Fanout exchange bound to multiple durable and transient queues ---------------------------------------------- - - FanOutExchange mbmFanout4("mbmFanout4", false, args0); // multiple bindings to durable/transient queues - Queue::shared_ptr dq7(new Queue("dq7", true, &testStore)); // durable no limit - dq7->configure(args0); - mbmFanout4.bind(dq7, "", 0); - Queue::shared_ptr dq8(new Queue("dq8", true, &testStore)); // durable w/ limit - dq8->configure(args1); - mbmFanout4.bind(dq8, "", 0); - Queue::shared_ptr tq9(new Queue("tq9", true)); // transient no limit - tq9->configure(args0); - mbmFanout4.bind(tq9, "", 0); - - intrusive_ptr<Message> msg20 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content - DeliverableMessage dmsg20(msg20); - mbmFanout4.route(dmsg20); // Brings queue 7 to capacity limit - msg20->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg20->isContentReleased(), false); - BOOST_CHECK_EQUAL(1u, dq7->getMessageCount()); - BOOST_CHECK_EQUAL(1u, dq8->getMessageCount()); - BOOST_CHECK_EQUAL(1u, tq9->getMessageCount()); - - intrusive_ptr<Message> msg21 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content - DeliverableMessage dmsg21(msg21); - mbmFanout4.route(dmsg21); - msg21->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg21->isContentReleased(), false); - BOOST_CHECK_EQUAL(2u, dq7->getMessageCount()); // over limit - BOOST_CHECK_EQUAL(2u, dq8->getMessageCount()); - BOOST_CHECK_EQUAL(2u, tq9->getMessageCount()); - - intrusive_ptr<Message> msg22 = mkMsg(testStore, std::string(5, 'X'), true); // durable w/ content - DeliverableMessage dmsg22(msg22); - mbmFanout4.route(dmsg22); - msg22->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg22->isContentReleased(), false); - BOOST_CHECK_EQUAL(3u, dq7->getMessageCount()); // over limit - BOOST_CHECK_EQUAL(3u, dq8->getMessageCount()); // over limit - BOOST_CHECK_EQUAL(3u, tq9->getMessageCount()); - - intrusive_ptr<Message> msg23 = mkMsg(testStore); // transient no content - DeliverableMessage dmsg23(msg23); - mbmFanout4.route(dmsg23); - msg23->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg23->isContentReleased(), false); - BOOST_CHECK_EQUAL(4u, dq7->getMessageCount()); // over limit - BOOST_CHECK_EQUAL(4u, dq8->getMessageCount()); // over limit - BOOST_CHECK_EQUAL(4u, tq9->getMessageCount()); - - intrusive_ptr<Message> msg24 = mkMsg(testStore, "", true); // durable no content - DeliverableMessage dmsg24(msg24); - mbmFanout4.route(dmsg24); - msg24->tryReleaseContent(); - BOOST_CHECK_EQUAL(msg24->isContentReleased(), false); - BOOST_CHECK_EQUAL(5u, dq7->getMessageCount()); // over limit - BOOST_CHECK_EQUAL(5u, dq8->getMessageCount()); // over limit - BOOST_CHECK_EQUAL(5u, tq9->getMessageCount()); -} - QPID_AUTO_TEST_CASE(testSetPositionFifo) { Queue::shared_ptr q(new Queue("my-queue", true)); BOOST_CHECK_EQUAL(q->getPosition(), SequenceNumber(0)); for (int i = 0; i < 10; ++i) - q->deliver(contentMessage(boost::lexical_cast<string>(i+1))); + q->deliver(MessageUtils::createMessage(qpid::types::Variant::Map(), boost::lexical_cast<string>(i+1))); // Verify the front of the queue TestConsumer::shared_ptr c(new TestConsumer("test", false)); // Don't acquire BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(1u, c->last.position); // Numbered from 1 - BOOST_CHECK_EQUAL("1", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(1u, c->lastMessage.getSequence()); // Numbered from 1 + BOOST_CHECK_EQUAL("1", c->lastMessage.getContent()); + // Verify the back of the queue - QueuedMessage qm; BOOST_CHECK_EQUAL(10u, q->getPosition()); - BOOST_CHECK(q->find(q->getPosition(), qm)); // Back of the queue - BOOST_CHECK_EQUAL("10", getContent(qm.payload)); BOOST_CHECK_EQUAL(10u, q->getMessageCount()); // Using setPosition to introduce a gap in sequence numbers. q->setPosition(15); BOOST_CHECK_EQUAL(10u, q->getMessageCount()); BOOST_CHECK_EQUAL(15u, q->getPosition()); - BOOST_CHECK(q->find(10, qm)); // Back of the queue - BOOST_CHECK_EQUAL("10", getContent(qm.payload)); - q->deliver(contentMessage("16")); - c->setPosition(9); + q->deliver(MessageUtils::createMessage(qpid::types::Variant::Map(), "16")); + + q->seek(*c, Queue::MessagePredicate(), 9); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(10u, c->last.position); - BOOST_CHECK_EQUAL("10", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(10u, c->lastMessage.getSequence()); + BOOST_CHECK_EQUAL("10", c->lastMessage.getContent()); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(16u, c->last.position); - BOOST_CHECK_EQUAL("16", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(16u, c->lastMessage.getSequence()); + BOOST_CHECK_EQUAL("16", c->lastMessage.getContent()); // Using setPosition to trunkcate the queue q->setPosition(5); BOOST_CHECK_EQUAL(5u, q->getMessageCount()); - q->deliver(contentMessage("6a")); - c->setPosition(4); + q->deliver(MessageUtils::createMessage(qpid::types::Variant::Map(), "6a")); + c = boost::shared_ptr<TestConsumer>(new TestConsumer("test", false)); // Don't acquire + q->seek(*c, Queue::MessagePredicate(), 4); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(5u, c->last.position); - BOOST_CHECK_EQUAL("5", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(5u, c->lastMessage.getSequence()); + BOOST_CHECK_EQUAL("5", c->lastMessage.getContent()); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(6u, c->last.position); - BOOST_CHECK_EQUAL("6a", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(6u, c->lastMessage.getSequence()); + BOOST_CHECK_EQUAL("6a", c->lastMessage.getContent()); BOOST_CHECK(!q->dispatch(c)); // No more messages. } QPID_AUTO_TEST_CASE(testSetPositionLvq) { - Queue::shared_ptr q(new Queue("my-queue", true)); + QueueSettings settings; string key="key"; - framing::FieldTable args; - args.setString("qpid.last_value_queue_key", "key"); - q->configure(args); + settings.lvqKey = key; + QueueFactory factory; + Queue::shared_ptr q(factory.create("my-queue", settings)); const char* values[] = { "a", "b", "c", "a", "b", "c" }; for (size_t i = 0; i < sizeof(values)/sizeof(values[0]); ++i) { - intrusive_ptr<Message> m = contentMessage(boost::lexical_cast<string>(i+1)); - m->insertCustomProperty(key, values[i]); - q->deliver(m); + qpid::types::Variant::Map properties; + properties[key] = values[i]; + q->deliver(MessageUtils::createMessage(properties, boost::lexical_cast<string>(i+1))); } BOOST_CHECK_EQUAL(3u, q->getMessageCount()); // Verify the front of the queue TestConsumer::shared_ptr c(new TestConsumer("test", false)); // Don't acquire BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(4u, c->last.position); // Numbered from 1 - BOOST_CHECK_EQUAL("4", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(4u, c->lastMessage.getSequence()); // Numbered from 1 + BOOST_CHECK_EQUAL("4", c->lastMessage.getContent()); // Verify the back of the queue - QueuedMessage qm; BOOST_CHECK_EQUAL(6u, q->getPosition()); - BOOST_CHECK(q->find(q->getPosition(), qm)); // Back of the queue - BOOST_CHECK_EQUAL("6", getContent(qm.payload)); q->setPosition(5); - c->setPosition(4); + + c = boost::shared_ptr<TestConsumer>(new TestConsumer("test", false)); // Don't acquire + q->seek(*c, Queue::MessagePredicate(), 4); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(5u, c->last.position); // Numbered from 1 + BOOST_CHECK_EQUAL(5u, c->lastMessage.getSequence()); // Numbered from 1 BOOST_CHECK(!q->dispatch(c)); } QPID_AUTO_TEST_CASE(testSetPositionPriority) { - Queue::shared_ptr q(new Queue("my-queue", true)); - framing::FieldTable args; - args.setInt("qpid.priorities", 10); - q->configure(args); + QueueSettings settings; + settings.priorities = 10; + QueueFactory factory; + Queue::shared_ptr q(factory.create("my-queue", settings)); const int priorities[] = { 1, 2, 3, 2, 1, 3 }; for (size_t i = 0; i < sizeof(priorities)/sizeof(priorities[0]); ++i) { - intrusive_ptr<Message> m = contentMessage(boost::lexical_cast<string>(i+1)); - m->getFrames().getHeaders()->get<DeliveryProperties>(true) - ->setPriority(priorities[i]); - q->deliver(m); + qpid::types::Variant::Map properties; + properties["priority"] = priorities[i]; + q->deliver(MessageUtils::createMessage(properties, boost::lexical_cast<string>(i+1))); } // Truncation removes messages in fifo order, not priority order. q->setPosition(3); - TestConsumer::shared_ptr c(new TestConsumer("test", false)); // Browse in FIFO order + TestConsumer::shared_ptr c(new TestConsumer("test", false)); // Browse in priority order BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(1u, c->last.position); + BOOST_CHECK_EQUAL(3u, c->lastMessage.getSequence()); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(2u, c->last.position); + BOOST_CHECK_EQUAL(2u, c->lastMessage.getSequence()); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(3u, c->last.position); + BOOST_CHECK_EQUAL(1u, c->lastMessage.getSequence()); BOOST_CHECK(!q->dispatch(c)); - intrusive_ptr<Message> m = contentMessage("4a"); - m->getFrames().getHeaders()->get<DeliveryProperties>(true) - ->setPriority(4); - q->deliver(m); + qpid::types::Variant::Map properties; + properties["priority"] = 4; + q->deliver(MessageUtils::createMessage(properties, "4a")); + BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(4u, c->last.position); - BOOST_CHECK_EQUAL("4a", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(4u, c->lastMessage.getSequence()); + BOOST_CHECK_EQUAL("4a", c->lastMessage.getContent()); // But consumers see priority order c.reset(new TestConsumer("test", true)); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(4u, c->last.position); - BOOST_CHECK_EQUAL("4a", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(4u, c->lastMessage.getSequence()); + BOOST_CHECK_EQUAL("4a", c->lastMessage.getContent()); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(3u, c->last.position); - BOOST_CHECK_EQUAL("3", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(3u, c->lastMessage.getSequence()); + BOOST_CHECK_EQUAL("3", c->lastMessage.getContent()); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(2u, c->last.position); - BOOST_CHECK_EQUAL("2", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(2u, c->lastMessage.getSequence()); + BOOST_CHECK_EQUAL("2", c->lastMessage.getContent()); BOOST_CHECK(q->dispatch(c)); - BOOST_CHECK_EQUAL(1u, c->last.position); - BOOST_CHECK_EQUAL("1", getContent(c->last.payload)); + BOOST_CHECK_EQUAL(1u, c->lastMessage.getSequence()); + BOOST_CHECK_EQUAL("1", c->lastMessage.getContent()); } QPID_AUTO_TEST_SUITE_END() diff --git a/qpid/cpp/src/tests/ReplicationTest.cpp b/qpid/cpp/src/tests/ReplicationTest.cpp deleted file mode 100644 index 055f06579f..0000000000 --- a/qpid/cpp/src/tests/ReplicationTest.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "unit_test.h" -#include "test_tools.h" -#include "config.h" -#include "BrokerFixture.h" - -#include "qpid/Plugin.h" -#include "qpid/broker/Broker.h" -#include "qpid/client/QueueOptions.h" -#include "qpid/framing/reply_exceptions.h" -#include "qpid/framing/SequenceNumber.h" -#include "qpid/replication/constants.h" -#include "qpid/sys/Shlib.h" - -#include <string> -#include <sstream> -#include <vector> - -#include <boost/assign.hpp> -#include <boost/bind.hpp> - -using namespace qpid::client; -using namespace qpid::framing; -using namespace qpid::replication::constants; -using boost::assign::list_of; - -namespace qpid { -namespace tests { - -QPID_AUTO_TEST_SUITE(ReplicationTestSuite) - -// FIXME aconway 2009-11-26: clean this up. -// The CMake-based build passes in the module suffix; if it's not there, this -// is a Linux/UNIX libtool-based build. -#if defined (QPID_MODULE_PREFIX) && defined (QPID_MODULE_SUFFIX) -static const char *default_shlib = - QPID_MODULE_PREFIX "replicating_listener" QPID_MODULE_POSTFIX QPID_MODULE_SUFFIX; -#else -static const char *default_shlib = ".libs/replicating_listener.so"; -#endif -qpid::sys::Shlib plugin(getLibPath("REPLICATING_LISTENER_LIB", default_shlib)); - -qpid::broker::Broker::Options getBrokerOpts(const std::vector<std::string>& args) -{ - std::vector<const char*> argv(args.size()); - transform(args.begin(), args.end(), argv.begin(), boost::bind(&std::string::c_str, _1)); - - qpid::broker::Broker::Options opts; - qpid::Plugin::addOptions(opts); - opts.parse(argv.size(), &argv[0], "", true); - return opts; -} - -QPID_AUTO_TEST_CASE(testReplicationExchange) -{ - qpid::broker::Broker::Options brokerOpts(getBrokerOpts(list_of<std::string>("qpidd") - ("--replication-exchange-name=qpid.replication"))); - SessionFixture f(brokerOpts); - - - std::string dataQ("queue-1"); - std::string eventQ("event-queue-1"); - std::string dataQ2("queue-2"); - std::string eventQ2("event-queue-2"); - FieldTable eventQopts; - eventQopts.setString("qpid.insert_sequence_numbers", REPLICATION_EVENT_SEQNO); - - f.session.queueDeclare(arg::queue=eventQ, arg::exclusive=true, arg::autoDelete=true, arg::arguments=eventQopts); - f.session.exchangeBind(arg::exchange="qpid.replication", arg::queue=eventQ, arg::bindingKey=dataQ); - - f.session.queueDeclare(arg::queue=eventQ2, arg::exclusive=true, arg::autoDelete=true, arg::arguments=eventQopts); - f.session.exchangeBind(arg::exchange="qpid.replication", arg::queue=eventQ2, arg::bindingKey=dataQ2); - - QueueOptions args; - args.enableQueueEvents(false); - f.session.queueDeclare(arg::queue=dataQ, arg::exclusive=true, arg::autoDelete=true, arg::arguments=args); - f.session.queueDeclare(arg::queue=dataQ2, arg::exclusive=true, arg::autoDelete=true, arg::arguments=args); - for (int i = 0; i < 10; i++) { - f.session.messageTransfer(arg::content=Message((boost::format("%1%_%2%") % "Message" % (i+1)).str(), dataQ)); - f.session.messageTransfer(arg::content=Message((boost::format("%1%_%2%") % "Message" % (i+1)).str(), dataQ2)); - } - Message msg; - LocalQueue incoming; - Subscription sub = f.subs.subscribe(incoming, dataQ); - for (int i = 0; i < 10; i++) { - BOOST_CHECK(incoming.get(msg, qpid::sys::TIME_SEC)); - BOOST_CHECK_EQUAL((boost::format("%1%_%2%") % "Message" % (i+1)).str(), msg.getData()); - } - BOOST_CHECK(!f.subs.get(msg, dataQ)); - - sub.cancel(); - sub = f.subs.subscribe(incoming, eventQ); - //check that we received enqueue events for first queue: - for (int i = 0; i < 10; i++) { - BOOST_CHECK(incoming.get(msg, qpid::sys::TIME_SEC)); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsString(REPLICATION_TARGET_QUEUE), dataQ); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsInt(REPLICATION_EVENT_TYPE), ENQUEUE); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsInt64(REPLICATION_EVENT_SEQNO), (i+1)); - BOOST_CHECK_EQUAL((boost::format("%1%_%2%") % "Message" % (i+1)).str(), msg.getData()); - } - //check that we received dequeue events for first queue: - for (int i = 0; i < 10; i++) { - BOOST_CHECK(incoming.get(msg, qpid::sys::TIME_SEC)); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsString(REPLICATION_TARGET_QUEUE), dataQ); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsInt(REPLICATION_EVENT_TYPE), DEQUEUE); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsInt(DEQUEUED_MESSAGE_POSITION), (i+1)); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsInt64(REPLICATION_EVENT_SEQNO), (i+11)); - } - - sub.cancel(); - sub = f.subs.subscribe(incoming, eventQ2); - //check that we received enqueue events for second queue: - for (int i = 0; i < 10; i++) { - BOOST_CHECK(incoming.get(msg, qpid::sys::TIME_SEC)); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsString(REPLICATION_TARGET_QUEUE), dataQ2); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsInt(REPLICATION_EVENT_TYPE), ENQUEUE); - BOOST_CHECK_EQUAL(msg.getHeaders().getAsInt64(REPLICATION_EVENT_SEQNO), (i+1)); - BOOST_CHECK_EQUAL((boost::format("%1%_%2%") % "Message" % (i+1)).str(), msg.getData()); - } -} - - -QPID_AUTO_TEST_SUITE_END() - -}} // namespace qpid::tests diff --git a/qpid/cpp/src/tests/TxMocks.h b/qpid/cpp/src/tests/TxMocks.h index 72cb50cd21..bf21104f70 100644 --- a/qpid/cpp/src/tests/TxMocks.h +++ b/qpid/cpp/src/tests/TxMocks.h @@ -119,8 +119,6 @@ public: assertEqualVector(expected, actual); } - void accept(TxOpConstVisitor&) const {} - ~MockTxOp(){} }; diff --git a/qpid/cpp/src/tests/TxPublishTest.cpp b/qpid/cpp/src/tests/TxPublishTest.cpp deleted file mode 100644 index a636646035..0000000000 --- a/qpid/cpp/src/tests/TxPublishTest.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -#include "qpid/broker/NullMessageStore.h" -#include "qpid/broker/RecoveryManager.h" -#include "qpid/broker/TxPublish.h" -#include "unit_test.h" -#include <iostream> -#include <list> -#include <vector> -#include "MessageUtils.h" -#include "TestMessageStore.h" - -using std::list; -using std::pair; -using std::vector; -using boost::intrusive_ptr; -using namespace qpid::broker; -using namespace qpid::framing; - -namespace qpid { -namespace tests { - -struct TxPublishTest -{ - - TestMessageStore store; - Queue::shared_ptr queue1; - Queue::shared_ptr queue2; - intrusive_ptr<Message> msg; - TxPublish op; - - TxPublishTest() : - queue1(new Queue("queue1", false, &store, 0)), - queue2(new Queue("queue2", false, &store, 0)), - msg(MessageUtils::createMessage("exchange", "routing_key", true)), - op(msg) - { - op.deliverTo(queue1); - op.deliverTo(queue2); - } -}; - - -QPID_AUTO_TEST_SUITE(TxPublishTestSuite) - -QPID_AUTO_TEST_CASE(testPrepare) -{ - TxPublishTest t; - - intrusive_ptr<PersistableMessage> pmsg = boost::static_pointer_cast<PersistableMessage>(t.msg); - //ensure messages are enqueued in store - t.op.prepare(0); - BOOST_CHECK_EQUAL((size_t) 2, t.store.enqueued.size()); - BOOST_CHECK_EQUAL(std::string("queue1"), t.store.enqueued[0].first); - BOOST_CHECK_EQUAL(pmsg, t.store.enqueued[0].second); - BOOST_CHECK_EQUAL(std::string("queue2"), t.store.enqueued[1].first); - BOOST_CHECK_EQUAL(pmsg, t.store.enqueued[1].second); - BOOST_CHECK_EQUAL( true, ( boost::static_pointer_cast<PersistableMessage>(t.msg))->isIngressComplete()); -} - -QPID_AUTO_TEST_CASE(testCommit) -{ - TxPublishTest t; - - //ensure messages are delivered to queue - t.op.prepare(0); - t.op.commit(); - BOOST_CHECK_EQUAL((uint32_t) 1, t.queue1->getMessageCount()); - intrusive_ptr<Message> msg_dequeue = t.queue1->get().payload; - - BOOST_CHECK_EQUAL( true, (boost::static_pointer_cast<PersistableMessage>(msg_dequeue))->isIngressComplete()); - BOOST_CHECK_EQUAL(t.msg, msg_dequeue); - - BOOST_CHECK_EQUAL((uint32_t) 1, t.queue2->getMessageCount()); - BOOST_CHECK_EQUAL(t.msg, t.queue2->get().payload); -} - -QPID_AUTO_TEST_SUITE_END() - -}} // namespace qpid::tests diff --git a/qpid/cpp/src/tests/ha_tests.py b/qpid/cpp/src/tests/ha_tests.py index 246b0ed423..310ef844bd 100755 --- a/qpid/cpp/src/tests/ha_tests.py +++ b/qpid/cpp/src/tests/ha_tests.py @@ -583,15 +583,7 @@ class ReplicationTests(BrokerTest): s = primary.connect().session().sender("q; {create:always, node:{x-declare:{arguments:{'qpid.policy_type':ring, 'qpid.max_count':5, 'qpid.priorities':10}}}}") priorities = [8,9,5,1,2,2,3,4,9,7,8,9,9,2] for p in priorities: s.send(Message(priority=p)) - - # FIXME aconway 2012-02-22: there is a bug in priority ring - # queues that allows a low priority message to displace a high - # one. The following commented-out assert_browse is for the - # correct result, the uncommented one is for the actualy buggy - # result. See https://issues.apache.org/jira/browse/QPID-3866 - # - # expect = sorted(priorities,reverse=True)[0:5] - expect = [9,9,9,9,2] + expect = sorted(priorities,reverse=True)[0:5] primary.assert_browse("q", expect, transform=lambda m: m.priority) backup.assert_browse_backup("q", expect, transform=lambda m: m.priority) diff --git a/qpid/cpp/src/tests/test_store.cpp b/qpid/cpp/src/tests/test_store.cpp index 257e77b6b4..83f6a5e4b1 100644 --- a/qpid/cpp/src/tests/test_store.cpp +++ b/qpid/cpp/src/tests/test_store.cpp @@ -34,6 +34,7 @@ #include "qpid/broker/NullMessageStore.h" #include "qpid/broker/Broker.h" +#include "qpid/broker/amqp_0_10/MessageTransfer.h" #include "qpid/framing/AMQFrame.h" #include "qpid/log/Statement.h" #include "qpid/Plugin.h" @@ -95,7 +96,7 @@ class TestStore : public NullMessageStore { const boost::intrusive_ptr<PersistableMessage>& pmsg, const PersistableQueue& ) { - Message* msg = dynamic_cast<Message*>(pmsg.get()); + qpid::broker::amqp_0_10::MessageTransfer* msg = dynamic_cast<qpid::broker::amqp_0_10::MessageTransfer*>(pmsg.get()); assert(msg); // Dump the message if there is a dump file. diff --git a/qpid/tests/src/py/qpid_tests/broker_0_10/alternate_exchange.py b/qpid/tests/src/py/qpid_tests/broker_0_10/alternate_exchange.py index 8cbb5793d9..2e2d5de13a 100644 --- a/qpid/tests/src/py/qpid_tests/broker_0_10/alternate_exchange.py +++ b/qpid/tests/src/py/qpid_tests/broker_0_10/alternate_exchange.py @@ -308,8 +308,6 @@ class AlternateExchangeTests(TestBase010): #create a queue using the intermediary as its alternate exchange: session.queue_declare(queue="delivery-queue", alternate_exchange="my-exchange", auto_delete=True) - #bind that queue to the dlq as well: - session.exchange_bind(exchange="dlq", queue="delivery-queue") #send it some messages: dp=self.session.delivery_properties(routing_key="delivery-queue") for m in ["One", "Two", "Three"]: @@ -349,5 +347,5 @@ class AlternateExchangeTests(TestBase010): def assertEmpty(self, queue): try: msg = queue.get(timeout=1) - self.fail("Queue not empty: " + msg) + self.fail("Queue not empty: " + str(msg)) except Empty: None diff --git a/qpid/tests/src/py/qpid_tests/broker_0_10/management.py b/qpid/tests/src/py/qpid_tests/broker_0_10/management.py index a1316ea854..4ec3e0dd03 100644 --- a/qpid/tests/src/py/qpid_tests/broker_0_10/management.py +++ b/qpid/tests/src/py/qpid_tests/broker_0_10/management.py @@ -498,7 +498,7 @@ class ManagementTest (TestBase010): session.queue_declare(queue="whatever", exclusive=True, auto_delete=True) def test_immediate_method(self): - url = "%s://%s:%d" % (self.broker.scheme or "amqp", self.broker.host, self.broker.port) + url = "%s://%s:%d" % (self.broker.scheme or "amqp", self.broker.host or "localhost", self.broker.port or 5672) conn = qpid.messaging.Connection(url) conn.open() sess = conn.session() @@ -659,7 +659,7 @@ class ManagementTest (TestBase010): self.assertEqual(rc.receive, True) # setup a connection & session to the broker - url = "%s://%s:%d" % (self.broker.scheme or "amqp", self.broker.host, self.broker.port) + url = "%s://%s:%d" % (self.broker.scheme or "amqp", self.broker.host or "localhost", self.broker.port or 5672) conn = qpid.messaging.Connection(url) conn.open() sess = conn.session() diff --git a/qpid/tests/src/py/qpid_tests/broker_0_10/msg_groups.py b/qpid/tests/src/py/qpid_tests/broker_0_10/msg_groups.py index ace7611a2f..ec015e1be4 100644 --- a/qpid/tests/src/py/qpid_tests/broker_0_10/msg_groups.py +++ b/qpid/tests/src/py/qpid_tests/broker_0_10/msg_groups.py @@ -787,7 +787,7 @@ class MultiConsumerMsgGroupTests(Base): except Empty: pass assert count == 3 # non-A's - assert a_count == 2 # pending acquired message included in browse results + assert a_count == 1 # assumes the acquired message was not the one purged and regular browsers don't get acquired messages s1.acknowledge() # ack the consumed A-0 self.qmf_session.delBroker(self.qmf_broker) diff --git a/qpid/tests/src/py/qpid_tests/broker_0_10/new_api.py b/qpid/tests/src/py/qpid_tests/broker_0_10/new_api.py index 05c4815e57..18a13e3ddf 100644 --- a/qpid/tests/src/py/qpid_tests/broker_0_10/new_api.py +++ b/qpid/tests/src/py/qpid_tests/broker_0_10/new_api.py @@ -106,10 +106,10 @@ class GeneralTests(Base): self.assertEqual(rx_alt.available(), 0, "No messages should have been routed to the alt_exchange") - # Close sess1; This will cause the queue to be deleted + # Close sess1; This will cause the queue to be deleted and all its messages (including those acquired) to be reouted to the alternate exchange sess1.close() sleep(1) - self.assertEqual(rx_alt.available(), 2, "2 of the messages should have been routed to the alt_exchange") + self.assertEqual(rx_alt.available(), 5, "All the messages should have been routed to the alt_exchange") # Close sess2; This will cause the acquired messages to be requeued and routed to the alternate sess2.close() diff --git a/qpid/tests/src/py/qpid_tests/broker_0_10/priority.py b/qpid/tests/src/py/qpid_tests/broker_0_10/priority.py index f99e908d76..bf4f1209b1 100644 --- a/qpid/tests/src/py/qpid_tests/broker_0_10/priority.py +++ b/qpid/tests/src/py/qpid_tests/broker_0_10/priority.py @@ -112,11 +112,11 @@ class PriorityTests (Base): #check all messages on the queue were received by the browser; don't relay on any specific ordering at present assert set([m.content for m in msgs]) == set([m.content for m in received]) - def ring_queue_check(self, msgs): + def ring_queue_check(self, msgs, count=10): """ Ensure that a ring queue removes lowest priority messages first. """ - snd = self.ssn.sender(address("priority-ring-queue", arguments="x-qpid-priorities:10, 'qpid.policy_type':ring, 'qpid.max_count':10"), + snd = self.ssn.sender(address("priority-ring-queue", arguments="x-qpid-priorities:10, 'qpid.policy_type':ring, 'qpid.max_count':%s" % count), durable=self.durable()) for m in msgs: snd.send(m) @@ -126,23 +126,31 @@ class PriorityTests (Base): while True: received.append(rcv.fetch(0)) except Empty: None - expected = [] - for m in msgs: - while len(expected) > 9: - expected=sorted_(expected, key=lambda x: priority_level(x.priority,10)) - expected.pop(0) - expected.append(m) - #print "sent %s; expected %s; got %s" % ([m.content for m in msgs], [m.content for m in expected], [m.content for m in received]) + expected = sorted_(msgs, key=lambda x: priority_level(x.priority,10))[len(msgs)-count:] + expected = sorted_(expected, key=lambda x: priority_level(x.priority,10), reverse=True) + #print "sent %s; expected %s; got %s" % ([m.priority for m in msgs], [m.priority for m in expected], [m.priority for m in received]) + #print "sent %s; expected %s; got %s" % ([m.content for m in msgs], [m.content for m in expected], [m.content for m in received]) assert [m.content for m in expected] == [m.content for m in received] def test_ring_queue_1(self): priorities = [4,5,3,6,9,9,2,9,2,9,9,1,9,9,9,3,3,3,9,9,3,9,3,9,9,9,9,9,9,2,3] - seq = content("msg") + seq = content("msg") self.ring_queue_check([Message(content=seq.next(), priority = p) for p in priorities]) def test_ring_queue_2(self): - priorities = [9,0,2,3,6,9,9,2,9,2,9,9,1,9,4,7,1,1,3,9,9,3,9,3,9,9,9,1,9,9,2,3,0,9] - seq = content("msg") + priorities = [9,0,2,3,6,3,4,2,9,2,9,9,1,9,4,7,1,1,3,9,7,3,9,3,9,1,5,1,9,7,2,3,0,9] + seq = content("msg") + self.ring_queue_check([Message(content=seq.next(), priority = p) for p in priorities]) + + def test_ring_queue_3(self): + #test case given for QPID-3866 + priorities = [8,9,5,1,2,2,3,4,9,7,8,9,9,2] + seq = content("msg") + self.ring_queue_check([Message(content=seq.next(), priority = p) for p in priorities], 5) + + def test_ring_queue_4(self): + priorities = [9,0,2,3,6,3,4,2,9,2,9,3,1,9,4,7,1,1,3,2,7,3,9,3,6,1,5,1,9,7,2,3,0,2] + seq = content("msg") self.ring_queue_check([Message(content=seq.next(), priority = p) for p in priorities]) def test_requeue(self): @@ -169,6 +177,7 @@ class PriorityTests (Base): for expected in sorted_(msgs, key=lambda m: priority_level(m.priority,10), reverse=True): msg = rcv.fetch(0) #print "expected priority %s got %s" % (expected.priority, msg.priority) + #print "expected content %s got %s" % (expected.content, msg.content) assert msg.content == expected.content self.ssn.acknowledge(msg) @@ -231,7 +240,7 @@ def sorted_(msgs, key=None, reverse=False): Workaround lack of sorted builtin function in python 2.3 and lack of keyword arguments to list.sort() """ - temp = msgs + temp = [m for m in msgs] temp.sort(key_to_cmp(key, reverse=reverse)) return temp diff --git a/qpid/tests/src/py/qpid_tests/broker_0_10/threshold.py b/qpid/tests/src/py/qpid_tests/broker_0_10/threshold.py index 6628ae8424..0ad2295822 100644 --- a/qpid/tests/src/py/qpid_tests/broker_0_10/threshold.py +++ b/qpid/tests/src/py/qpid_tests/broker_0_10/threshold.py @@ -41,7 +41,7 @@ class ThresholdTests (Base): snd.send(m) count = count + 1 size = size + len(m.content) - event = rcv.fetch() + event = rcv.fetch(timeout=1) schema = event.content[0]["_schema_id"] assert schema["_class_name"] == "queueThresholdExceeded" values = event.content[0]["_values"] |