diff options
Diffstat (limited to 'qpid/cpp/src/qpid/sys')
130 files changed, 17382 insertions, 0 deletions
diff --git a/qpid/cpp/src/qpid/sys/AggregateOutput.cpp b/qpid/cpp/src/qpid/sys/AggregateOutput.cpp new file mode 100644 index 0000000000..fc95f46fb9 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/AggregateOutput.cpp @@ -0,0 +1,88 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/AggregateOutput.h" +#include "qpid/log/Statement.h" +#include <algorithm> + +namespace qpid { +namespace sys { + +AggregateOutput::AggregateOutput(OutputControl& c) : busy(false), control(c) {} + +void AggregateOutput::abort() { control.abort(); } + +void AggregateOutput::activateOutput() { control.activateOutput(); } + +void AggregateOutput::giveReadCredit(int32_t credit) { control.giveReadCredit(credit); } + +namespace { +// Clear the busy flag and notify waiting threads in destructor. +struct ScopedBusy { + bool& flag; + Monitor& monitor; + ScopedBusy(bool& f, Monitor& m) : flag(f), monitor(m) { f = true; } + ~ScopedBusy() { flag = false; monitor.notifyAll(); } +}; +} + +bool AggregateOutput::doOutput() { + Mutex::ScopedLock l(lock); + ScopedBusy sb(busy, lock); + + while (!tasks.empty()) { + OutputTask* t=tasks.front(); + tasks.pop_front(); + bool didOutput; + { + // Allow concurrent call to addOutputTask. + // removeOutputTask will wait till !busy before removing a task. + Mutex::ScopedUnlock u(lock); + didOutput = t->doOutput(); + } + if (didOutput) { + tasks.push_back(t); + return true; + } + } + return false; +} + +void AggregateOutput::addOutputTask(OutputTask* task) { + Mutex::ScopedLock l(lock); + tasks.push_back(task); +} + +void AggregateOutput::removeOutputTask(OutputTask* task) { + Mutex::ScopedLock l(lock); + while (busy) lock.wait(); + tasks.erase(std::remove(tasks.begin(), tasks.end(), task), tasks.end()); +} + +void AggregateOutput::removeAll() +{ + Mutex::ScopedLock l(lock); + while (busy) lock.wait(); + tasks.clear(); +} + + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/AggregateOutput.h b/qpid/cpp/src/qpid/sys/AggregateOutput.h new file mode 100644 index 0000000000..d7c0ff29e3 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/AggregateOutput.h @@ -0,0 +1,77 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef _AggregateOutput_ +#define _AggregateOutput_ + +#include "qpid/sys/Monitor.h" +#include "qpid/sys/OutputControl.h" +#include "qpid/sys/OutputTask.h" +#include "qpid/CommonImportExport.h" + +#include <algorithm> +#include <deque> + +namespace qpid { +namespace sys { + +/** + * Holds a collection of output tasks, doOutput picks the next one to execute. + * + * Tasks are automatically removed if their doOutput() or hasOutput() returns false. + * + * Thread safe. addOutputTask may be called in one connection thread while + * doOutput is called in another. + */ + +class QPID_COMMON_CLASS_EXTERN AggregateOutput : public OutputTask, public OutputControl +{ + typedef std::deque<OutputTask*> TaskList; + + Monitor lock; + TaskList tasks; + bool busy; + OutputControl& control; + + public: + QPID_COMMON_EXTERN AggregateOutput(OutputControl& c); + + // These may be called concurrently with any function. + QPID_COMMON_EXTERN void abort(); + QPID_COMMON_EXTERN void activateOutput(); + QPID_COMMON_EXTERN void giveReadCredit(int32_t); + QPID_COMMON_EXTERN void addOutputTask(OutputTask* t); + + // These functions must not be called concurrently with each other. + QPID_COMMON_EXTERN bool doOutput(); + QPID_COMMON_EXTERN void removeOutputTask(OutputTask* t); + QPID_COMMON_EXTERN void removeAll(); + + /** Apply f to each OutputTask* in the tasks list */ + template <class F> void eachOutput(F f) { + Mutex::ScopedLock l(lock); + std::for_each(tasks.begin(), tasks.end(), f); + } +}; + +}} // namespace qpid::sys + + +#endif diff --git a/qpid/cpp/src/qpid/sys/AsynchIO.h b/qpid/cpp/src/qpid/sys/AsynchIO.h new file mode 100644 index 0000000000..41f74f7ed0 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/AsynchIO.h @@ -0,0 +1,160 @@ +#ifndef _sys_AsynchIO +#define _sys_AsynchIO +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IntegerTypes.h" +#include "qpid/CommonImportExport.h" + +#include <string.h> + +#include <boost/function.hpp> +#include <boost/shared_ptr.hpp> + +namespace qpid { +namespace sys { + +class Socket; +class Poller; + +/* + * Asynchronous acceptor: accepts connections then does a callback with the + * accepted fd + */ +class AsynchAcceptor { +public: + typedef boost::function1<void, const Socket&> Callback; + + QPID_COMMON_EXTERN static AsynchAcceptor* create(const Socket& s, Callback callback); + virtual ~AsynchAcceptor() {}; + virtual void start(boost::shared_ptr<Poller> poller) = 0; +}; + +/* + * Asynchronous connector: starts the process of initiating a connection and + * invokes a callback when completed or failed. + */ +class AsynchConnector { +public: + typedef boost::function1<void, const Socket&> ConnectedCallback; + typedef boost::function3<void, const Socket&, int, const std::string&> FailedCallback; + + // Call create() to allocate a new AsynchConnector object with the + // specified poller, addressing, and callbacks. + // This method is implemented in platform-specific code to + // create a correctly typed object. The platform code also manages + // deletes. To correctly manage heaps when needed, the allocate and + // delete should both be done from the same class/library. + QPID_COMMON_EXTERN static AsynchConnector* create(const Socket& s, + const std::string& hostname, + const std::string& port, + ConnectedCallback connCb, + FailedCallback failCb); + virtual void start(boost::shared_ptr<Poller> poller) = 0; + virtual void stop() {}; +protected: + AsynchConnector() {} + virtual ~AsynchConnector() {} +}; + +struct AsynchIOBufferBase { + char* const bytes; + const int32_t byteCount; + int32_t dataStart; + int32_t dataCount; + + AsynchIOBufferBase(char* const b, const int32_t s) : + bytes(b), + byteCount(s), + dataStart(0), + dataCount(0) + {} + + virtual ~AsynchIOBufferBase() + {} + + void squish() { + if (dataStart != 0) { + ::memmove(bytes, bytes + dataStart, dataCount); + dataStart = 0; + } + } +}; + +/* + * Asychronous reader/writer: + * Reader accepts buffers to read into; reads into the provided buffers + * and then does a callback with the buffer and amount read. Optionally it + * can callback when there is something to read but no buffer to read it into. + * + * Writer accepts a buffer and queues it for writing; can also be given + * a callback for when writing is "idle" (ie fd is writable, but nothing + * to write). + */ +class AsynchIO { +public: + typedef AsynchIOBufferBase BufferBase; + + typedef boost::function2<void, AsynchIO&, BufferBase*> ReadCallback; + typedef boost::function1<void, AsynchIO&> EofCallback; + typedef boost::function1<void, AsynchIO&> DisconnectCallback; + typedef boost::function2<void, AsynchIO&, const Socket&> ClosedCallback; + typedef boost::function1<void, AsynchIO&> BuffersEmptyCallback; + typedef boost::function1<void, AsynchIO&> IdleCallback; + typedef boost::function1<void, AsynchIO&> RequestCallback; + + // Call create() to allocate a new AsynchIO object with the specified + // callbacks. This method is implemented in platform-specific code to + // create a correctly typed object. The platform code also manages + // deletes. To correctly manage heaps when needed, the allocate and + // delete should both be done from the same class/library. + QPID_COMMON_EXTERN static AsynchIO* create(const Socket& s, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb = 0, + BuffersEmptyCallback eCb = 0, + IdleCallback iCb = 0); +public: + virtual void queueForDeletion() = 0; + + virtual void start(boost::shared_ptr<Poller> poller) = 0; + virtual void queueReadBuffer(BufferBase* buff) = 0; + virtual void unread(BufferBase* buff) = 0; + virtual void queueWrite(BufferBase* buff) = 0; + virtual void notifyPendingWrite() = 0; + virtual void queueWriteClose() = 0; + virtual bool writeQueueEmpty() = 0; + virtual void startReading() = 0; + virtual void stopReading() = 0; + virtual void requestCallback(RequestCallback) = 0; + virtual BufferBase* getQueuedBuffer() = 0; + +protected: + // Derived class manages lifetime; must be constructed using the + // static create() method. Deletes not allowed from outside. + AsynchIO() {} + virtual ~AsynchIO() {} +}; + +}} + +#endif // _sys_AsynchIO diff --git a/qpid/cpp/src/qpid/sys/AsynchIOHandler.cpp b/qpid/cpp/src/qpid/sys/AsynchIOHandler.cpp new file mode 100644 index 0000000000..30a87d9d44 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/AsynchIOHandler.cpp @@ -0,0 +1,228 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/AsynchIOHandler.h" +#include "qpid/sys/AsynchIO.h" +#include "qpid/sys/Socket.h" +#include "qpid/sys/SecuritySettings.h" +#include "qpid/framing/AMQP_HighestVersion.h" +#include "qpid/framing/ProtocolInitiation.h" +#include "qpid/log/Statement.h" + +#include <boost/bind.hpp> + +namespace qpid { +namespace sys { + +// Buffer definition +struct Buff : public AsynchIO::BufferBase { + Buff() : + AsynchIO::BufferBase(new char[65536], 65536) + {} + ~Buff() + { delete [] bytes;} +}; + +AsynchIOHandler::AsynchIOHandler(std::string id, ConnectionCodec::Factory* f) : + identifier(id), + aio(0), + factory(f), + codec(0), + readError(false), + isClient(false), + readCredit(InfiniteCredit) +{} + +AsynchIOHandler::~AsynchIOHandler() { + if (codec) + codec->closed(); + delete codec; +} + +void AsynchIOHandler::init(AsynchIO* a, int numBuffs) { + aio = a; + + // Give connection some buffers to use + for (int i = 0; i < numBuffs; i++) { + aio->queueReadBuffer(new Buff); + } +} + +void AsynchIOHandler::write(const framing::ProtocolInitiation& data) +{ + QPID_LOG(debug, "SENT [" << identifier << "] INIT(" << data << ")"); + AsynchIO::BufferBase* buff = aio->getQueuedBuffer(); + if (!buff) + buff = new Buff; + framing::Buffer out(buff->bytes, buff->byteCount); + data.encode(out); + buff->dataCount = data.encodedSize(); + aio->queueWrite(buff); +} + +void AsynchIOHandler::abort() { + // Don't disconnect if we're already disconnecting + if (!readError) { + aio->requestCallback(boost::bind(&AsynchIOHandler::eof, this, _1)); + } +} + +void AsynchIOHandler::activateOutput() { + aio->notifyPendingWrite(); +} + +// Input side +void AsynchIOHandler::giveReadCredit(int32_t credit) { + // Check whether we started in the don't about credit state + if (readCredit.boolCompareAndSwap(InfiniteCredit, credit)) + return; + // TODO In theory should be able to use an atomic operation before taking the lock + // but in practice there seems to be an unexplained race in that case + ScopedLock<Mutex> l(creditLock); + if (readCredit.fetchAndAdd(credit) != 0) + return; + assert(readCredit.get() >= 0); + if (readCredit.get() != 0) + aio->startReading(); +} + +void AsynchIOHandler::readbuff(AsynchIO& , AsynchIO::BufferBase* buff) { + if (readError) { + return; + } + + // Check here for read credit + if (readCredit.get() != InfiniteCredit) { + if (readCredit.get() == 0) { + // FIXME aconway 2009-10-01: Workaround to avoid "false wakeups". + // readbuff is sometimes called with no credit. + // This should be fixed somewhere else to avoid such calls. + aio->unread(buff); + return; + } + // TODO In theory should be able to use an atomic operation before taking the lock + // but in practice there seems to be an unexplained race in that case + ScopedLock<Mutex> l(creditLock); + if (--readCredit == 0) { + assert(readCredit.get() >= 0); + if (readCredit.get() == 0) { + aio->stopReading(); + } + } + } + + size_t decoded = 0; + if (codec) { // Already initiated + try { + decoded = codec->decode(buff->bytes+buff->dataStart, buff->dataCount); + }catch(const std::exception& e){ + QPID_LOG(error, e.what()); + readError = true; + aio->queueWriteClose(); + } + }else{ + framing::Buffer in(buff->bytes+buff->dataStart, buff->dataCount); + framing::ProtocolInitiation protocolInit; + if (protocolInit.decode(in)) { + decoded = in.getPosition(); + QPID_LOG(debug, "RECV [" << identifier << "] INIT(" << protocolInit << ")"); + try { + codec = factory->create(protocolInit.getVersion(), *this, identifier, SecuritySettings()); + if (!codec) { + //TODO: may still want to revise this... + //send valid version header & close connection. + write(framing::ProtocolInitiation(framing::highestProtocolVersion)); + readError = true; + aio->queueWriteClose(); + } + } catch (const std::exception& e) { + QPID_LOG(error, e.what()); + readError = true; + aio->queueWriteClose(); + } + } + } + // TODO: unreading needs to go away, and when we can cope + // with multiple sub-buffers in the general buffer scheme, it will + if (decoded != size_t(buff->dataCount)) { + // Adjust buffer for used bytes and then "unread them" + buff->dataStart += decoded; + buff->dataCount -= decoded; + aio->unread(buff); + } else { + // Give whole buffer back to aio subsystem + aio->queueReadBuffer(buff); + } +} + +void AsynchIOHandler::eof(AsynchIO& a) { + disconnect(a); + readError = true; + aio->queueWriteClose(); +} + +void AsynchIOHandler::closedSocket(AsynchIO&, const Socket& s) { + // If we closed with data still to send log a warning + if (!aio->writeQueueEmpty()) { + QPID_LOG(warning, "CLOSING [" << identifier << "] unsent data (probably due to client disconnect)"); + } + delete &s; + aio->queueForDeletion(); + delete this; +} + +void AsynchIOHandler::disconnect(AsynchIO&) { + QPID_LOG(debug, "DISCONNECTED [" << identifier << "]"); + if (codec) codec->closed(); +} + +// Notifications +void AsynchIOHandler::nobuffs(AsynchIO&) { +} + +void AsynchIOHandler::idle(AsynchIO&){ + if (isClient && codec == 0) { + codec = factory->create(*this, identifier, SecuritySettings()); + write(framing::ProtocolInitiation(codec->getVersion())); + return; + } + if (codec == 0) return; + try { + if (codec->canEncode()) { + // Try and get a queued buffer if not then construct new one + AsynchIO::BufferBase* buff = aio->getQueuedBuffer(); + if (!buff) buff = new Buff; + size_t encoded=codec->encode(buff->bytes, buff->byteCount); + buff->dataCount = encoded; + aio->queueWrite(buff); + } + if (codec->isClosed()) { + readError = true; + aio->queueWriteClose(); + } + } catch (const std::exception& e) { + QPID_LOG(error, e.what()); + readError = true; + aio->queueWriteClose(); + } +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/AsynchIOHandler.h b/qpid/cpp/src/qpid/sys/AsynchIOHandler.h new file mode 100644 index 0000000000..b9867606c4 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/AsynchIOHandler.h @@ -0,0 +1,80 @@ +#ifndef _sys_AsynchIOHandler_h +#define _sys_AsynchIOHandler_h +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/OutputControl.h" +#include "qpid/sys/ConnectionCodec.h" +#include "qpid/sys/AtomicValue.h" +#include "qpid/sys/Mutex.h" +#include "qpid/CommonImportExport.h" + +namespace qpid { + +namespace framing { + class ProtocolInitiation; +} + +namespace sys { + +class AsynchIO; +struct AsynchIOBufferBase; +class Socket; + +class AsynchIOHandler : public OutputControl { + std::string identifier; + AsynchIO* aio; + ConnectionCodec::Factory* factory; + ConnectionCodec* codec; + bool readError; + bool isClient; + AtomicValue<int32_t> readCredit; + static const int32_t InfiniteCredit = -1; + Mutex creditLock; + + void write(const framing::ProtocolInitiation&); + + public: + QPID_COMMON_EXTERN AsynchIOHandler(std::string id, ConnectionCodec::Factory* f); + QPID_COMMON_EXTERN ~AsynchIOHandler(); + QPID_COMMON_EXTERN void init(AsynchIO* a, int numBuffs); + + QPID_COMMON_INLINE_EXTERN void setClient() { isClient = true; } + + // Output side + QPID_COMMON_EXTERN void abort(); + QPID_COMMON_EXTERN void activateOutput(); + QPID_COMMON_EXTERN void giveReadCredit(int32_t credit); + + // Input side + QPID_COMMON_EXTERN void readbuff(AsynchIO& aio, AsynchIOBufferBase* buff); + QPID_COMMON_EXTERN void eof(AsynchIO& aio); + QPID_COMMON_EXTERN void disconnect(AsynchIO& aio); + + // Notifications + QPID_COMMON_EXTERN void nobuffs(AsynchIO& aio); + QPID_COMMON_EXTERN void idle(AsynchIO& aio); + QPID_COMMON_EXTERN void closedSocket(AsynchIO& aio, const Socket& s); +}; + +}} // namespace qpid::sys + +#endif // _sys_AsynchIOHandler_h diff --git a/qpid/cpp/src/qpid/sys/AtomicCount.h b/qpid/cpp/src/qpid/sys/AtomicCount.h new file mode 100644 index 0000000000..94580c61f3 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/AtomicCount.h @@ -0,0 +1,52 @@ +#ifndef _posix_AtomicCount_h +#define _posix_AtomicCount_h + +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <boost/detail/atomic_count.hpp> +#include "qpid/sys/ScopedIncrement.h" + +namespace qpid { +namespace sys { + +/** + * Atomic counter. + */ +class AtomicCount { + public: + typedef ::qpid::sys::ScopedDecrement<AtomicCount> ScopedDecrement; + typedef ::qpid::sys::ScopedIncrement<AtomicCount> ScopedIncrement; + + AtomicCount(long value = 0) : count(value) {} + + void operator++() { ++count ; } + + long operator--() { return --count; } + + operator long() const { return count; } + + private: + boost::detail::atomic_count count; +}; + + +}} + + +#endif // _posix_AtomicCount_h diff --git a/qpid/cpp/src/qpid/sys/AtomicValue.h b/qpid/cpp/src/qpid/sys/AtomicValue.h new file mode 100644 index 0000000000..bf995f991e --- /dev/null +++ b/qpid/cpp/src/qpid/sys/AtomicValue.h @@ -0,0 +1,39 @@ +#ifndef QPID_SYS_ATOMICVALUE_H +#define QPID_SYS_ATOMICVALUE_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +// Have to check for clang before gcc as clang pretends to be gcc too +#if defined( __clang__ ) +// Use the clang doesn't support atomic builtins for 64 bit values, so use the slow versions +#include "qpid/sys/AtomicValue_mutex.h" + +#elif defined( __GNUC__ ) && __GNUC__ >= 4 && ( defined( __i686__ ) || defined( __x86_64__ ) ) +// Use the Gnu C built-in atomic operations if compiling with gcc on a suitable platform. +#include "qpid/sys/AtomicValue_gcc.h" + +#else +// Fall-back to mutex locked operations if we don't have atomic ops. +#include "qpid/sys/AtomicValue_mutex.h" +#endif + +#endif /*!QPID_SYS_ATOMICVALUE_GCC_H*/ diff --git a/qpid/cpp/src/qpid/sys/AtomicValue_gcc.h b/qpid/cpp/src/qpid/sys/AtomicValue_gcc.h new file mode 100644 index 0000000000..d022b07c1d --- /dev/null +++ b/qpid/cpp/src/qpid/sys/AtomicValue_gcc.h @@ -0,0 +1,68 @@ +#ifndef QPID_SYS_ATOMICVALUE_GCC_H +#define QPID_SYS_ATOMICVALUE_GCC_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#if !defined(QPID_SYS_ATOMICVALUE_H) +#error "This file should only be included via AtomicValue.h." +#endif + +namespace qpid { +namespace sys { + +/** + * Atomic value of type T. T must be an integral type of size 1,2,4 or 8 bytes. + * All operations are atomic and preform a full memory barrier unless otherwise noted. + */ +template <class T> +class AtomicValue +{ + public: + AtomicValue(T init=0) : value(init) {} + + // Update and return new value. + inline T operator+=(T n) { return __sync_add_and_fetch(&value, n); } + inline T operator-=(T n) { return __sync_sub_and_fetch(&value, n); } + inline T operator++() { return *this += 1; } + inline T operator--() { return *this -= 1; } + + // Update and return old value. + inline T fetchAndAdd(T n) { return __sync_fetch_and_add(&value, n); } + inline T fetchAndSub(T n) { return __sync_fetch_and_sub(&value, n); } + inline T operator++(int) { return fetchAndAdd(1); } + inline T operator--(int) { return fetchAndSub(1); } + + /** If current value == testval then set to newval. Returns the old value. */ + T valueCompareAndSwap(T testval, T newval) { return __sync_val_compare_and_swap(&value, testval, newval); } + + /** If current value == testval then set to newval. Returns true if the swap was performed. */ + bool boolCompareAndSwap(T testval, T newval) { return __sync_bool_compare_and_swap(&value, testval, newval); } + + T get() const { return const_cast<AtomicValue<T>*>(this)->fetchAndAdd(static_cast<T>(0)); } + + private: + T value; +}; + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_ATOMICVALUE_GCC_H*/ diff --git a/qpid/cpp/src/qpid/sys/AtomicValue_mutex.h b/qpid/cpp/src/qpid/sys/AtomicValue_mutex.h new file mode 100644 index 0000000000..e4d433e7f5 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/AtomicValue_mutex.h @@ -0,0 +1,83 @@ +#ifndef QPID_SYS_ATOMICVALUE_MUTEX_H +#define QPID_SYS_ATOMICVALUE_MUTEX_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#if !defined(QPID_SYS_ATOMICVALUE_H) +#error "This file should only be included via AtomicValue.h." +#endif + +#include "qpid/sys/Mutex.h" + +namespace qpid { +namespace sys { + +/** + * Atomic value of type T. T must be an integral type of size 1,2,4 or 8 bytes. + * All operations are atomic and preform a full memory barrier unless otherwise noted. + */ +template <class T> +class AtomicValue +{ + public: + AtomicValue(T init=0) : value(init) {} + + // Update and return new value. + inline T operator+=(T n) { Lock l(lock); return value += n; } + inline T operator-=(T n) { Lock l(lock); return value -= n; } + inline T operator++() { return *this += 1; } + inline T operator--() { return *this -= 1; } + + // Update and return old value. + inline T fetchAndAdd(T n) { Lock l(lock); T old=value; value += n; return old; } + inline T fetchAndSub(T n) { Lock l(lock); T old=value; value -= n; return old; } + inline T operator++(int) { return fetchAndAdd(1); } + inline T operator--(int) { return fetchAndSub(1); } + + AtomicValue& operator=(T newval) { Lock l(lock); value = newval; return *this; } + + /** If current value == testval then set to newval. Returns the old value. */ + T valueCompareAndSwap(T testval, T newval) { + Lock l(lock); + T old=value; + if (value == testval) value = newval; + return old; + } + + /** If current value == testval then set to newval. Returns true if the swap was performed. */ + bool boolCompareAndSwap(T testval, T newval) { + Lock l(lock); + if (value == testval) { value = newval; return true; } + return false; + } + + T get() const { Lock l(lock); return value; } + + private: + typedef Mutex::ScopedLock Lock; + T value; + mutable Mutex lock; +}; + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_ATOMICVALUE_MUTEX_H*/ diff --git a/qpid/cpp/src/qpid/sys/BlockingQueue.h b/qpid/cpp/src/qpid/sys/BlockingQueue.h new file mode 100644 index 0000000000..ca6b529930 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/BlockingQueue.h @@ -0,0 +1,129 @@ +#ifndef QPID_SYS_BLOCKINGQUEUE_H +#define QPID_SYS_BLOCKINGQUEUE_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Waitable.h" + +#include <queue> + +namespace qpid { +namespace sys { + +/** + * A simple blocking queue template + */ +template <class T> +class BlockingQueue +{ + mutable sys::Waitable waitable; + std::queue<T> queue; + +public: + BlockingQueue() {} + ~BlockingQueue() { close(); } + + /** Pop from the queue, block up to timeout if empty. + *@param result Set to value popped from queue. + *@param timeout Defaults to infinite. + *@return true if result was set, false if queue empty after timeout. + */ + bool pop(T& result, Duration timeout=TIME_INFINITE) { + Mutex::ScopedLock l(waitable); + { + Waitable::ScopedWait w(waitable); + if (timeout == TIME_INFINITE) { + while (queue.empty()) waitable.wait(); + } else if (timeout) { + AbsTime deadline(now(),timeout); + while (queue.empty() && deadline > now()) waitable.wait(deadline); + } else { + //ensure zero timeout pop does not miss the fact that + //queue is closed + waitable.checkException(); + } + } + if (queue.empty()) return false; + result = queue.front(); + queue.pop(); + if (!queue.empty()) + waitable.notify(); // Notify another waiter. + return true; + } + + T pop(Duration timeout=TIME_INFINITE) { + T result; + bool ok = pop(result, timeout); + if (!ok) + throw Exception("Timed out waiting on a blocking queue"); + return result; + } + + /** Push a value onto the queue. + * Note it is not an error to push onto a closed queue. + */ + void push(const T& t) { + Mutex::ScopedLock l(waitable); + queue.push(t); + waitable.notify(); // Notify a waiter. + } + + /** + * Close the queue. + *@ex exception to throw to waiting threads. ClosedException by default. + */ + void close(const ExceptionHolder& ex=ExceptionHolder(new ClosedException())) + { + Mutex::ScopedLock l(waitable); + if (!waitable.hasException()) { + waitable.setException(ex); + waitable.notifyAll(); + waitable.waitWaiters(); // Ensure no threads are still waiting. + } + } + + /** Open a closed queue. */ + void open() { + Mutex::ScopedLock l(waitable); + waitable.resetException(); + } + + bool isClosed() const { + Mutex::ScopedLock l(waitable); + return waitable.hasException(); + } + + bool empty() const { + Mutex::ScopedLock l(waitable); + return queue.empty(); + } + size_t size() const { + Mutex::ScopedLock l(waitable); + return queue.size(); + } +}; + +}} + + + +#endif /*!QPID_SYS_BLOCKINGQUEUE_H*/ diff --git a/qpid/cpp/src/qpid/sys/ClusterSafe.cpp b/qpid/cpp/src/qpid/sys/ClusterSafe.cpp new file mode 100644 index 0000000000..dd37615145 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ClusterSafe.cpp @@ -0,0 +1,66 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "ClusterSafe.h" +#include "qpid/log/Statement.h" +#include "qpid/sys/Thread.h" +#include <stdlib.h> + +namespace qpid { +namespace sys { + +namespace { +bool inCluster = false; +QPID_TSS bool inContext = false; +} + +bool isClusterSafe() { return !inCluster || inContext; } + +void assertClusterSafe() { + if (!isClusterSafe()) { + QPID_LOG(critical, "Modified cluster state outside of cluster context"); + ::abort(); + } +} + +ClusterSafeScope::ClusterSafeScope() { + save = inContext; + inContext = true; +} + +ClusterSafeScope::~ClusterSafeScope() { + assert(inContext); + inContext = save; +} + +ClusterUnsafeScope::ClusterUnsafeScope() { + save = inContext; + inContext = false; +} + +ClusterUnsafeScope::~ClusterUnsafeScope() { + assert(!inContext); + inContext = save; +} + +void enableClusterSafe() { inCluster = true; } + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/ClusterSafe.h b/qpid/cpp/src/qpid/sys/ClusterSafe.h new file mode 100644 index 0000000000..27e4eb46a5 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ClusterSafe.h @@ -0,0 +1,87 @@ +#ifndef QPID_SYS_CLUSTERSAFE_H +#define QPID_SYS_CLUSTERSAFE_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/CommonImportExport.h" + +namespace qpid { +namespace sys { + +/** + * Assertion to add to code that modifies clustered state. + * + * In a non-clustered broker this is a no-op. + * + * In a clustered broker, checks that it is being called + * in a context where it is safe to modify clustered state. + * If not it aborts the process as this is a serious bug. + * + * This function is in the common library rather than the cluster + * library because it is called by code in the broker library. + */ +QPID_COMMON_EXTERN void assertClusterSafe(); + +/** + * In a non-clustered broker, returns true. + * + * In a clustered broker returns true if we are in a context where it + * is safe to modify cluster state. + * + * This function is in the common library rather than the cluster + * library because it is called by code in the broker library. + */ +QPID_COMMON_EXTERN bool isClusterSafe(); + +/** + * Mark a scope as cluster safe. Sets isClusterSafe in constructor and resets + * to previous value in destructor. + */ +class ClusterSafeScope { + public: + ClusterSafeScope(); + ~ClusterSafeScope(); + private: + bool save; +}; + +/** + * Mark a scope as cluster unsafe. Clears isClusterSafe in constructor and resets + * to previous value in destructor. + */ +class ClusterUnsafeScope { + public: + QPID_COMMON_EXTERN ClusterUnsafeScope(); + QPID_COMMON_EXTERN ~ClusterUnsafeScope(); + private: + bool save; +}; + +/** + * Enable cluster-safe assertions. By default they are no-ops. + * Called by cluster code. + */ +void enableClusterSafe(); + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_CLUSTERSAFE_H*/ diff --git a/qpid/cpp/src/qpid/sys/Codec.h b/qpid/cpp/src/qpid/sys/Codec.h new file mode 100644 index 0000000000..ace721fbcc --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Codec.h @@ -0,0 +1,52 @@ +#ifndef QPID_SYS_CODEC_H +#define QPID_SYS_CODEC_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include <cstddef> + +namespace qpid { +namespace sys { + +/** + * Generic codec interface + */ +class Codec +{ + public: + virtual ~Codec() {} + + /** Decode from buffer, return number of bytes decoded. + * @return may be less than size if there was incomplete + * data at the end of the buffer. + */ + virtual std::size_t decode(const char* buffer, std::size_t size) = 0; + + + /** Encode into buffer, return number of bytes encoded */ + virtual std::size_t encode(const char* buffer, std::size_t size) = 0; + + /** Return true if we have data to encode */ + virtual bool canEncode() = 0; +}; +}} // namespace qpid::sys + +#endif /*!QPID_SYS_CODEC_H*/ diff --git a/qpid/cpp/src/qpid/sys/ConnectionCodec.h b/qpid/cpp/src/qpid/sys/ConnectionCodec.h new file mode 100644 index 0000000000..c2890f06dc --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ConnectionCodec.h @@ -0,0 +1,68 @@ +#ifndef QPID_SYS_CONNECTION_CODEC_H +#define QPID_SYS_CONNECTION_CODEC_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/Codec.h" +#include "qpid/framing/ProtocolVersion.h" + +namespace qpid { + +namespace sys { + +class InputHandlerFactory; +class OutputControl; +struct SecuritySettings; + +/** + * Interface of coder/decoder for a connection of a specific protocol + * version. + */ +class ConnectionCodec : public Codec { + public: + virtual ~ConnectionCodec() {} + + /** Network connection was closed from other end. */ + virtual void closed() = 0; + + virtual bool isClosed() const = 0; + + virtual framing::ProtocolVersion getVersion() const = 0; + + struct Factory { + virtual ~Factory() {} + + /** Return 0 if version unknown */ + virtual ConnectionCodec* create( + framing::ProtocolVersion, OutputControl&, const std::string& id, + const SecuritySettings& + ) = 0; + + /** Return "preferred" codec for outbound connections. */ + virtual ConnectionCodec* create( + OutputControl&, const std::string& id, const SecuritySettings& + ) = 0; + }; +}; + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_CONNECTION_CODEC_H*/ diff --git a/qpid/cpp/src/qpid/sys/ConnectionInputHandler.h b/qpid/cpp/src/qpid/sys/ConnectionInputHandler.h new file mode 100644 index 0000000000..92de808308 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ConnectionInputHandler.h @@ -0,0 +1,52 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef _ConnectionInputHandler_ +#define _ConnectionInputHandler_ + +#include "qpid/framing/InputHandler.h" +#include "qpid/sys/OutputTask.h" +#include "qpid/sys/TimeoutHandler.h" + +namespace qpid { +namespace sys { + + +/** + * ConnectionInputHandler provides methods to process incoming frames + * using InputHandler::receive() and to generate outgoing messages in + * OutputTask::doOutput() + * + */ + + class ConnectionInputHandler : + public qpid::framing::InputHandler, + public TimeoutHandler, public OutputTask + { + public: + + virtual void closed() = 0; + }; + +} +} + + +#endif diff --git a/qpid/cpp/src/qpid/sys/ConnectionInputHandlerFactory.h b/qpid/cpp/src/qpid/sys/ConnectionInputHandlerFactory.h new file mode 100644 index 0000000000..9bb7e13686 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ConnectionInputHandlerFactory.h @@ -0,0 +1,54 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef _ConnectionInputHandlerFactory_ +#define _ConnectionInputHandlerFactory_ + +#include <boost/noncopyable.hpp> +#include <string> + +namespace qpid { +namespace sys { + +class ConnectionOutputHandler; +class ConnectionInputHandler; + +/** + * Callback interface used by the Acceptor to + * create a ConnectionInputHandler for each new connection. + */ +class ConnectionInputHandlerFactory : private boost::noncopyable +{ + public: + /** + *@param out handler for connection output. + *@param id identify the connection for management purposes. + */ + virtual ConnectionInputHandler* create(ConnectionOutputHandler* out, + const std::string& id, + bool isClient) = 0; + + virtual ~ConnectionInputHandlerFactory(){} +}; + +}} + + +#endif diff --git a/qpid/cpp/src/qpid/sys/ConnectionOutputHandler.h b/qpid/cpp/src/qpid/sys/ConnectionOutputHandler.h new file mode 100644 index 0000000000..421dd7c269 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ConnectionOutputHandler.h @@ -0,0 +1,43 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef _ConnectionOutputHandler_ +#define _ConnectionOutputHandler_ + +#include "qpid/framing/OutputHandler.h" +#include "qpid/sys/OutputControl.h" + +namespace qpid { +namespace sys { + +/** + * Provides the output handler associated with a connection. + */ +class ConnectionOutputHandler : public virtual qpid::framing::OutputHandler, public OutputControl +{ + public: + virtual void close() = 0; + virtual size_t getBuffered() const { return 0; } +}; + +}} + + +#endif diff --git a/qpid/cpp/src/qpid/sys/ConnectionOutputHandlerPtr.h b/qpid/cpp/src/qpid/sys/ConnectionOutputHandlerPtr.h new file mode 100644 index 0000000000..95a08d15ae --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ConnectionOutputHandlerPtr.h @@ -0,0 +1,56 @@ +#ifndef QPID_SYS_CONNECTIONOUTPUTHANDLERPTR_H +#define QPID_SYS_CONNECTIONOUTPUTHANDLERPTR_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ConnectionOutputHandler.h" + +namespace qpid { +namespace sys { + +/** + * A ConnectionOutputHandler that delegates to another + * ConnectionOutputHandler. Allows the "real" ConnectionOutputHandler + * to be changed without updating all the pointers/references + * using the ConnectionOutputHandlerPtr + */ +class ConnectionOutputHandlerPtr : public ConnectionOutputHandler +{ + public: + ConnectionOutputHandlerPtr(ConnectionOutputHandler* p) : next(p) { assert(next); } + void set(ConnectionOutputHandler* p) { next = p; assert(next); } + ConnectionOutputHandler* get() { return next; } + const ConnectionOutputHandler* get() const { return next; } + + void close() { next->close(); } + size_t getBuffered() const { return next->getBuffered(); } + void abort() { next->abort(); } + void activateOutput() { next->activateOutput(); } + void giveReadCredit(int32_t credit) { next->giveReadCredit(credit); } + void send(framing::AMQFrame& f) { next->send(f); } + + private: + ConnectionOutputHandler* next; +}; +}} // namespace qpid::sys + +#endif /*!QPID_SYS_CONNECTIONOUTPUTHANDLERPTR_H*/ diff --git a/qpid/cpp/src/qpid/sys/CopyOnWriteArray.h b/qpid/cpp/src/qpid/sys/CopyOnWriteArray.h new file mode 100644 index 0000000000..45a231dfd8 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/CopyOnWriteArray.h @@ -0,0 +1,156 @@ +#ifndef QPID_SYS_COPYONWRITEARRAY_H +#define QPID_SYS_COPYONWRITEARRAY_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Mutex.h" +#include <algorithm> +#include <vector> +#include <boost/shared_ptr.hpp> + +namespace qpid { +namespace sys { + +/** + * An array that copies on adding/removing element allowing lock-free + * iteration. + */ +template <class T> +class CopyOnWriteArray +{ +public: + typedef boost::shared_ptr<const std::vector<T> > ConstPtr; + + CopyOnWriteArray() {} + CopyOnWriteArray(const CopyOnWriteArray& c) : array(c.array) {} + + void add(T& t) + { + Mutex::ScopedLock l(lock); + ArrayPtr copy(array ? new std::vector<T>(*array) : new std::vector<T>()); + copy->push_back(t); + array = copy; + } + + bool remove(T& t) + { + Mutex::ScopedLock l(lock); + if (array && std::find(array->begin(), array->end(), t) != array->end()) { + ArrayPtr copy(new std::vector<T>(*array)); + copy->erase(std::find(copy->begin(), copy->end(), t)); + array = copy; + return true; + } else { + return false; + } + } + + bool clear() + { + Mutex::ScopedLock l(lock); + if (array && !array->empty()) { + ArrayPtr copy; + array = copy; + return true; + } else { + return false; + } + } + + template <class F> + bool add_unless(T& t, F f) + { + Mutex::ScopedLock l(lock); + if (array && std::find_if(array->begin(), array->end(), f) != array->end()) { + return false; + } else { + ArrayPtr copy(array ? new std::vector<T>(*array) : new std::vector<T>()); + copy->push_back(t); + array = copy; + return true; + } + } + + template <class F> + bool remove_if(F f) + { + Mutex::ScopedLock l(lock); + if (array && std::find_if(array->begin(), array->end(), f) != array->end()) { + ArrayPtr copy(new std::vector<T>(*array)); + copy->erase(std::remove_if(copy->begin(), copy->end(), f), copy->end()); + array = copy; + return true; + } + return false; + } + + template <class TestFn, class ModifierFn> + bool modify_if(TestFn f, ModifierFn & m) + { + if (!array) + return false; + { + Mutex::ScopedLock l(lock); + if (std::find_if(array->begin(), array->end(), f) != array->end()) + { + ArrayPtr copy(new std::vector<T>(*array)); + m(*std::find_if(copy->begin(), copy->end(), f)); + array = copy; + return true; + } + } + return false; + } + + template <class F> + F for_each(F f) + { + ArrayPtr a; + { + Mutex::ScopedLock l(lock); + a = array; + } + if (!a) return f; + return std::for_each(a->begin(), a->end(), f); + } + + ConstPtr snapshot() + { + ConstPtr a; + { + Mutex::ScopedLock l(lock); + a = array; + } + return a; + } + +private: + typedef boost::shared_ptr< std::vector<T> > ArrayPtr; + Mutex lock; + ArrayPtr array; +}; + +}} + + + +#endif /*!QPID_SYS_COPYONWRITEARRAY_H*/ diff --git a/qpid/cpp/src/qpid/sys/DeletionManager.h b/qpid/cpp/src/qpid/sys/DeletionManager.h new file mode 100644 index 0000000000..c1fea19f30 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/DeletionManager.h @@ -0,0 +1,162 @@ +#ifndef _sys_DeletionManager_h +#define _sys_DeletionManager_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <vector> +#include <algorithm> +#include <boost/shared_ptr.hpp> + +namespace qpid { +namespace sys { + +struct deleter +{ + template <typename T> + void operator()(T* ptr){ delete ptr;} +}; + +/** + * DeletionManager keeps track of handles that need to be deleted but may still be + * in use by one of the threads concurrently. + * + * The mode of operation is like this: + * - When we want to delete but we might still be using the handle we + * * Transfer ownership of the handle to this class + * * Mark the handle as (potentially) in use by every thread + * - Then subsequently at points where the thread code knows it isn't + * using any handles it declares that it is using no handles + * - When the last thread declares no use of a handle it automatically + * gets deleted by the shared_ptr implementation + * + * The class only has static members and data and so can only be used once for + * any particular handle type + */ +template <typename H> +class DeletionManager +{ + struct ThreadStatus; + +public: + // Mark every thread as using the handle - it will be deleted + // below after every thread marks the handle as unused + static void markForDeletion(H* handle) { + allThreadsStatuses.addHandle(shared_ptr(handle)); + } + + // Mark this thread is not using any handle - + // handles get deleted here when no one else + // is using them either + static void markAllUnusedInThisThread() { + ThreadStatus* threadStatus = getThreadStatus(); + ScopedLock<Mutex> l(threadStatus->lock); + + // The actual deletions will happen here when all the shared_ptr + // ref counts hit 0 (that is when every thread marks the handle unused) + threadStatus->handles.clear(); + } + + static void destroyThreadState() { + ThreadStatus* threadStatus = getThreadStatus(); + allThreadsStatuses.delThreadStatus(threadStatus); + delete threadStatus; + threadStatus = 0; + } + +private: + + static ThreadStatus*& getThreadStatus() { + static __thread ThreadStatus* threadStatus = 0; + + // Thread local vars can't be dynamically constructed so we need + // to check whether we've made it yet and construct it if not + // (no locking necessary for the check as it's thread local!) + if (!threadStatus) { + threadStatus = new ThreadStatus; + allThreadsStatuses.addThreadStatus(threadStatus); + } + + return threadStatus; + } + + typedef boost::shared_ptr<H> shared_ptr; + + // In theory we know that we never need more handles than the number of + // threads runnning so we could use a fixed size array. However at this point + // in the code we don't have easy access to this information. + struct ThreadStatus + { + Mutex lock; + std::vector<shared_ptr> handles; + }; + + class AllThreadsStatuses + { + Mutex lock; + std::vector<ThreadStatus*> statuses; + + struct handleAdder + { + shared_ptr handle; + + handleAdder(shared_ptr h): handle(h) {} + + void operator()(ThreadStatus* ptr) { + ScopedLock<Mutex> l(ptr->lock); + ptr->handles.push_back(handle); + } + }; + + public: + // Need this to be able to do static initialisation + explicit AllThreadsStatuses(int) {} + + ~AllThreadsStatuses() { + ScopedLock<Mutex> l(lock); + std::for_each(statuses.begin(), statuses.end(), deleter()); + } + + void addThreadStatus(ThreadStatus* t) { + ScopedLock<Mutex> l(lock); + statuses.push_back(t); + } + + void delThreadStatus(ThreadStatus* t) { + ScopedLock<Mutex> l(lock); + typename std::vector<ThreadStatus*>::iterator it = + std::find(statuses.begin(),statuses.end(), t); + if (it != statuses.end()) { + statuses.erase(it); + } + } + + void addHandle(shared_ptr h) { + ScopedLock<Mutex> l(lock); + std::for_each(statuses.begin(), statuses.end(), handleAdder(h)); + } + }; + + static AllThreadsStatuses allThreadsStatuses; +}; + +}} +#endif // _sys_DeletionManager_h diff --git a/qpid/cpp/src/qpid/sys/DispatchHandle.cpp b/qpid/cpp/src/qpid/sys/DispatchHandle.cpp new file mode 100644 index 0000000000..5d6fc4e72f --- /dev/null +++ b/qpid/cpp/src/qpid/sys/DispatchHandle.cpp @@ -0,0 +1,352 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/DispatchHandle.h" +#include "qpid/log/Statement.h" + +#include <algorithm> + +#include <boost/cast.hpp> + +#include <assert.h> + +namespace qpid { +namespace sys { + +DispatchHandle::DispatchHandle(const IOHandle& h, Callback rCb, Callback wCb, Callback dCb) : + PollerHandle(h), + readableCallback(rCb), + writableCallback(wCb), + disconnectedCallback(dCb), + state(IDLE) +{ +} + + +DispatchHandle::~DispatchHandle() { +} + +void DispatchHandle::startWatch(Poller::shared_ptr poller0) { + bool r = readableCallback; + bool w = writableCallback; + + ScopedLock<Mutex> lock(stateLock); + assert(state == IDLE); + + poller = poller0; + poller->registerHandle(*this); + state = WAITING; + Poller::Direction dir = r ? + ( w ? Poller::INOUT : Poller::INPUT ) : + ( w ? Poller::OUTPUT : Poller::NONE ); + poller->monitorHandle(*this, dir); +} + +void DispatchHandle::rewatch() { + bool r = readableCallback; + bool w = writableCallback; + if (!r && !w) { + return; + } + Poller::Direction dir = r ? + ( w ? Poller::INOUT : Poller::INPUT ) : + ( w ? Poller::OUTPUT : Poller::NONE ); + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->monitorHandle(*this, dir); +} + +void DispatchHandle::rewatchRead() { + if (!readableCallback) { + return; + } + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->monitorHandle(*this, Poller::INPUT); +} + +void DispatchHandle::rewatchWrite() { + if (!writableCallback) { + return; + } + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->monitorHandle(*this, Poller::OUTPUT); +} + +void DispatchHandle::unwatchRead() { + if (!readableCallback) { + return; + } + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->unmonitorHandle(*this, Poller::INPUT); +} + +void DispatchHandle::unwatchWrite() { + if (!writableCallback) { + return; + } + + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->unmonitorHandle(*this, Poller::OUTPUT); +} + +void DispatchHandle::unwatch() { + ScopedLock<Mutex> lock(stateLock); + switch(state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + break; + } + assert(poller); + poller->unmonitorHandle(*this, Poller::INOUT); +} + +void DispatchHandle::stopWatch() { + ScopedLock<Mutex> lock(stateLock); + switch (state) { + case IDLE: + assert(state != IDLE); + return; + case STOPPING: + assert(state != STOPPING); + return; + case CALLING: + state = STOPPING; + break; + case WAITING: + state = IDLE; + break; + case DELETING: + return; + } + assert(poller); + poller->unregisterHandle(*this); + poller.reset(); +} + +// If we are in the IDLE/STOPPING state we can't do the callback as we've +// not/no longer got the fd registered in any poller +void DispatchHandle::call(Callback iCb) { + assert(iCb); + ScopedLock<Mutex> lock(stateLock); + switch (state) { + case IDLE: + case STOPPING: + case DELETING: + return; + default: + interruptedCallbacks.push(iCb); + assert(poller); + (void) poller->interrupt(*this); + } +} + +// The slightly strange switch structure +// is to ensure that the lock is released before +// we do the delete +void DispatchHandle::doDelete() { + { + ScopedLock<Mutex> lock(stateLock); + // Ensure that we're no longer watching anything + switch (state) { + case IDLE: + state = DELETING; + break; + case STOPPING: + state = DELETING; + return; + case WAITING: + state = DELETING; + assert(poller); + (void) poller->interrupt(*this); + poller->unregisterHandle(*this); + return; + case CALLING: + state = DELETING; + assert(poller); + poller->unregisterHandle(*this); + return; + case DELETING: + return; + } + } + // If we're IDLE we can do this right away + delete this; +} + +void DispatchHandle::processEvent(Poller::EventType type) { + + // Phase I + { + ScopedLock<Mutex> lock(stateLock); + + switch(state) { + case IDLE: + // Can get here if a non connection thread stops watching + // whilst we were stuck in the above lock + return; + case WAITING: + state = CALLING; + break; + case CALLING: + assert(state!=CALLING); + return; + case STOPPING: + assert(state!=STOPPING); + return; + case DELETING: + // Need to make sure we clean up any pending callbacks in this case + std::swap(callbacks, interruptedCallbacks); + goto saybyebye; + } + + std::swap(callbacks, interruptedCallbacks); + } + + // Do callbacks - whilst we are doing the callbacks we are prevented from processing + // the same handle until we re-enable it. To avoid rentering the callbacks for a single + // handle re-enabling in the callbacks is actually deferred until they are complete. + try { + switch (type) { + case Poller::READABLE: + readableCallback(*this); + break; + case Poller::WRITABLE: + writableCallback(*this); + break; + case Poller::READ_WRITABLE: + readableCallback(*this); + writableCallback(*this); + break; + case Poller::DISCONNECTED: + if (disconnectedCallback) { + disconnectedCallback(*this); + } + break; + case Poller::INTERRUPTED: + { + // We'll actually do the interrupt below + } + break; + default: + assert(false); + } + + // If we have any callbacks do them now - + // (because we use a copy from before the previous callbacks we won't + // do anything yet that was just added) + while (callbacks.size() > 0) { + { + ScopedLock<Mutex> lock(stateLock); + switch (state) { + case DELETING: + goto finishcallbacks; + default: + break; + } + } + Callback cb = callbacks.front(); + assert(cb); + cb(*this); + callbacks.pop(); + } + } catch (std::exception& e) { + // One of the callbacks threw an exception - that's not allowed + QPID_LOG(error, "Caught exception in state: " << state << " with event: " << type << ": " << e.what()); + // It would be nice to clean up and delete ourselves here, but we can't + } + +finishcallbacks: + { + ScopedLock<Mutex> lock(stateLock); + switch (state) { + case IDLE: + assert(state!=IDLE); + return; + case STOPPING: + state = IDLE; + return; + case WAITING: + assert(state!=WAITING); + return; + case CALLING: + state = WAITING; + return; + case DELETING: + break; + } + } + +saybyebye: + delete this; +} + +}} diff --git a/qpid/cpp/src/qpid/sys/DispatchHandle.h b/qpid/cpp/src/qpid/sys/DispatchHandle.h new file mode 100644 index 0000000000..115a3c44f7 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/DispatchHandle.h @@ -0,0 +1,150 @@ +#ifndef _sys_DispatchHandle_h +#define _sys_DispatchHandle_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Poller.h" +#include "qpid/sys/Mutex.h" +#include "qpid/CommonImportExport.h" +#include <boost/function.hpp> + +#include <queue> + +namespace qpid { +namespace sys { + +class DispatchHandleRef; +/** + * In order to have your own handle (file descriptor on Unix) watched by the poller + * you need to: + * + * - Subclass IOHandle, in the constructor supply an appropriate + * IOHandlerPrivate object for the platform. + * + * - Construct a DispatchHandle passing it your IOHandle and + * callback functions for read, write and disconnect events. + * + * - Ensure the DispatchHandle is not deleted until the poller is no longer using it. + * TODO: astitcher document DispatchHandleRef to simplify this. + * + * When an event occurs on the handle, the poller calls the relevant callback and + * stops watching that handle. Your callback can call rewatch() or related functions + * to re-enable polling. + */ +class DispatchHandle : public PollerHandle { + friend class DispatchHandleRef; +public: + typedef boost::function1<void, DispatchHandle&> Callback; + typedef std::queue<Callback> CallbackQueue; + +private: + Callback readableCallback; + Callback writableCallback; + Callback disconnectedCallback; + CallbackQueue interruptedCallbacks; + CallbackQueue callbacks; // Double buffer + Poller::shared_ptr poller; + Mutex stateLock; + enum { + IDLE, + STOPPING, + WAITING, + CALLING, + DELETING + } state; + +public: + /** + * Provide a handle to poll and a set of callbacks. Note + * callbacks can be 0, meaning you are not interested in that + * event. + * + *@param h: the handle to watch. The IOHandle encapsulates a + * platfrom-specific handle to an IO object (e.g. a file descriptor + * on Unix.) + *@param rCb Callback called when the handle is readable. + *@param wCb Callback called when the handle is writable. + *@param dCb Callback called when the handle is disconnected. + */ + QPID_COMMON_EXTERN DispatchHandle(const IOHandle& h, Callback rCb, Callback wCb, Callback dCb); + QPID_COMMON_EXTERN ~DispatchHandle(); + + /** Add this DispatchHandle to the poller to be watched. */ + QPID_COMMON_EXTERN void startWatch(Poller::shared_ptr poller); + + /** Resume watching for all non-0 callbacks. */ + QPID_COMMON_EXTERN void rewatch(); + /** Resume watching for read only. */ + QPID_COMMON_EXTERN void rewatchRead(); + + /** Resume watching for write only. */ + QPID_COMMON_EXTERN void rewatchWrite(); + + /** Stop watching temporarily. The DispatchHandle remains + associated with the poller and can be re-activated using + rewatch. */ + QPID_COMMON_EXTERN void unwatch(); + /** Stop watching for read */ + QPID_COMMON_EXTERN void unwatchRead(); + /** Stop watching for write */ + QPID_COMMON_EXTERN void unwatchWrite(); + + /** Stop watching permanently. Disassociates from the poller. */ + QPID_COMMON_EXTERN void stopWatch(); + + /** Interrupt watching this handle and make a serialised callback that respects the + * same exclusivity guarantees as the other callbacks + */ + QPID_COMMON_EXTERN void call(Callback iCb); + +protected: + QPID_COMMON_EXTERN void doDelete(); + +private: + QPID_COMMON_EXTERN void processEvent(Poller::EventType dir); +}; + +class DispatchHandleRef { + DispatchHandle* ref; + +public: + typedef boost::function1<void, DispatchHandle&> Callback; + DispatchHandleRef(const IOHandle& h, Callback rCb, Callback wCb, Callback dCb) : + ref(new DispatchHandle(h, rCb, wCb, dCb)) + {} + + ~DispatchHandleRef() { ref->doDelete(); } + + void startWatch(Poller::shared_ptr poller) { ref->startWatch(poller); } + void rewatch() { ref->rewatch(); } + void rewatchRead() { ref->rewatchRead(); } + void rewatchWrite() { ref->rewatchWrite(); } + void unwatch() { ref->unwatch(); } + void unwatchRead() { ref->unwatchRead(); } + void unwatchWrite() { ref->unwatchWrite(); } + void stopWatch() { ref->stopWatch(); } + void call(Callback iCb) { ref->call(iCb); } +}; + +}} + +#endif // _sys_DispatchHandle_h diff --git a/qpid/cpp/src/qpid/sys/Dispatcher.cpp b/qpid/cpp/src/qpid/sys/Dispatcher.cpp new file mode 100644 index 0000000000..5f52dcd990 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Dispatcher.cpp @@ -0,0 +1,40 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Dispatcher.h" + +#include <assert.h> + +namespace qpid { +namespace sys { + +Dispatcher::Dispatcher(Poller::shared_ptr poller0) : + poller(poller0) { +} + +Dispatcher::~Dispatcher() { +} + +void Dispatcher::run() { + poller->run(); +} + +}} diff --git a/qpid/cpp/src/qpid/sys/Dispatcher.h b/qpid/cpp/src/qpid/sys/Dispatcher.h new file mode 100644 index 0000000000..e8213d0579 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Dispatcher.h @@ -0,0 +1,44 @@ +#ifndef _sys_Dispatcher_h +#define _sys_Dispatcher_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Poller.h" +#include "qpid/sys/Runnable.h" +#include "qpid/CommonImportExport.h" + +namespace qpid { +namespace sys { + +class Dispatcher : public Runnable { + const Poller::shared_ptr poller; + +public: + QPID_COMMON_EXTERN Dispatcher(Poller::shared_ptr poller); + QPID_COMMON_EXTERN ~Dispatcher(); + + QPID_COMMON_EXTERN void run(); +}; + +}} + +#endif // _sys_Dispatcher_h diff --git a/qpid/cpp/src/qpid/sys/FileSysDir.h b/qpid/cpp/src/qpid/sys/FileSysDir.h new file mode 100755 index 0000000000..ffe7823f0a --- /dev/null +++ b/qpid/cpp/src/qpid/sys/FileSysDir.h @@ -0,0 +1,62 @@ +#ifndef QPID_SYS_FILESYSDIR_H +#define QPID_SYS_FILESYSDIR_H + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <string> + +namespace qpid { +namespace sys { + +/** + * @class FileSysDir + * + * Represents a filesystem directory accessible from the local host. + * This class simply checks existence of, and creates, a directory. It could + * be added to later to list contents, etc. + */ +class FileSysDir +{ + const std::string dirPath; + + public: + + FileSysDir (std::string path) : dirPath(path) {} + ~FileSysDir () {} + + /** + * Check to see if the directory exists and is a directory. Throws an + * exception if there is an error checking existence or if the path + * exists but is not a directory. + * + * @retval true if the path exists and is a directory. + * @retval false if the path does not exist. + */ + bool exists (void) const; + + void mkdir(void); + + std::string getPath () { return dirPath; } +}; + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_FILESYSDIR_H*/ diff --git a/qpid/cpp/src/qpid/sys/Fork.h b/qpid/cpp/src/qpid/sys/Fork.h new file mode 100644 index 0000000000..4ec061f7bc --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Fork.h @@ -0,0 +1,24 @@ +#ifndef QPID_SYS_FORK_H +#define QPID_SYS_FORK_H + +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "posix/Fork.h" + +#endif /*!QPID_SYS_FORK_H*/ diff --git a/qpid/cpp/src/qpid/sys/LockFile.h b/qpid/cpp/src/qpid/sys/LockFile.h new file mode 100644 index 0000000000..14a76cbf3e --- /dev/null +++ b/qpid/cpp/src/qpid/sys/LockFile.h @@ -0,0 +1,64 @@ +#ifndef _sys_LockFile_h +#define _sys_LockFile_h + +/* + * + * Copyright (c) 2008 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <string> + +#include "qpid/CommonImportExport.h" +#include "qpid/sys/IntegerTypes.h" + +namespace qpid { +namespace sys { + +class LockFilePrivate; + +/** + * @class LockFile + * + * LockFile represents a locked file suitable for a coarse-grain system + * lock. For example, the broker uses this to ensure that only one broker + * runs. A common usage idiom is to store the current "owner" process ID + * in the lock file - if the lock file exists, but the stored process ID + * doesn't, the old owner has probably died without cleaning up the lock + * file. + */ +class LockFile : private boost::noncopyable +{ + std::string path; + bool created; + boost::shared_ptr<LockFilePrivate> impl; + +protected: + int read(void*, size_t) const; + int write(void*, size_t) const; + +public: + QPID_COMMON_EXTERN LockFile(const std::string& path_, bool create); + QPID_COMMON_EXTERN ~LockFile(); +}; + +}} /* namespace qpid::sys */ + +#endif /*!_sys_LockFile_h*/ + + + diff --git a/qpid/cpp/src/qpid/sys/LockPtr.h b/qpid/cpp/src/qpid/sys/LockPtr.h new file mode 100644 index 0000000000..738a864317 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/LockPtr.h @@ -0,0 +1,89 @@ +#ifndef QPID_SYS_LOCKPTR_H +#define QPID_SYS_LOCKPTR_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Mutex.h" +#include <boost/noncopyable.hpp> + +namespace qpid { +namespace sys { + +class Mutex; + +/** + * LockPtr is a smart pointer to T. It is constructed from a volatile + * T* and a Lock (by default a Mutex). It const_casts away the + * volatile qualifier and locks the Lock for the duration of its + * + * Used in conjuntion with the "volatile" keyword to get the compiler + * to help enforce correct concurrent use of mutli-threaded objects. + * See ochttp://www.ddj.com/cpp/184403766 for a detailed discussion. + * + * To summarize the convention: + * - Declare thread-safe member functions as volatile. + * - Declare instances of the class that may be called concurrently as volatile. + * - Use LockPtr to cast away the volatile qualifier while taking a lock. + * + * This means that code calling on a concurrently-used object + * (declared volatile) can only call thread-safe (volatile) member + * functions. Code that needs to use thread-unsafe members must use a + * LockPtr, thereby acquiring the lock and making it safe to do so. + * + * A good type-safe pattern is the internally-locked object: + * - It has it's own private lock member. + * - All public functions are thread safe and declared volatile. + * - Any thread-unsafe, non-volatile functions are private. + * - Only member function implementations use LockPtr to access private functions. + * + * This encapsulates all the locking logic inside the class. + * + * One nice feature of this convention is the common case where you + * need a public, locked version of some function foo() and also a + * private unlocked version to avoid recursive locks. They can be declared as + * volatile and non-volatile overloads of the same function: + * + * // public + * void Thing::foo() volatile { LockPtr<Thing>(this, myLock)->foo(); } + * // private + * void Thing::foo() { ... do stuff ...} + */ + +template <class T, class Lock> class LockPtr : public boost::noncopyable { + public: + LockPtr(volatile T* p, Lock& l) : ptr(const_cast<T*>(p)), lock(l) { lock.lock(); } + LockPtr(volatile T* p, volatile Lock& l) : ptr(const_cast<T*>(p)), lock(const_cast<Lock&>(l)) { lock.lock(); } + ~LockPtr() { lock.unlock(); } + + T& operator*() { return *ptr; } + T* operator->() { return ptr; } + + private: + T* ptr; + Lock& lock; +}; + + +}} // namespace qpid::sys + + +#endif /*!QPID_SYS_LOCKPTR_H*/ diff --git a/qpid/cpp/src/qpid/sys/OutputControl.h b/qpid/cpp/src/qpid/sys/OutputControl.h new file mode 100644 index 0000000000..eae99beb0f --- /dev/null +++ b/qpid/cpp/src/qpid/sys/OutputControl.h @@ -0,0 +1,43 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IntegerTypes.h" + +#ifndef _OutputControl_ +#define _OutputControl_ + +namespace qpid { +namespace sys { + + class OutputControl + { + public: + virtual ~OutputControl() {} + virtual void abort() = 0; + virtual void activateOutput() = 0; + virtual void giveReadCredit(int32_t credit) = 0; + }; + +} +} + + +#endif diff --git a/qpid/cpp/src/qpid/sys/OutputTask.h b/qpid/cpp/src/qpid/sys/OutputTask.h new file mode 100644 index 0000000000..fb08a63cd0 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/OutputTask.h @@ -0,0 +1,41 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef _OutputTask_ +#define _OutputTask_ + +namespace qpid { +namespace sys { + + class OutputTask + { + public: + virtual ~OutputTask() {} + /** Generate some output. + *@return true if output was generated, false if there is no work to do. + */ + virtual bool doOutput() = 0; + }; + +} +} + + +#endif diff --git a/qpid/cpp/src/qpid/sys/PipeHandle.h b/qpid/cpp/src/qpid/sys/PipeHandle.h new file mode 100755 index 0000000000..8aac76996b --- /dev/null +++ b/qpid/cpp/src/qpid/sys/PipeHandle.h @@ -0,0 +1,51 @@ +#ifndef _sys_PipeHandle_h +#define _sys_PipeHandle_h + +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ + +#include "qpid/sys/IntegerTypes.h" +#include "qpid/CommonImportExport.h" +#include <string> + +// This class is a portability wrapper around pipe fds. +// It currently exists primarily and solely for the purpose of +// integration with single-threaded components that require QMF +// integration through a signalling fd. + +namespace qpid { +namespace sys { + + class PipeHandle { + private: + int writeFd; + int readFd; + public: + QPID_COMMON_EXTERN PipeHandle(bool nonBlocking=true); + QPID_COMMON_EXTERN ~PipeHandle(); + QPID_COMMON_EXTERN int read(void* buf, size_t bufSize); + QPID_COMMON_EXTERN int write(const void* buf, size_t bufSize); + QPID_COMMON_EXTERN int getReadHandle(); + }; + +}} + +#endif /*!_sys_PipeHandle_h*/ diff --git a/qpid/cpp/src/qpid/sys/PollableCondition.h b/qpid/cpp/src/qpid/sys/PollableCondition.h new file mode 100644 index 0000000000..2eb6f2d947 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/PollableCondition.h @@ -0,0 +1,64 @@ +#ifndef QPID_SYS_POLLABLECONDITION_H +#define QPID_SYS_POLLABLECONDITION_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Poller.h" +#include "qpid/CommonImportExport.h" +#include <boost/function.hpp> +#include <boost/shared_ptr.hpp> + + +namespace qpid { +namespace sys { + +class PollableConditionPrivate; + +class PollableCondition { +public: + typedef boost::function1<void, PollableCondition&> Callback; + + QPID_COMMON_EXTERN PollableCondition(const Callback& cb, + const boost::shared_ptr<sys::Poller>& poller); + + QPID_COMMON_EXTERN ~PollableCondition(); + + /** + * Set the condition. Triggers callback to Callback from Poller. + */ + QPID_COMMON_EXTERN void set(); + + /** + * Clear the condition. Stops callbacks from Poller. + */ + QPID_COMMON_EXTERN void clear(); + + private: + PollableConditionPrivate *impl; + + Callback callback; + boost::shared_ptr<sys::Poller> poller; +}; + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_POLLABLECONDITION_H*/ diff --git a/qpid/cpp/src/qpid/sys/PollableQueue.h b/qpid/cpp/src/qpid/sys/PollableQueue.h new file mode 100644 index 0000000000..81c2301c1e --- /dev/null +++ b/qpid/cpp/src/qpid/sys/PollableQueue.h @@ -0,0 +1,176 @@ +#ifndef QPID_SYS_POLLABLEQUEUE_H +#define QPID_SYS_POLLABLEQUEUE_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/PollableCondition.h" +#include "qpid/sys/Monitor.h" +#include "qpid/sys/Thread.h" +#include <boost/function.hpp> +#include <boost/bind.hpp> +#include <algorithm> +#include <vector> + +namespace qpid { +namespace sys { + +class Poller; + +/** + * A queue whose item processing is dispatched by sys::Poller. + * Any thread can push to the queue; items pushed trigger an event the Poller + * recognizes. When a Poller I/O thread dispatches the event, a + * user-specified callback is invoked with all items on the queue. + */ +template <class T> +class PollableQueue { + public: + typedef std::vector<T> Batch; + typedef T value_type; + + /** + * Callback to process a batch of items from the queue. + * + * @param batch Queue of values to process. Any items remaining + * on return from Callback are put back on the queue. + * @return iterator pointing to the first un-processed item in batch. + * Items from this point up to batch.end() are put back on the queue. + */ + typedef boost::function<typename Batch::const_iterator (const Batch& batch)> Callback; + + /** + * Constructor; sets necessary parameters. + * + * @param cb Callback that will be called to process items on the + * queue. Will be called from a Poller I/O thread. + * @param poller Poller to use for dispatching queue events. + */ + PollableQueue(const Callback& cb, + const boost::shared_ptr<sys::Poller>& poller); + + ~PollableQueue(); + + /** Push a value onto the queue. Thread safe */ + void push(const T& t); + + /** Start polling. */ + void start(); + + /** Stop polling and wait for the current callback, if any, to complete. */ + void stop(); + + /** Are we currently stopped?*/ + bool isStopped() const { ScopedLock l(lock); return stopped; } + + size_t size() { ScopedLock l(lock); return queue.size(); } + bool empty() { ScopedLock l(lock); return queue.empty(); } + + /** + * Allow any queued events to be processed; intended for calling + * after all dispatch threads exit the Poller loop in order to + * ensure clean shutdown with no events left on the queue. + */ + void shutdown(); + + private: + typedef sys::Monitor::ScopedLock ScopedLock; + typedef sys::Monitor::ScopedUnlock ScopedUnlock; + + void dispatch(PollableCondition& cond); + void process(); + + mutable sys::Monitor lock; + Callback callback; + PollableCondition condition; + Batch queue, batch; + Thread dispatcher; + bool stopped; +}; + +template <class T> PollableQueue<T>::PollableQueue( + const Callback& cb, const boost::shared_ptr<sys::Poller>& p) + : callback(cb), + condition(boost::bind(&PollableQueue<T>::dispatch, this, _1), p), + stopped(true) +{ +} + +template <class T> void PollableQueue<T>::start() { + ScopedLock l(lock); + if (!stopped) return; + stopped = false; + if (!queue.empty()) condition.set(); +} + +template <class T> PollableQueue<T>::~PollableQueue() { +} + +template <class T> void PollableQueue<T>::push(const T& t) { + ScopedLock l(lock); + if (queue.empty() && !stopped) condition.set(); + queue.push_back(t); +} + +template <class T> void PollableQueue<T>::dispatch(PollableCondition& cond) { + ScopedLock l(lock); + assert(!dispatcher); + dispatcher = Thread::current(); + process(); + dispatcher = Thread(); + if (queue.empty()) cond.clear(); + if (stopped) lock.notifyAll(); +} + +template <class T> void PollableQueue<T>::process() { + // Called with lock held + while (!stopped && !queue.empty()) { + assert(batch.empty()); + batch.swap(queue); + typename Batch::const_iterator putBack; + { + ScopedUnlock u(lock); // Allow concurrent push to queue. + putBack = callback(batch); + } + // put back unprocessed items. + queue.insert(queue.begin(), putBack, typename Batch::const_iterator(batch.end())); + batch.clear(); + } +} + +template <class T> void PollableQueue<T>::shutdown() { + ScopedLock l(lock); + process(); +} + +template <class T> void PollableQueue<T>::stop() { + ScopedLock l(lock); + if (stopped) return; + condition.clear(); + stopped = true; + // Avoid deadlock if stop is called from the dispatch thread + if (dispatcher && dispatcher != Thread::current()) + while (dispatcher) lock.wait(); +} + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_POLLABLEQUEUE_H*/ diff --git a/qpid/cpp/src/qpid/sys/Poller.h b/qpid/cpp/src/qpid/sys/Poller.h new file mode 100644 index 0000000000..01ee139ee6 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Poller.h @@ -0,0 +1,135 @@ +#ifndef _sys_Poller_h +#define _sys_Poller_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Time.h" +#include "qpid/sys/Runnable.h" +#include "qpid/CommonImportExport.h" +#include <boost/shared_ptr.hpp> + +namespace qpid { +namespace sys { + +/** + * Poller is an abstract base class that registers callbacks to be + * called when there is IO activity. Concrete derived classes + * implement polling APIs such as epoll or equivalents on other + * operating systems. + * + * On the broker, Connection::received() is called with incoming + * frames from clients, and Connection::doOutput() is called when a + * connection is writeable. + * + * @see DispatchHandler for more details of normal use. + */ +class PollerHandle; +class PollerPrivate; +class Poller : public Runnable { + PollerPrivate* const impl; + +public: + typedef boost::shared_ptr<Poller> shared_ptr; + + enum Direction { + NONE = 0, + INPUT, + OUTPUT, + INOUT + }; + + enum EventType { + INVALID = 0, + READABLE, + WRITABLE, + READ_WRITABLE, + DISCONNECTED, + SHUTDOWN, + TIMEOUT, + INTERRUPTED + }; + + struct Event { + PollerHandle* handle; + EventType type; + + Event(PollerHandle* handle0, EventType type0) : + handle(handle0), + type(type0) { + } + + void process(); + }; + + QPID_COMMON_EXTERN Poller(); + QPID_COMMON_EXTERN ~Poller(); + /** Note: this function is async-signal safe */ + QPID_COMMON_EXTERN void shutdown(); + + // Interrupt waiting for a specific poller handle + // returns true if we could interrupt the handle + // - in this case on return the handle is no longer being monitored, + // but we will receive an event from some invocation of poller::wait + // with the handle and the INTERRUPTED event type + // if it returns false then the handle is not being monitored by the poller + // - This can either be because it has just received an event which has been + // reported and has not been reenabled since. + // - Because it was removed from the monitoring set + // - Or because it is already being interrupted + QPID_COMMON_EXTERN bool interrupt(PollerHandle& handle); + + // Poller run loop + QPID_COMMON_EXTERN void run(); + + QPID_COMMON_EXTERN void registerHandle(PollerHandle& handle); + QPID_COMMON_EXTERN void unregisterHandle(PollerHandle& handle); + QPID_COMMON_EXTERN void monitorHandle(PollerHandle& handle, Direction dir); + QPID_COMMON_EXTERN void unmonitorHandle(PollerHandle& handle, Direction dir); + QPID_COMMON_EXTERN Event wait(Duration timeout = TIME_INFINITE); + + QPID_COMMON_EXTERN bool hasShutdown(); +}; + +/** + * Handle class to use for polling + */ +class IOHandle; +class PollerHandlePrivate; +class PollerHandle { + friend class Poller; + friend class PollerPrivate; + friend struct Poller::Event; + + PollerHandlePrivate* const impl; + QPID_COMMON_INLINE_EXTERN virtual void processEvent(Poller::EventType) {}; + +public: + QPID_COMMON_EXTERN PollerHandle(const IOHandle& h); + QPID_COMMON_EXTERN virtual ~PollerHandle(); +}; + +inline void Poller::Event::process() { + handle->processEvent(type); +} + +}} +#endif // _sys_Poller_h diff --git a/qpid/cpp/src/qpid/sys/ProtocolFactory.h b/qpid/cpp/src/qpid/sys/ProtocolFactory.h new file mode 100644 index 0000000000..4d198a92da --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ProtocolFactory.h @@ -0,0 +1,57 @@ +#ifndef _sys_ProtocolFactory_h +#define _sys_ProtocolFactory_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IntegerTypes.h" +#include "qpid/SharedObject.h" +#include "qpid/sys/ConnectionCodec.h" +#include <boost/function.hpp> + +namespace qpid { +namespace sys { + +class Poller; + +class ProtocolFactory : public qpid::SharedObject<ProtocolFactory> +{ + public: + typedef boost::function2<void, int, std::string> ConnectFailedCallback; + + virtual ~ProtocolFactory() = 0; + virtual uint16_t getPort() const = 0; + virtual void accept(boost::shared_ptr<Poller>, ConnectionCodec::Factory*) = 0; + virtual void connect( + boost::shared_ptr<Poller>, + const std::string& host, const std::string& port, + ConnectionCodec::Factory* codec, + ConnectFailedCallback failed) = 0; + virtual bool supports(const std::string& /*capability*/) { return false; } +}; + +inline ProtocolFactory::~ProtocolFactory() {} + +}} + + + +#endif //!_sys_ProtocolFactory_h diff --git a/qpid/cpp/src/qpid/sys/RdmaIOPlugin.cpp b/qpid/cpp/src/qpid/sys/RdmaIOPlugin.cpp new file mode 100644 index 0000000000..631d116b41 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/RdmaIOPlugin.cpp @@ -0,0 +1,399 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ProtocolFactory.h" + +#include "qpid/Plugin.h" +#include "qpid/broker/Broker.h" +#include "qpid/framing/AMQP_HighestVersion.h" +#include "qpid/log/Statement.h" +#include "qpid/sys/rdma/RdmaIO.h" +#include "qpid/sys/rdma/rdma_exception.h" +#include "qpid/sys/OutputControl.h" +#include "qpid/sys/SecuritySettings.h" + +#include <boost/bind.hpp> +#include <memory> + +#include <netdb.h> + +using std::auto_ptr; +using std::string; +using std::stringstream; + +namespace qpid { +namespace sys { + +class RdmaIOHandler : public OutputControl { + std::string identifier; + ConnectionCodec::Factory* factory; + ConnectionCodec* codec; + bool readError; + + sys::Mutex pollingLock; + bool polling; + + Rdma::AsynchIO* aio; + Rdma::Connection::intrusive_ptr connection; + + void write(const framing::ProtocolInitiation&); + void disconnectAction(); + + public: + RdmaIOHandler(Rdma::Connection::intrusive_ptr c, ConnectionCodec::Factory* f); + ~RdmaIOHandler(); + void init(Rdma::AsynchIO* a); + void start(Poller::shared_ptr poller); + + // Output side + void close(); + void abort(); + void activateOutput(); + void giveReadCredit(int32_t credit); + void initProtocolOut(); + + // Input side + void readbuff(Rdma::AsynchIO& aio, Rdma::Buffer* buff); + void initProtocolIn(Rdma::Buffer* buff); + + // Notifications + void full(Rdma::AsynchIO& aio); + void idle(Rdma::AsynchIO& aio); + void error(Rdma::AsynchIO& aio); + void disconnected(); + void drained(); +}; + +RdmaIOHandler::RdmaIOHandler(Rdma::Connection::intrusive_ptr c, qpid::sys::ConnectionCodec::Factory* f) : + identifier(c->getFullName()), + factory(f), + codec(0), + readError(false), + polling(false), + connection(c) +{ +} + +RdmaIOHandler::~RdmaIOHandler() { + if (codec) + codec->closed(); + delete codec; + delete aio; +} + +void RdmaIOHandler::init(Rdma::AsynchIO* a) { + aio = a; +} + +void RdmaIOHandler::start(Poller::shared_ptr poller) { + Mutex::ScopedLock l(pollingLock); + assert(!polling); + + polling = true; + + aio->start(poller); +} + +void RdmaIOHandler::write(const framing::ProtocolInitiation& data) +{ + QPID_LOG(debug, "Rdma: SENT [" << identifier << "] INIT(" << data << ")"); + Rdma::Buffer* buff = aio->getSendBuffer(); + assert(buff); + framing::Buffer out(buff->bytes(), buff->byteCount()); + data.encode(out); + buff->dataCount(data.encodedSize()); + aio->queueWrite(buff); +} + +void RdmaIOHandler::close() { + aio->drainWriteQueue(boost::bind(&RdmaIOHandler::drained, this)); +} + +// TODO: Dummy implementation, need to fill this in for heartbeat timeout to work +void RdmaIOHandler::abort() { +} + +void RdmaIOHandler::activateOutput() { + aio->notifyPendingWrite(); +} + +void RdmaIOHandler::idle(Rdma::AsynchIO&) { + // TODO: Shouldn't need this test as idle() should only ever be called when + // the connection is writable anyway + if ( !aio->writable() ) { + return; + } + if (codec == 0) return; + if (!codec->canEncode()) { + return; + } + Rdma::Buffer* buff = aio->getSendBuffer(); + if (buff) { + size_t encoded=codec->encode(buff->bytes(), buff->byteCount()); + buff->dataCount(encoded); + aio->queueWrite(buff); + if (codec->isClosed()) { + close(); + } + } +} + +void RdmaIOHandler::initProtocolOut() { + // We mustn't have already started the conversation + // but we must be able to send + assert( codec == 0 ); + assert( aio->writable() ); + codec = factory->create(*this, identifier, SecuritySettings()); + write(framing::ProtocolInitiation(codec->getVersion())); +} + +void RdmaIOHandler::error(Rdma::AsynchIO&) { + disconnected(); +} + +namespace { + void stopped(RdmaIOHandler* async) { + delete async; + } +} + +void RdmaIOHandler::disconnectAction() { + { + Mutex::ScopedLock l(pollingLock); + // If we're closed already then we'll get to drained() anyway + if (!polling) return; + polling = false; + } + aio->stop(boost::bind(&stopped, this)); +} + +void RdmaIOHandler::disconnected() { + aio->requestCallback(boost::bind(&RdmaIOHandler::disconnectAction, this)); +} + +void RdmaIOHandler::drained() { + // We know we've drained the write queue now, but we don't have to do anything + // because we can rely on the client to disconnect to trigger the connection + // cleanup. +} + +void RdmaIOHandler::full(Rdma::AsynchIO&) { + QPID_LOG(debug, "Rdma: buffer full [" << identifier << "]"); +} + +// TODO: Dummy implementation of read throttling +void RdmaIOHandler::giveReadCredit(int32_t) { +} + +// The logic here is subtly different from TCP as RDMA is message oriented +// so we define that an RDMA message is a frame - in this case there is no putting back +// of any message remainder - there shouldn't be any. And what we read here can't be +// smaller than a frame +void RdmaIOHandler::readbuff(Rdma::AsynchIO&, Rdma::Buffer* buff) { + if (readError) { + return; + } + size_t decoded = 0; + try { + if (codec) { + decoded = codec->decode(buff->bytes(), buff->dataCount()); + }else{ + // Need to start protocol processing + initProtocolIn(buff); + } + }catch(const std::exception& e){ + QPID_LOG(error, e.what()); + readError = true; + close(); + } +} + +void RdmaIOHandler::initProtocolIn(Rdma::Buffer* buff) { + framing::Buffer in(buff->bytes(), buff->dataCount()); + framing::ProtocolInitiation protocolInit; + size_t decoded = 0; + if (protocolInit.decode(in)) { + decoded = in.getPosition(); + QPID_LOG(debug, "Rdma: RECV [" << identifier << "] INIT(" << protocolInit << ")"); + + codec = factory->create(protocolInit.getVersion(), *this, identifier, SecuritySettings()); + + // If we failed to create the codec then we don't understand the offered protocol version + if (!codec) { + // send valid version header & close connection. + write(framing::ProtocolInitiation(framing::highestProtocolVersion)); + readError = true; + close(); + } + } +} + +class RdmaIOProtocolFactory : public ProtocolFactory { + auto_ptr<Rdma::Listener> listener; + const uint16_t listeningPort; + + public: + RdmaIOProtocolFactory(int16_t port, int backlog); + void accept(Poller::shared_ptr, ConnectionCodec::Factory*); + void connect(Poller::shared_ptr, const string& host, const std::string& port, ConnectionCodec::Factory*, ConnectFailedCallback); + + uint16_t getPort() const; + + private: + bool request(Rdma::Connection::intrusive_ptr, const Rdma::ConnectionParams&, ConnectionCodec::Factory*); + void established(Poller::shared_ptr, Rdma::Connection::intrusive_ptr); + void connected(Poller::shared_ptr, Rdma::Connection::intrusive_ptr, const Rdma::ConnectionParams&, ConnectionCodec::Factory*); + void connectionError(Rdma::Connection::intrusive_ptr, Rdma::ErrorType); + void disconnected(Rdma::Connection::intrusive_ptr); + void rejected(Rdma::Connection::intrusive_ptr, const Rdma::ConnectionParams&, ConnectFailedCallback); +}; + +// Static instance to initialise plugin +static class RdmaIOPlugin : public Plugin { + void earlyInitialize(Target&) { + } + + void initialize(Target& target) { + // Check whether we actually have any rdma devices + if ( Rdma::deviceCount() == 0 ) { + QPID_LOG(info, "Rdma: Disabled: no rdma devices found"); + return; + } + + broker::Broker* broker = dynamic_cast<broker::Broker*>(&target); + // Only provide to a Broker + if (broker) { + const broker::Broker::Options& opts = broker->getOptions(); + ProtocolFactory::shared_ptr protocol(new RdmaIOProtocolFactory(opts.port, opts.connectionBacklog)); + QPID_LOG(notice, "Rdma: Listening on RDMA port " << protocol->getPort()); + broker->registerProtocolFactory("rdma", protocol); + } + } +} rdmaPlugin; + +RdmaIOProtocolFactory::RdmaIOProtocolFactory(int16_t port, int /*backlog*/) : + listeningPort(port) +{} + +void RdmaIOProtocolFactory::established(Poller::shared_ptr poller, Rdma::Connection::intrusive_ptr ci) { + RdmaIOHandler* async = ci->getContext<RdmaIOHandler>(); + async->start(poller); +} + +bool RdmaIOProtocolFactory::request(Rdma::Connection::intrusive_ptr ci, const Rdma::ConnectionParams& cp, + ConnectionCodec::Factory* f) { + try { + if (cp.rdmaProtocolVersion == 0) { + QPID_LOG(warning, "Rdma: connection from protocol version 0 client"); + } + RdmaIOHandler* async = new RdmaIOHandler(ci, f); + Rdma::AsynchIO* aio = + new Rdma::AsynchIO(ci->getQueuePair(), + cp.rdmaProtocolVersion, + cp.maxRecvBufferSize, cp.initialXmitCredit, Rdma::DEFAULT_WR_ENTRIES, + boost::bind(&RdmaIOHandler::readbuff, async, _1, _2), + boost::bind(&RdmaIOHandler::idle, async, _1), + 0, // boost::bind(&RdmaIOHandler::full, async, _1), + boost::bind(&RdmaIOHandler::error, async, _1)); + async->init(aio); + + // Record aio so we can get it back from a connection + ci->addContext(async); + return true; + } catch (const Rdma::Exception& e) { + QPID_LOG(error, "Rdma: Cannot accept new connection (Rdma exception): " << e.what()); + } catch (const std::exception& e) { + QPID_LOG(error, "Rdma: Cannot accept new connection (unknown exception): " << e.what()); + } + + // If we get here we caught an exception so reject connection + return false; +} + +void RdmaIOProtocolFactory::connectionError(Rdma::Connection::intrusive_ptr, Rdma::ErrorType) { +} + +void RdmaIOProtocolFactory::disconnected(Rdma::Connection::intrusive_ptr ci) { + // If we've got a connection already tear it down, otherwise ignore + RdmaIOHandler* async = ci->getContext<RdmaIOHandler>(); + if (async) { + // Make sure we don't disconnect more than once + ci->removeContext(); + async->disconnected(); + } +} + +uint16_t RdmaIOProtocolFactory::getPort() const { + return listeningPort; // Immutable no need for lock. +} + +void RdmaIOProtocolFactory::accept(Poller::shared_ptr poller, ConnectionCodec::Factory* fact) { + ::sockaddr_in sin; + + sin.sin_family = AF_INET; + sin.sin_port = htons(listeningPort); + sin.sin_addr.s_addr = INADDR_ANY; + + listener.reset( + new Rdma::Listener( + Rdma::ConnectionParams(65536, Rdma::DEFAULT_WR_ENTRIES), + boost::bind(&RdmaIOProtocolFactory::established, this, poller, _1), + boost::bind(&RdmaIOProtocolFactory::connectionError, this, _1, _2), + boost::bind(&RdmaIOProtocolFactory::disconnected, this, _1), + boost::bind(&RdmaIOProtocolFactory::request, this, _1, _2, fact))); + + SocketAddress sa("",boost::lexical_cast<std::string>(listeningPort)); + listener->start(poller, sa); +} + +// Only used for outgoing connections (in federation) +void RdmaIOProtocolFactory::rejected(Rdma::Connection::intrusive_ptr, const Rdma::ConnectionParams&, ConnectFailedCallback failed) { + failed(-1, "Connection rejected"); +} + +// Do the same as connection request and established but mark a client too +void RdmaIOProtocolFactory::connected(Poller::shared_ptr poller, Rdma::Connection::intrusive_ptr ci, const Rdma::ConnectionParams& cp, + ConnectionCodec::Factory* f) { + (void) request(ci, cp, f); + established(poller, ci); + RdmaIOHandler* async = ci->getContext<RdmaIOHandler>(); + async->initProtocolOut(); +} + +void RdmaIOProtocolFactory::connect( + Poller::shared_ptr poller, + const std::string& host, const std::string& port, + ConnectionCodec::Factory* f, + ConnectFailedCallback failed) +{ + Rdma::Connector* c = + new Rdma::Connector( + Rdma::ConnectionParams(8000, Rdma::DEFAULT_WR_ENTRIES), + boost::bind(&RdmaIOProtocolFactory::connected, this, poller, _1, _2, f), + boost::bind(&RdmaIOProtocolFactory::connectionError, this, _1, _2), + boost::bind(&RdmaIOProtocolFactory::disconnected, this, _1), + boost::bind(&RdmaIOProtocolFactory::rejected, this, _1, _2, failed)); + + SocketAddress sa(host, port); + c->start(poller, sa); +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/Runnable.cpp b/qpid/cpp/src/qpid/sys/Runnable.cpp new file mode 100644 index 0000000000..325d87c91b --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Runnable.cpp @@ -0,0 +1,32 @@ +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/Runnable.h" +#include <boost/bind.hpp> + +namespace qpid { +namespace sys { + +Runnable::~Runnable() {} + +Runnable::Functor Runnable::functor() +{ + return boost::bind(&Runnable::run, this); +} + +}} diff --git a/qpid/cpp/src/qpid/sys/ScopedIncrement.h b/qpid/cpp/src/qpid/sys/ScopedIncrement.h new file mode 100644 index 0000000000..8645ab2484 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ScopedIncrement.h @@ -0,0 +1,67 @@ +#ifndef _posix_ScopedIncrement_h +#define _posix_ScopedIncrement_h + +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <boost/noncopyable.hpp> +#include <boost/function.hpp> + +namespace qpid { +namespace sys { + +/** + * Increment counter in constructor and decrement in destructor. + * Optionally call a function if the decremented counter value is 0. + * Note the function must not throw, it is called in the destructor. + */ +template <class T, class F=boost::function<void()> > +class ScopedIncrement : boost::noncopyable +{ + public: + ScopedIncrement(T& c, F f=0) + : count(c), callback(f) { ++count; } + ~ScopedIncrement() { if (--count == 0 && callback) callback(); } + + private: + T& count; + F callback; +}; + + +/** Decrement counter in constructor and increment in destructor. */ +template <class T> +class ScopedDecrement : boost::noncopyable +{ + public: + ScopedDecrement(T& c) : count(c) { value = --count; } + ~ScopedDecrement() { ++count; } + + /** Return the value after the decrement. */ + operator long() { return value; } + + private: + T& count; + long value; +}; + + +}} + + +#endif // _posix_ScopedIncrement_h diff --git a/qpid/cpp/src/qpid/sys/SecurityLayer.h b/qpid/cpp/src/qpid/sys/SecurityLayer.h new file mode 100644 index 0000000000..52bc40e352 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/SecurityLayer.h @@ -0,0 +1,42 @@ +#ifndef QPID_SYS_SECURITYLAYER_H +#define QPID_SYS_SECURITYLAYER_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/Codec.h" + +namespace qpid { +namespace sys { + +/** + * Defines interface to a SASL negotiated Security Layer (for + * encryption/integrity) + */ +class SecurityLayer : public Codec +{ + public: + virtual void init(Codec*) = 0; + virtual ~SecurityLayer() {} +}; + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_SECURITYLAYER_H*/ diff --git a/qpid/cpp/src/qpid/sys/SecuritySettings.h b/qpid/cpp/src/qpid/sys/SecuritySettings.h new file mode 100644 index 0000000000..bfcd08fd0f --- /dev/null +++ b/qpid/cpp/src/qpid/sys/SecuritySettings.h @@ -0,0 +1,58 @@ +#ifndef QPID_SYS_SECURITYSETTINGS_H +#define QPID_SYS_SECURITYSETTINGS_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +namespace qpid { +namespace sys { + +/** + * Conveys security information from a given transport to the upper + * layers. + */ +struct SecuritySettings +{ + /** + * Security Strength Factor (SSF). Possible values are: + * + * @li 0 No security + * @li 1 Integrity checking only + * @li >1 Integrity and confidentiality with the number + * giving the encryption key length. + */ + unsigned int ssf; + /** + * An authorisation id + */ + std::string authid; + + /** + * Disables SASL mechanisms that are vulnerable to passive + * dictionary-based password attacks + */ + bool nodict; + + SecuritySettings() : ssf(0), nodict(false) {} +}; + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_SECURITYSETTINGS_H*/ diff --git a/qpid/cpp/src/qpid/sys/Semaphore.h b/qpid/cpp/src/qpid/sys/Semaphore.h new file mode 100644 index 0000000000..9d70f89aeb --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Semaphore.h @@ -0,0 +1,79 @@ +#ifndef _sys_Semaphore_h +#define _sys_Semaphore_h + +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/Monitor.h" + +namespace qpid { +namespace sys { + +class Semaphore +{ +public: + Semaphore(uint c = 1) : count(c) {} + + void lock() { acquire(); } + void unlock() { release(); } + bool trylock() { return tryAcquire(); } + + bool tryAcquire() + { + Monitor::ScopedLock l(monitor); + if (count) { + count--; + return true; + } else { + return false; + } + } + + void acquire() + { + Monitor::ScopedLock l(monitor); + while (count == 0) monitor.wait(); + count--; + } + + void release(uint n) + { + Monitor::ScopedLock l(monitor); + if (count==0) monitor.notifyAll(); + count+=n; + } + + void release() + { + release(1); + } + + void forceLock() + { + Monitor::ScopedLock l(monitor); + count = 0; + } + +private: + Monitor monitor; + uint count; +}; + +}} + +#endif /*!_sys_Semaphore_h*/ diff --git a/qpid/cpp/src/qpid/sys/Shlib.cpp b/qpid/cpp/src/qpid/sys/Shlib.cpp new file mode 100644 index 0000000000..342d726876 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Shlib.cpp @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Shlib.h" + +#include "qpid/log/Statement.h" + +namespace qpid { +namespace sys { + +AutoShlib::~AutoShlib() throw() { + try { + unload(); + } catch(const std::exception& e) { + QPID_LOG(error, "Unloading shared library: " << e.what()); + } +} + +// Note: Other functions are defined in apr/Shlib.cpp or posix/Shlib.cpp. + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/Shlib.h b/qpid/cpp/src/qpid/sys/Shlib.h new file mode 100644 index 0000000000..7f66cfec14 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Shlib.h @@ -0,0 +1,76 @@ +#ifndef QPID_SYS_SHLIB_H +#define QPID_SYS_SHLIB_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/CommonImportExport.h" +#include <boost/noncopyable.hpp> +#include <iostream> + +namespace qpid { +namespace sys { + +/** Encapsulates a shared library handle. + *@see AutoShlib + */ +class Shlib { + public: + /** Load a shared library */ + Shlib(const char* libname) { load(libname); } + + /** Load a shared library */ + Shlib(const std::string& libname) { load(libname.c_str()); } + + /** Unload shared library. */ + QPID_COMMON_EXTERN void unload(); + + /** Look up symbol. */ + QPID_COMMON_EXTERN void* getSymbol(const char* symbol); + + /** Look up symbol in shared library, cast it to the desired + * pointer type, void* by default. + */ + template <class T> + T getSymbol(const char* symbol) { + // Double cast avoids warning about casting object to function pointer + return reinterpret_cast<T>(reinterpret_cast<intptr_t>( + this->getSymbol(symbol))); + } + + private: + void* handle; + QPID_COMMON_EXTERN void load(const char* libname); +}; + +/** A shared library handle that unloads the shlib in it's dtor */ +class AutoShlib : public Shlib { + public: + /** Load shared library */ + AutoShlib(const std::string& libname) : Shlib(libname) {} + /** Calls unload() */ + QPID_COMMON_EXTERN ~AutoShlib() throw(); +}; + + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_SHLIB_H*/ diff --git a/qpid/cpp/src/qpid/sys/ShutdownHandler.h b/qpid/cpp/src/qpid/sys/ShutdownHandler.h new file mode 100644 index 0000000000..88baecb5b6 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ShutdownHandler.h @@ -0,0 +1,37 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef _ShutdownHandler_ +#define _ShutdownHandler_ + +namespace qpid { +namespace sys { + + class ShutdownHandler + { + public: + virtual void shutdown() = 0; + virtual ~ShutdownHandler(){} + }; + +} +} + +#endif diff --git a/qpid/cpp/src/qpid/sys/Socket.h b/qpid/cpp/src/qpid/sys/Socket.h new file mode 100644 index 0000000000..9f62f3be1c --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Socket.h @@ -0,0 +1,103 @@ +#ifndef _sys_Socket_h +#define _sys_Socket_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/IntegerTypes.h" +#include "qpid/CommonImportExport.h" +#include <string> + +namespace qpid { +namespace sys { + +class Duration; +class SocketAddress; + +class QPID_COMMON_CLASS_EXTERN Socket : public IOHandle +{ +public: + /** Create a socket wrapper for descriptor. */ + QPID_COMMON_EXTERN Socket(); + + /** Set socket non blocking */ + void setNonblocking() const; + + QPID_COMMON_EXTERN void setTcpNoDelay() const; + + QPID_COMMON_EXTERN void connect(const std::string& host, const std::string& port) const; + QPID_COMMON_EXTERN void connect(const SocketAddress&) const; + + QPID_COMMON_EXTERN void close() const; + + /** Bind to a port and start listening. + *@param port 0 means choose an available port. + *@param backlog maximum number of pending connections. + *@return The bound port. + */ + QPID_COMMON_EXTERN int listen(const std::string& host = "", const std::string& port = "0", int backlog = 10) const; + QPID_COMMON_EXTERN int listen(const SocketAddress&, int backlog = 10) const; + + /** + * Returns an address (host and port) for the remote end of the + * socket + */ + QPID_COMMON_EXTERN std::string getPeerAddress() const; + /** + * Returns an address (host and port) for the local end of the + * socket + */ + QPID_COMMON_EXTERN std::string getLocalAddress() const; + + /** + * Returns the full address of the connection: local and remote host and port. + */ + QPID_COMMON_INLINE_EXTERN std::string getFullAddress() const { return getLocalAddress()+"-"+getPeerAddress(); } + + /** + * Returns the error code stored in the socket. This may be used + * to determine the result of a non-blocking connect. + */ + int getError() const; + + /** Accept a connection from a socket that is already listening + * and has an incoming connection + */ + QPID_COMMON_EXTERN Socket* accept() const; + + // TODO The following are raw operations, maybe they need better wrapping? + QPID_COMMON_EXTERN int read(void *buf, size_t count) const; + QPID_COMMON_EXTERN int write(const void *buf, size_t count) const; + +private: + /** Create socket */ + void createSocket(const SocketAddress&) const; + + Socket(IOHandlePrivate*); + mutable std::string localname; + mutable std::string peername; + mutable bool nonblocking; + mutable bool nodelay; +}; + +}} +#endif /*!_sys_Socket_h*/ diff --git a/qpid/cpp/src/qpid/sys/SocketAddress.h b/qpid/cpp/src/qpid/sys/SocketAddress.h new file mode 100644 index 0000000000..c2120338cf --- /dev/null +++ b/qpid/cpp/src/qpid/sys/SocketAddress.h @@ -0,0 +1,53 @@ +#ifndef _sys_SocketAddress_h +#define _sys_SocketAddress_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IntegerTypes.h" +#include "qpid/CommonImportExport.h" +#include <string> + +struct addrinfo; + +namespace qpid { +namespace sys { + +class SocketAddress { + friend const ::addrinfo& getAddrInfo(const SocketAddress&); + +public: + /** Create a SocketAddress from hostname and port*/ + QPID_COMMON_EXTERN SocketAddress(const std::string& host, const std::string& port); + QPID_COMMON_EXTERN SocketAddress(const SocketAddress&); + QPID_COMMON_EXTERN SocketAddress& operator=(const SocketAddress&); + QPID_COMMON_EXTERN ~SocketAddress(); + + std::string asString(bool numeric=true) const; + +private: + std::string host; + std::string port; + mutable ::addrinfo* addrInfo; +}; + +}} +#endif /*!_sys_SocketAddress_h*/ diff --git a/qpid/cpp/src/qpid/sys/SslPlugin.cpp b/qpid/cpp/src/qpid/sys/SslPlugin.cpp new file mode 100644 index 0000000000..471a0cef60 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/SslPlugin.cpp @@ -0,0 +1,186 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ProtocolFactory.h" + +#include "qpid/Plugin.h" +#include "qpid/sys/ssl/check.h" +#include "qpid/sys/ssl/util.h" +#include "qpid/sys/ssl/SslHandler.h" +#include "qpid/sys/ssl/SslIo.h" +#include "qpid/sys/ssl/SslSocket.h" +#include "qpid/broker/Broker.h" +#include "qpid/log/Statement.h" + +#include <boost/bind.hpp> +#include <memory> + + +namespace qpid { +namespace sys { + +struct SslServerOptions : ssl::SslOptions +{ + uint16_t port; + bool clientAuth; + bool nodict; + + SslServerOptions() : port(5671), + clientAuth(false), + nodict(false) + { + addOptions() + ("ssl-port", optValue(port, "PORT"), "Port on which to listen for SSL connections") + ("ssl-require-client-authentication", optValue(clientAuth), + "Forces clients to authenticate in order to establish an SSL connection") + ("ssl-sasl-no-dict", optValue(nodict), + "Disables SASL mechanisms that are vulnerable to passive dictionary-based password attacks"); + } +}; + +class SslProtocolFactory : public ProtocolFactory { + const bool tcpNoDelay; + qpid::sys::ssl::SslSocket listener; + const uint16_t listeningPort; + std::auto_ptr<qpid::sys::ssl::SslAcceptor> acceptor; + bool nodict; + + public: + SslProtocolFactory(const SslServerOptions&, int backlog, bool nodelay); + void accept(Poller::shared_ptr, ConnectionCodec::Factory*); + void connect(Poller::shared_ptr, const std::string& host, const std::string& port, + ConnectionCodec::Factory*, + boost::function2<void, int, std::string> failed); + + uint16_t getPort() const; + bool supports(const std::string& capability); + + private: + void established(Poller::shared_ptr, const qpid::sys::ssl::SslSocket&, ConnectionCodec::Factory*, + bool isClient); +}; + +// Static instance to initialise plugin +static struct SslPlugin : public Plugin { + SslServerOptions options; + + Options* getOptions() { return &options; } + + ~SslPlugin() { ssl::shutdownNSS(); } + + void earlyInitialize(Target&) { + } + + void initialize(Target& target) { + broker::Broker* broker = dynamic_cast<broker::Broker*>(&target); + // Only provide to a Broker + if (broker) { + if (options.certDbPath.empty()) { + QPID_LOG(notice, "SSL plugin not enabled, you must set --ssl-cert-db to enable it."); + } else { + try { + ssl::initNSS(options, true); + + const broker::Broker::Options& opts = broker->getOptions(); + ProtocolFactory::shared_ptr protocol(new SslProtocolFactory(options, + opts.connectionBacklog, + opts.tcpNoDelay)); + QPID_LOG(notice, "Listening for SSL connections on TCP port " << protocol->getPort()); + broker->registerProtocolFactory("ssl", protocol); + } catch (const std::exception& e) { + QPID_LOG(error, "Failed to initialise SSL plugin: " << e.what()); + } + } + } + } +} sslPlugin; + +SslProtocolFactory::SslProtocolFactory(const SslServerOptions& options, int backlog, bool nodelay) : + tcpNoDelay(nodelay), listeningPort(listener.listen(options.port, backlog, options.certName, options.clientAuth)), + nodict(options.nodict) +{} + +void SslProtocolFactory::established(Poller::shared_ptr poller, const qpid::sys::ssl::SslSocket& s, + ConnectionCodec::Factory* f, bool isClient) { + qpid::sys::ssl::SslHandler* async = new qpid::sys::ssl::SslHandler(s.getFullAddress(), f, nodict); + + if (tcpNoDelay) { + s.setTcpNoDelay(tcpNoDelay); + QPID_LOG(info, "Set TCP_NODELAY on connection to " << s.getPeerAddress()); + } + + if (isClient) + async->setClient(); + qpid::sys::ssl::SslIO* aio = new qpid::sys::ssl::SslIO(s, + boost::bind(&qpid::sys::ssl::SslHandler::readbuff, async, _1, _2), + boost::bind(&qpid::sys::ssl::SslHandler::eof, async, _1), + boost::bind(&qpid::sys::ssl::SslHandler::disconnect, async, _1), + boost::bind(&qpid::sys::ssl::SslHandler::closedSocket, async, _1, _2), + boost::bind(&qpid::sys::ssl::SslHandler::nobuffs, async, _1), + boost::bind(&qpid::sys::ssl::SslHandler::idle, async, _1)); + + async->init(aio, 4); + aio->start(poller); +} + +uint16_t SslProtocolFactory::getPort() const { + return listeningPort; // Immutable no need for lock. +} + +void SslProtocolFactory::accept(Poller::shared_ptr poller, + ConnectionCodec::Factory* fact) { + acceptor.reset( + new qpid::sys::ssl::SslAcceptor(listener, + boost::bind(&SslProtocolFactory::established, this, poller, _1, fact, false))); + acceptor->start(poller); +} + +void SslProtocolFactory::connect( + Poller::shared_ptr poller, + const std::string& host, const std::string& port, + ConnectionCodec::Factory* fact, + ConnectFailedCallback failed) +{ + // Note that the following logic does not cause a memory leak. + // The allocated Socket is freed either by the SslConnector + // upon connection failure or by the SslIoHandle upon connection + // shutdown. The allocated SslConnector frees itself when it + // is no longer needed. + + qpid::sys::ssl::SslSocket* socket = new qpid::sys::ssl::SslSocket(); + new qpid::sys::ssl::SslConnector (*socket, poller, host, port, + boost::bind(&SslProtocolFactory::established, this, poller, _1, fact, true), + failed); +} + +namespace +{ +const std::string SSL = "ssl"; +} + +bool SslProtocolFactory::supports(const std::string& capability) +{ + std::string s = capability; + transform(s.begin(), s.end(), s.begin(), tolower); + return s == SSL; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/StateMonitor.h b/qpid/cpp/src/qpid/sys/StateMonitor.h new file mode 100644 index 0000000000..eac37a8543 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/StateMonitor.h @@ -0,0 +1,78 @@ +#ifndef QPID_SYS_STATEMONITOR_H +#define QPID_SYS_STATEMONITOR_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/Waitable.h" + +#include <bitset> + +namespace qpid { +namespace sys { + +/** + * A monitor with an enum state value. + * + *@param Enum: enum type to use for states. + *@param EnumMax: Highest enum value. + */ +template <class Enum, size_t MaxEnum> +class StateMonitor : public Waitable +{ + public: + struct Set : public std::bitset<MaxEnum + 1> { + Set() {} + Set(Enum s) { set(s); } + Set(Enum s, Enum t) { std::bitset<MaxEnum + 1>::set(s).set(t); } + Set(Enum s, Enum t, Enum u) { std::bitset<MaxEnum + 1>::set(s).set(t).set(u); } + Set(Enum s, Enum t, Enum u, Enum v) { std::bitset<MaxEnum + 1>::set(s).set(t).set(u).set(v); } + }; + + + StateMonitor(Enum initial) { state=initial; } + + /** @pre Caller holds a ScopedLock. */ + void set(Enum s) { state=s; notifyAll(); } + /** @pre Caller holds a ScopedLock. */ + StateMonitor& operator=(Enum s) { set(s); return *this; } + + /** @pre Caller holds a ScopedLock. */ + Enum get() const { return state; } + /** @pre Caller holds a ScopedLock. */ + operator Enum() const { return state; } + + /** @pre Caller holds a ScopedLock */ + void waitFor(Enum s) { ScopedWait w(*this); while (s != state) wait(); } + /** @pre Caller holds a ScopedLock */ + void waitFor(Set s) { ScopedWait w(*this); while (!s.test(state)) wait(); } + /** @pre Caller holds a ScopedLock */ + void waitNot(Enum s) { ScopedWait w(*this); while (s == state) wait(); } + /** @pre Caller holds a ScopedLock */ + void waitNot(Set s) { ScopedWait w(*this); while (s.test(state)) wait(); } + + private: + Enum state; +}; + +}} + + +#endif /*!QPID_SYS_STATEMONITOR_H*/ diff --git a/qpid/cpp/src/qpid/sys/TCPIOPlugin.cpp b/qpid/cpp/src/qpid/sys/TCPIOPlugin.cpp new file mode 100644 index 0000000000..34338ce434 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/TCPIOPlugin.cpp @@ -0,0 +1,152 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ProtocolFactory.h" +#include "qpid/sys/AsynchIOHandler.h" +#include "qpid/sys/AsynchIO.h" + +#include "qpid/Plugin.h" +#include "qpid/sys/Socket.h" +#include "qpid/sys/Poller.h" +#include "qpid/broker/Broker.h" +#include "qpid/log/Statement.h" + +#include <boost/bind.hpp> +#include <memory> + +namespace qpid { +namespace sys { + +class AsynchIOProtocolFactory : public ProtocolFactory { + const bool tcpNoDelay; + Socket listener; + const uint16_t listeningPort; + std::auto_ptr<AsynchAcceptor> acceptor; + + public: + AsynchIOProtocolFactory(const std::string& host, const std::string& port, int backlog, bool nodelay); + void accept(Poller::shared_ptr, ConnectionCodec::Factory*); + void connect(Poller::shared_ptr, const std::string& host, const std::string& port, + ConnectionCodec::Factory*, + ConnectFailedCallback); + + uint16_t getPort() const; + + private: + void established(Poller::shared_ptr, const Socket&, ConnectionCodec::Factory*, + bool isClient); + void connectFailed(const Socket&, int, const std::string&, ConnectFailedCallback); +}; + +// Static instance to initialise plugin +static class TCPIOPlugin : public Plugin { + void earlyInitialize(Target&) { + } + + void initialize(Target& target) { + broker::Broker* broker = dynamic_cast<broker::Broker*>(&target); + // Only provide to a Broker + if (broker) { + const broker::Broker::Options& opts = broker->getOptions(); + ProtocolFactory::shared_ptr protocolt( + new AsynchIOProtocolFactory( + "", boost::lexical_cast<std::string>(opts.port), + opts.connectionBacklog, + opts.tcpNoDelay)); + QPID_LOG(notice, "Listening on TCP port " << protocolt->getPort()); + broker->registerProtocolFactory("tcp", protocolt); + } + } +} tcpPlugin; + +AsynchIOProtocolFactory::AsynchIOProtocolFactory(const std::string& host, const std::string& port, int backlog, bool nodelay) : + tcpNoDelay(nodelay), listeningPort(listener.listen(host, port, backlog)) +{} + +void AsynchIOProtocolFactory::established(Poller::shared_ptr poller, const Socket& s, + ConnectionCodec::Factory* f, bool isClient) { + AsynchIOHandler* async = new AsynchIOHandler(s.getFullAddress(), f); + + if (tcpNoDelay) { + s.setTcpNoDelay(); + QPID_LOG(info, "Set TCP_NODELAY on connection to " << s.getPeerAddress()); + } + + if (isClient) + async->setClient(); + AsynchIO* aio = AsynchIO::create + (s, + boost::bind(&AsynchIOHandler::readbuff, async, _1, _2), + boost::bind(&AsynchIOHandler::eof, async, _1), + boost::bind(&AsynchIOHandler::disconnect, async, _1), + boost::bind(&AsynchIOHandler::closedSocket, async, _1, _2), + boost::bind(&AsynchIOHandler::nobuffs, async, _1), + boost::bind(&AsynchIOHandler::idle, async, _1)); + + async->init(aio, 4); + aio->start(poller); +} + +uint16_t AsynchIOProtocolFactory::getPort() const { + return listeningPort; // Immutable no need for lock. +} + +void AsynchIOProtocolFactory::accept(Poller::shared_ptr poller, + ConnectionCodec::Factory* fact) { + acceptor.reset( + AsynchAcceptor::create(listener, + boost::bind(&AsynchIOProtocolFactory::established, this, poller, _1, fact, false))); + acceptor->start(poller); +} + +void AsynchIOProtocolFactory::connectFailed( + const Socket& s, int ec, const std::string& emsg, + ConnectFailedCallback failedCb) +{ + failedCb(ec, emsg); + s.close(); + delete &s; +} + +void AsynchIOProtocolFactory::connect( + Poller::shared_ptr poller, + const std::string& host, const std::string& port, + ConnectionCodec::Factory* fact, + ConnectFailedCallback failed) +{ + // Note that the following logic does not cause a memory leak. + // The allocated Socket is freed either by the AsynchConnector + // upon connection failure or by the AsynchIO upon connection + // shutdown. The allocated AsynchConnector frees itself when it + // is no longer needed. + Socket* socket = new Socket(); + AsynchConnector* c = AsynchConnector::create( + *socket, + host, + port, + boost::bind(&AsynchIOProtocolFactory::established, + this, poller, _1, fact, true), + boost::bind(&AsynchIOProtocolFactory::connectFailed, + this, _1, _2, _3, failed)); + c->start(poller); +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/TimeoutHandler.h b/qpid/cpp/src/qpid/sys/TimeoutHandler.h new file mode 100644 index 0000000000..0c10709bbf --- /dev/null +++ b/qpid/cpp/src/qpid/sys/TimeoutHandler.h @@ -0,0 +1,39 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef _TimeoutHandler_ +#define _TimeoutHandler_ + +namespace qpid { +namespace sys { + + class TimeoutHandler + { + public: + virtual void idleOut() = 0; + virtual void idleIn() = 0; + virtual ~TimeoutHandler(){} + }; + +} +} + + +#endif diff --git a/qpid/cpp/src/qpid/sys/Timer.cpp b/qpid/cpp/src/qpid/sys/Timer.cpp new file mode 100644 index 0000000000..fdb2e8c6bb --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Timer.cpp @@ -0,0 +1,205 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/Timer.h" +#include "qpid/sys/Mutex.h" +#include "qpid/log/Statement.h" + +#include <numeric> + +using boost::intrusive_ptr; +using std::max; + +namespace qpid { +namespace sys { + +TimerTask::TimerTask(Duration timeout, const std::string& n) : + name(n), + sortTime(AbsTime::FarFuture()), + period(timeout), + nextFireTime(AbsTime::now(), timeout), + cancelled(false) +{} + +TimerTask::TimerTask(AbsTime time, const std::string& n) : + name(n), + sortTime(AbsTime::FarFuture()), + period(0), + nextFireTime(time), + cancelled(false) +{} + +TimerTask::~TimerTask() {} + +bool TimerTask::readyToFire() const { + return !(nextFireTime > AbsTime::now()); +} + +void TimerTask::fireTask() { + cancelled = true; + fire(); +} + +// This can only be used to setup the next fire time. After the Timer has already fired +void TimerTask::setupNextFire() { + if (period && readyToFire()) { + nextFireTime = max(AbsTime::now(), AbsTime(nextFireTime, period)); + cancelled = false; + } else { + QPID_LOG(error, name << " couldn't setup next timer firing: " << Duration(nextFireTime, AbsTime::now()) << "[" << period << "]"); + } +} + +// Only allow tasks to be delayed +void TimerTask::restart() { nextFireTime = max(nextFireTime, AbsTime(AbsTime::now(), period)); } + +void TimerTask::cancel() { + ScopedLock<Mutex> l(callbackLock); + cancelled = true; +} + +Timer::Timer() : + active(false), + late(50 * TIME_MSEC), + overran(2 * TIME_MSEC), + lateCancel(500 * TIME_MSEC), + warn(5 * TIME_SEC) +{ + start(); +} + +Timer::~Timer() +{ + stop(); +} + +// TODO AStitcher 21/08/09 The threshholds for emitting warnings are a little arbitrary +void Timer::run() +{ + Monitor::ScopedLock l(monitor); + while (active) { + if (tasks.empty()) { + monitor.wait(); + } else { + intrusive_ptr<TimerTask> t = tasks.top(); + tasks.pop(); + assert(!(t->nextFireTime < t->sortTime)); + + // warn on extreme lateness + AbsTime start(AbsTime::now()); + Duration delay(t->sortTime, start); + { + ScopedLock<Mutex> l(t->callbackLock); + if (t->cancelled) { + { + Monitor::ScopedUnlock u(monitor); + drop(t); + } + if (delay > lateCancel) { + QPID_LOG(debug, t->name << " cancelled timer woken up " << + delay / TIME_MSEC << "ms late"); + } + continue; + } else if(Duration(t->nextFireTime, start) >= 0) { + { + Monitor::ScopedUnlock u(monitor); + fire(t); + } + // Warn if callback overran next timer's start. + AbsTime end(AbsTime::now()); + Duration overrun (0); + if (!tasks.empty()) { + overrun = Duration(tasks.top()->nextFireTime, end); + } + bool warningsEnabled; + QPID_LOG_TEST(warning, warningsEnabled); + if (warningsEnabled) { + if (overrun > overran) { + if (delay > overran) // if delay is significant to an overrun. + warn.lateAndOverran(t->name, delay, overrun, Duration(start, end)); + else + warn.overran(t->name, overrun, Duration(start, end)); + } + else if (delay > late) + warn.late(t->name, delay); + } + continue; + } else { + // If the timer was adjusted into the future it might no longer + // be the next event, so push and then get top to make sure + // You can only push events into the future + t->sortTime = t->nextFireTime; + tasks.push(t); + } + } + assert(!tasks.empty()); + monitor.wait(tasks.top()->sortTime); + } + } +} + +void Timer::add(intrusive_ptr<TimerTask> task) +{ + Monitor::ScopedLock l(monitor); + task->sortTime = task->nextFireTime; + tasks.push(task); + monitor.notify(); +} + +void Timer::start() +{ + Monitor::ScopedLock l(monitor); + if (!active) { + active = true; + runner = Thread(this); + } +} + +void Timer::stop() +{ + { + Monitor::ScopedLock l(monitor); + if (!active) return; + active = false; + monitor.notifyAll(); + } + runner.join(); +} + +// Allow subclasses to override behavior when firing a task. +void Timer::fire(boost::intrusive_ptr<TimerTask> t) { + try { + t->fireTask(); + } catch (const std::exception& e) { + QPID_LOG(error, "Exception thrown by timer task " << t->getName() << ": " << e.what()); + } +} + +// Provided for subclasses: called when a task is droped. +void Timer::drop(boost::intrusive_ptr<TimerTask>) {} + +bool operator<(const intrusive_ptr<TimerTask>& a, + const intrusive_ptr<TimerTask>& b) +{ + // Lower priority if time is later + return a.get() && b.get() && a->sortTime > b->sortTime; +} + +}} diff --git a/qpid/cpp/src/qpid/sys/Timer.h b/qpid/cpp/src/qpid/sys/Timer.h new file mode 100644 index 0000000000..98ba39ce38 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Timer.h @@ -0,0 +1,107 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef sys_Timer +#define sys_Timer + +#include "qpid/sys/TimerWarnings.h" +#include "qpid/sys/Monitor.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/Thread.h" +#include "qpid/sys/Runnable.h" +#include "qpid/RefCounted.h" +#include "qpid/CommonImportExport.h" +#include <memory> +#include <queue> + +#include <boost/intrusive_ptr.hpp> + +namespace qpid { +namespace sys { + +class Timer; + +class TimerTask : public RefCounted { + friend class Timer; + friend bool operator<(const boost::intrusive_ptr<TimerTask>&, + const boost::intrusive_ptr<TimerTask>&); + + std::string name; + AbsTime sortTime; + Duration period; + AbsTime nextFireTime; + Mutex callbackLock; + volatile bool cancelled; + + bool readyToFire() const; + void fireTask(); + + public: + QPID_COMMON_EXTERN TimerTask(Duration period, const std::string& name); + QPID_COMMON_EXTERN TimerTask(AbsTime fireTime, const std::string& name); + QPID_COMMON_EXTERN virtual ~TimerTask(); + + QPID_COMMON_EXTERN void setupNextFire(); + QPID_COMMON_EXTERN void restart(); + QPID_COMMON_EXTERN void cancel(); + + std::string getName() const { return name; } + + protected: + // Must be overridden with callback + virtual void fire() = 0; +}; + +// For the priority_queue order +bool operator<(const boost::intrusive_ptr<TimerTask>& a, + const boost::intrusive_ptr<TimerTask>& b); + +class Timer : private Runnable { + qpid::sys::Monitor monitor; + std::priority_queue<boost::intrusive_ptr<TimerTask> > tasks; + qpid::sys::Thread runner; + bool active; + + // Runnable interface + void run(); + + public: + QPID_COMMON_EXTERN Timer(); + QPID_COMMON_EXTERN virtual ~Timer(); + + QPID_COMMON_EXTERN virtual void add(boost::intrusive_ptr<TimerTask> task); + QPID_COMMON_EXTERN virtual void start(); + QPID_COMMON_EXTERN virtual void stop(); + + protected: + QPID_COMMON_EXTERN virtual void fire(boost::intrusive_ptr<TimerTask> task); + QPID_COMMON_EXTERN virtual void drop(boost::intrusive_ptr<TimerTask> task); + // Allow derived classes to change the late/overran thresholds. + Duration late; + Duration overran; + Duration lateCancel; + TimerWarnings warn; +}; + + +}} + + +#endif diff --git a/qpid/cpp/src/qpid/sys/TimerWarnings.cpp b/qpid/cpp/src/qpid/sys/TimerWarnings.cpp new file mode 100644 index 0000000000..87c3169456 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/TimerWarnings.cpp @@ -0,0 +1,82 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "TimerWarnings.h" +#include "qpid/log/Statement.h" + +namespace qpid { +namespace sys { + +TimerWarnings::TimerWarnings(Duration reportInterval) : + interval(reportInterval), + nextReport(now(), reportInterval) +{} + +void TimerWarnings::late(const std::string& task, Duration delay) { + taskStats[task].lateDelay.add(delay); + log(); +} + +void TimerWarnings::overran(const std::string& task, Duration overrun, Duration time) +{ + taskStats[task].overranOverrun.add(overrun); + taskStats[task].overranTime.add(time); + log(); +} + +void TimerWarnings::lateAndOverran( + const std::string& task, Duration delay, Duration overrun, Duration time) +{ + taskStats[task].lateAndOverranDelay.add(delay); + taskStats[task].lateAndOverranOverrun.add(overrun); + taskStats[task].lateAndOverranTime.add(time); + log(); +} + +void TimerWarnings::log() { + if (!taskStats.empty() && nextReport < now()) { + for (TaskStatsMap::iterator i = taskStats.begin(); i != taskStats.end(); ++i) { + std::string task = i->first; + TaskStats& stats = i->second; + if (stats.lateDelay.count) + QPID_LOG(warning, task << " task late " + << stats.lateDelay.count << " times by " + << stats.lateDelay.average()/TIME_MSEC << "ms on average."); + + if (stats.overranOverrun.count) + QPID_LOG(warning, task << " task overran " + << stats.overranOverrun.count << " times by " + << stats.overranOverrun.average()/TIME_MSEC << "ms (taking " + << stats.overranTime.average() << "ns) on average."); + + if (stats.lateAndOverranOverrun.count) + QPID_LOG(warning, task << " task late and overran " + << stats.lateAndOverranOverrun.count << " times: late " + << stats.lateAndOverranDelay.average()/TIME_MSEC << "ms, overran " + << stats.lateAndOverranOverrun.average()/TIME_MSEC << "ms (taking " + << stats.lateAndOverranTime.average() << "ns) on average."); + + } + nextReport = AbsTime(now(), interval); + taskStats.clear(); + } +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/TimerWarnings.h b/qpid/cpp/src/qpid/sys/TimerWarnings.h new file mode 100644 index 0000000000..337a434ab5 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/TimerWarnings.h @@ -0,0 +1,81 @@ +#ifndef QPID_SYS_TIMERWARNINGS_H +#define QPID_SYS_TIMERWARNINGS_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Time.h" +#include <map> +#include <string> + +namespace qpid { +namespace sys { + +/** + * The Timer class logs warnings when timer tasks are late and/or overrun. + * + * It is too expensive to log a warning for every late/overrun + * incident, doing so aggravates the problem of tasks over-running and + * being late. + * + * This class collects statistical data about each incident and prints + * an aggregated report at regular intervals. + */ +class TimerWarnings +{ + public: + TimerWarnings(Duration reportInterval); + + void late(const std::string& task, Duration delay); + + void overran(const std::string& task, Duration overrun, Duration time); + + void lateAndOverran(const std::string& task, + Duration delay, Duration overrun, Duration time); + + private: + struct Statistic { + Statistic() : total(0), count(0) {} + void add(int64_t value) { total += value; ++count; } + int64_t average() const { return count ? total/count : 0; } + int64_t total; + int64_t count; + }; + + // Keep statistics for 3 classes of incident: late, overrun and both. + struct TaskStats { + Statistic lateDelay; // Just late + Statistic overranOverrun, overranTime; // Just overrun + // Both + Statistic lateAndOverranDelay, lateAndOverranOverrun, lateAndOverranTime; + }; + + typedef std::map<std::string, TaskStats> TaskStatsMap; + + void log(); + + Duration interval; + AbsTime nextReport; + TaskStatsMap taskStats; +}; +}} // namespace qpid::sys + +#endif /*!QPID_SYS_TIMERWARNINGS_H*/ diff --git a/qpid/cpp/src/qpid/sys/Waitable.h b/qpid/cpp/src/qpid/sys/Waitable.h new file mode 100644 index 0000000000..8f6bd17049 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/Waitable.h @@ -0,0 +1,117 @@ +#ifndef QPID_SYS_WAITABLE_H +#define QPID_SYS_WAITABLE_H + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Monitor.h" +#include "qpid/sys/ExceptionHolder.h" +#include <assert.h> + +namespace qpid { +namespace sys { + +/** + * A monitor that keeps track of waiting threads. Threads declare a + * ScopedWait around wait() inside a ScopedLock to be considered + * waiters. + * + * Allows waiting threads to be interrupted by an exception. + */ +class Waitable : public Monitor { + public: + Waitable() : waiters(0) {} + + ~Waitable() { assert(waiters == 0); } + + /** Use this inside a scoped lock around the + * call to wait() to be counted as a waiter. + */ + struct ScopedWait { + Waitable& w; + ScopedWait(Waitable& w_) : w(w_) { ++w.waiters; } + ~ScopedWait() { if (--w.waiters==0) w.notifyAll(); } + }; + + /** Block till there are no more waiters in ScopedWaits. + * waitWaiters() does not raise an exception even if waiters + * were interrupted by one. + *@pre Must be called inside a ScopedLock but NOT a ScopedWait. + */ + void waitWaiters() { + while (waiters != 0) + Monitor::wait(); + } + + /** Returns the number of outstanding ScopedWaits. + * Must be called with the lock held. + */ + size_t hasWaiters() const { + return waiters; + } + + /** Set an execption to interrupt waiters in ScopedWait. + * Must be called with the lock held. + */ + void setException(const ExceptionHolder& e) { + exception = e; + notifyAll(); + + } + + /** True if the waitable has an exception */ + bool hasException() const { return exception; } + + /** Throws if the waitable has an exception */ + void checkException() const { exception.raise(); } + + /** Clear the exception if any */ + void resetException() { exception.reset(); } + + /** Throws an exception if one is set before or during the wait. */ + void wait() { + ExCheck e(exception); + Monitor::wait(); + } + + /** Throws an exception if one is set before or during the wait. */ + bool wait(const AbsTime& absoluteTime) { + ExCheck e(exception); + return Monitor::wait(absoluteTime); + } + + private: + struct ExCheck { + const ExceptionHolder& exception; + ExCheck(const ExceptionHolder& e) : exception(e) { e.raise(); } + ~ExCheck() { exception.raise(); } + }; + + size_t waiters; + ExceptionHolder exception; + + friend struct ScopedWait; +}; + +}} // namespace qpid::sys + + + +#endif /*!QPID_SYS_WAITABLE_H*/ diff --git a/qpid/cpp/src/qpid/sys/alloca.h b/qpid/cpp/src/qpid/sys/alloca.h new file mode 100644 index 0000000000..0f58920908 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/alloca.h @@ -0,0 +1,42 @@ +#ifndef QPID_SYS_ALLOCA_H +#define QPID_SYS_ALLOCA_H + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#if (defined(_WINDOWS) || defined (WIN32)) +# include <malloc.h> + +# if defined(_MSC_VER) +# ifdef alloc +# undef alloc +# endif +# define alloc _alloc +# ifdef alloca +# undef alloca +# endif +# define alloca _alloca +# endif +# if !defined _WINDOWS && !defined WIN32 +# include <alloca.h> +# endif +#endif + +#endif /*!QPID_SYS_ALLOCA_H*/ diff --git a/qpid/cpp/src/qpid/sys/apr/APRBase.cpp b/qpid/cpp/src/qpid/sys/apr/APRBase.cpp new file mode 100644 index 0000000000..8bdba66bdc --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/APRBase.cpp @@ -0,0 +1,89 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include <iostream> +#include "qpid/log/Statement.h" +#include "qpid/sys/apr/APRBase.h" + +using namespace qpid::sys; + +APRBase* APRBase::instance = 0; + +APRBase* APRBase::getInstance(){ + if(instance == 0){ + instance = new APRBase(); + } + return instance; +} + + +APRBase::APRBase() : count(0){ + apr_initialize(); + CHECK_APR_SUCCESS(apr_pool_create(&pool, 0)); + CHECK_APR_SUCCESS(apr_thread_mutex_create(&mutex, APR_THREAD_MUTEX_NESTED, pool)); +} + +APRBase::~APRBase(){ + CHECK_APR_SUCCESS(apr_thread_mutex_destroy(mutex)); + apr_pool_destroy(pool); + apr_terminate(); +} + +bool APRBase::_increment(){ + bool deleted(false); + CHECK_APR_SUCCESS(apr_thread_mutex_lock(mutex)); + if(this == instance){ + count++; + }else{ + deleted = true; + } + CHECK_APR_SUCCESS(apr_thread_mutex_unlock(mutex)); + return !deleted; +} + +void APRBase::_decrement(){ + APRBase* copy = 0; + CHECK_APR_SUCCESS(apr_thread_mutex_lock(mutex)); + if(--count == 0){ + copy = instance; + instance = 0; + } + CHECK_APR_SUCCESS(apr_thread_mutex_unlock(mutex)); + if(copy != 0){ + delete copy; + } +} + +void APRBase::increment(){ + int count = 0; + while(count++ < 2 && !getInstance()->_increment()) + QPID_LOG(warning, "APR initialization triggered concurrently with termination."); +} + +void APRBase::decrement(){ + getInstance()->_decrement(); +} + +std::string qpid::sys::get_desc(apr_status_t status){ + const int size = 50; + char tmp[size]; + return std::string(apr_strerror(status, tmp, size)); +} + diff --git a/qpid/cpp/src/qpid/sys/apr/APRBase.h b/qpid/cpp/src/qpid/sys/apr/APRBase.h new file mode 100644 index 0000000000..7b5644a129 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/APRBase.h @@ -0,0 +1,74 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef _APRBase_ +#define _APRBase_ + +#include <string> +#include <apr_thread_mutex.h> +#include <apr_errno.h> + +namespace qpid { +namespace sys { + + /** + * Use of APR libraries necessitates explicit init and terminate + * calls. Any class using APR libs should obtain the reference to + * this singleton and increment on construction, decrement on + * destruction. This class can then correctly initialise apr + * before the first use and terminate after the last use. + */ + class APRBase{ + static APRBase* instance; + apr_pool_t* pool; + apr_thread_mutex_t* mutex; + int count; + + APRBase(); + ~APRBase(); + static APRBase* getInstance(); + bool _increment(); + void _decrement(); + public: + static void increment(); + static void decrement(); + }; + + //this is also a convenient place for a helper function for error checking: + void check(apr_status_t status, const char* file, const int line); + std::string get_desc(apr_status_t status); + +#define CHECK_APR_SUCCESS(A) qpid::sys::check(A, __FILE__, __LINE__); + +} +} + +// Inlined as it is called *a lot* +void inline qpid::sys::check(apr_status_t status, const char* file, const int line){ + if (status != APR_SUCCESS){ + char tmp[256]; + throw Exception(QPID_MSG(apr_strerror(status, tmp, size))) + } +} + + + + +#endif diff --git a/qpid/cpp/src/qpid/sys/apr/APRPool.cpp b/qpid/cpp/src/qpid/sys/apr/APRPool.cpp new file mode 100644 index 0000000000..e221bfc2f1 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/APRPool.cpp @@ -0,0 +1,41 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/apr/APRPool.h" +#include "qpid/sys/apr/APRBase.h" +#include <boost/pool/detail/singleton.hpp> + +using namespace qpid::sys; + +APRPool::APRPool(){ + APRBase::increment(); + CHECK_APR_SUCCESS(apr_pool_create(&pool, NULL)); +} + +APRPool::~APRPool(){ + apr_pool_destroy(pool); + APRBase::decrement(); +} + +apr_pool_t* APRPool::get() { + return boost::details::pool::singleton_default<APRPool>::instance().pool; +} + diff --git a/qpid/cpp/src/qpid/sys/apr/APRPool.h b/qpid/cpp/src/qpid/sys/apr/APRPool.h new file mode 100644 index 0000000000..da7661fcfa --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/APRPool.h @@ -0,0 +1,50 @@ +#ifndef _APRPool_ +#define _APRPool_ + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include <boost/noncopyable.hpp> +#include <apr_pools.h> + +namespace qpid { +namespace sys { +/** + * Singleton APR memory pool. + */ +class APRPool : private boost::noncopyable { + public: + APRPool(); + ~APRPool(); + + /** Get singleton instance */ + static apr_pool_t* get(); + + private: + apr_pool_t* pool; +}; + +}} + + + + + +#endif /*!_APRPool_*/ diff --git a/qpid/cpp/src/qpid/sys/apr/Condition.h b/qpid/cpp/src/qpid/sys/apr/Condition.h new file mode 100644 index 0000000000..66d465ca75 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/Condition.h @@ -0,0 +1,84 @@ +#ifndef _sys_apr_Condition_h +#define _sys_apr_Condition_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/apr/APRPool.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/Time.h" + +#include <sys/errno.h> +#include <boost/noncopyable.hpp> +#include <apr_thread_cond.h> + +namespace qpid { +namespace sys { + +/** + * A condition variable for thread synchronization. + */ +class Condition +{ + public: + inline Condition(); + inline ~Condition(); + inline void wait(Mutex&); + inline bool wait(Mutex&, const AbsTime& absoluteTime); + inline void notify(); + inline void notifyAll(); + + private: + apr_thread_cond_t* condition; +}; + + +Condition::Condition() { + CHECK_APR_SUCCESS(apr_thread_cond_create(&condition, APRPool::get())); +} + +Condition::~Condition() { + CHECK_APR_SUCCESS(apr_thread_cond_destroy(condition)); +} + +void Condition::wait(Mutex& mutex) { + CHECK_APR_SUCCESS(apr_thread_cond_wait(condition, mutex.mutex)); +} + +bool Condition::wait(Mutex& mutex, const AbsTime& absoluteTime){ + // APR uses microseconds. + apr_status_t status = + apr_thread_cond_timedwait( + condition, mutex.mutex, Duration(now(), absoluteTime)/TIME_USEC); + if(status != APR_TIMEUP) CHECK_APR_SUCCESS(status); + return status == 0; +} + +void Condition::notify(){ + CHECK_APR_SUCCESS(apr_thread_cond_signal(condition)); +} + +void Condition::notifyAll(){ + CHECK_APR_SUCCESS(apr_thread_cond_broadcast(condition)); +} + +}} +#endif /*!_sys_apr_Condition_h*/ diff --git a/qpid/cpp/src/qpid/sys/apr/Mutex.h b/qpid/cpp/src/qpid/sys/apr/Mutex.h new file mode 100644 index 0000000000..cb75f5b339 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/Mutex.h @@ -0,0 +1,124 @@ +#ifndef _sys_apr_Mutex_h +#define _sys_apr_Mutex_h + +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/apr/APRBase.h" +#include "qpid/sys/apr/APRPool.h" + +#include <boost/noncopyable.hpp> +#include <apr_thread_mutex.h> + +namespace qpid { +namespace sys { + +class Condition; + +/** + * Mutex lock. + */ +class Mutex : private boost::noncopyable { + public: + typedef ScopedLock<Mutex> ScopedLock; + typedef ScopedUnlock<Mutex> ScopedUnlock; + + inline Mutex(); + inline ~Mutex(); + inline void lock(); + inline void unlock(); + inline bool trylock(); + + protected: + apr_thread_mutex_t* mutex; + friend class Condition; +}; + +Mutex::Mutex() { + CHECK_APR_SUCCESS(apr_thread_mutex_create(&mutex, APR_THREAD_MUTEX_NESTED, APRPool::get())); +} + +Mutex::~Mutex(){ + CHECK_APR_SUCCESS(apr_thread_mutex_destroy(mutex)); +} + +void Mutex::lock() { + CHECK_APR_SUCCESS(apr_thread_mutex_lock(mutex)); +} +void Mutex::unlock() { + CHECK_APR_SUCCESS(apr_thread_mutex_unlock(mutex)); +} + +bool Mutex::trylock() { + return apr_thread_mutex_trylock(mutex) == 0; +} + + +/** + * RW lock. + */ +class RWlock : private boost::noncopyable { + friend class Condition; + +public: + typedef ScopedRlock<RWlock> ScopedRlock; + typedef ScopedWlock<RWlock> ScopedWlock; + + inline RWlock(); + inline ~RWlock(); + inline void wlock(); // will write-lock + inline void rlock(); // will read-lock + inline void unlock(); + inline bool trywlock(); // will write-try + inline bool tryrlock(); // will read-try + + protected: + apr_thread_mutex_t* mutex; +}; + +RWlock::RWlock() { + CHECK_APR_SUCCESS(apr_thread_mutex_create(&mutex, APR_THREAD_MUTEX_NESTED, APRPool::get())); +} + +RWlock::~RWlock(){ + CHECK_APR_SUCCESS(apr_thread_mutex_destroy(mutex)); +} + +void RWlock::wlock() { + CHECK_APR_SUCCESS(apr_thread_mutex_lock(mutex)); +} + +void RWlock::rlock() { + CHECK_APR_SUCCESS(apr_thread_mutex_lock(mutex)); +} + +void RWlock::unlock() { + CHECK_APR_SUCCESS(apr_thread_mutex_unlock(mutex)); +} + +bool RWlock::trywlock() { + return apr_thread_mutex_trylock(mutex) == 0; +} + +bool RWlock::tryrlock() { + return apr_thread_mutex_trylock(mutex) == 0; +} + + +}} +#endif /*!_sys_apr_Mutex_h*/ diff --git a/qpid/cpp/src/qpid/sys/apr/Shlib.cpp b/qpid/cpp/src/qpid/sys/apr/Shlib.cpp new file mode 100644 index 0000000000..b7ee13a03b --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/Shlib.cpp @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Shlib.h" +#include "qpid/sys/apr/APRBase.h" +#include "qpid/sys/apr/APRPool.h" +#include <apr_dso.h> + +namespace qpid { +namespace sys { + +void Shlib::load(const char* libname) { + apr_dso_handle_t* aprHandle; + CHECK_APR_SUCCESS( + apr_dso_load(&aprHandle, libname, APRPool::get())); + handle=aprHandle; +} + +void Shlib::unload() { + CHECK_APR_SUCCESS( + apr_dso_unload(static_cast<apr_dso_handle_t*>(handle))); +} + +void* Shlib::getSymbol(const char* name) { + apr_dso_handle_sym_t symbol; + CHECK_APR_SUCCESS(apr_dso_sym(&symbol, + static_cast<apr_dso_handle_t*>(handle), + name)); + return (void*) symbol; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/apr/Socket.cpp b/qpid/cpp/src/qpid/sys/apr/Socket.cpp new file mode 100644 index 0000000000..d9024d11c1 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/Socket.cpp @@ -0,0 +1,114 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + + +#include "qpid/sys/Socket.h" + +#include "qpid/sys/apr/APRBase.h" +#include "qpid/sys/apr/APRPool.h" + +#include <apr_network_io.h> + +namespace qpid { +namespace sys { + +class SocketPrivate { +public: + SocketPrivate(apr_socket_t* s = 0) : + socket(s) + {} + + apr_socket_t* socket; +}; + +Socket::Socket() : + impl(new SocketPrivate) +{ + createTcp(); +} + +Socket::Socket(SocketPrivate* sp) : + impl(sp) +{} + +Socket::~Socket() { + delete impl; +} + +void Socket::createTcp() const { + apr_socket_t*& socket = impl->socket; + apr_socket_t* s; + CHECK_APR_SUCCESS( + apr_socket_create( + &s, APR_INET, SOCK_STREAM, APR_PROTO_TCP, + APRPool::get())); + socket = s; +} + +void Socket::setTimeout(const Duration& interval) const { + apr_socket_t*& socket = impl->socket; + apr_socket_timeout_set(socket, interval/TIME_USEC); +} + +void Socket::connect(const std::string& host, int port) const { + apr_socket_t*& socket = impl->socket; + apr_sockaddr_t* address; + CHECK_APR_SUCCESS( + apr_sockaddr_info_get( + &address, host.c_str(), APR_UNSPEC, port, APR_IPV4_ADDR_OK, + APRPool::get())); + CHECK_APR_SUCCESS(apr_socket_connect(socket, address)); +} + +void Socket::close() const { + apr_socket_t*& socket = impl->socket; + if (socket == 0) return; + CHECK_APR_SUCCESS(apr_socket_close(socket)); + socket = 0; +} + +ssize_t Socket::send(const void* data, size_t size) const +{ + apr_socket_t*& socket = impl->socket; + apr_size_t sent = size; + apr_status_t status = + apr_socket_send(socket, reinterpret_cast<const char*>(data), &sent); + if (APR_STATUS_IS_TIMEUP(status)) return SOCKET_TIMEOUT; + if (APR_STATUS_IS_EOF(status)) return SOCKET_EOF; + CHECK_APR_SUCCESS(status); + return sent; +} + +ssize_t Socket::recv(void* data, size_t size) const +{ + apr_socket_t*& socket = impl->socket; + apr_size_t received = size; + apr_status_t status = + apr_socket_recv(socket, reinterpret_cast<char*>(data), &received); + if (APR_STATUS_IS_TIMEUP(status)) + return SOCKET_TIMEOUT; + if (APR_STATUS_IS_EOF(status)) + return SOCKET_EOF; + CHECK_APR_SUCCESS(status); + return received; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/apr/Thread.cpp b/qpid/cpp/src/qpid/sys/apr/Thread.cpp new file mode 100644 index 0000000000..b52d0e6ace --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/Thread.cpp @@ -0,0 +1,34 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/apr/Thread.h" +#include "qpid/sys/Runnable.h" + +using namespace qpid::sys; +using qpid::sys::Runnable; + +void* APR_THREAD_FUNC Thread::runRunnable(apr_thread_t* thread, void *data) { + reinterpret_cast<Runnable*>(data)->run(); + CHECK_APR_SUCCESS(apr_thread_exit(thread, APR_SUCCESS)); + return NULL; +} + + diff --git a/qpid/cpp/src/qpid/sys/apr/Thread.h b/qpid/cpp/src/qpid/sys/apr/Thread.h new file mode 100644 index 0000000000..6cc63db5c9 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/Thread.h @@ -0,0 +1,106 @@ +#ifndef _sys_apr_Thread_h +#define _sys_apr_Thread_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/apr/APRPool.h" +#include "qpid/sys/apr/APRBase.h" + +#include <apr_thread_proc.h> +#include <apr_portable.h> + +namespace qpid { +namespace sys { + +class Runnable; + +class Thread +{ + public: + inline static Thread current(); + + /** ID of current thread for logging. + * Workaround for broken Thread::current() in APR + */ + inline static long logId(); + + inline static void yield(); + + inline Thread(); + inline explicit Thread(qpid::sys::Runnable*); + inline explicit Thread(qpid::sys::Runnable&); + + inline void join(); + + inline long id(); + + private: + static void* APR_THREAD_FUNC runRunnable(apr_thread_t* thread, void *data); + inline Thread(apr_thread_t* t); + apr_thread_t* thread; +}; + +Thread::Thread() : thread(0) {} + +Thread::Thread(Runnable* runnable) { + CHECK_APR_SUCCESS( + apr_thread_create(&thread, 0, runRunnable, runnable, APRPool::get())); +} + +Thread::Thread(Runnable& runnable) { + CHECK_APR_SUCCESS( + apr_thread_create(&thread, 0, runRunnable, &runnable, APRPool::get())); +} + +void Thread::join(){ + apr_status_t status; + if (thread != 0) + CHECK_APR_SUCCESS(apr_thread_join(&status, thread)); +} + +long Thread::id() { + return long(thread); +} + +/** ID of current thread for logging. + * Workaround for broken Thread::current() in APR + */ +long Thread::logId() { + return static_cast<long>(apr_os_thread_current()); +} + +Thread::Thread(apr_thread_t* t) : thread(t) {} + +Thread Thread::current(){ + apr_thread_t* thr; + apr_os_thread_t osthr = apr_os_thread_current(); + CHECK_APR_SUCCESS(apr_os_thread_put(&thr, &osthr, APRPool::get())); + return Thread(thr); +} + +void Thread::yield() +{ + apr_thread_yield(); +} + +}} +#endif /*!_sys_apr_Thread_h*/ diff --git a/qpid/cpp/src/qpid/sys/apr/Time.cpp b/qpid/cpp/src/qpid/sys/apr/Time.cpp new file mode 100644 index 0000000000..34e740b144 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/apr/Time.cpp @@ -0,0 +1,36 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Time.h" + +#include <apr_time.h> + +namespace qpid { +namespace sys { + +AbsTime AbsTime::now() { + AbsTime time_now; + time_now.time_ns = apr_time_now() * TIME_USEC; + return time_now; +} + +}} + diff --git a/qpid/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.cpp b/qpid/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.cpp new file mode 100644 index 0000000000..3d868da64b --- /dev/null +++ b/qpid/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.cpp @@ -0,0 +1,127 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/cyrus/CyrusSecurityLayer.h" +#include <algorithm> +#include "qpid/framing/reply_exceptions.h" +#include "qpid/log/Statement.h" +#include <string.h> + +namespace qpid { +namespace sys { +namespace cyrus { + +CyrusSecurityLayer::CyrusSecurityLayer(sasl_conn_t* c, uint16_t maxFrameSize) : + conn(c), decrypted(0), decryptedSize(0), encrypted(0), encryptedSize(0), codec(0), maxInputSize(0), + decodeBuffer(maxFrameSize), encodeBuffer(maxFrameSize), encoded(0) +{ + const void* value(0); + int result = sasl_getprop(conn, SASL_MAXOUTBUF, &value); + if (result != SASL_OK) { + throw framing::InternalErrorException(QPID_MSG("SASL encode error: " << sasl_errdetail(conn))); + } + maxInputSize = *(reinterpret_cast<const unsigned*>(value)); +} + +size_t CyrusSecurityLayer::decode(const char* input, size_t size) +{ + size_t inStart = 0; + do { + size_t inSize = std::min(size - inStart, maxInputSize); + int result = sasl_decode(conn, input + inStart, inSize, &decrypted, &decryptedSize); + if (result != SASL_OK) { + throw framing::InternalErrorException(QPID_MSG("SASL decode error: " << sasl_errdetail(conn))); + } + inStart += inSize; + size_t copied = 0; + do { + size_t count = std::min(decryptedSize - copied, decodeBuffer.size - decodeBuffer.position); + ::memcpy(decodeBuffer.data + decodeBuffer.position, decrypted + copied, count); + copied += count; + decodeBuffer.position += count; + size_t decodedSize = codec->decode(decodeBuffer.data, decodeBuffer.position); + if (decodedSize < decodeBuffer.position) { + ::memmove(decodeBuffer.data, decodeBuffer.data + decodedSize, decodeBuffer.position - decodedSize); + } + decodeBuffer.position -= decodedSize; + } while (copied < decryptedSize); + } while (inStart < size); + return size; +} + +size_t CyrusSecurityLayer::encode(const char* buffer, size_t size) +{ + size_t processed = 0;//records how many bytes have been written to buffer + do { + if (!encrypted) { + if (!encoded) { + encodeBuffer.position = 0; + encoded = codec->encode(encodeBuffer.data, encodeBuffer.size); + if (!encoded) break;//nothing more to do + } + + size_t encryptable = std::min(encoded, maxInputSize); + int result = sasl_encode(conn, encodeBuffer.data + encodeBuffer.position, encryptable, &encrypted, &encryptedSize); + if (result != SASL_OK) { + throw framing::InternalErrorException(QPID_MSG("SASL encode error: " << sasl_errdetail(conn))); + } + encodeBuffer.position += encryptable; + encoded -= encryptable; + } + size_t remaining = size - processed; + if (remaining < encryptedSize) { + //can't fit all encrypted data in the buffer we've + //been given, copy in what we can and hold on to the + //rest until the next call + ::memcpy(const_cast<char*>(buffer + processed), encrypted, remaining); + processed += remaining; + encrypted += remaining; + encryptedSize -= remaining; + } else { + ::memcpy(const_cast<char*>(buffer + processed), encrypted, encryptedSize); + processed += encryptedSize; + encrypted = 0; + encryptedSize = 0; + } + } while (processed < size); + return processed; +} + +bool CyrusSecurityLayer::canEncode() +{ + return codec && (encrypted || codec->canEncode()); +} + +void CyrusSecurityLayer::init(qpid::sys::Codec* c) +{ + codec = c; +} + +CyrusSecurityLayer::DataBuffer::DataBuffer(size_t s) : position(0), size(s) +{ + data = new char[size]; +} + +CyrusSecurityLayer::DataBuffer::~DataBuffer() +{ + delete[] data; +} + +}}} // namespace qpid::sys::cyrus diff --git a/qpid/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.h b/qpid/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.h new file mode 100644 index 0000000000..1645cf1a58 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/cyrus/CyrusSecurityLayer.h @@ -0,0 +1,68 @@ +#ifndef QPID_SYS_CYRUS_CYRUSSECURITYLAYER_H +#define QPID_SYS_CYRUS_CYRUSSECURITYLAYER_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IntegerTypes.h" +#include "qpid/sys/SecurityLayer.h" +#include <sasl/sasl.h> + +namespace qpid { +namespace sys { +namespace cyrus { + + +/** + * Implementation of SASL security layer using cyrus-sasl library + */ +class CyrusSecurityLayer : public qpid::sys::SecurityLayer +{ + public: + CyrusSecurityLayer(sasl_conn_t*, uint16_t maxFrameSize); + size_t decode(const char* buffer, size_t size); + size_t encode(const char* buffer, size_t size); + bool canEncode(); + void init(qpid::sys::Codec*); + private: + struct DataBuffer + { + char* data; + size_t position; + const size_t size; + DataBuffer(size_t); + ~DataBuffer(); + }; + + sasl_conn_t* conn; + const char* decrypted; + unsigned decryptedSize; + const char* encrypted; + unsigned encryptedSize; + qpid::sys::Codec* codec; + size_t maxInputSize; + DataBuffer decodeBuffer; + DataBuffer encodeBuffer; + size_t encoded; +}; +}}} // namespace qpid::sys::cyrus + +#endif /*!QPID_SYS_CYRUS_CYRUSSECURITYLAYER_H*/ diff --git a/qpid/cpp/src/qpid/sys/epoll/EpollPoller.cpp b/qpid/cpp/src/qpid/sys/epoll/EpollPoller.cpp new file mode 100644 index 0000000000..9ad05c71a3 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/epoll/EpollPoller.cpp @@ -0,0 +1,674 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Poller.h" +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/AtomicCount.h" +#include "qpid/sys/DeletionManager.h" +#include "qpid/sys/posix/check.h" +#include "qpid/sys/posix/PrivatePosix.h" +#include "qpid/log/Statement.h" + +#include <sys/epoll.h> +#include <errno.h> +#include <signal.h> + +#include <assert.h> +#include <queue> +#include <set> +#include <exception> + +namespace qpid { +namespace sys { + +// Deletion manager to handle deferring deletion of PollerHandles to when they definitely aren't being used +DeletionManager<PollerHandlePrivate> PollerHandleDeletionManager; + +// Instantiate (and define) class static for DeletionManager +template <> +DeletionManager<PollerHandlePrivate>::AllThreadsStatuses DeletionManager<PollerHandlePrivate>::allThreadsStatuses(0); + +class PollerHandlePrivate { + friend class Poller; + friend class PollerPrivate; + friend class PollerHandle; + + enum FDStat { + ABSENT, + MONITORED, + INACTIVE, + HUNGUP, + MONITORED_HUNGUP, + INTERRUPTED, + INTERRUPTED_HUNGUP, + DELETED + }; + + ::__uint32_t events; + const IOHandlePrivate* ioHandle; + PollerHandle* pollerHandle; + FDStat stat; + Mutex lock; + + PollerHandlePrivate(const IOHandlePrivate* h, PollerHandle* p) : + events(0), + ioHandle(h), + pollerHandle(p), + stat(ABSENT) { + } + + int fd() const { + return toFd(ioHandle); + } + + bool isActive() const { + return stat == MONITORED || stat == MONITORED_HUNGUP; + } + + void setActive() { + stat = (stat == HUNGUP || stat == INTERRUPTED_HUNGUP) + ? MONITORED_HUNGUP + : MONITORED; + } + + bool isInactive() const { + return stat == INACTIVE || stat == HUNGUP; + } + + void setInactive() { + stat = INACTIVE; + } + + bool isIdle() const { + return stat == ABSENT; + } + + void setIdle() { + stat = ABSENT; + } + + bool isHungup() const { + return + stat == MONITORED_HUNGUP || + stat == HUNGUP || + stat == INTERRUPTED_HUNGUP; + } + + void setHungup() { + assert(stat == MONITORED); + stat = HUNGUP; + } + + bool isInterrupted() const { + return stat == INTERRUPTED || stat == INTERRUPTED_HUNGUP; + } + + void setInterrupted() { + stat = (stat == MONITORED_HUNGUP || stat == HUNGUP) + ? INTERRUPTED_HUNGUP + : INTERRUPTED; + } + + bool isDeleted() const { + return stat == DELETED; + } + + void setDeleted() { + stat = DELETED; + } +}; + +PollerHandle::PollerHandle(const IOHandle& h) : + impl(new PollerHandlePrivate(h.impl, this)) +{} + +PollerHandle::~PollerHandle() { + { + ScopedLock<Mutex> l(impl->lock); + if (impl->isDeleted()) { + return; + } + impl->pollerHandle = 0; + if (impl->isInterrupted()) { + impl->setDeleted(); + return; + } + assert(impl->isIdle()); + impl->setDeleted(); + } + PollerHandleDeletionManager.markForDeletion(impl); +} + +class HandleSet +{ + Mutex lock; + std::set<PollerHandle*> handles; + public: + void add(PollerHandle*); + void remove(PollerHandle*); + void cleanup(); +}; + +void HandleSet::add(PollerHandle* h) +{ + ScopedLock<Mutex> l(lock); + handles.insert(h); +} +void HandleSet::remove(PollerHandle* h) +{ + ScopedLock<Mutex> l(lock); + handles.erase(h); +} +void HandleSet::cleanup() +{ + // Inform all registered handles of disconnection + std::set<PollerHandle*> copy; + handles.swap(copy); + for (std::set<PollerHandle*>::const_iterator i = copy.begin(); i != copy.end(); ++i) { + Poller::Event event(*i, Poller::DISCONNECTED); + event.process(); + } +} + +/** + * Concrete implementation of Poller to use the Linux specific epoll + * interface + */ +class PollerPrivate { + friend class Poller; + + static const int DefaultFds = 256; + + struct ReadablePipe { + int fds[2]; + + /** + * This encapsulates an always readable pipe which we can add + * to the epoll set to force epoll_wait to return + */ + ReadablePipe() { + QPID_POSIX_CHECK(::pipe(fds)); + // Just write the pipe's fds to the pipe + QPID_POSIX_CHECK(::write(fds[1], fds, 2)); + } + + ~ReadablePipe() { + ::close(fds[0]); + ::close(fds[1]); + } + + int getFD() { + return fds[0]; + } + }; + + static ReadablePipe alwaysReadable; + static int alwaysReadableFd; + + class InterruptHandle: public PollerHandle { + std::queue<PollerHandle*> handles; + + void processEvent(Poller::EventType) { + PollerHandle* handle = handles.front(); + handles.pop(); + assert(handle); + + // Synthesise event + Poller::Event event(handle, Poller::INTERRUPTED); + + // Process synthesised event + event.process(); + } + + public: + InterruptHandle() : + PollerHandle(DummyIOHandle) + {} + + void addHandle(PollerHandle& h) { + handles.push(&h); + } + + PollerHandle* getHandle() { + PollerHandle* handle = handles.front(); + handles.pop(); + return handle; + } + + bool queuedHandles() { + return handles.size() > 0; + } + }; + + const int epollFd; + bool isShutdown; + InterruptHandle interruptHandle; + HandleSet registeredHandles; + AtomicCount threadCount; + + static ::__uint32_t directionToEpollEvent(Poller::Direction dir) { + switch (dir) { + case Poller::INPUT: return ::EPOLLIN; + case Poller::OUTPUT: return ::EPOLLOUT; + case Poller::INOUT: return ::EPOLLIN | ::EPOLLOUT; + default: return 0; + } + } + + static Poller::EventType epollToDirection(::__uint32_t events) { + // POLLOUT & POLLHUP are mutually exclusive really, but at least socketpairs + // can give you both! + events = (events & ::EPOLLHUP) ? events & ~::EPOLLOUT : events; + ::__uint32_t e = events & (::EPOLLIN | ::EPOLLOUT); + switch (e) { + case ::EPOLLIN: return Poller::READABLE; + case ::EPOLLOUT: return Poller::WRITABLE; + case ::EPOLLIN | ::EPOLLOUT: return Poller::READ_WRITABLE; + default: + return (events & (::EPOLLHUP | ::EPOLLERR)) ? + Poller::DISCONNECTED : Poller::INVALID; + } + } + + PollerPrivate() : + epollFd(::epoll_create(DefaultFds)), + isShutdown(false) { + QPID_POSIX_CHECK(epollFd); + // Add always readable fd into our set (but not listening to it yet) + ::epoll_event epe; + epe.events = 0; + epe.data.u64 = 1; + QPID_POSIX_CHECK(::epoll_ctl(epollFd, EPOLL_CTL_ADD, alwaysReadableFd, &epe)); + } + + ~PollerPrivate() { + // It's probably okay to ignore any errors here as there can't be data loss + ::close(epollFd); + + // Need to put the interruptHandle in idle state to delete it + static_cast<PollerHandle&>(interruptHandle).impl->setIdle(); + } + + void resetMode(PollerHandlePrivate& handle); + + void interrupt() { + ::epoll_event epe; + // Use EPOLLONESHOT so we only wake a single thread + epe.events = ::EPOLLIN | ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy + epe.data.ptr = &static_cast<PollerHandle&>(interruptHandle); + QPID_POSIX_CHECK(::epoll_ctl(epollFd, EPOLL_CTL_MOD, alwaysReadableFd, &epe)); + } + + void interruptAll() { + ::epoll_event epe; + // Not EPOLLONESHOT, so we eventually get all threads + epe.events = ::EPOLLIN; + epe.data.u64 = 2; // Keep valgrind happy + QPID_POSIX_CHECK(::epoll_ctl(epollFd, EPOLL_CTL_MOD, alwaysReadableFd, &epe)); + } +}; + +PollerPrivate::ReadablePipe PollerPrivate::alwaysReadable; +int PollerPrivate::alwaysReadableFd = alwaysReadable.getFD(); + +void Poller::registerHandle(PollerHandle& handle) { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + assert(eh.isIdle()); + + ::epoll_event epe; + epe.events = ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy + epe.data.ptr = &eh; + + impl->registeredHandles.add(&handle); + QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_ADD, eh.fd(), &epe)); + + eh.setActive(); +} + +void Poller::unregisterHandle(PollerHandle& handle) { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + assert(!eh.isIdle()); + + impl->registeredHandles.remove(&handle); + int rc = ::epoll_ctl(impl->epollFd, EPOLL_CTL_DEL, eh.fd(), 0); + // Ignore EBADF since deleting a nonexistent fd has the overall required result! + // And allows the case where a sloppy program closes the fd and then does the delFd() + if (rc == -1 && errno != EBADF) { + QPID_POSIX_CHECK(rc); + } + + eh.setIdle(); +} + +void PollerPrivate::resetMode(PollerHandlePrivate& eh) { + PollerHandle* ph; + { + ScopedLock<Mutex> l(eh.lock); + assert(!eh.isActive()); + + if (eh.isIdle() || eh.isDeleted()) { + return; + } + + if (eh.events==0) { + eh.setActive(); + return; + } + + if (!eh.isInterrupted()) { + ::epoll_event epe; + epe.events = eh.events | ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy + epe.data.ptr = &eh; + + QPID_POSIX_CHECK(::epoll_ctl(epollFd, EPOLL_CTL_MOD, eh.fd(), &epe)); + + eh.setActive(); + return; + } + ph = eh.pollerHandle; + } + + PollerHandlePrivate& ihp = *static_cast<PollerHandle&>(interruptHandle).impl; + ScopedLock<Mutex> l(ihp.lock); + interruptHandle.addHandle(*ph); + ihp.setActive(); + interrupt(); +} + +void Poller::monitorHandle(PollerHandle& handle, Direction dir) { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + assert(!eh.isIdle()); + + ::__uint32_t oldEvents = eh.events; + eh.events |= PollerPrivate::directionToEpollEvent(dir); + + // If no change nothing more to do - avoid unnecessary system call + if (oldEvents==eh.events) { + return; + } + + // If we're not actually listening wait till we are to perform change + if (!eh.isActive()) { + return; + } + + ::epoll_event epe; + epe.events = eh.events | ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy + epe.data.ptr = &eh; + + QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_MOD, eh.fd(), &epe)); +} + +void Poller::unmonitorHandle(PollerHandle& handle, Direction dir) { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + assert(!eh.isIdle()); + + ::__uint32_t oldEvents = eh.events; + eh.events &= ~PollerPrivate::directionToEpollEvent(dir); + + // If no change nothing more to do - avoid unnecessary system call + if (oldEvents==eh.events) { + return; + } + + // If we're not actually listening wait till we are to perform change + if (!eh.isActive()) { + return; + } + + ::epoll_event epe; + epe.events = eh.events | ::EPOLLONESHOT; + epe.data.u64 = 0; // Keep valgrind happy + epe.data.ptr = &eh; + + QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_MOD, eh.fd(), &epe)); +} + +void Poller::shutdown() { + // NB: this function must be async-signal safe, it must not + // call any function that is not async-signal safe. + + // Allow sloppy code to shut us down more than once + if (impl->isShutdown) + return; + + // Don't use any locking here - isShutdown will be visible to all + // after the epoll_ctl() anyway (it's a memory barrier) + impl->isShutdown = true; + + impl->interruptAll(); +} + +bool Poller::interrupt(PollerHandle& handle) { + { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + if (eh.isIdle() || eh.isDeleted()) { + return false; + } + + if (eh.isInterrupted()) { + return true; + } + + // Stop monitoring handle for read or write + ::epoll_event epe; + epe.events = 0; + epe.data.u64 = 0; // Keep valgrind happy + epe.data.ptr = &eh; + QPID_POSIX_CHECK(::epoll_ctl(impl->epollFd, EPOLL_CTL_MOD, eh.fd(), &epe)); + + if (eh.isInactive()) { + eh.setInterrupted(); + return true; + } + eh.setInterrupted(); + } + + PollerPrivate::InterruptHandle& ih = impl->interruptHandle; + PollerHandlePrivate& eh = *static_cast<PollerHandle&>(ih).impl; + ScopedLock<Mutex> l(eh.lock); + ih.addHandle(handle); + + impl->interrupt(); + eh.setActive(); + return true; +} + +void Poller::run() { + // Ensure that we exit thread responsibly under all circumstances + try { + // Make sure we can't be interrupted by signals at a bad time + ::sigset_t ss; + ::sigfillset(&ss); + ::pthread_sigmask(SIG_SETMASK, &ss, 0); + + ++(impl->threadCount); + do { + Event event = wait(); + + // If can read/write then dispatch appropriate callbacks + if (event.handle) { + event.process(); + } else { + // Handle shutdown + switch (event.type) { + case SHUTDOWN: + PollerHandleDeletionManager.destroyThreadState(); + //last thread to respond to shutdown cleans up: + if (--(impl->threadCount) == 0) impl->registeredHandles.cleanup(); + return; + default: + // This should be impossible + assert(false); + } + } + } while (true); + } catch (const std::exception& e) { + QPID_LOG(error, "IO worker thread exiting with unhandled exception: " << e.what()); + } + PollerHandleDeletionManager.destroyThreadState(); + --(impl->threadCount); +} + +bool Poller::hasShutdown() +{ + return impl->isShutdown; +} + +Poller::Event Poller::wait(Duration timeout) { + static __thread PollerHandlePrivate* lastReturnedHandle = 0; + epoll_event epe; + int timeoutMs = (timeout == TIME_INFINITE) ? -1 : timeout / TIME_MSEC; + AbsTime targetTimeout = + (timeout == TIME_INFINITE) ? + FAR_FUTURE : + AbsTime(now(), timeout); + + if (lastReturnedHandle) { + impl->resetMode(*lastReturnedHandle); + lastReturnedHandle = 0; + } + + // Repeat until we weren't interrupted by signal + do { + PollerHandleDeletionManager.markAllUnusedInThisThread(); + int rc = ::epoll_wait(impl->epollFd, &epe, 1, timeoutMs); + if (rc ==-1 && errno != EINTR) { + QPID_POSIX_CHECK(rc); + } else if (rc > 0) { + assert(rc == 1); + void* dataPtr = epe.data.ptr; + + // Check if this is an interrupt + PollerPrivate::InterruptHandle& interruptHandle = impl->interruptHandle; + if (dataPtr == &interruptHandle) { + // If we are shutting down we need to rearm the shutdown interrupt to + // ensure everyone still sees it. It's okay that this might be overridden + // below as we will be back here if it is. + if (impl->isShutdown) { + impl->interruptAll(); + } + PollerHandle* wrappedHandle = 0; + { + ScopedLock<Mutex> l(interruptHandle.impl->lock); + if (interruptHandle.impl->isActive()) { + wrappedHandle = interruptHandle.getHandle(); + // If there is an interrupt queued behind this one we need to arm it + // We do it this way so that another thread can pick it up + if (interruptHandle.queuedHandles()) { + impl->interrupt(); + interruptHandle.impl->setActive(); + } else { + interruptHandle.impl->setInactive(); + } + } + } + if (wrappedHandle) { + PollerHandlePrivate& eh = *wrappedHandle->impl; + { + ScopedLock<Mutex> l(eh.lock); + if (!eh.isDeleted()) { + if (!eh.isIdle()) { + eh.setInactive(); + } + lastReturnedHandle = &eh; + assert(eh.pollerHandle == wrappedHandle); + return Event(wrappedHandle, INTERRUPTED); + } + } + PollerHandleDeletionManager.markForDeletion(&eh); + } + continue; + } + + // Check for shutdown + if (impl->isShutdown) { + PollerHandleDeletionManager.markAllUnusedInThisThread(); + return Event(0, SHUTDOWN); + } + + PollerHandlePrivate& eh = *static_cast<PollerHandlePrivate*>(dataPtr); + ScopedLock<Mutex> l(eh.lock); + + // the handle could have gone inactive since we left the epoll_wait + if (eh.isActive()) { + PollerHandle* handle = eh.pollerHandle; + assert(handle); + + // If the connection has been hungup we could still be readable + // (just not writable), allow us to readable until we get here again + if (epe.events & ::EPOLLHUP) { + if (eh.isHungup()) { + eh.setInactive(); + // Don't set up last Handle so that we don't reset this handle + // on re-entering Poller::wait. This means that we will never + // be set active again once we've returned disconnected, and so + // can never be returned again. + return Event(handle, DISCONNECTED); + } + eh.setHungup(); + } else { + eh.setInactive(); + } + lastReturnedHandle = &eh; + return Event(handle, PollerPrivate::epollToDirection(epe.events)); + } + } + // We only get here if one of the following: + // * epoll_wait was interrupted by a signal + // * epoll_wait timed out + // * the state of the handle changed after being returned by epoll_wait + // + // The only things we can do here are return a timeout or wait more. + // Obviously if we timed out we return timeout; if the wait was meant to + // be indefinite then we should never return with a time out so we go again. + // If the wait wasn't indefinite, we check whether we are after the target wait + // time or not + if (timeoutMs == -1) { + continue; + } + if (rc == 0 && now() > targetTimeout) { + PollerHandleDeletionManager.markAllUnusedInThisThread(); + return Event(0, TIMEOUT); + } + } while (true); +} + +// Concrete constructors +Poller::Poller() : + impl(new PollerPrivate()) +{} + +Poller::~Poller() { + delete impl; +} + +}} diff --git a/qpid/cpp/src/qpid/sys/posix/AsynchIO.cpp b/qpid/cpp/src/qpid/sys/posix/AsynchIO.cpp new file mode 100644 index 0000000000..b5a0b0bf32 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/AsynchIO.cpp @@ -0,0 +1,611 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/AsynchIO.h" +#include "qpid/sys/Socket.h" +#include "qpid/sys/SocketAddress.h" +#include "qpid/sys/Poller.h" +#include "qpid/sys/DispatchHandle.h" +#include "qpid/sys/Time.h" +#include "qpid/log/Statement.h" + +#include "qpid/sys/posix/check.h" + +// TODO The basic algorithm here is not really POSIX specific and with a +// bit more abstraction could (should) be promoted to be platform portable +#include <unistd.h> +#include <sys/socket.h> +#include <signal.h> +#include <errno.h> +#include <string.h> + +#include <boost/bind.hpp> +#include <boost/lexical_cast.hpp> + +using namespace qpid::sys; + +namespace { + +struct StaticInit { + StaticInit() { + /** + * Make *process* not generate SIGPIPE when writing to closed + * pipe/socket (necessary as default action is to terminate process) + */ + ::signal(SIGPIPE, SIG_IGN); + }; +} init; + +/* + * We keep per thread state to avoid locking overhead. The assumption is that + * on average all the connections are serviced by all the threads so the state + * recorded in each thread is about the same. If this turns out not to be the + * case we could rebalance the info occasionally. + */ +__thread int threadReadTotal = 0; +__thread int threadMaxRead = 0; +__thread int threadReadCount = 0; +__thread int threadWriteTotal = 0; +__thread int threadWriteCount = 0; +__thread int64_t threadMaxReadTimeNs = 2 * 1000000; // start at 2ms +} + +/* + * Asynch Acceptor + */ +namespace qpid { +namespace sys { +namespace posix { + +class AsynchAcceptor : public qpid::sys::AsynchAcceptor { +public: + AsynchAcceptor(const Socket& s, AsynchAcceptor::Callback callback); + ~AsynchAcceptor(); + void start(Poller::shared_ptr poller); + +private: + void readable(DispatchHandle& handle); + +private: + AsynchAcceptor::Callback acceptedCallback; + DispatchHandle handle; + const Socket& socket; + +}; + +AsynchAcceptor::AsynchAcceptor(const Socket& s, + AsynchAcceptor::Callback callback) : + acceptedCallback(callback), + handle(s, boost::bind(&AsynchAcceptor::readable, this, _1), 0, 0), + socket(s) { + + s.setNonblocking(); +} + +AsynchAcceptor::~AsynchAcceptor() { + handle.stopWatch(); +} + +void AsynchAcceptor::start(Poller::shared_ptr poller) { + handle.startWatch(poller); +} + +/* + * We keep on accepting as long as there is something to accept + */ +void AsynchAcceptor::readable(DispatchHandle& h) { + Socket* s; + do { + errno = 0; + // TODO: Currently we ignore the peers address, perhaps we should + // log it or use it for connection acceptance. + try { + s = socket.accept(); + if (s) { + acceptedCallback(*s); + } else { + break; + } + } catch (const std::exception& e) { + QPID_LOG(error, "Could not accept socket: " << e.what()); + break; + } + } while (true); + + h.rewatch(); +} + +/* + * POSIX version of AsynchIO TCP socket connector. + * + * The class is implemented in terms of DispatchHandle to allow it to be + * deleted by deleting the contained DispatchHandle. + */ +class AsynchConnector : public qpid::sys::AsynchConnector, + private DispatchHandle { + +private: + void connComplete(DispatchHandle& handle); + +private: + ConnectedCallback connCallback; + FailedCallback failCallback; + const Socket& socket; + +public: + AsynchConnector(const Socket& socket, + const std::string& hostname, + const std::string& port, + ConnectedCallback connCb, + FailedCallback failCb); + void start(Poller::shared_ptr poller); + void stop(); +}; + +AsynchConnector::AsynchConnector(const Socket& s, + const std::string& hostname, + const std::string& port, + ConnectedCallback connCb, + FailedCallback failCb) : + DispatchHandle(s, + 0, + boost::bind(&AsynchConnector::connComplete, this, _1), + boost::bind(&AsynchConnector::connComplete, this, _1)), + connCallback(connCb), + failCallback(failCb), + socket(s) +{ + socket.setNonblocking(); + SocketAddress sa(hostname, port); + // Note, not catching any exceptions here, also has effect of destructing + socket.connect(sa); +} + +void AsynchConnector::start(Poller::shared_ptr poller) +{ + startWatch(poller); +} + +void AsynchConnector::stop() +{ + stopWatch(); +} + +void AsynchConnector::connComplete(DispatchHandle& h) +{ + h.stopWatch(); + int errCode = socket.getError(); + if (errCode == 0) { + connCallback(socket); + } else { + failCallback(socket, errCode, strError(errCode)); + } + DispatchHandle::doDelete(); +} + +/* + * POSIX version of AsynchIO reader/writer + * + * The class is implemented in terms of DispatchHandle to allow it to be + * deleted by deleting the contained DispatchHandle. + */ +class AsynchIO : public qpid::sys::AsynchIO, private DispatchHandle { + +public: + AsynchIO(const Socket& s, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb = 0, + BuffersEmptyCallback eCb = 0, + IdleCallback iCb = 0); + + // Methods inherited from qpid::sys::AsynchIO + + virtual void queueForDeletion(); + + virtual void start(Poller::shared_ptr poller); + virtual void queueReadBuffer(BufferBase* buff); + virtual void unread(BufferBase* buff); + virtual void queueWrite(BufferBase* buff); + virtual void notifyPendingWrite(); + virtual void queueWriteClose(); + virtual bool writeQueueEmpty(); + virtual void startReading(); + virtual void stopReading(); + virtual void requestCallback(RequestCallback); + virtual BufferBase* getQueuedBuffer(); + +private: + ~AsynchIO(); + + // Methods that are callback targets from Dispatcher. + void readable(DispatchHandle& handle); + void writeable(DispatchHandle& handle); + void disconnected(DispatchHandle& handle); + void requestedCall(RequestCallback); + void close(DispatchHandle& handle); + +private: + ReadCallback readCallback; + EofCallback eofCallback; + DisconnectCallback disCallback; + ClosedCallback closedCallback; + BuffersEmptyCallback emptyCallback; + IdleCallback idleCallback; + const Socket& socket; + std::deque<BufferBase*> bufferQueue; + std::deque<BufferBase*> writeQueue; + bool queuedClose; + /** + * This flag is used to detect and handle concurrency between + * calls to notifyPendingWrite() (which can be made from any thread) and + * the execution of the writeable() method (which is always on the + * thread processing this handle. + */ + volatile bool writePending; + /** + * This records whether we've been reading is flow controlled: + * it's safe as a simple boolean as the only way to be stopped + * is in calls only allowed in the callback context, the only calls + * checking it are also in calls only allowed in callback context. + */ + volatile bool readingStopped; +}; + +AsynchIO::AsynchIO(const Socket& s, + ReadCallback rCb, EofCallback eofCb, DisconnectCallback disCb, + ClosedCallback cCb, BuffersEmptyCallback eCb, IdleCallback iCb) : + + DispatchHandle(s, + boost::bind(&AsynchIO::readable, this, _1), + boost::bind(&AsynchIO::writeable, this, _1), + boost::bind(&AsynchIO::disconnected, this, _1)), + readCallback(rCb), + eofCallback(eofCb), + disCallback(disCb), + closedCallback(cCb), + emptyCallback(eCb), + idleCallback(iCb), + socket(s), + queuedClose(false), + writePending(false), + readingStopped(false) { + + s.setNonblocking(); +} + +struct deleter +{ + template <typename T> + void operator()(T *ptr){ delete ptr;} +}; + +AsynchIO::~AsynchIO() { + std::for_each( bufferQueue.begin(), bufferQueue.end(), deleter()); + std::for_each( writeQueue.begin(), writeQueue.end(), deleter()); +} + +void AsynchIO::queueForDeletion() { + DispatchHandle::doDelete(); +} + +void AsynchIO::start(Poller::shared_ptr poller) { + DispatchHandle::startWatch(poller); +} + +void AsynchIO::queueReadBuffer(BufferBase* buff) { + assert(buff); + buff->dataStart = 0; + buff->dataCount = 0; + + bool queueWasEmpty = bufferQueue.empty(); + bufferQueue.push_back(buff); + if (queueWasEmpty && !readingStopped) + DispatchHandle::rewatchRead(); +} + +void AsynchIO::unread(BufferBase* buff) { + assert(buff); + buff->squish(); + + bool queueWasEmpty = bufferQueue.empty(); + bufferQueue.push_front(buff); + if (queueWasEmpty && !readingStopped) + DispatchHandle::rewatchRead(); +} + +void AsynchIO::queueWrite(BufferBase* buff) { + assert(buff); + // If we've already closed the socket then throw the write away + if (queuedClose) { + queueReadBuffer(buff); + return; + } else { + writeQueue.push_front(buff); + } + writePending = false; + DispatchHandle::rewatchWrite(); +} + +// This can happen outside the callback context +void AsynchIO::notifyPendingWrite() { + writePending = true; + DispatchHandle::rewatchWrite(); +} + +void AsynchIO::queueWriteClose() { + queuedClose = true; + DispatchHandle::rewatchWrite(); +} + +bool AsynchIO::writeQueueEmpty() { + return writeQueue.empty(); +} + +// This can happen outside the callback context +void AsynchIO::startReading() { + readingStopped = false; + DispatchHandle::rewatchRead(); +} + +void AsynchIO::stopReading() { + readingStopped = true; + DispatchHandle::unwatchRead(); +} + +void AsynchIO::requestCallback(RequestCallback callback) { + // TODO creating a function object every time isn't all that + // efficient - if this becomes heavily used do something better (what?) + assert(callback); + DispatchHandle::call(boost::bind(&AsynchIO::requestedCall, this, callback)); +} + +void AsynchIO::requestedCall(RequestCallback callback) { + assert(callback); + callback(*this); +} + +/** Return a queued buffer if there are enough + * to spare + */ +AsynchIO::BufferBase* AsynchIO::getQueuedBuffer() { + // Always keep at least one buffer (it might have data that was "unread" in it) + if (bufferQueue.size()<=1) + return 0; + BufferBase* buff = bufferQueue.back(); + assert(buff); + buff->dataStart = 0; + buff->dataCount = 0; + bufferQueue.pop_back(); + return buff; +} + +/* + * We keep on reading as long as we have something to read, a buffer + * to put it in and reading is not stopped by flow control. + */ +void AsynchIO::readable(DispatchHandle& h) { + if (readingStopped) { + // We have been flow controlled. + return; + } + int readTotal = 0; + AbsTime readStartTime = AbsTime::now(); + do { + // (Try to) get a buffer + if (!bufferQueue.empty()) { + // Read into buffer + BufferBase* buff = bufferQueue.front(); + assert(buff); + bufferQueue.pop_front(); + errno = 0; + int readCount = buff->byteCount-buff->dataCount; + int rc = socket.read(buff->bytes + buff->dataCount, readCount); + if (rc > 0) { + buff->dataCount += rc; + threadReadTotal += rc; + readTotal += rc; + + readCallback(*this, buff); + if (readingStopped) { + // We have been flow controlled. + break; + } + + if (rc != readCount) { + // If we didn't fill the read buffer then time to stop reading + break; + } + + // Stop reading if we've overrun our timeslot + if (Duration(readStartTime, AbsTime::now()) > threadMaxReadTimeNs) { + break; + } + + } else { + // Put buffer back (at front so it doesn't interfere with unread buffers) + bufferQueue.push_front(buff); + assert(buff); + + // Eof or other side has gone away + if (rc == 0 || errno == ECONNRESET) { + eofCallback(*this); + h.unwatchRead(); + break; + } else if (errno == EAGAIN) { + // We have just put a buffer back so we know + // we can carry on watching for reads + break; + } else { + // Report error then just treat as a socket disconnect + QPID_LOG(error, "Error reading socket: " << qpid::sys::strError(errno) << "(" << errno << ")" ); + eofCallback(*this); + h.unwatchRead(); + break; + } + } + } else { + // Something to read but no buffer + if (emptyCallback) { + emptyCallback(*this); + } + // If we still have no buffers we can't do anything more + if (bufferQueue.empty()) { + h.unwatchRead(); + break; + } + + } + } while (true); + + ++threadReadCount; + threadMaxRead = std::max(threadMaxRead, readTotal); + return; +} + +/* + * We carry on writing whilst we have data to write and we can write + */ +void AsynchIO::writeable(DispatchHandle& h) { + int writeTotal = 0; + do { + // See if we've got something to write + if (!writeQueue.empty()) { + // Write buffer + BufferBase* buff = writeQueue.back(); + writeQueue.pop_back(); + errno = 0; + assert(buff->dataStart+buff->dataCount <= buff->byteCount); + int rc = socket.write(buff->bytes+buff->dataStart, buff->dataCount); + if (rc >= 0) { + threadWriteTotal += rc; + writeTotal += rc; + + // If we didn't write full buffer put rest back + if (rc != buff->dataCount) { + buff->dataStart += rc; + buff->dataCount -= rc; + writeQueue.push_back(buff); + break; + } + + // Recycle the buffer + queueReadBuffer(buff); + + // If we've already written more than the max for reading then stop + // (this is to stop writes dominating reads) + if (writeTotal > threadMaxRead) + break; + } else { + // Put buffer back + writeQueue.push_back(buff); + if (errno == ECONNRESET || errno == EPIPE) { + // Just stop watching for write here - we'll get a + // disconnect callback soon enough + h.unwatchWrite(); + break; + } else if (errno == EAGAIN) { + // We have just put a buffer back so we know + // we can carry on watching for writes + break; + } else { + // Report error then just treat as a socket disconnect + QPID_LOG(error, "Error writing socket: " << qpid::sys::strError(errno) << "(" << errno << ")" ); + h.unwatchWrite(); + break; + } + } + } else { + // If we're waiting to close the socket then can do it now as there is nothing to write + if (queuedClose) { + close(h); + break; + } + // Fd is writable, but nothing to write + if (idleCallback) { + writePending = false; + idleCallback(*this); + } + // If we still have no buffers to write we can't do anything more + if (writeQueue.empty() && !writePending && !queuedClose) { + h.unwatchWrite(); + // The following handles the case where writePending is + // set to true after the test above; in this case its + // possible that the unwatchWrite overwrites the + // desired rewatchWrite so we correct that here + if (writePending) + h.rewatchWrite(); + break; + } + } + } while (true); + + ++threadWriteCount; + return; +} + +void AsynchIO::disconnected(DispatchHandle& h) { + // If we have not already queued close then call disconnected callback before closing + if (!queuedClose && disCallback) disCallback(*this); + close(h); +} + +/* + * Close the socket and callback to say we've done it + */ +void AsynchIO::close(DispatchHandle& h) { + h.stopWatch(); + socket.close(); + if (closedCallback) { + closedCallback(*this, socket); + } +} + +} // namespace posix + +AsynchAcceptor* AsynchAcceptor::create(const Socket& s, + Callback callback) +{ + return new posix::AsynchAcceptor(s, callback); +} + +AsynchConnector* AsynchConnector::create(const Socket& s, + const std::string& hostname, + const std::string& port, + ConnectedCallback connCb, + FailedCallback failCb) +{ + return new posix::AsynchConnector(s, hostname, port, connCb, failCb); +} + +AsynchIO* AsynchIO::create(const Socket& s, + AsynchIO::ReadCallback rCb, + AsynchIO::EofCallback eofCb, + AsynchIO::DisconnectCallback disCb, + AsynchIO::ClosedCallback cCb, + AsynchIO::BuffersEmptyCallback eCb, + AsynchIO::IdleCallback iCb) +{ + return new posix::AsynchIO(s, rCb, eofCb, disCb, cCb, eCb, iCb); +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/posix/FileSysDir.cpp b/qpid/cpp/src/qpid/sys/posix/FileSysDir.cpp new file mode 100755 index 0000000000..22dc487e74 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/FileSysDir.cpp @@ -0,0 +1,54 @@ +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/FileSysDir.h" +#include "qpid/sys/StrError.h" +#include "qpid/Exception.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <cerrno> +#include <unistd.h> + +namespace qpid { +namespace sys { + +bool FileSysDir::exists (void) const +{ + const char *cpath = dirPath.c_str (); + struct stat s; + if (::stat(cpath, &s)) { + if (errno == ENOENT) { + return false; + } + throw qpid::Exception (strError(errno) + + ": Can't check directory: " + dirPath); + } + if (S_ISDIR(s.st_mode)) + return true; + throw qpid::Exception(dirPath + " is not a directory"); +} + +void FileSysDir::mkdir(void) +{ + if (::mkdir(dirPath.c_str(), 0755)) + throw Exception ("Can't create directory: " + dirPath); +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/posix/Fork.cpp b/qpid/cpp/src/qpid/sys/posix/Fork.cpp new file mode 100644 index 0000000000..a0d404a16e --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/Fork.cpp @@ -0,0 +1,129 @@ +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include "qpid/sys/Fork.h" +#include "qpid/log/Statement.h" +#include "qpid/Exception.h" + +#include <errno.h> +#include <fcntl.h> +#include <signal.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/select.h> +#include <sys/types.h> +#include <unistd.h> + +namespace qpid { +namespace sys { + +using namespace std; + +namespace { + +void writeStr(int fd, const std::string& str) { + const char* WRITE_ERR = "Error writing to parent process"; + int size = str.size(); + if (int(sizeof(size)) > ::write(fd, &size, sizeof(size))) throw ErrnoException(WRITE_ERR); + if (size > ::write(fd, str.data(), size)) throw ErrnoException(WRITE_ERR); +} + +string readStr(int fd) { + string value; + const char* READ_ERR = "Error reading from forked process"; + int size; + if (int(sizeof(size)) > ::read(fd, &size, sizeof(size))) throw ErrnoException(READ_ERR); + if (size > 0) { // Read string message + value.resize(size); + if (size > ::read(fd, const_cast<char*>(value.data()), size)) throw ErrnoException(READ_ERR); + } + return value; +} + +} // namespace + +Fork::Fork() {} +Fork::~Fork() {} + +void Fork::fork() { + pid_t pid = ::fork(); + if (pid < 0) throw ErrnoException("Failed to fork the process"); + if (pid == 0) child(); + else parent(pid); +} + +ForkWithMessage::ForkWithMessage() { + pipeFds[0] = pipeFds[1] = -1; +} + +struct AutoCloseFd { + int fd; + AutoCloseFd(int d) : fd(d) {} + ~AutoCloseFd() { ::close(fd); } +}; + +void ForkWithMessage::fork() { + if(::pipe(pipeFds) < 0) throw ErrnoException("Can't create pipe"); + pid_t pid = ::fork(); + if(pid < 0) throw ErrnoException("Fork fork failed"); + if (pid == 0) { // Child + AutoCloseFd ac(pipeFds[1]); // Write side. + ::close(pipeFds[0]); // Read side + try { + child(); + } + catch (const std::exception& e) { + QPID_LOG(error, "Error in forked child: " << e.what()); + std::string msg = e.what(); + if (msg.empty()) msg = " "; // Make sure we send a non-empty error string. + writeStr(pipeFds[1], msg); + } + } + else { // Parent + close(pipeFds[1]); // Write side. + AutoCloseFd ac(pipeFds[0]); // Read side + parent(pid); + } +} + +string ForkWithMessage::wait(int timeout) { // parent waits for child. + errno = 0; + struct timeval tv; + tv.tv_sec = timeout; + tv.tv_usec = 0; + + fd_set fds; + FD_ZERO(&fds); + FD_SET(pipeFds[0], &fds); + int n=select(FD_SETSIZE, &fds, 0, 0, &tv); + if(n<0) throw ErrnoException("Error waiting for fork"); + if (n==0) throw Exception("Timed out waiting for fork"); + + string error = readStr(pipeFds[0]); + if (error.empty()) return readStr(pipeFds[0]); + else throw Exception("Error in forked process: " + error); +} + +// Write empty error string followed by value string to pipe. +void ForkWithMessage::ready(const string& value) { // child + // Write empty string for error followed by value. + writeStr(pipeFds[1], string()); // No error + writeStr(pipeFds[1], value); +} + + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/posix/Fork.h b/qpid/cpp/src/qpid/sys/posix/Fork.h new file mode 100644 index 0000000000..698c61ed30 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/Fork.h @@ -0,0 +1,82 @@ +#ifndef QPID_SYS_POSIX_FORK_H +#define QPID_SYS_POSIX_FORK_H + +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <string> +#include <sys/types.h> + +namespace qpid { +namespace sys { + +/** + * Fork the process. Call parent() in parent and child() in child. + */ +class Fork { + public: + Fork(); + virtual ~Fork(); + + /** + * Fork the process. + * Calls parent() in the parent process, child() in the child. + */ + virtual void fork(); + + protected: + + /** Called in parent process. + *@child pid of child process + */ + virtual void parent(pid_t child) = 0; + + /** Called in child process */ + virtual void child() = 0; +}; + +/** + * Like Fork but also allows the child to send a string message + * or throw an exception to the parent. + */ +class ForkWithMessage : public Fork { + public: + ForkWithMessage(); + void fork(); + + protected: + /** Call from parent(): wait for child to send a value or throw exception. + * @timeout in seconds to wait for response. + * @return value passed by child to ready(). + */ + std::string wait(int timeout); + + /** Call from child(): Send a value to the parent. + *@param value returned by parent call to wait(). + */ + void ready(const std::string& value); + + private: + int pipeFds[2]; +}; + +}} // namespace qpid::sys + + + +#endif /*!QPID_SYS_POSIX_FORK_H*/ diff --git a/qpid/cpp/src/qpid/sys/posix/IOHandle.cpp b/qpid/cpp/src/qpid/sys/posix/IOHandle.cpp new file mode 100644 index 0000000000..9c049ee1de --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/IOHandle.cpp @@ -0,0 +1,44 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IOHandle.h" + +#include "qpid/sys/posix/PrivatePosix.h" + +namespace qpid { +namespace sys { + +int toFd(const IOHandlePrivate* h) +{ + return h->fd; +} + +NullIOHandle DummyIOHandle; + +IOHandle::IOHandle(IOHandlePrivate* h) : + impl(h) +{} + +IOHandle::~IOHandle() { + delete impl; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/posix/LockFile.cpp b/qpid/cpp/src/qpid/sys/posix/LockFile.cpp new file mode 100755 index 0000000000..1862ff6ac9 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/LockFile.cpp @@ -0,0 +1,108 @@ +/* + * + * Copyright (c) 2008 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/LockFile.h" +#include "qpid/sys/posix/PidFile.h" + +#include <string> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include "qpid/sys/posix/check.h" + +namespace qpid { +namespace sys { + +class LockFilePrivate { + friend class LockFile; + friend class PidFile; + + int fd; + +public: + LockFilePrivate(int f) : fd(f) {} +}; + +LockFile::LockFile(const std::string& path_, bool create) + : path(path_), created(create) { + + errno = 0; + int flags=create ? O_WRONLY|O_CREAT|O_NOFOLLOW : O_RDWR; + int fd = ::open(path.c_str(), flags, 0644); + if (fd < 0) throw ErrnoException("Cannot open " + path, errno); + if (::lockf(fd, F_TLOCK, 0) < 0) { + ::close(fd); + throw ErrnoException("Cannot lock " + path, errno); + } + impl.reset(new LockFilePrivate(fd)); +} + +LockFile::~LockFile() { + if (impl) { + int f = impl->fd; + if (f >= 0) { + int unused_ret; + unused_ret = ::lockf(f, F_ULOCK, 0); // Suppress warnings about ignoring return value. + ::close(f); + impl->fd = -1; + } + } +} + +int LockFile::read(void* bytes, size_t len) const { + if (!impl) + throw Exception("Lock file not open: " + path); + + ssize_t rc = ::read(impl->fd, bytes, len); + if ((ssize_t)len > rc) { + throw Exception("Cannot read lock file: " + path); + } + return rc; +} + +int LockFile::write(void* bytes, size_t len) const { + if (!impl) + throw Exception("Lock file not open: " + path); + + ssize_t rc = ::write(impl->fd, bytes, len); + if ((ssize_t)len > rc) { + throw Exception("Cannot write lock file: " + path); + } + return rc; +} + +PidFile::PidFile(const std::string& path_, bool create): + LockFile(path_, create) +{} + +pid_t PidFile::readPid(void) const { + pid_t pid; + int desired_read = sizeof(pid_t); + read(&pid, desired_read); + return pid; +} + +void PidFile::writePid(void) { + pid_t pid = getpid(); + int desired_write = sizeof(pid_t); + write(&pid, desired_write); +} + +}} /* namespace qpid::sys */ diff --git a/qpid/cpp/src/qpid/sys/posix/Mutex.cpp b/qpid/cpp/src/qpid/sys/posix/Mutex.cpp new file mode 100644 index 0000000000..0e1f0d30c2 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/Mutex.cpp @@ -0,0 +1,46 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/Mutex.h" + +namespace qpid { +namespace sys { + +/** + * Initialise a recursive mutex attr for use in creating mutexes later + * (we use pthread_once to make sure it is initialised exactly once) + */ + +namespace { +pthread_once_t onceControl = PTHREAD_ONCE_INIT; +pthread_mutexattr_t mutexattr; + +void initMutexattr() { + pthread_mutexattr_init(&mutexattr); + pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE); +} +} + +const pthread_mutexattr_t* Mutex::getAttribute() { + pthread_once(&onceControl, initMutexattr); + return &mutexattr; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/posix/PidFile.h b/qpid/cpp/src/qpid/sys/posix/PidFile.h new file mode 100644 index 0000000000..fb19d407f4 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/PidFile.h @@ -0,0 +1,62 @@ +#ifndef _sys_PidFile_h +#define _sys_PidFile_h + +/* + * + * Copyright (c) 2008 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/LockFile.h" + +#include "qpid/CommonImportExport.h" +#include "qpid/sys/IntegerTypes.h" + +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <string> + +namespace qpid { +namespace sys { + +class PidFile : public LockFile +{ +public: + QPID_COMMON_EXTERN PidFile(const std::string& path_, bool create); + + /** + * Read the process ID from the lock file. This method assumes that + * if there is a process ID in the file, it was written there by + * writePid(); thus, it's at the start of the file. + * + * Throws an exception if there is an error reading the file. + * + * @returns The stored process ID. No validity check is done on it. + */ + QPID_COMMON_EXTERN pid_t readPid(void) const; + + /** + * Write the current process's ID to the lock file. It's written at + * the start of the file and will overwrite any other content that + * may be in the file. + * + * Throws an exception if the write fails. + */ + QPID_COMMON_EXTERN void writePid(void); +}; + +}} /* namespace qpid::sys */ + +#endif /*!_sys_PidFile_h*/ diff --git a/qpid/cpp/src/qpid/sys/posix/PipeHandle.cpp b/qpid/cpp/src/qpid/sys/posix/PipeHandle.cpp new file mode 100755 index 0000000000..4b19783338 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/PipeHandle.cpp @@ -0,0 +1,64 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +#include "qpid/sys/PipeHandle.h" +#include "qpid/sys/posix/check.h" +#include <unistd.h> +#include <fcntl.h> +#include <sys/socket.h> + +namespace qpid { +namespace sys { + +PipeHandle::PipeHandle(bool nonBlocking) { + + int pair[2]; + pair[0] = pair[1] = -1; + + if (socketpair(PF_UNIX, SOCK_STREAM, 0, pair) == -1) + throw qpid::Exception(QPID_MSG("Creation of pipe failed")); + + writeFd = pair[0]; + readFd = pair[1]; + + // Set the socket to non-blocking + if (nonBlocking) { + int flags = fcntl(readFd, F_GETFL); + fcntl(readFd, F_SETFL, flags | O_NONBLOCK); + } +} + +PipeHandle::~PipeHandle() { + close(readFd); + close(writeFd); +} + +int PipeHandle::read(void* buf, size_t bufSize) { + return ::read(readFd,buf,bufSize); +} + +int PipeHandle::write(const void* buf, size_t bufSize) { + return ::write(writeFd,buf,bufSize); +} + +int PipeHandle::getReadHandle() { + return readFd; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/posix/PollableCondition.cpp b/qpid/cpp/src/qpid/sys/posix/PollableCondition.cpp new file mode 100644 index 0000000000..b22a615a54 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/PollableCondition.cpp @@ -0,0 +1,124 @@ +#ifndef QPID_SYS_LINUX_POLLABLECONDITION_CPP +#define QPID_SYS_LINUX_POLLABLECONDITION_CPP + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/PollableCondition.h" +#include "qpid/sys/DispatchHandle.h" +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/posix/PrivatePosix.h" +#include "qpid/Exception.h" + +#include <boost/bind.hpp> + +#include <unistd.h> +#include <fcntl.h> + +namespace qpid { +namespace sys { + +class PollableConditionPrivate : public sys::IOHandle { + friend class PollableCondition; + +private: + PollableConditionPrivate(const sys::PollableCondition::Callback& cb, + sys::PollableCondition& parent, + const boost::shared_ptr<sys::Poller>& poller); + ~PollableConditionPrivate(); + + void dispatch(sys::DispatchHandle& h); + void set(); + void clear(); + +private: + PollableCondition::Callback cb; + PollableCondition& parent; + boost::shared_ptr<sys::Poller> poller; + int writeFd; + std::auto_ptr<DispatchHandleRef> handle; +}; + +PollableConditionPrivate::PollableConditionPrivate( + const sys::PollableCondition::Callback& cb, + sys::PollableCondition& parent, + const boost::shared_ptr<sys::Poller>& poller +) : IOHandle(new sys::IOHandlePrivate), cb(cb), parent(parent) +{ + int fds[2]; + if (::pipe(fds) == -1) + throw ErrnoException(QPID_MSG("Can't create PollableCondition")); + impl->fd = fds[0]; + writeFd = fds[1]; + if (::fcntl(impl->fd, F_SETFL, O_NONBLOCK) == -1) + throw ErrnoException(QPID_MSG("Can't create PollableCondition")); + if (::fcntl(writeFd, F_SETFL, O_NONBLOCK) == -1) + throw ErrnoException(QPID_MSG("Can't create PollableCondition")); + handle.reset (new DispatchHandleRef( + *this, + boost::bind(&sys::PollableConditionPrivate::dispatch, this, _1), + 0, 0)); + handle->startWatch(poller); + handle->unwatch(); + + // Make the read FD readable + static const char dummy=0; + ssize_t n = ::write(writeFd, &dummy, 1); + if (n == -1 && errno != EAGAIN) + throw ErrnoException("Error setting PollableCondition"); +} + +PollableConditionPrivate::~PollableConditionPrivate() { + handle->stopWatch(); + close(writeFd); +} + +void PollableConditionPrivate::dispatch(sys::DispatchHandle&) { + cb(parent); +} + +void PollableConditionPrivate::set() { + handle->rewatch(); +} + +void PollableConditionPrivate::clear() { + handle->unwatch(); +} + + +PollableCondition::PollableCondition(const Callback& cb, + const boost::shared_ptr<sys::Poller>& poller +) : impl(new PollableConditionPrivate(cb, *this, poller)) +{ +} + +PollableCondition::~PollableCondition() +{ + delete impl; +} + +void PollableCondition::set() { impl->set(); } + +void PollableCondition::clear() { impl->clear(); } + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_LINUX_POLLABLECONDITION_CPP*/ diff --git a/qpid/cpp/src/qpid/sys/posix/Shlib.cpp b/qpid/cpp/src/qpid/sys/posix/Shlib.cpp new file mode 100644 index 0000000000..3fb685d5b8 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/Shlib.cpp @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Shlib.h" +#include "qpid/Exception.h" +#include "qpid/Msg.h" +#include <dlfcn.h> + + +namespace qpid { +namespace sys { + +void Shlib::load(const char* name) { + ::dlerror(); + handle = ::dlopen(name, RTLD_NOW); + const char* error = ::dlerror(); + if (error) { + throw Exception(QPID_MSG(error << ": " << name)); + } +} + +void Shlib::unload() { + if (handle) { + ::dlerror(); + ::dlclose(handle); + const char* error = ::dlerror(); + if (error) { + throw Exception(QPID_MSG(error)); + } + handle = 0; + } +} + +void* Shlib::getSymbol(const char* name) { + ::dlerror(); + void* sym = ::dlsym(handle, name); + const char* error = ::dlerror(); + if (error) + throw Exception(QPID_MSG(error << ": " << name)); + return sym; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/posix/Socket.cpp b/qpid/cpp/src/qpid/sys/posix/Socket.cpp new file mode 100644 index 0000000000..aa25f8062d --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/Socket.cpp @@ -0,0 +1,247 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Socket.h" + +#include "qpid/sys/SocketAddress.h" +#include "qpid/sys/posix/check.h" +#include "qpid/sys/posix/PrivatePosix.h" + +#include <fcntl.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <sys/errno.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <netdb.h> +#include <cstdlib> +#include <string.h> +#include <iostream> + +#include <boost/format.hpp> + +namespace qpid { +namespace sys { + +namespace { +std::string getName(int fd, bool local) +{ + ::sockaddr_storage name; // big enough for any socket address + ::socklen_t namelen = sizeof(name); + + int result = -1; + if (local) { + result = ::getsockname(fd, (::sockaddr*)&name, &namelen); + } else { + result = ::getpeername(fd, (::sockaddr*)&name, &namelen); + } + QPID_POSIX_CHECK(result); + + char servName[NI_MAXSERV]; + char dispName[NI_MAXHOST]; + if (int rc=::getnameinfo((::sockaddr*)&name, namelen, dispName, sizeof(dispName), + servName, sizeof(servName), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + throw QPID_POSIX_ERROR(rc); + return std::string(dispName) + ":" + std::string(servName); +} +} + +Socket::Socket() : + IOHandle(new IOHandlePrivate), + nonblocking(false), + nodelay(false) +{} + +Socket::Socket(IOHandlePrivate* h) : + IOHandle(h), + nonblocking(false), + nodelay(false) +{} + +void Socket::createSocket(const SocketAddress& sa) const +{ + int& socket = impl->fd; + if (socket != -1) Socket::close(); + int s = ::socket(getAddrInfo(sa).ai_family, getAddrInfo(sa).ai_socktype, 0); + if (s < 0) throw QPID_POSIX_ERROR(errno); + socket = s; + + try { + if (nonblocking) setNonblocking(); + if (nodelay) setTcpNoDelay(); + } catch (std::exception&) { + ::close(s); + socket = -1; + throw; + } +} + +void Socket::setNonblocking() const { + int& socket = impl->fd; + nonblocking = true; + if (socket != -1) { + QPID_POSIX_CHECK(::fcntl(socket, F_SETFL, O_NONBLOCK)); + } +} + +void Socket::setTcpNoDelay() const +{ + int& socket = impl->fd; + nodelay = true; + if (socket != -1) { + int flag = 1; + int result = setsockopt(impl->fd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(flag)); + QPID_POSIX_CHECK(result); + } +} + +void Socket::connect(const std::string& host, const std::string& port) const +{ + SocketAddress sa(host, port); + connect(sa); +} + +void Socket::connect(const SocketAddress& addr) const +{ + // The display name for an outbound connection needs to be the name that was specified + // for the address rather than a resolved IP address as we don't know which of + // the IP addresses is actually the one that will be connected to. + peername = addr.asString(false); + + // However the string we compare with the local port must be numeric or it might not + // match when it should as getLocalAddress() will always be numeric + std::string connectname = addr.asString(); + + createSocket(addr); + + const int& socket = impl->fd; + // TODO the correct thing to do here is loop on failure until you've used all the returned addresses + if ((::connect(socket, getAddrInfo(addr).ai_addr, getAddrInfo(addr).ai_addrlen) < 0) && + (errno != EINPROGRESS)) { + throw Exception(QPID_MSG(strError(errno) << ": " << peername)); + } + // When connecting to a port on the same host which no longer has + // a process associated with it, the OS occasionally chooses the + // remote port (which is unoccupied) as the port to bind the local + // end of the socket, resulting in a "circular" connection. + // + // This seems like something the OS should prevent but I have + // confirmed that sporadic hangs in + // cluster_tests.LongTests.test_failover on RHEL5 are caused by + // such a circular connection. + // + // Raise an error if we see such a connection, since we know there is + // no listener on the peer address. + // + if (getLocalAddress() == connectname) { + close(); + throw Exception(QPID_MSG("Connection refused: " << peername)); + } +} + +void +Socket::close() const +{ + int& socket = impl->fd; + if (socket == -1) return; + if (::close(socket) < 0) throw QPID_POSIX_ERROR(errno); + socket = -1; +} + +int Socket::listen(const std::string& host, const std::string& port, int backlog) const +{ + SocketAddress sa(host, port); + return listen(sa, backlog); +} + +int Socket::listen(const SocketAddress& sa, int backlog) const +{ + createSocket(sa); + + const int& socket = impl->fd; + int yes=1; + QPID_POSIX_CHECK(setsockopt(socket,SOL_SOCKET,SO_REUSEADDR,&yes,sizeof(yes))); + + if (::bind(socket, getAddrInfo(sa).ai_addr, getAddrInfo(sa).ai_addrlen) < 0) + throw Exception(QPID_MSG("Can't bind to port " << sa.asString() << ": " << strError(errno))); + if (::listen(socket, backlog) < 0) + throw Exception(QPID_MSG("Can't listen on port " << sa.asString() << ": " << strError(errno))); + + struct sockaddr_in name; + socklen_t namelen = sizeof(name); + if (::getsockname(socket, (struct sockaddr*)&name, &namelen) < 0) + throw QPID_POSIX_ERROR(errno); + + return ntohs(name.sin_port); +} + +Socket* Socket::accept() const +{ + int afd = ::accept(impl->fd, 0, 0); + if ( afd >= 0) { + Socket* s = new Socket(new IOHandlePrivate(afd)); + s->localname = localname; + return s; + } + else if (errno == EAGAIN) + return 0; + else throw QPID_POSIX_ERROR(errno); +} + +int Socket::read(void *buf, size_t count) const +{ + return ::read(impl->fd, buf, count); +} + +int Socket::write(const void *buf, size_t count) const +{ + return ::write(impl->fd, buf, count); +} + +std::string Socket::getPeerAddress() const +{ + if (peername.empty()) { + peername = getName(impl->fd, false); + } + return peername; +} + +std::string Socket::getLocalAddress() const +{ + if (localname.empty()) { + localname = getName(impl->fd, true); + } + return localname; +} + +int Socket::getError() const +{ + int result; + socklen_t rSize = sizeof (result); + + if (::getsockopt(impl->fd, SOL_SOCKET, SO_ERROR, &result, &rSize) < 0) + throw QPID_POSIX_ERROR(errno); + + return result; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/posix/SocketAddress.cpp b/qpid/cpp/src/qpid/sys/posix/SocketAddress.cpp new file mode 100644 index 0000000000..10f1c8a563 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/SocketAddress.cpp @@ -0,0 +1,107 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/SocketAddress.h" + +#include "qpid/sys/posix/check.h" + +#include <sys/socket.h> +#include <string.h> +#include <netdb.h> + +#include <algorithm> + +namespace qpid { +namespace sys { + +SocketAddress::SocketAddress(const std::string& host0, const std::string& port0) : + host(host0), + port(port0), + addrInfo(0) +{ +} + +SocketAddress::SocketAddress(const SocketAddress& sa) : + host(sa.host), + port(sa.port), + addrInfo(0) +{ +} + +SocketAddress& SocketAddress::operator=(const SocketAddress& sa) +{ + SocketAddress temp(sa); + + std::swap(temp, *this); + return *this; +} + +SocketAddress::~SocketAddress() +{ + if (addrInfo) { + ::freeaddrinfo(addrInfo); + } +} + +std::string SocketAddress::asString(bool numeric) const +{ + if (!numeric) + return host + ":" + port; + // Canonicalise into numeric id + const ::addrinfo& ai = getAddrInfo(*this); + char servName[NI_MAXSERV]; + char dispName[NI_MAXHOST]; + if (int rc=::getnameinfo(ai.ai_addr, ai.ai_addrlen, + dispName, sizeof(dispName), + servName, sizeof(servName), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + throw QPID_POSIX_ERROR(rc); + std::string s(dispName); + s += ":"; + s += servName; + return s; +} + +const ::addrinfo& getAddrInfo(const SocketAddress& sa) +{ + if (!sa.addrInfo) { + ::addrinfo hints; + ::memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; // Change this to support IPv6 + hints.ai_socktype = SOCK_STREAM; + + const char* node = 0; + if (sa.host.empty()) { + hints.ai_flags |= AI_PASSIVE; + } else { + node = sa.host.c_str(); + } + const char* service = sa.port.empty() ? "0" : sa.port.c_str(); + + int n = ::getaddrinfo(node, service, &hints, &sa.addrInfo); + if (n != 0) + throw Exception(QPID_MSG("Cannot resolve " << sa.asString(false) << ": " << ::gai_strerror(n))); + } + + return *sa.addrInfo; +} + +}} diff --git a/qpid/cpp/src/qpid/sys/posix/StrError.cpp b/qpid/cpp/src/qpid/sys/posix/StrError.cpp new file mode 100644 index 0000000000..633e20213c --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/StrError.cpp @@ -0,0 +1,41 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/StrError.h" + +#include <string.h> + +namespace qpid { +namespace sys { + +std::string strError(int err) { + char buf[512] = "Unknown error"; +#ifdef _GNU_SOURCE + // GNU strerror_r returns the message + return ::strerror_r(err, buf, sizeof(buf)); +#else + // POSIX strerror_r doesn't return the buffer + ::strerror_r(err, buf, sizeof(buf)); + return std::string(buf); +#endif +} + +}} diff --git a/qpid/cpp/src/qpid/sys/posix/SystemInfo.cpp b/qpid/cpp/src/qpid/sys/posix/SystemInfo.cpp new file mode 100755 index 0000000000..a19ab6885c --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/SystemInfo.cpp @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/SystemInfo.h" + +#include "qpid/sys/posix/check.h" + +#include <sys/ioctl.h> +#include <sys/utsname.h> +#include <sys/types.h> // For FreeBSD +#include <sys/socket.h> // For FreeBSD +#include <netinet/in.h> // For FreeBSD +#include <ifaddrs.h> +#include <iostream> +#include <fstream> +#include <sstream> +#include <netdb.h> + +#ifndef HOST_NAME_MAX +# define HOST_NAME_MAX 256 +#endif + +using namespace std; + +namespace qpid { +namespace sys { + +long SystemInfo::concurrency() { +#ifdef _SC_NPROCESSORS_ONLN // Linux specific. + return sysconf(_SC_NPROCESSORS_ONLN); +#else + return -1; +#endif +} + +bool SystemInfo::getLocalHostname (Address &address) { + char name[HOST_NAME_MAX]; + if (::gethostname(name, sizeof(name)) != 0) + return false; + address.host = name; + return true; +} + +static const string LOCALHOST("127.0.0.1"); +static const string TCP("tcp"); + +void SystemInfo::getLocalIpAddresses (uint16_t port, + std::vector<Address> &addrList) { + ::ifaddrs* ifaddr = 0; + QPID_POSIX_CHECK(::getifaddrs(&ifaddr)); + for (::ifaddrs* ifap = ifaddr; ifap != 0; ifap = ifap->ifa_next) { + if (ifap->ifa_addr == 0) continue; + + int family = ifap->ifa_addr->sa_family; + switch (family) { + case AF_INET: { + char dispName[NI_MAXHOST]; + int rc = ::getnameinfo( + ifap->ifa_addr, + (family == AF_INET) + ? sizeof(struct sockaddr_in) + : sizeof(struct sockaddr_in6), + dispName, sizeof(dispName), + 0, 0, NI_NUMERICHOST); + if (rc != 0) { + throw QPID_POSIX_ERROR(rc); + } + string addr(dispName); + if (addr != LOCALHOST) { + addrList.push_back(Address(TCP, addr, port)); + } + break; + } + // TODO: Url parsing currently can't cope with IPv6 addresses so don't return them + // when it can cope move this line to above "case AF_INET:" + case AF_INET6: + default: + continue; + } + } + freeifaddrs(ifaddr); + + if (addrList.empty()) { + addrList.push_back(Address(TCP, LOCALHOST, port)); + } +} + +void SystemInfo::getSystemId (std::string &osName, + std::string &nodeName, + std::string &release, + std::string &version, + std::string &machine) +{ + struct utsname _uname; + if (uname (&_uname) == 0) + { + osName = _uname.sysname; + nodeName = _uname.nodename; + release = _uname.release; + version = _uname.version; + machine = _uname.machine; + } +} + +uint32_t SystemInfo::getProcessId() +{ + return (uint32_t) ::getpid(); +} + +uint32_t SystemInfo::getParentProcessId() +{ + return (uint32_t) ::getppid(); +} + +// Linux specific (Solaris has quite different stuff in /proc) +string SystemInfo::getProcessName() +{ + string value; + + ifstream input("/proc/self/status"); + if (input.good()) { + while (!input.eof()) { + string key; + input >> key; + if (key == "Name:") { + input >> value; + break; + } + } + input.close(); + } + + return value; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/posix/Thread.cpp b/qpid/cpp/src/qpid/sys/posix/Thread.cpp new file mode 100644 index 0000000000..a1d6396763 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/Thread.cpp @@ -0,0 +1,88 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Thread.h" + +#include "qpid/sys/Runnable.h" +#include "qpid/sys/posix/check.h" + +#include <pthread.h> + +namespace qpid { +namespace sys { + +namespace { +void* runRunnable(void* p) +{ + static_cast<Runnable*>(p)->run(); + return 0; +} +} + +class ThreadPrivate { +public: + pthread_t thread; + + ThreadPrivate(Runnable* runnable) { + QPID_POSIX_ASSERT_THROW_IF(::pthread_create(&thread, NULL, runRunnable, runnable)); + } + + ThreadPrivate() : thread(::pthread_self()) {} +}; + +Thread::Thread() {} + +Thread::Thread(Runnable* runnable) : impl(new ThreadPrivate(runnable)) {} + +Thread::Thread(Runnable& runnable) : impl(new ThreadPrivate(&runnable)) {} + +Thread::operator bool() { + return impl; +} + +bool Thread::operator==(const Thread& t) const { + return ::pthread_equal(impl->thread, t.impl->thread) != 0; +} + +bool Thread::operator!=(const Thread& t) const { + return !(*this==t); +} + +void Thread::join(){ + if (impl) { + QPID_POSIX_ASSERT_THROW_IF(::pthread_join(impl->thread, 0)); + } +} + +unsigned long Thread::logId() { + // This does need to be the C cast operator as + // pthread_t could be either a pointer or an integer + // and so we can't know static_cast<> or reinterpret_cast<> + return (unsigned long) ::pthread_self(); +} + +Thread Thread::current() { + Thread t; + t.impl.reset(new ThreadPrivate()); + return t; +} + +}} diff --git a/qpid/cpp/src/qpid/sys/posix/Time.cpp b/qpid/cpp/src/qpid/sys/posix/Time.cpp new file mode 100644 index 0000000000..9661f0c5e8 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/posix/Time.cpp @@ -0,0 +1,121 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/posix/PrivatePosix.h" + +#include "qpid/sys/Time.h" +#include <ostream> +#include <time.h> +#include <stdio.h> +#include <sys/time.h> +#include <unistd.h> +#include <iomanip> + +namespace { +int64_t max_abstime() { return std::numeric_limits<int64_t>::max(); } +} + +namespace qpid { +namespace sys { + +AbsTime::AbsTime(const AbsTime& t, const Duration& d) : + timepoint(d == Duration::max() ? max_abstime() : t.timepoint+d.nanosecs) +{} + +AbsTime AbsTime::Epoch() { + AbsTime epoch; epoch.timepoint = 0; + return epoch; +} + +AbsTime AbsTime::FarFuture() { + AbsTime ff; ff.timepoint = max_abstime(); return ff; +} + +AbsTime AbsTime::now() { + struct timespec ts; + ::clock_gettime(CLOCK_REALTIME, &ts); + AbsTime time_now; + time_now.timepoint = toTime(ts).nanosecs; + return time_now; +} + +Duration::Duration(const AbsTime& start, const AbsTime& finish) : + nanosecs(finish.timepoint - start.timepoint) +{} + +struct timespec& toTimespec(struct timespec& ts, const Duration& t) { + ts.tv_sec = t / TIME_SEC; + ts.tv_nsec = t % TIME_SEC; + return ts; +} + +struct timeval& toTimeval(struct timeval& tv, const Duration& t) { + tv.tv_sec = t/TIME_SEC; + tv.tv_usec = (t%TIME_SEC)/TIME_USEC; + return tv; +} + +Duration toTime(const struct timespec& ts) { + return ts.tv_sec*TIME_SEC + ts.tv_nsec; +} + +std::ostream& operator<<(std::ostream& o, const Duration& d) { + return o << int64_t(d) << "ns"; +} + +namespace { +inline std::ostream& outputFormattedTime(std::ostream& o, const ::time_t* time) { + ::tm timeinfo; + char time_string[100]; + ::strftime(time_string, 100, + "%Y-%m-%d %H:%M:%S", + localtime_r(time, &timeinfo)); + return o << time_string; +} +} + +std::ostream& operator<<(std::ostream& o, const AbsTime& t) { + ::time_t rawtime(t.timepoint/TIME_SEC); + return outputFormattedTime(o, &rawtime); +} + +void outputFormattedNow(std::ostream& o) { + ::time_t rawtime; + ::time(&rawtime); + outputFormattedTime(o, &rawtime); + o << " "; +} + +void outputHiresNow(std::ostream& o) { + ::timespec time; + ::clock_gettime(CLOCK_REALTIME, &time); + o << time.tv_sec << "." << std::setw(9) << std::setfill('0') << time.tv_nsec << "s "; +} + +void sleep(int secs) { + ::sleep(secs); +} + +void usleep(uint64_t usecs) { + ::usleep(usecs); +} + +}} diff --git a/qpid/cpp/src/qpid/sys/rdma/RdmaClient.cpp b/qpid/cpp/src/qpid/sys/rdma/RdmaClient.cpp new file mode 100644 index 0000000000..38e9b59541 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/rdma/RdmaClient.cpp @@ -0,0 +1,247 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/rdma/RdmaIO.h" +#include "qpid/sys/rdma/rdma_exception.h" +#include "qpid/sys/Time.h" +#include "qpid/sys/Thread.h" + +#include <netdb.h> +#include <arpa/inet.h> + +#include <vector> +#include <string> +#include <iostream> +#include <algorithm> +#include <cmath> +#include <boost/bind.hpp> + +using std::vector; +using std::string; +using std::cout; +using std::cerr; +using std::copy; +using std::rand; + +using qpid::sys::Thread; +using qpid::sys::Poller; +using qpid::sys::Dispatcher; +using qpid::sys::SocketAddress; +using qpid::sys::AbsTime; +using qpid::sys::Duration; +using qpid::sys::TIME_SEC; +using qpid::sys::TIME_INFINITE; + +namespace qpid { +namespace tests { + +// count of messages +int64_t smsgs = 0; +int64_t sbytes = 0; +int64_t rmsgs = 0; +int64_t rbytes = 0; + +int target = 1000000; +int msgsize = 200; +AbsTime startTime; +Duration sendingDuration(TIME_INFINITE); +Duration fullTestDuration(TIME_INFINITE); + +// Random generator +// This is an RNG designed by George Marsaglia see http://en.wikipedia.org/wiki/Xorshift +class Xor128Generator { + uint32_t x; + uint32_t y; + uint32_t z; + uint32_t w; + +public: + Xor128Generator() : + x(123456789),y(362436069),z(521288629),w(88675123) + {++(*this);} + + Xor128Generator& operator++() { + uint32_t t = x ^ (x << 11); + x = y; y = z; z = w; + w = w ^ (w >> 19) ^ t ^ (t >> 8); + return *this; + } + + uint32_t operator*() { + return w; + } +}; + +Xor128Generator output; +Xor128Generator input; + +void write(Rdma::AsynchIO& aio) { + while (aio.writable() && smsgs < target) { + Rdma::Buffer* b = aio.getSendBuffer(); + if (!b) break; + b->dataCount(msgsize); + uint32_t* ip = reinterpret_cast<uint32_t*>(b->bytes()); + uint32_t* lip = ip + b->dataCount() / sizeof(uint32_t); + while (ip != lip) {*ip++ = *output; ++output;} + aio.queueWrite(b); + ++smsgs; + sbytes += msgsize; + } +} + +void dataError(Rdma::AsynchIO&) { + cout << "Data error:\n"; +} + +void data(Poller::shared_ptr p, Rdma::AsynchIO& aio, Rdma::Buffer* b) { + ++rmsgs; + rbytes += b->dataCount(); + + // Check message is unaltered + bool match = true; + uint32_t* ip = reinterpret_cast<uint32_t*>(b->bytes()); + uint32_t* lip = ip + b->dataCount() / sizeof(uint32_t); + while (ip != lip) { if (*ip++ != *input) {match = false; break;} ++input;} + if (!match) { + cout << "Data doesn't match: at msg " << rmsgs << " byte " << rbytes-b->dataCount() << " (ish)\n"; + exit(1); + } + + // When all messages have been recvd stop + if (rmsgs < target) { + write(aio); + } else { + fullTestDuration = std::min(fullTestDuration, Duration(startTime, AbsTime::now())); + if (aio.incompletedWrites() == 0) + p->shutdown(); + } +} + +void full(Rdma::AsynchIO& a, Rdma::Buffer* b) { + // Warn as we shouldn't get here anymore + cerr << "!"; + + // Don't need to keep buffer just adjust the counts + --smsgs; + sbytes -= b->dataCount(); + + // Give buffer back + a.returnSendBuffer(b); +} + +void idle(Poller::shared_ptr p, Rdma::AsynchIO& aio) { + if (smsgs < target) { + write(aio); + } else { + sendingDuration = std::min(sendingDuration, Duration(startTime, AbsTime::now())); + if (rmsgs >= target && aio.incompletedWrites() == 0) + p->shutdown(); + } +} + +void drained(Rdma::AsynchIO&) { + cout << "Drained:\n"; +} + +void connected(Poller::shared_ptr poller, Rdma::Connection::intrusive_ptr& ci, const Rdma::ConnectionParams& cp) { + cout << "Connected\n"; + Rdma::QueuePair::intrusive_ptr q = ci->getQueuePair(); + + Rdma::AsynchIO* aio = new Rdma::AsynchIO(ci->getQueuePair(), + cp.rdmaProtocolVersion, + cp.maxRecvBufferSize, cp.initialXmitCredit , Rdma::DEFAULT_WR_ENTRIES, + boost::bind(&data, poller, _1, _2), + boost::bind(&idle, poller, _1), + &full, + dataError); + + startTime = AbsTime::now(); + write(*aio); + + aio->start(poller); +} + +void disconnected(boost::shared_ptr<Poller> p, Rdma::Connection::intrusive_ptr&) { + cout << "Disconnected\n"; + p->shutdown(); +} + +void connectionError(boost::shared_ptr<Poller> p, Rdma::Connection::intrusive_ptr&, const Rdma::ErrorType) { + cout << "Connection error\n"; + p->shutdown(); +} + +void rejected(boost::shared_ptr<Poller> p, Rdma::Connection::intrusive_ptr&, const Rdma::ConnectionParams&) { + cout << "Connection rejected\n"; + p->shutdown(); +} + +}} // namespace qpid::tests + +using namespace qpid::tests; + +int main(int argc, char* argv[]) { + vector<string> args(&argv[0], &argv[argc]); + + string host = args[1]; + string port = (args.size() < 3) ? "20079" : args[2]; + + if (args.size() > 3) + msgsize = atoi(args[3].c_str()); + cout << "Message size: " << msgsize << "\n"; + + try { + boost::shared_ptr<Poller> p(new Poller()); + + Rdma::Connector c( + Rdma::ConnectionParams(msgsize, Rdma::DEFAULT_WR_ENTRIES), + boost::bind(&connected, p, _1, _2), + boost::bind(&connectionError, p, _1, _2), + boost::bind(&disconnected, p, _1), + boost::bind(&rejected, p, _1, _2)); + + SocketAddress sa(host, port); + cout << "Connecting to: " << sa.asString() <<"\n"; + c.start(p, sa); + + // The poller loop blocks all signals so run in its own thread + Thread t(*p); + t.join(); + } catch (Rdma::Exception& e) { + int err = e.getError(); + cerr << "Error: " << e.what() << "(" << err << ")\n"; + } + + cout + << "Sent: " << smsgs + << "msgs (" << sbytes + << "bytes) in: " << double(sendingDuration)/TIME_SEC + << "s: " << double(smsgs)*TIME_SEC/sendingDuration + << "msgs/s(" << double(sbytes)*TIME_SEC/sendingDuration + << "bytes/s)\n"; + cout + << "Recd: " << rmsgs + << "msgs (" << rbytes + << "bytes) in: " << double(fullTestDuration)/TIME_SEC + << "s: " << double(rmsgs)*TIME_SEC/fullTestDuration + << "msgs/s(" << double(rbytes)*TIME_SEC/fullTestDuration + << "bytes/s)\n"; + +} diff --git a/qpid/cpp/src/qpid/sys/rdma/RdmaIO.cpp b/qpid/cpp/src/qpid/sys/rdma/RdmaIO.cpp new file mode 100644 index 0000000000..78bcdec68e --- /dev/null +++ b/qpid/cpp/src/qpid/sys/rdma/RdmaIO.cpp @@ -0,0 +1,720 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/rdma/RdmaIO.h" + +#include "qpid/log/Statement.h" + +#include <string> +#include <boost/bind.hpp> + +using qpid::sys::SocketAddress; +using qpid::sys::DispatchHandle; +using qpid::sys::Poller; +using qpid::sys::ScopedLock; +using qpid::sys::Mutex; + +namespace Rdma { + // Set packing as these are 'on the wire' structures +# pragma pack(push, 1) + + // Header structure for each transmitted frame + struct FrameHeader { + const static uint32_t FlagsMask = 0xf0000000; + uint32_t data; // written in network order + + FrameHeader() {} + FrameHeader(uint32_t credit, uint32_t flags = 0) { + data = htonl((credit & ~FlagsMask) | (flags & FlagsMask)); + } + + uint32_t credit() const { + return ntohl(data) & ~FlagsMask; + } + + uint32_t flags() const { + return ntohl(data) & FlagsMask; + } + }; + + const size_t FrameHeaderSize = sizeof(FrameHeader); + + // Structure for Connection Parameters on the network + // + // The original version (now called 0) of these parameters had a couple of mistakes: + // * No way to version the protocol (need to introduce a new protocol for iWarp) + // * Used host order int32 (but only deployed on LE archs as far as we know) + // so effectively was LE on the wire which is the opposite of network order. + // + // Fortunately the values sent were sufficiently restricted that a 16 bit short could + // be carved out to indicate the protocol version as these bits were always sent as 0. + // + // So the current version of parameters uses the last 2 bytes to indicate the protocol + // version, if this is 0 then we interpret the rest of the struct without byte swapping + // to remain compatible with the previous protocol. + struct NConnectionParams { + uint32_t maxRecvBufferSize; + uint16_t initialXmitCredit; + uint16_t rdmaProtocolVersion; + + NConnectionParams(const ConnectionParams& c) : + maxRecvBufferSize(c.rdmaProtocolVersion ? htonl(c.maxRecvBufferSize) : c.maxRecvBufferSize), + initialXmitCredit(c.rdmaProtocolVersion ? htons(c.initialXmitCredit) : c.initialXmitCredit), + // 0 is the same with/without byteswapping! + rdmaProtocolVersion(htons(c.rdmaProtocolVersion)) + {} + + operator ConnectionParams() const { + return + ConnectionParams( + rdmaProtocolVersion ? ntohl(maxRecvBufferSize) : maxRecvBufferSize, + rdmaProtocolVersion ? ntohs(initialXmitCredit) : initialXmitCredit, + ntohs(rdmaProtocolVersion)); + } + }; +# pragma pack(pop) + + class IOException : public std::exception { + std::string s; + + public: + IOException(std::string s0): s(s0) {} + ~IOException() throw() {} + + const char* what() const throw() { + return s.c_str(); + } + }; + + AsynchIO::AsynchIO( + QueuePair::intrusive_ptr q, + int version, + int size, + int xCredit, + int rCount, + ReadCallback rc, + IdleCallback ic, + FullCallback fc, + ErrorCallback ec + ) : + protocolVersion(version), + bufferSize(size), + recvCredit(0), + xmitCredit(xCredit), + recvBufferCount(rCount), + xmitBufferCount(xCredit), + outstandingWrites(0), + draining(false), + state(IDLE), + qp(q), + dataHandle(*qp, boost::bind(&AsynchIO::dataEvent, this), 0, 0), + readCallback(rc), + idleCallback(ic), + fullCallback(fc), + errorCallback(ec), + pendingWriteAction(boost::bind(&AsynchIO::writeEvent, this)) + { + if (protocolVersion > maxSupportedProtocolVersion) + throw IOException("Unsupported Rdma Protocol"); + qp->nonblocking(); + qp->notifyRecv(); + qp->notifySend(); + + // Prepost recv buffers before we go any further + qp->allocateRecvBuffers(recvBufferCount, bufferSize+FrameHeaderSize); + + // Create xmit buffers, reserve space for frame header. + qp->createSendBuffers(xmitBufferCount, bufferSize, FrameHeaderSize); + } + + AsynchIO::~AsynchIO() { + // Warn if we are deleting whilst there are still unreclaimed write buffers + if ( outstandingWrites>0 ) + QPID_LOG(error, "RDMA: qp=" << qp << ": Deleting queue before all write buffers finished"); + + // Turn off callbacks if necessary (before doing the deletes) + if (state != STOPPED) { + QPID_LOG(error, "RDMA: qp=" << qp << ": Deleting queue whilst not shutdown"); + dataHandle.stopWatch(); + } + // TODO: It might turn out to be more efficient in high connection loads to reuse the + // buffers rather than having to reregister them all the time (this would be straightforward if all + // connections haver the same buffer size and harder otherwise) + } + + void AsynchIO::start(Poller::shared_ptr poller) { + dataHandle.startWatch(poller); + } + + // State constraints + // On entry: None + // On exit: STOPPED + // Mark for deletion/Delete this object when we have no outstanding writes + void AsynchIO::stop(NotifyCallback nc) { + ScopedLock<Mutex> l(stateLock); + state = STOPPED; + notifyCallback = nc; + dataHandle.call(boost::bind(&AsynchIO::doStoppedCallback, this)); + } + + namespace { + void requestedCall(AsynchIO* aio, AsynchIO::RequestCallback callback) { + assert(callback); + callback(*aio); + } + } + + void AsynchIO::requestCallback(RequestCallback callback) { + // TODO creating a function object every time isn't all that + // efficient - if this becomes heavily used do something better (what?) + assert(callback); + dataHandle.call(boost::bind(&requestedCall, this, callback)); + } + + // Mark writing closed (so we don't accept any more writes or make any idle callbacks) + void AsynchIO::drainWriteQueue(NotifyCallback nc) { + draining = true; + notifyCallback = nc; + } + + void AsynchIO::queueBuffer(Buffer* buff, int credit) { + switch (protocolVersion) { + case 0: + if (!buff) { + Buffer* ob = getSendBuffer(); + // Have to send something as adapters hate it when you try to transfer 0 bytes + *reinterpret_cast< uint32_t* >(ob->bytes()) = htonl(credit); + ob->dataCount(sizeof(uint32_t)); + qp->postSend(credit | IgnoreData, ob); + } else if (credit > 0) { + qp->postSend(credit, buff); + } else { + qp->postSend(buff); + } + break; + case 1: + if (!buff) + buff = getSendBuffer(); + // Add FrameHeader after frame data + FrameHeader header(credit); + assert(buff->dataCount() <= buff->byteCount()); // ensure app data doesn't impinge on reserved space. + ::memcpy(buff->bytes()+buff->dataCount(), &header, FrameHeaderSize); + buff->dataCount(buff->dataCount()+FrameHeaderSize); + qp->postSend(buff); + break; + } + } + + Buffer* AsynchIO::extractBuffer(const QueuePairEvent& e) { + Buffer* b = e.getBuffer(); + switch (protocolVersion) { + case 0: { + bool dataPresent = true; + // Get our xmitCredit if it was sent + if (e.immPresent() ) { + assert(xmitCredit>=0); + xmitCredit += (e.getImm() & ~FlagsMask); + dataPresent = ((e.getImm() & IgnoreData) == 0); + assert(xmitCredit>0); + } + if (!dataPresent) { + b->dataCount(0); + } + break; + } + case 1: + b->dataCount(b->dataCount()-FrameHeaderSize); + FrameHeader header; + ::memcpy(&header, b->bytes()+b->dataCount(), FrameHeaderSize); + assert(xmitCredit>=0); + xmitCredit += header.credit(); + assert(xmitCredit>=0); + break; + } + + return b; + } + + void AsynchIO::queueWrite(Buffer* buff) { + // Make sure we don't overrun our available buffers + // either at our end or the known available at the peers end + if (writable()) { + // TODO: We might want to batch up sending credit + int creditSent = recvCredit & ~FlagsMask; + queueBuffer(buff, creditSent); + recvCredit -= creditSent; + ++outstandingWrites; + --xmitCredit; + assert(xmitCredit>=0); + } else { + if (fullCallback) { + fullCallback(*this, buff); + } else { + QPID_LOG(error, "RDMA: qp=" << qp << ": Write queue full, but no callback, throwing buffer away"); + returnSendBuffer(buff); + } + } + } + + // State constraints + // On entry: None + // On exit: NOTIFY_PENDING || STOPPED + void AsynchIO::notifyPendingWrite() { + ScopedLock<Mutex> l(stateLock); + switch (state) { + case IDLE: + dataHandle.call(pendingWriteAction); + // Fall Thru + case NOTIFY: + state = NOTIFY_PENDING; + break; + case NOTIFY_PENDING: + case STOPPED: + break; + } + } + + // State constraints + // On entry: IDLE || STOPPED + // On exit: IDLE || STOPPED + void AsynchIO::dataEvent() { + { + ScopedLock<Mutex> l(stateLock); + + if (state == STOPPED) return; + + state = NOTIFY_PENDING; + } + processCompletions(); + + writeEvent(); + } + + // State constraints + // On entry: NOTIFY_PENDING || STOPPED + // On exit: IDLE || STOPPED + void AsynchIO::writeEvent() { + State newState; + do { + { + ScopedLock<Mutex> l(stateLock); + + switch (state) { + case STOPPED: + return; + default: + state = NOTIFY; + } + } + + doWriteCallback(); + + { + ScopedLock<Mutex> l(stateLock); + + newState = state; + switch (newState) { + case NOTIFY_PENDING: + case STOPPED: + break; + default: + state = IDLE; + } + } + } while (newState == NOTIFY_PENDING); + } + + void AsynchIO::processCompletions() { + QueuePair::intrusive_ptr q = qp->getNextChannelEvent(); + + // Re-enable notification for queue: + // This needs to happen before we could do anything that could generate more work completion + // events (ie the callbacks etc. in the following). + // This can't make us reenter this code as the handle attached to the completion queue will still be + // disabled by the poller until we leave this code + qp->notifyRecv(); + qp->notifySend(); + + int recvEvents = 0; + int sendEvents = 0; + + // If no event do nothing + if (!q) + return; + + assert(q == qp); + + // Repeat until no more events + do { + QueuePairEvent e(qp->getNextEvent()); + if (!e) + break; + + ::ibv_wc_status status = e.getEventStatus(); + if (status != IBV_WC_SUCCESS) { + // Need special check for IBV_WC_WR_FLUSH_ERR here + // we will get this for every send/recv queue entry that was pending + // when disconnected, these aren't real errors and mostly need to be ignored + if (status == IBV_WC_WR_FLUSH_ERR) { + QueueDirection dir = e.getDirection(); + if (dir == SEND) { + Buffer* b = e.getBuffer(); + ++sendEvents; + returnSendBuffer(b); + --outstandingWrites; + } else { + ++recvEvents; + } + continue; + } + errorCallback(*this); + // TODO: Probably need to flush queues at this point + return; + } + + // Test if recv (or recv with imm) + //::ibv_wc_opcode eventType = e.getEventType(); + QueueDirection dir = e.getDirection(); + if (dir == RECV) { + ++recvEvents; + + Buffer* b = extractBuffer(e); + + // if there was no data sent then the message was only to update our credit + if ( b->dataCount() > 0 ) { + readCallback(*this, b); + } + + // At this point the buffer has been consumed so put it back on the recv queue + // TODO: Is this safe to do if the connection is disconnected already? + qp->postRecv(b); + + // Received another message + ++recvCredit; + + // Send recvCredit if it is large enough (it will have got this large because we've not sent anything recently) + if (recvCredit > recvBufferCount/2) { + // TODO: This should use RDMA write with imm as there might not ever be a buffer to receive this message + // but this is a little unlikely, as to get in this state we have to have received messages without sending any + // for a while so its likely we've received an credit update from the far side. + if (writable()) { + int creditSent = recvCredit & ~FlagsMask; + queueBuffer(0, creditSent); + recvCredit -= creditSent; + ++outstandingWrites; + --xmitCredit; + assert(xmitCredit>=0); + } else { + QPID_LOG(warning, "RDMA: qp=" << qp << ": Unable to send unsolicited credit"); + } + } + } else { + Buffer* b = e.getBuffer(); + ++sendEvents; + returnSendBuffer(b); + --outstandingWrites; + } + } while (true); + + // Not sure if this is expected or not + if (recvEvents == 0 && sendEvents == 0) { + QPID_LOG(debug, "RDMA: qp=" << qp << ": Got channel event with no recv/send completions"); + } + } + + void AsynchIO::doWriteCallback() { + // TODO: maybe don't call idle unless we're low on write buffers + // Keep on calling the idle routine as long as we are writable and we got something to write last call + + // Do callback even if there are no available free buffers as the application itself might be + // holding onto buffers + while (writable()) { + int xc = xmitCredit; + idleCallback(*this); + // Check whether we actually wrote anything + if (xmitCredit == xc) { + QPID_LOG(debug, "RDMA: qp=" << qp << ": Called for data, but got none: xmitCredit=" << xmitCredit); + return; + } + } + + checkDrained(); + } + + void AsynchIO::checkDrained() { + // If we've got all the write confirmations and we're draining + // We might get deleted in the drained callback so return immediately + if (draining) { + if (outstandingWrites == 0) { + draining = false; + NotifyCallback nc; + nc.swap(notifyCallback); + nc(*this); + } + return; + } + } + + void AsynchIO::doStoppedCallback() { + // Ensure we can't get any more callbacks (except for the stopped callback) + dataHandle.stopWatch(); + + NotifyCallback nc; + nc.swap(notifyCallback); + nc(*this); + } + + ConnectionManager::ConnectionManager( + ErrorCallback errc, + DisconnectedCallback dc + ) : + state(IDLE), + ci(Connection::make()), + handle(*ci, boost::bind(&ConnectionManager::event, this, _1), 0, 0), + errorCallback(errc), + disconnectedCallback(dc) + { + QPID_LOG(debug, "RDMA: ci=" << ci << ": Creating ConnectionManager"); + ci->nonblocking(); + } + + ConnectionManager::~ConnectionManager() + { + QPID_LOG(debug, "RDMA: ci=" << ci << ": Deleting ConnectionManager"); + } + + void ConnectionManager::start(Poller::shared_ptr poller, const qpid::sys::SocketAddress& addr) { + startConnection(ci, addr); + handle.startWatch(poller); + } + + void ConnectionManager::doStoppedCallback() { + // Ensure we can't get any more callbacks (except for the stopped callback) + handle.stopWatch(); + + NotifyCallback nc; + nc.swap(notifyCallback); + nc(*this); + } + + void ConnectionManager::stop(NotifyCallback nc) { + state = STOPPED; + notifyCallback = nc; + handle.call(boost::bind(&ConnectionManager::doStoppedCallback, this)); + } + + void ConnectionManager::event(DispatchHandle&) { + if (state.get() == STOPPED) return; + connectionEvent(ci); + } + + Listener::Listener( + const ConnectionParams& cp, + EstablishedCallback ec, + ErrorCallback errc, + DisconnectedCallback dc, + ConnectionRequestCallback crc + ) : + ConnectionManager(errc, dc), + checkConnectionParams(cp), + connectionRequestCallback(crc), + establishedCallback(ec) + { + } + + void Listener::startConnection(Connection::intrusive_ptr ci, const qpid::sys::SocketAddress& addr) { + ci->bind(addr); + ci->listen(); + } + + namespace { + const int64_t PoisonContext = -1; + } + + void Listener::connectionEvent(Connection::intrusive_ptr ci) { + ConnectionEvent e(ci->getNextEvent()); + + // If (for whatever reason) there was no event do nothing + if (!e) + return; + + // Important documentation ommision the new rdma_cm_id + // you get from CONNECT_REQUEST has the same context info + // as its parent listening rdma_cm_id + ::rdma_cm_event_type eventType = e.getEventType(); + ::rdma_conn_param conn_param = e.getConnectionParam(); + Rdma::Connection::intrusive_ptr id = e.getConnection(); + + // Check for previous disconnection (it appears that you actually can get connection + // request events after a disconnect event in rare circumstances) + if (reinterpret_cast<int64_t>(id->getContext<void*>())==PoisonContext) + return; + + switch (eventType) { + case RDMA_CM_EVENT_CONNECT_REQUEST: { + // Make sure peer has sent params we can use + if (!conn_param.private_data || conn_param.private_data_len < sizeof(NConnectionParams)) { + QPID_LOG(warning, "Rdma: rejecting connection attempt: unusable connection parameters"); + id->reject(); + break; + } + + const NConnectionParams* rcp = static_cast<const NConnectionParams*>(conn_param.private_data); + ConnectionParams cp = *rcp; + + // Reject if requested msg size is bigger than we allow + if ( + cp.maxRecvBufferSize > checkConnectionParams.maxRecvBufferSize || + cp.initialXmitCredit > checkConnectionParams.initialXmitCredit + ) { + QPID_LOG(warning, "Rdma: rejecting connection attempt: connection parameters out of range: (" + << cp.maxRecvBufferSize << ">" << checkConnectionParams.maxRecvBufferSize << " || " + << cp.initialXmitCredit << ">" << checkConnectionParams.initialXmitCredit + << ")"); + id->reject(&checkConnectionParams); + break; + } + + bool accept = true; + if (connectionRequestCallback) + accept = connectionRequestCallback(id, cp); + + if (accept) { + // Accept connection + cp.initialXmitCredit = checkConnectionParams.initialXmitCredit; + id->accept(conn_param, rcp); + } else { + // Reject connection + QPID_LOG(warning, "Rdma: rejecting connection attempt: application policy"); + id->reject(); + } + break; + } + case RDMA_CM_EVENT_ESTABLISHED: + establishedCallback(id); + break; + case RDMA_CM_EVENT_DISCONNECTED: + disconnectedCallback(id); + // Poison the id context so that we do no more callbacks on it + id->removeContext(); + id->addContext(reinterpret_cast<void*>(PoisonContext)); + break; + case RDMA_CM_EVENT_CONNECT_ERROR: + errorCallback(id, CONNECT_ERROR); + break; + default: + // Unexpected response + errorCallback(id, UNKNOWN); + //std::cerr << "Warning: unexpected response to listen - " << eventType << "\n"; + } + } + + Connector::Connector( + const ConnectionParams& cp, + ConnectedCallback cc, + ErrorCallback errc, + DisconnectedCallback dc, + RejectedCallback rc + ) : + ConnectionManager(errc, dc), + connectionParams(cp), + rejectedCallback(rc), + connectedCallback(cc) + { + } + + void Connector::startConnection(Connection::intrusive_ptr ci, const qpid::sys::SocketAddress& addr) { + ci->resolve_addr(addr); + } + + void Connector::connectionEvent(Connection::intrusive_ptr ci) { + ConnectionEvent e(ci->getNextEvent()); + + // If (for whatever reason) there was no event do nothing + if (!e) + return; + + ::rdma_cm_event_type eventType = e.getEventType(); + ::rdma_conn_param conn_param = e.getConnectionParam(); + Rdma::Connection::intrusive_ptr id = e.getConnection(); + switch (eventType) { + case RDMA_CM_EVENT_ADDR_RESOLVED: + // RESOLVE_ADDR + ci->resolve_route(); + break; + case RDMA_CM_EVENT_ADDR_ERROR: + // RESOLVE_ADDR + errorCallback(ci, ADDR_ERROR); + break; + case RDMA_CM_EVENT_ROUTE_RESOLVED: { + // RESOLVE_ROUTE: + NConnectionParams rcp(connectionParams); + ci->connect(&rcp); + break; + } + case RDMA_CM_EVENT_ROUTE_ERROR: + // RESOLVE_ROUTE: + errorCallback(ci, ROUTE_ERROR); + break; + case RDMA_CM_EVENT_CONNECT_ERROR: + // CONNECTING + errorCallback(ci, CONNECT_ERROR); + break; + case RDMA_CM_EVENT_UNREACHABLE: + // CONNECTING + errorCallback(ci, UNREACHABLE); + break; + case RDMA_CM_EVENT_REJECTED: { + // CONNECTING + + // We can get this event if our peer is not running on the other side + // in this case we could get nearly anything in the private data: + // From private_data == 0 && private_data_len == 0 (Chelsio iWarp) + // to 148 bytes of zeros (Mellanox IB) + // + // So assume that if the the private data is absent or not the size of + // the connection parameters it isn't valid + ConnectionParams cp(0, 0, 0); + if (conn_param.private_data && conn_param.private_data_len == sizeof(NConnectionParams)) { + // Extract private data from event + const NConnectionParams* rcp = static_cast<const NConnectionParams*>(conn_param.private_data); + cp = *rcp; + } + rejectedCallback(ci, cp); + break; + } + case RDMA_CM_EVENT_ESTABLISHED: { + // CONNECTING + // Extract private data from event + assert(conn_param.private_data && conn_param.private_data_len >= sizeof(NConnectionParams)); + const NConnectionParams* rcp = static_cast<const NConnectionParams*>(conn_param.private_data); + ConnectionParams cp = *rcp; + connectedCallback(ci, cp); + break; + } + case RDMA_CM_EVENT_DISCONNECTED: + // ESTABLISHED + disconnectedCallback(ci); + break; + default: + QPID_LOG(warning, "RDMA: Unexpected event in connect: " << eventType); + } + } +} diff --git a/qpid/cpp/src/qpid/sys/rdma/RdmaIO.h b/qpid/cpp/src/qpid/sys/rdma/RdmaIO.h new file mode 100644 index 0000000000..ec9caaf08d --- /dev/null +++ b/qpid/cpp/src/qpid/sys/rdma/RdmaIO.h @@ -0,0 +1,250 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef Rdma_Acceptor_h +#define Rdma_Acceptor_h + +#include "qpid/sys/rdma/rdma_wrap.h" + +#include "qpid/sys/AtomicValue.h" +#include "qpid/sys/Dispatcher.h" +#include "qpid/sys/DispatchHandle.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/SocketAddress.h" + +#include <netinet/in.h> + +#include <boost/function.hpp> + +namespace Rdma { + + class Connection; + + class AsynchIO + { + typedef boost::function1<void, AsynchIO&> ErrorCallback; + typedef boost::function2<void, AsynchIO&, Buffer*> ReadCallback; + typedef boost::function1<void, AsynchIO&> IdleCallback; + typedef boost::function2<void, AsynchIO&, Buffer*> FullCallback; + typedef boost::function1<void, AsynchIO&> NotifyCallback; + + int protocolVersion; + int bufferSize; + int recvCredit; + int xmitCredit; + int recvBufferCount; + int xmitBufferCount; + int outstandingWrites; + bool draining; + enum State {IDLE, NOTIFY, NOTIFY_PENDING, STOPPED}; + State state; + qpid::sys::Mutex stateLock; + QueuePair::intrusive_ptr qp; + qpid::sys::DispatchHandleRef dataHandle; + + ReadCallback readCallback; + IdleCallback idleCallback; + FullCallback fullCallback; + ErrorCallback errorCallback; + NotifyCallback notifyCallback; + qpid::sys::DispatchHandle::Callback pendingWriteAction; + + public: + typedef boost::function1<void, AsynchIO&> RequestCallback; + + // TODO: Instead of specifying a buffer size specify the amount of memory the AsynchIO class can use + // for buffers both read and write (allocate half to each up front) and fail if we cannot allocate that much + // locked memory + AsynchIO( + QueuePair::intrusive_ptr q, + int version, + int size, + int xCredit, + int rCount, + ReadCallback rc, + IdleCallback ic, + FullCallback fc, + ErrorCallback ec + ); + ~AsynchIO(); + + void start(qpid::sys::Poller::shared_ptr poller); + bool writable() const; + void queueWrite(Buffer* buff); + void notifyPendingWrite(); + void drainWriteQueue(NotifyCallback); + void stop(NotifyCallback); + void requestCallback(RequestCallback); + int incompletedWrites() const; + Buffer* getSendBuffer(); + void returnSendBuffer(Buffer*); + + private: + const static int maxSupportedProtocolVersion = 1; + + // Constants for the peer-peer command messages + // These are sent in the high bits if the imm data of an rdma message + // The low bits are used to send the credit + const static int FlagsMask = 0xF0000000; // Mask for all flag bits - be sure to update this if you add more command bits + const static int IgnoreData = 0x10000000; // Message contains no application data + + void dataEvent(); + void writeEvent(); + void processCompletions(); + void doWriteCallback(); + void checkDrained(); + void doStoppedCallback(); + + void queueBuffer(Buffer* buff, int credit); + Buffer* extractBuffer(const QueuePairEvent& e); + }; + + // We're only writable if: + // * not draining write queue + // * we've got space in the transmit queue + // * we've got credit to transmit + // * if there's only 1 transmit credit we must send some credit + inline bool AsynchIO::writable() const { + assert(xmitCredit>=0); + return !draining && + outstandingWrites < xmitBufferCount && + xmitCredit > 0 && + ( xmitCredit > 1 || recvCredit > 0); + } + + inline int AsynchIO::incompletedWrites() const { + return outstandingWrites; + } + + inline Buffer* AsynchIO::getSendBuffer() { + return qp->getSendBuffer(); + } + + inline void AsynchIO::returnSendBuffer(Buffer* b) { + qp->returnSendBuffer(b); + } + + // These are the parameters necessary to start the conversation + // * Each peer HAS to allocate buffers of the size of the maximum receive from its peer + // * Each peer HAS to know the initial "credit" it has for transmitting to its peer + struct ConnectionParams { + uint32_t maxRecvBufferSize; + uint16_t initialXmitCredit; + uint16_t rdmaProtocolVersion; + + // Default to protocol version 1 + ConnectionParams(uint32_t s, uint16_t c, uint16_t v = 1) : + maxRecvBufferSize(s), + initialXmitCredit(c), + rdmaProtocolVersion(v) + {} + }; + + enum ErrorType { + ADDR_ERROR, + ROUTE_ERROR, + CONNECT_ERROR, + UNREACHABLE, + UNKNOWN + }; + + typedef boost::function2<void, Rdma::Connection::intrusive_ptr, ErrorType> ErrorCallback; + typedef boost::function1<void, Rdma::Connection::intrusive_ptr> DisconnectedCallback; + + class ConnectionManager { + typedef boost::function1<void, ConnectionManager&> NotifyCallback; + + enum State {IDLE, STOPPED}; + qpid::sys::AtomicValue<State> state; + Connection::intrusive_ptr ci; + qpid::sys::DispatchHandleRef handle; + NotifyCallback notifyCallback; + + protected: + ErrorCallback errorCallback; + DisconnectedCallback disconnectedCallback; + + public: + ConnectionManager( + ErrorCallback errc, + DisconnectedCallback dc + ); + + virtual ~ConnectionManager(); + + void start(qpid::sys::Poller::shared_ptr poller, const qpid::sys::SocketAddress& addr); + void stop(NotifyCallback); + + private: + void event(qpid::sys::DispatchHandle& handle); + void doStoppedCallback(); + + virtual void startConnection(Connection::intrusive_ptr ci, const qpid::sys::SocketAddress& addr) = 0; + virtual void connectionEvent(Connection::intrusive_ptr ci) = 0; + }; + + typedef boost::function2<bool, Rdma::Connection::intrusive_ptr, const ConnectionParams&> ConnectionRequestCallback; + typedef boost::function1<void, Rdma::Connection::intrusive_ptr> EstablishedCallback; + + class Listener : public ConnectionManager + { + ConnectionParams checkConnectionParams; + ConnectionRequestCallback connectionRequestCallback; + EstablishedCallback establishedCallback; + + public: + Listener( + const ConnectionParams& cp, + EstablishedCallback ec, + ErrorCallback errc, + DisconnectedCallback dc, + ConnectionRequestCallback crc = 0 + ); + + private: + void startConnection(Connection::intrusive_ptr ci, const qpid::sys::SocketAddress& addr); + void connectionEvent(Connection::intrusive_ptr ci); + }; + + typedef boost::function2<void, Rdma::Connection::intrusive_ptr, const ConnectionParams&> RejectedCallback; + typedef boost::function2<void, Rdma::Connection::intrusive_ptr, const ConnectionParams&> ConnectedCallback; + + class Connector : public ConnectionManager + { + ConnectionParams connectionParams; + RejectedCallback rejectedCallback; + ConnectedCallback connectedCallback; + + public: + Connector( + const ConnectionParams& cp, + ConnectedCallback cc, + ErrorCallback errc, + DisconnectedCallback dc, + RejectedCallback rc = 0 + ); + + private: + void startConnection(Connection::intrusive_ptr ci, const qpid::sys::SocketAddress& addr); + void connectionEvent(Connection::intrusive_ptr ci); + }; +} + +#endif // Rdma_Acceptor_h diff --git a/qpid/cpp/src/qpid/sys/rdma/RdmaServer.cpp b/qpid/cpp/src/qpid/sys/rdma/RdmaServer.cpp new file mode 100644 index 0000000000..9b0710fd8f --- /dev/null +++ b/qpid/cpp/src/qpid/sys/rdma/RdmaServer.cpp @@ -0,0 +1,210 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/Thread.h" +#include "qpid/sys/rdma/RdmaIO.h" +#include "qpid/sys/rdma/rdma_exception.h" + +#include <arpa/inet.h> + +#include <vector> +#include <queue> +#include <string> +#include <iostream> + +#include <boost/bind.hpp> + +using std::vector; +using std::queue; +using std::string; +using std::cout; +using std::cerr; + +using qpid::sys::Thread; +using qpid::sys::SocketAddress; +using qpid::sys::Poller; + +// All the accepted connections +namespace qpid { +namespace tests { + +struct Buffer { + char* bytes() const {return bytes_;} + int32_t byteCount() const {return size;} + + Buffer(const int32_t s): + bytes_(new char[s]), + size(s) + { + } + + ~Buffer() { + delete [] bytes_; + } +private: + char* bytes_; + int32_t size; +}; + +struct ConRec { + Rdma::Connection::intrusive_ptr connection; + Rdma::AsynchIO* data; + queue<Buffer*> queuedWrites; + + ConRec(Rdma::Connection::intrusive_ptr c) : + connection(c) + {} +}; + +void dataError(Rdma::AsynchIO&) { + cout << "Data error:\n"; +} + +void idle(ConRec* cr, Rdma::AsynchIO& a) { + // Need to make sure full is not called as it would reorder messages + while (!cr->queuedWrites.empty() && a.writable()) { + Rdma::Buffer* rbuf = a.getSendBuffer(); + if (!rbuf) break; + Buffer* buf = cr->queuedWrites.front(); + cr->queuedWrites.pop(); + std::copy(buf->bytes(), buf->bytes()+buf->byteCount(), rbuf->bytes()); + rbuf->dataCount(buf->byteCount()); + delete buf; + a.queueWrite(rbuf); + } +} + +void data(ConRec* cr, Rdma::AsynchIO& a, Rdma::Buffer* b) { + // Echo data back + Rdma::Buffer* buf = 0; + if (cr->queuedWrites.empty() && a.writable()) { + buf = a.getSendBuffer(); + } + if (buf) { + std::copy(b->bytes(), b->bytes()+b->dataCount(), buf->bytes()); + buf->dataCount(b->dataCount()); + a.queueWrite(buf); + } else { + Buffer* buf = new Buffer(b->dataCount()); + std::copy(b->bytes(), b->bytes()+b->dataCount(), buf->bytes()); + cr->queuedWrites.push(buf); + // Try to empty queue + idle(cr, a); + } +} + +void full(ConRec*, Rdma::AsynchIO&, Rdma::Buffer*) { + // Shouldn't ever be called + cout << "!"; +} + +void drained(Rdma::AsynchIO&) { + cout << "Drained:\n"; +} + +void disconnected(Rdma::Connection::intrusive_ptr& ci) { + ConRec* cr = ci->getContext<ConRec>(); + cr->connection->disconnect(); + cr->data->drainWriteQueue(drained); + delete cr; + cout << "Disconnected: " << cr << "\n"; +} + +void connectionError(Rdma::Connection::intrusive_ptr& ci, Rdma::ErrorType) { + ConRec* cr = ci->getContext<ConRec>(); + cr->connection->disconnect(); + if (cr) { + cr->data->drainWriteQueue(drained); + delete cr; + } + cout << "Connection error: " << cr << "\n"; +} + +bool connectionRequest(Rdma::Connection::intrusive_ptr& ci, const Rdma::ConnectionParams& cp) { + cout << "Incoming connection: "; + + // For fun reject alternate connection attempts + static bool x = false; + x = true; + + // Must create aio here so as to prepost buffers *before* we accept connection + if (x) { + ConRec* cr = new ConRec(ci); + Rdma::AsynchIO* aio = + new Rdma::AsynchIO(ci->getQueuePair(), + cp.rdmaProtocolVersion, + cp.maxRecvBufferSize, cp.initialXmitCredit, Rdma::DEFAULT_WR_ENTRIES, + boost::bind(data, cr, _1, _2), + boost::bind(idle, cr, _1), + boost::bind(full, cr, _1, _2), + dataError); + ci->addContext(cr); + cr->data = aio; + cout << "Accept=>" << cr << "\n"; + } else { + cout << "Reject\n"; + } + + return x; +} + +void connected(Poller::shared_ptr poller, Rdma::Connection::intrusive_ptr& ci) { + static int cnt = 0; + ConRec* cr = ci->getContext<ConRec>(); + cout << "Connected: " << cr << "(" << ++cnt << ")\n"; + + cr->data->start(poller); +} + +}} // namespace qpid::tests + +using namespace qpid::tests; + +int main(int argc, char* argv[]) { + vector<string> args(&argv[0], &argv[argc]); + + std::string port = (args.size() < 2) ? "20079" : args[1]; + cout << "Listening on port: " << port << "\n"; + + try { + boost::shared_ptr<Poller> p(new Poller()); + + Rdma::Listener a( + Rdma::ConnectionParams(16384, Rdma::DEFAULT_WR_ENTRIES), + boost::bind(connected, p, _1), + connectionError, + disconnected, + connectionRequest); + + + SocketAddress sa("", port); + a.start(p, sa); + + // The poller loop blocks all signals so run in its own thread + Thread t(*p); + + ::pause(); + p->shutdown(); + t.join(); + } catch (Rdma::Exception& e) { + int err = e.getError(); + cerr << "Error: " << e.what() << "(" << err << ")\n"; + } +} diff --git a/qpid/cpp/src/qpid/sys/rdma/rdma_exception.h b/qpid/cpp/src/qpid/sys/rdma/rdma_exception.h new file mode 100644 index 0000000000..a3a289e38a --- /dev/null +++ b/qpid/cpp/src/qpid/sys/rdma/rdma_exception.h @@ -0,0 +1,69 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef RDMA_EXCEPTION_H +#define RDMA_EXCEPTION_H + +#include <exception> + +#include <errno.h> +#include <string.h> + +namespace Rdma { + static __thread char s[50]; + class Exception : public std::exception { + int err; + + public: + Exception(int e) : err(e) {} + int getError() { return err; } + const char* what() const throw() { + return ::strerror_r(err, s, 50); + } + }; + + inline void THROW_ERRNO() { + throw Rdma::Exception(errno); + } + + inline void CHECK(int rc) { + if (rc != 0) + throw Rdma::Exception((rc == -1) ? errno : rc >0 ? rc : -rc); + } + + inline int GETERR(int rc) { + return (rc == -1) ? errno : rc > 0 ? rc : -rc; + } + + inline void CHECK_IBV(int rc) { + if (rc != 0) + throw Rdma::Exception(rc); + } + + template <typename T> + inline + T* CHECK_NULL(T* rc) { + if (rc == 0) + THROW_ERRNO(); + return rc; + } +} + +#endif // RDMA_EXCEPTION_H diff --git a/qpid/cpp/src/qpid/sys/rdma/rdma_factories.cpp b/qpid/cpp/src/qpid/sys/rdma/rdma_factories.cpp new file mode 100644 index 0000000000..a66f5b4035 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/rdma/rdma_factories.cpp @@ -0,0 +1,105 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/rdma/rdma_factories.h" + +#include "qpid/sys/rdma/rdma_exception.h" + + +namespace Rdma { + // Intentionally ignore return values for these functions + // - we can't do anything about then anyway + void acker(::rdma_cm_event* e) throw () { + if (e) (void) ::rdma_ack_cm_event(e); + } + + void destroyEChannel(::rdma_event_channel* c) throw () { + if (c) (void) ::rdma_destroy_event_channel(c); + } + + void destroyId(::rdma_cm_id* i) throw () { + if (i) (void) ::rdma_destroy_id(i); + } + + void deallocPd(::ibv_pd* p) throw () { + if (p) (void) ::ibv_dealloc_pd(p); + } + + void deregMr(::ibv_mr* mr) throw () { + if (mr) (void) ::ibv_dereg_mr(mr); + } + + void destroyCChannel(::ibv_comp_channel* c) throw () { + if (c) (void) ::ibv_destroy_comp_channel(c); + } + + void destroyCq(::ibv_cq* cq) throw () { + if (cq) (void) ::ibv_destroy_cq(cq); + } + + void destroyQp(::ibv_qp* qp) throw () { + if (qp) (void) ::ibv_destroy_qp(qp); + } + + boost::shared_ptr< ::rdma_cm_id > mkId(::rdma_cm_id* i) { + return boost::shared_ptr< ::rdma_cm_id >(i, destroyId); + } + + boost::shared_ptr< ::rdma_cm_event > mkEvent(::rdma_cm_event* e) { + return boost::shared_ptr< ::rdma_cm_event >(e, acker); + } + + boost::shared_ptr< ::ibv_qp > mkQp(::ibv_qp* qp) { + return boost::shared_ptr< ::ibv_qp > (qp, destroyQp); + } + + boost::shared_ptr< ::rdma_event_channel > mkEChannel() { + ::rdma_event_channel* c = CHECK_NULL(::rdma_create_event_channel()); + return boost::shared_ptr< ::rdma_event_channel >(c, destroyEChannel); + } + + boost::shared_ptr< ::rdma_cm_id > + mkId(::rdma_event_channel* ec, void* context, ::rdma_port_space ps) { + ::rdma_cm_id* i; + CHECK(::rdma_create_id(ec, &i, context, ps)); + return mkId(i); + } + + boost::shared_ptr< ::ibv_pd > allocPd(::ibv_context* c) { + ::ibv_pd* pd = CHECK_NULL(::ibv_alloc_pd(c)); + return boost::shared_ptr< ::ibv_pd >(pd, deallocPd); + } + + boost::shared_ptr< ::ibv_mr > regMr(::ibv_pd* pd, void* addr, size_t length, ::ibv_access_flags access) { + ::ibv_mr* mr = CHECK_NULL(::ibv_reg_mr(pd, addr, length, access)); + return boost::shared_ptr< ::ibv_mr >(mr, deregMr); + } + + boost::shared_ptr< ::ibv_comp_channel > mkCChannel(::ibv_context* c) { + ::ibv_comp_channel* cc = CHECK_NULL(::ibv_create_comp_channel(c)); + return boost::shared_ptr< ::ibv_comp_channel >(cc, destroyCChannel); + } + + boost::shared_ptr< ::ibv_cq > + mkCq(::ibv_context* c, int cqe, void* context, ::ibv_comp_channel* cc) { + ::ibv_cq* cq = CHECK_NULL(::ibv_create_cq(c, cqe, context, cc, 0)); + return boost::shared_ptr< ::ibv_cq >(cq, destroyCq); + } +} diff --git a/qpid/cpp/src/qpid/sys/rdma/rdma_factories.h b/qpid/cpp/src/qpid/sys/rdma/rdma_factories.h new file mode 100644 index 0000000000..bfca71fc7e --- /dev/null +++ b/qpid/cpp/src/qpid/sys/rdma/rdma_factories.h @@ -0,0 +1,40 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef RDMA_FACTORIES_H +#define RDMA_FACTORIES_H + +#include <rdma/rdma_cma.h> + +#include <boost/shared_ptr.hpp> + +namespace Rdma { + boost::shared_ptr< ::rdma_event_channel > mkEChannel(); + boost::shared_ptr< ::rdma_cm_id > mkId(::rdma_event_channel* ec, void* context, ::rdma_port_space ps); + boost::shared_ptr< ::rdma_cm_id > mkId(::rdma_cm_id* i); + boost::shared_ptr< ::rdma_cm_event > mkEvent(::rdma_cm_event* e); + boost::shared_ptr< ::ibv_qp > mkQp(::ibv_qp* qp); + boost::shared_ptr< ::ibv_pd > allocPd(::ibv_context* c); + boost::shared_ptr< ::ibv_mr > regMr(::ibv_pd* pd, void* addr, size_t length, ::ibv_access_flags access); + boost::shared_ptr< ::ibv_comp_channel > mkCChannel(::ibv_context* c); + boost::shared_ptr< ::ibv_cq > mkCq(::ibv_context* c, int cqe, void* context, ::ibv_comp_channel* cc); +} + +#endif // RDMA_FACTORIES_H diff --git a/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.cpp b/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.cpp new file mode 100644 index 0000000000..efe454c5be --- /dev/null +++ b/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.cpp @@ -0,0 +1,566 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/rdma/rdma_wrap.h" + +#include "qpid/sys/rdma/rdma_factories.h" +#include "qpid/sys/rdma/rdma_exception.h" + +#include "qpid/sys/posix/PrivatePosix.h" + +#include <fcntl.h> +#include <netdb.h> + +#include <iostream> +#include <stdexcept> + +namespace Rdma { + const ::rdma_conn_param DEFAULT_CONNECT_PARAM = { + 0, // .private_data + 0, // .private_data_len + 4, // .responder_resources + 4, // .initiator_depth + 0, // .flow_control + 5, // .retry_count + 7 // .rnr_retry_count + }; + + // This is moderately inefficient so don't use in a critical path + int deviceCount() { + int count; + ::ibv_free_device_list(::ibv_get_device_list(&count)); + return count; + } + + Buffer::Buffer(uint32_t lkey, char* bytes, const int32_t byteCount, + const int32_t reserve) : + bufferSize(byteCount + reserve), reserved(reserve) + { + sge.addr = (uintptr_t) bytes; + sge.length = 0; + sge.lkey = lkey; + } + + QueuePairEvent::QueuePairEvent() : + dir(NONE) + {} + + QueuePairEvent::QueuePairEvent( + const ::ibv_wc& w, + boost::shared_ptr< ::ibv_cq > c, + QueueDirection d) : + cq(c), + wc(w), + dir(d) + { + assert(dir != NONE); + } + + QueuePairEvent::operator bool() const { + return dir != NONE; + } + + bool QueuePairEvent::immPresent() const { + return wc.wc_flags & IBV_WC_WITH_IMM; + } + + uint32_t QueuePairEvent::getImm() const { + return ntohl(wc.imm_data); + } + + QueueDirection QueuePairEvent::getDirection() const { + return dir; + } + + ::ibv_wc_opcode QueuePairEvent::getEventType() const { + return wc.opcode; + } + + ::ibv_wc_status QueuePairEvent::getEventStatus() const { + return wc.status; + } + + Buffer* QueuePairEvent::getBuffer() const { + Buffer* b = reinterpret_cast<Buffer*>(wc.wr_id); + b->dataCount(wc.byte_len); + return b; + } + + QueuePair::QueuePair(boost::shared_ptr< ::rdma_cm_id > i) : + qpid::sys::IOHandle(new qpid::sys::IOHandlePrivate), + pd(allocPd(i->verbs)), + cchannel(mkCChannel(i->verbs)), + scq(mkCq(i->verbs, DEFAULT_CQ_ENTRIES, 0, cchannel.get())), + rcq(mkCq(i->verbs, DEFAULT_CQ_ENTRIES, 0, cchannel.get())), + outstandingSendEvents(0), + outstandingRecvEvents(0) + { + impl->fd = cchannel->fd; + + // Set cq context to this QueuePair object so we can find + // ourselves again + scq->cq_context = this; + rcq->cq_context = this; + + ::ibv_device_attr dev_attr; + CHECK(::ibv_query_device(i->verbs, &dev_attr)); + + ::ibv_qp_init_attr qp_attr = {}; + + // TODO: make a default struct for this + qp_attr.cap.max_send_wr = DEFAULT_WR_ENTRIES; + qp_attr.cap.max_send_sge = 1; + qp_attr.cap.max_recv_wr = DEFAULT_WR_ENTRIES; + qp_attr.cap.max_recv_sge = 1; + + qp_attr.send_cq = scq.get(); + qp_attr.recv_cq = rcq.get(); + qp_attr.qp_type = IBV_QPT_RC; + + CHECK(::rdma_create_qp(i.get(), pd.get(), &qp_attr)); + qp = mkQp(i->qp); + + // Set the qp context to this so we can find ourselves again + qp->qp_context = this; + } + + QueuePair::~QueuePair() { + // Reset back pointer in case someone else has the qp + qp->qp_context = 0; + + // Dispose queue pair before we ack events + qp.reset(); + + if (outstandingSendEvents > 0) + ::ibv_ack_cq_events(scq.get(), outstandingSendEvents); + if (outstandingRecvEvents > 0) + ::ibv_ack_cq_events(rcq.get(), outstandingRecvEvents); + + // Deallocate recv buffer memory + if (rmr) delete [] static_cast<char*>(rmr->addr); + + // Deallocate recv buffer memory + if (smr) delete [] static_cast<char*>(smr->addr); + + // The buffers vectors automatically deletes all the buffers we've allocated + } + + // Create buffers to use for writing + void QueuePair::createSendBuffers(int sendBufferCount, int bufferSize, int reserved) + { + assert(!smr); + + // Round up buffersize to cacheline (64 bytes) + int dataLength = (bufferSize+reserved+63) & (~63); + + // Allocate memory block for all receive buffers + char* mem = new char [sendBufferCount * dataLength]; + smr = regMr(pd.get(), mem, sendBufferCount * dataLength, ::IBV_ACCESS_LOCAL_WRITE); + sendBuffers.reserve(sendBufferCount); + freeBuffers.reserve(sendBufferCount); + for (int i = 0; i<sendBufferCount; ++i) { + // Allocate xmit buffer + sendBuffers.push_back(Buffer(smr->lkey, &mem[i*dataLength], bufferSize, reserved)); + freeBuffers.push_back(i); + } + } + + Buffer* QueuePair::getSendBuffer() { + qpid::sys::ScopedLock<qpid::sys::Mutex> l(bufferLock); + if (freeBuffers.empty()) + return 0; + int i = freeBuffers.back(); + freeBuffers.pop_back(); + assert(i >= 0 && i < int(sendBuffers.size())); + Buffer* b = &sendBuffers[i]; + b->dataCount(0); + return b; + } + + void QueuePair::returnSendBuffer(Buffer* b) { + qpid::sys::ScopedLock<qpid::sys::Mutex> l(bufferLock); + int i = b - &sendBuffers[0]; + assert(i >= 0 && i < int(sendBuffers.size())); + freeBuffers.push_back(i); + } + + void QueuePair::allocateRecvBuffers(int recvBufferCount, int bufferSize) + { + assert(!rmr); + + // Round up buffersize to cacheline (64 bytes) + bufferSize = (bufferSize+63) & (~63); + + // Allocate memory block for all receive buffers + char* mem = new char [recvBufferCount * bufferSize]; + rmr = regMr(pd.get(), mem, recvBufferCount * bufferSize, ::IBV_ACCESS_LOCAL_WRITE); + recvBuffers.reserve(recvBufferCount); + for (int i = 0; i<recvBufferCount; ++i) { + // Allocate recv buffer + recvBuffers.push_back(Buffer(rmr->lkey, &mem[i*bufferSize], bufferSize)); + postRecv(&recvBuffers[i]); + } + } + + // Make channel non-blocking by making + // associated fd nonblocking + void QueuePair::nonblocking() { + ::fcntl(cchannel->fd, F_SETFL, O_NONBLOCK); + } + + // If we get EAGAIN because the channel has been set non blocking + // and we'd have to wait then return an empty event + QueuePair::intrusive_ptr QueuePair::getNextChannelEvent() { + // First find out which cq has the event + ::ibv_cq* cq; + void* ctx; + int rc = ::ibv_get_cq_event(cchannel.get(), &cq, &ctx); + if (rc == -1 && errno == EAGAIN) + return 0; + CHECK(rc); + + // Batch acknowledge the event + if (cq == scq.get()) { + if (++outstandingSendEvents > DEFAULT_CQ_ENTRIES / 2) { + ::ibv_ack_cq_events(cq, outstandingSendEvents); + outstandingSendEvents = 0; + } + } else if (cq == rcq.get()) { + if (++outstandingRecvEvents > DEFAULT_CQ_ENTRIES / 2) { + ::ibv_ack_cq_events(cq, outstandingRecvEvents); + outstandingRecvEvents = 0; + } + } + + return static_cast<QueuePair*>(ctx); + } + + QueuePairEvent QueuePair::getNextEvent() { + ::ibv_wc w; + if (::ibv_poll_cq(scq.get(), 1, &w) == 1) + return QueuePairEvent(w, scq, SEND); + else if (::ibv_poll_cq(rcq.get(), 1, &w) == 1) + return QueuePairEvent(w, rcq, RECV); + else + return QueuePairEvent(); + } + + void QueuePair::notifyRecv() { + CHECK_IBV(ibv_req_notify_cq(rcq.get(), 0)); + } + + void QueuePair::notifySend() { + CHECK_IBV(ibv_req_notify_cq(scq.get(), 0)); + } + + void QueuePair::postRecv(Buffer* buf) { + ::ibv_recv_wr rwr = {}; + + rwr.wr_id = reinterpret_cast<uint64_t>(buf); + // We are given the whole buffer + buf->dataCount(buf->byteCount()); + rwr.sg_list = &buf->sge; + rwr.num_sge = 1; + + ::ibv_recv_wr* badrwr = 0; + CHECK(::ibv_post_recv(qp.get(), &rwr, &badrwr)); + if (badrwr) + throw std::logic_error("ibv_post_recv(): Bad rwr"); + } + + void QueuePair::postSend(Buffer* buf) { + ::ibv_send_wr swr = {}; + + swr.wr_id = reinterpret_cast<uint64_t>(buf); + swr.opcode = IBV_WR_SEND; + swr.send_flags = IBV_SEND_SIGNALED; + swr.sg_list = &buf->sge; + swr.num_sge = 1; + + ::ibv_send_wr* badswr = 0; + CHECK(::ibv_post_send(qp.get(), &swr, &badswr)); + if (badswr) + throw std::logic_error("ibv_post_send(): Bad swr"); + } + + void QueuePair::postSend(uint32_t imm, Buffer* buf) { + ::ibv_send_wr swr = {}; + + swr.wr_id = reinterpret_cast<uint64_t>(buf); + swr.imm_data = htonl(imm); + swr.opcode = IBV_WR_SEND_WITH_IMM; + swr.send_flags = IBV_SEND_SIGNALED; + swr.sg_list = &buf->sge; + swr.num_sge = 1; + + ::ibv_send_wr* badswr = 0; + CHECK(::ibv_post_send(qp.get(), &swr, &badswr)); + if (badswr) + throw std::logic_error("ibv_post_send(): Bad swr"); + } + + ConnectionEvent::ConnectionEvent(::rdma_cm_event* e) : + id((e->event != RDMA_CM_EVENT_CONNECT_REQUEST) ? + Connection::find(e->id) : new Connection(e->id)), + listen_id(Connection::find(e->listen_id)), + event(mkEvent(e)) + {} + + ConnectionEvent::operator bool() const { + return event; + } + + ::rdma_cm_event_type ConnectionEvent::getEventType() const { + return event->event; + } + + ::rdma_conn_param ConnectionEvent::getConnectionParam() const { + // It's badly documented, but it seems from the librdma source code that all the following + // event types have a valid param.conn + switch (event->event) { + case RDMA_CM_EVENT_CONNECT_REQUEST: + case RDMA_CM_EVENT_ESTABLISHED: + case RDMA_CM_EVENT_REJECTED: + case RDMA_CM_EVENT_DISCONNECTED: + case RDMA_CM_EVENT_CONNECT_ERROR: + return event->param.conn; + default: + ::rdma_conn_param p = {}; + return p; + } + } + + boost::intrusive_ptr<Connection> ConnectionEvent::getConnection () const { + return id; + } + + boost::intrusive_ptr<Connection> ConnectionEvent::getListenId() const { + return listen_id; + } + + // Wrap the passed in rdma_cm_id with a Connection + // this basically happens only on connection request + Connection::Connection(::rdma_cm_id* i) : + qpid::sys::IOHandle(new qpid::sys::IOHandlePrivate), + id(mkId(i)), + context(0) + { + impl->fd = id->channel->fd; + + // Just overwrite the previous context as it will + // have come from the listening connection + if (i) + i->context = this; + } + + Connection::Connection() : + qpid::sys::IOHandle(new qpid::sys::IOHandlePrivate), + channel(mkEChannel()), + id(mkId(channel.get(), this, RDMA_PS_TCP)), + context(0) + { + impl->fd = channel->fd; + } + + Connection::~Connection() { + // Reset the id context in case someone else has it + id->context = 0; + } + + void Connection::ensureQueuePair() { + assert(id.get()); + + // Only allocate a queue pair if there isn't one already + if (qp) + return; + + qp = new QueuePair(id); + } + + Connection::intrusive_ptr Connection::make() { + return new Connection(); + } + + Connection::intrusive_ptr Connection::find(::rdma_cm_id* i) { + if (!i) + return 0; + Connection* id = static_cast< Connection* >(i->context); + if (!id) + throw std::logic_error("Couldn't find existing Connection"); + return id; + } + + // Make channel non-blocking by making + // associated fd nonblocking + void Connection::nonblocking() { + assert(id.get()); + ::fcntl(id->channel->fd, F_SETFL, O_NONBLOCK); + } + + // If we get EAGAIN because the channel has been set non blocking + // and we'd have to wait then return an empty event + ConnectionEvent Connection::getNextEvent() { + assert(id.get()); + ::rdma_cm_event* e; + int rc = ::rdma_get_cm_event(id->channel, &e); + if (GETERR(rc) == EAGAIN) + return ConnectionEvent(); + CHECK(rc); + return ConnectionEvent(e); + } + + void Connection::bind(const qpid::sys::SocketAddress& src_addr) const { + assert(id.get()); + CHECK(::rdma_bind_addr(id.get(), getAddrInfo(src_addr).ai_addr)); + } + + void Connection::listen(int backlog) const { + assert(id.get()); + CHECK(::rdma_listen(id.get(), backlog)); + } + + void Connection::resolve_addr( + const qpid::sys::SocketAddress& dst_addr, + int timeout_ms) const + { + assert(id.get()); + CHECK(::rdma_resolve_addr(id.get(), 0, getAddrInfo(dst_addr).ai_addr, timeout_ms)); + } + + void Connection::resolve_route(int timeout_ms) const { + assert(id.get()); + CHECK(::rdma_resolve_route(id.get(), timeout_ms)); + } + + void Connection::disconnect() const { + assert(id.get()); + int rc = ::rdma_disconnect(id.get()); + // iWarp doesn't let you disconnect a disconnected connection + // but Infiniband can do so it's okay to call rdma_disconnect() + // in response to a disconnect event, but we may get an error + if (GETERR(rc) == EINVAL) + return; + CHECK(rc); + } + + // TODO: Currently you can only connect with the default connection parameters + void Connection::connect(const void* data, size_t len) { + assert(id.get()); + // Need to have a queue pair before we can connect + ensureQueuePair(); + + ::rdma_conn_param p = DEFAULT_CONNECT_PARAM; + p.private_data = data; + p.private_data_len = len; + CHECK(::rdma_connect(id.get(), &p)); + } + + void Connection::connect() { + connect(0, 0); + } + + void Connection::accept(const ::rdma_conn_param& param, const void* data, size_t len) { + assert(id.get()); + // Need to have a queue pair before we can accept + ensureQueuePair(); + + ::rdma_conn_param p = param; + p.private_data = data; + p.private_data_len = len; + CHECK(::rdma_accept(id.get(), &p)); + } + + void Connection::accept(const ::rdma_conn_param& param) { + accept(param, 0, 0); + } + + void Connection::reject(const void* data, size_t len) const { + assert(id.get()); + CHECK(::rdma_reject(id.get(), data, len)); + } + + void Connection::reject() const { + assert(id.get()); + CHECK(::rdma_reject(id.get(), 0, 0)); + } + + QueuePair::intrusive_ptr Connection::getQueuePair() { + assert(id.get()); + + ensureQueuePair(); + + return qp; + } + + std::string Connection::getLocalName() const { + ::sockaddr* addr = ::rdma_get_local_addr(id.get()); + char hostName[NI_MAXHOST]; + char portName[NI_MAXSERV]; + CHECK_IBV(::getnameinfo( + addr, sizeof(::sockaddr_storage), + hostName, sizeof(hostName), + portName, sizeof(portName), + NI_NUMERICHOST | NI_NUMERICSERV)); + std::string r(hostName); + r += ":"; + r += portName; + return r; + } + + std::string Connection::getPeerName() const { + ::sockaddr* addr = ::rdma_get_peer_addr(id.get()); + char hostName[NI_MAXHOST]; + char portName[NI_MAXSERV]; + CHECK_IBV(::getnameinfo( + addr, sizeof(::sockaddr_storage), + hostName, sizeof(hostName), + portName, sizeof(portName), + NI_NUMERICHOST | NI_NUMERICSERV)); + std::string r(hostName); + r += ":"; + r += portName; + return r; + } +} + +std::ostream& operator<<(std::ostream& o, ::rdma_cm_event_type t) { +# define CHECK_TYPE(t) case t: o << #t; break; + switch(t) { + CHECK_TYPE(RDMA_CM_EVENT_ADDR_RESOLVED) + CHECK_TYPE(RDMA_CM_EVENT_ADDR_ERROR) + CHECK_TYPE(RDMA_CM_EVENT_ROUTE_RESOLVED) + CHECK_TYPE(RDMA_CM_EVENT_ROUTE_ERROR) + CHECK_TYPE(RDMA_CM_EVENT_CONNECT_REQUEST) + CHECK_TYPE(RDMA_CM_EVENT_CONNECT_RESPONSE) + CHECK_TYPE(RDMA_CM_EVENT_CONNECT_ERROR) + CHECK_TYPE(RDMA_CM_EVENT_UNREACHABLE) + CHECK_TYPE(RDMA_CM_EVENT_REJECTED) + CHECK_TYPE(RDMA_CM_EVENT_ESTABLISHED) + CHECK_TYPE(RDMA_CM_EVENT_DISCONNECTED) + CHECK_TYPE(RDMA_CM_EVENT_DEVICE_REMOVAL) + CHECK_TYPE(RDMA_CM_EVENT_MULTICAST_JOIN) + CHECK_TYPE(RDMA_CM_EVENT_MULTICAST_ERROR) + default: + o << "UNKNOWN_EVENT"; + } +# undef CHECK_TYPE + return o; +} diff --git a/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.h b/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.h new file mode 100644 index 0000000000..8e3429027b --- /dev/null +++ b/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.h @@ -0,0 +1,287 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#ifndef RDMA_WRAP_H +#define RDMA_WRAP_H + +#include <rdma/rdma_cma.h> + +#include "qpid/RefCounted.h" +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/Mutex.h" + +#include <boost/shared_ptr.hpp> +#include <boost/intrusive_ptr.hpp> +#include <boost/ptr_container/ptr_deque.hpp> + +#include <vector> + +namespace qpid { +namespace sys { + class SocketAddress; +}} + +namespace Rdma { + const int DEFAULT_TIMEOUT = 2000; // 2 secs + const int DEFAULT_BACKLOG = 100; + const int DEFAULT_CQ_ENTRIES = 256; + const int DEFAULT_WR_ENTRIES = 64; + extern const ::rdma_conn_param DEFAULT_CONNECT_PARAM; + + int deviceCount(); + + struct Buffer { + friend class QueuePair; + friend class QueuePairEvent; + + char* bytes() const; + int32_t byteCount() const; + int32_t dataCount() const; + void dataCount(int32_t); + + private: + Buffer(uint32_t lkey, char* bytes, const int32_t byteCount, const int32_t reserve=0); + int32_t bufferSize; + int32_t reserved; // for framing header + ::ibv_sge sge; + }; + + inline char* Buffer::bytes() const { + return (char*) sge.addr; + } + + /** return the number of bytes available for application data */ + inline int32_t Buffer::byteCount() const { + return bufferSize - reserved; + } + + inline int32_t Buffer::dataCount() const { + return sge.length; + } + + inline void Buffer::dataCount(int32_t s) { + // catch any attempt to overflow a buffer + assert(s <= bufferSize + reserved); + sge.length = s; + } + + class Connection; + + enum QueueDirection { + NONE, + SEND, + RECV + }; + + class QueuePairEvent { + boost::shared_ptr< ::ibv_cq > cq; + ::ibv_wc wc; + QueueDirection dir; + + friend class QueuePair; + + QueuePairEvent(); + QueuePairEvent( + const ::ibv_wc& w, + boost::shared_ptr< ::ibv_cq > c, + QueueDirection d); + + public: + operator bool() const; + bool immPresent() const; + uint32_t getImm() const; + QueueDirection getDirection() const; + ::ibv_wc_opcode getEventType() const; + ::ibv_wc_status getEventStatus() const; + Buffer* getBuffer() const; + }; + + // Wrapper for a queue pair - this has the functionality for + // putting buffers on the receive queue and for sending buffers + // to the other end of the connection. + class QueuePair : public qpid::sys::IOHandle, public qpid::RefCounted { + friend class Connection; + + boost::shared_ptr< ::ibv_pd > pd; + boost::shared_ptr< ::ibv_mr > smr; + boost::shared_ptr< ::ibv_mr > rmr; + boost::shared_ptr< ::ibv_comp_channel > cchannel; + boost::shared_ptr< ::ibv_cq > scq; + boost::shared_ptr< ::ibv_cq > rcq; + boost::shared_ptr< ::ibv_qp > qp; + int outstandingSendEvents; + int outstandingRecvEvents; + std::vector<Buffer> sendBuffers; + std::vector<Buffer> recvBuffers; + qpid::sys::Mutex bufferLock; + std::vector<int> freeBuffers; + + QueuePair(boost::shared_ptr< ::rdma_cm_id > id); + ~QueuePair(); + + public: + typedef boost::intrusive_ptr<QueuePair> intrusive_ptr; + + // Create a buffers to use for writing + void createSendBuffers(int sendBufferCount, int dataSize, int headerSize); + + // Get a send buffer + Buffer* getSendBuffer(); + + // Return buffer to pool after use + void returnSendBuffer(Buffer* b); + + // Create and post recv buffers + void allocateRecvBuffers(int recvBufferCount, int bufferSize); + + // Make channel non-blocking by making + // associated fd nonblocking + void nonblocking(); + + // If we get EAGAIN because the channel has been set non blocking + // and we'd have to wait then return an empty event + QueuePair::intrusive_ptr getNextChannelEvent(); + + QueuePairEvent getNextEvent(); + + void postRecv(Buffer* buf); + void postSend(Buffer* buf); + void postSend(uint32_t imm, Buffer* buf); + void notifyRecv(); + void notifySend(); + }; + + class ConnectionEvent { + friend class Connection; + + // The order of the members is important as we have to acknowledge + // the event before destroying the ids on destruction + boost::intrusive_ptr<Connection> id; + boost::intrusive_ptr<Connection> listen_id; + boost::shared_ptr< ::rdma_cm_event > event; + + ConnectionEvent() {} + ConnectionEvent(::rdma_cm_event* e); + + // Default copy, assignment and destructor ok + public: + operator bool() const; + ::rdma_cm_event_type getEventType() const; + ::rdma_conn_param getConnectionParam() const; + boost::intrusive_ptr<Connection> getConnection () const; + boost::intrusive_ptr<Connection> getListenId() const; + }; + + // For the moment this is a fairly simple wrapper for rdma_cm_id. + // + // NB: It allocates a protection domain (pd) per connection which means that + // registered buffers can't be shared between different connections + // (this can only happen between connections on the same controller in any case, + // so needs careful management if used) + class Connection : public qpid::sys::IOHandle, public qpid::RefCounted { + boost::shared_ptr< ::rdma_event_channel > channel; + boost::shared_ptr< ::rdma_cm_id > id; + QueuePair::intrusive_ptr qp; + + void* context; + + friend class ConnectionEvent; + friend class QueuePair; + + // Wrap the passed in rdma_cm_id with a Connection + // this basically happens only on connection request + Connection(::rdma_cm_id* i); + Connection(); + ~Connection(); + + void ensureQueuePair(); + + public: + typedef boost::intrusive_ptr<Connection> intrusive_ptr; + + static intrusive_ptr make(); + static intrusive_ptr find(::rdma_cm_id* i); + + template <typename T> + void addContext(T* c) { + // Don't allow replacing context + if (!context) + context = c; + } + + void removeContext() { + context = 0; + } + + template <typename T> + T* getContext() { + return static_cast<T*>(context); + } + + // Make channel non-blocking by making + // associated fd nonblocking + void nonblocking(); + + // If we get EAGAIN because the channel has been set non blocking + // and we'd have to wait then return an empty event + ConnectionEvent getNextEvent(); + + void bind(const qpid::sys::SocketAddress& src_addr) const; + void listen(int backlog = DEFAULT_BACKLOG) const; + void resolve_addr( + const qpid::sys::SocketAddress& dst_addr, + int timeout_ms = DEFAULT_TIMEOUT) const; + void resolve_route(int timeout_ms = DEFAULT_TIMEOUT) const; + void disconnect() const; + + // TODO: Currently you can only connect with the default connection parameters + void connect(const void* data, size_t len); + void connect(); + template <typename T> + void connect(const T* data) { + connect(data, sizeof(T)); + } + + // TODO: Not sure how to default accept params - they come from the connection request + // event + void accept(const ::rdma_conn_param& param, const void* data, size_t len); + void accept(const ::rdma_conn_param& param); + template <typename T> + void accept(const ::rdma_conn_param& param, const T* data) { + accept(param, data, sizeof(T)); + } + + void reject(const void* data, size_t len) const; + void reject() const; + template <typename T> + void reject(const T* data) const { + reject(data, sizeof(T)); + } + + QueuePair::intrusive_ptr getQueuePair(); + std::string getLocalName() const; + std::string getPeerName() const; + std::string getFullName() const { return getLocalName()+"-"+getPeerName(); } + }; +} + +std::ostream& operator<<(std::ostream& o, ::rdma_cm_event_type t); + +#endif // RDMA_WRAP_H diff --git a/qpid/cpp/src/qpid/sys/solaris/ECFPoller.cpp b/qpid/cpp/src/qpid/sys/solaris/ECFPoller.cpp new file mode 100644 index 0000000000..06d542c938 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/solaris/ECFPoller.cpp @@ -0,0 +1,444 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/log/Logger.h" +#include "qpid/sys/Poller.h" +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/DeletionManager.h" +#include "qpid/sys/posix/check.h" +#include "qpid/sys/posix/PrivatePosix.h" + +#include <port.h> +#include <poll.h> +#include <errno.h> +#include <pthread.h> +#include <signal.h> + +#include <assert.h> +#include <queue> +#include <exception> + + +//TODO: Remove this +#include "qpid/sys/Dispatcher.h" + +namespace qpid { +namespace sys { + +// Deletion manager to handle deferring deletion of PollerHandles to when they definitely aren't being used +DeletionManager<PollerHandlePrivate> PollerHandleDeletionManager; + +// Instantiate (and define) class static for DeletionManager +template <> +DeletionManager<PollerHandlePrivate>::AllThreadsStatuses DeletionManager<PollerHandlePrivate>::allThreadsStatuses(0); + +class PollerHandlePrivate { + friend class Poller; + friend class PollerHandle; + + enum FDStat { + ABSENT, + MONITORED, + INACTIVE, + HUNGUP, + MONITORED_HUNGUP, + DELETED + }; + + int fd; + uint32_t events; + FDStat stat; + Mutex lock; + + PollerHandlePrivate(int f) : + fd(f), + events(0), + stat(ABSENT) { + } + + bool isActive() const { + return stat == MONITORED || stat == MONITORED_HUNGUP; + } + + void setActive() { + stat = (stat == HUNGUP) ? MONITORED_HUNGUP : MONITORED; + } + + bool isInactive() const { + return stat == INACTIVE || stat == HUNGUP; + } + + void setInactive() { + stat = INACTIVE; + } + + bool isIdle() const { + return stat == ABSENT; + } + + void setIdle() { + stat = ABSENT; + } + + bool isHungup() const { + return stat == MONITORED_HUNGUP || stat == HUNGUP; + } + + void setHungup() { + assert(stat == MONITORED); + stat = HUNGUP; + } + + bool isDeleted() const { + return stat == DELETED; + } + + void setDeleted() { + stat = DELETED; + } +}; + +PollerHandle::PollerHandle(const IOHandle& h) : + impl(new PollerHandlePrivate(toFd(h.impl))) +{} + +PollerHandle::~PollerHandle() { + { + ScopedLock<Mutex> l(impl->lock); + if (impl->isDeleted()) { + return; + } + if (impl->isActive()) { + impl->setDeleted(); + } + } + PollerHandleDeletionManager.markForDeletion(impl); +} + +/** + * Concrete implementation of Poller to use the Solaris Event Completion + * Framework interface + */ +class PollerPrivate { + friend class Poller; + + class InterruptHandle: public PollerHandle { + std::queue<PollerHandle*> handles; + + void processEvent(Poller::EventType) { + PollerHandle* handle = handles.front(); + handles.pop(); + assert(handle); + + //Synthesise event + Poller::Event event(handle, Poller::INTERRUPTED); + + //Process synthesised event + event.process(); + } + + public: + InterruptHandle() : PollerHandle(DummyIOHandle) {} + + void addHandle(PollerHandle& h) { + handles.push(&h); + } + + PollerHandle *getHandle() { + PollerHandle* handle = handles.front(); + handles.pop(); + return handle; + } + + bool queuedHandles() { + return handles.size() > 0; + } + }; + + const int portId; + bool isShutdown; + InterruptHandle interruptHandle; + + static uint32_t directionToPollEvent(Poller::Direction dir) { + switch (dir) { + case Poller::INPUT: return POLLIN; + case Poller::OUTPUT: return POLLOUT; + case Poller::INOUT: return POLLIN | POLLOUT; + default: return 0; + } + } + + static Poller::EventType pollToDirection(uint32_t events) { + uint32_t e = events & (POLLIN | POLLOUT); + switch (e) { + case POLLIN: return Poller::READABLE; + case POLLOUT: return Poller::WRITABLE; + case POLLIN | POLLOUT: return Poller::READ_WRITABLE; + default: + return (events & (POLLHUP | POLLERR)) ? + Poller::DISCONNECTED : Poller::INVALID; + } + } + + PollerPrivate() : + portId(::port_create()), + isShutdown(false) { + QPID_POSIX_CHECK(portId); + QPID_LOG(trace, "port_create returned port Id: " << portId); + } + + ~PollerPrivate() { + } + + void interrupt() { + //Send an Alarm to the port + //We need to send a nonzero event mask, using POLLHUP, + //nevertheless the wait method will only look for a PORT_ALERT_SET + QPID_LOG(trace, "Sending a port_alert to " << portId); + QPID_POSIX_CHECK(::port_alert(portId, PORT_ALERT_SET, POLLHUP, + &static_cast<PollerHandle&>(interruptHandle))); + } +}; + +void Poller::addFd(PollerHandle& handle, Direction dir) { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + + uint32_t events = 0; + + if (eh.isIdle()) { + events = PollerPrivate::directionToPollEvent(dir); + } else { + assert(eh.isActive()); + events = eh.events | PollerPrivate::directionToPollEvent(dir); + } + + //port_associate can be used to add an association or modify an + //existing one + QPID_POSIX_CHECK(::port_associate(impl->portId, PORT_SOURCE_FD, (uintptr_t) eh.fd, events, &handle)); + eh.events = events; + eh.setActive(); + QPID_LOG(trace, "Poller::addFd(handle=" << &handle + << "[" << typeid(&handle).name() + << "], fd=" << eh.fd << ")"); +} + +void Poller::delFd(PollerHandle& handle) { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + assert(!eh.isIdle()); + int rc = ::port_dissociate(impl->portId, PORT_SOURCE_FD, (uintptr_t) eh.fd); + //Allow closing an invalid fd, allowing users to close fd before + //doing delFd() + if (rc == -1 && errno != EBADFD) { + QPID_POSIX_CHECK(rc); + } + eh.setIdle(); + QPID_LOG(trace, "Poller::delFd(handle=" << &handle + << ", fd=" << eh.fd << ")"); +} + +// modFd is equivalent to delFd followed by addFd +void Poller::modFd(PollerHandle& handle, Direction dir) { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + assert(!eh.isIdle()); + + eh.events = PollerPrivate::directionToPollEvent(dir); + + //If fd is already associated, events and user arguments are updated + //So, no need to check if fd is already associated + QPID_POSIX_CHECK(::port_associate(impl->portId, PORT_SOURCE_FD, (uintptr_t) eh.fd, eh.events, &handle)); + eh.setActive(); + QPID_LOG(trace, "Poller::modFd(handle=" << &handle + << ", fd=" << eh.fd << ")"); +} + +void Poller::rearmFd(PollerHandle& handle) { + PollerHandlePrivate& eh = *handle.impl; + ScopedLock<Mutex> l(eh.lock); + assert(eh.isInactive()); + + QPID_POSIX_CHECK(::port_associate(impl->portId, PORT_SOURCE_FD, (uintptr_t) eh.fd, eh.events, &handle)); + eh.setActive(); + QPID_LOG(trace, "Poller::rearmdFd(handle=" << &handle + << ", fd=" << eh.fd << ")"); +} + +void Poller::shutdown() { + //Allow sloppy code to shut us down more than once + if (impl->isShutdown) + return; + + impl->isShutdown = true; + impl->interrupt(); +} + +bool Poller::hasShutdown() +{ + return impl->isShutdown; +} + +bool Poller::interrupt(PollerHandle& handle) { + PollerPrivate::InterruptHandle& ih = impl->interruptHandle; + PollerHandlePrivate& eh = *static_cast<PollerHandle&>(ih).impl; + ScopedLock<Mutex> l(eh.lock); + ih.addHandle(handle); + impl->interrupt(); + eh.setActive(); + return true; +} + +void Poller::run() { + // Make sure we can't be interrupted by signals at a bad time + ::sigset_t ss; + ::sigfillset(&ss); + ::pthread_sigmask(SIG_SETMASK, &ss, 0); + + do { + Event event = wait(); + + // If can read/write then dispatch appropriate callbacks + if (event.handle) { + event.process(); + } else { + // Handle shutdown + switch (event.type) { + case SHUTDOWN: + return; + default: + // This should be impossible + assert(false); + } + } + } while (true); +} + +Poller::Event Poller::wait(Duration timeout) { + timespec_t tout; + timespec_t* ptout = NULL; + port_event_t pe; + + AbsTime targetTimeout = (timeout == TIME_INFINITE) ? FAR_FUTURE : + AbsTime(now(), timeout); + + if (timeout != TIME_INFINITE) { + tout.tv_sec = 0; + tout.tv_nsec = timeout; + ptout = &tout; + } + + do { + PollerHandleDeletionManager.markAllUnusedInThisThread(); + QPID_LOG(trace, "About to enter port_get on " << impl->portId + << ". Thread " << pthread_self() + << ", timeout=" << timeout); + + + int rc = ::port_get(impl->portId, &pe, ptout); + + QPID_LOG(trace, "port_get on " << impl->portId + << " returned " << rc); + + if (impl->isShutdown) { + PollerHandleDeletionManager.markAllUnusedInThisThread(); + return Event(0, SHUTDOWN); + } + + if (rc < 0) { + switch (errno) { + case EINTR: + continue; + case ETIME: + return Event(0, TIMEOUT); + default: + QPID_POSIX_CHECK(rc); + } + } else { + PollerHandle* handle = static_cast<PollerHandle*>(pe.portev_user); + PollerHandlePrivate& eh = *handle->impl; + ScopedLock<Mutex> l(eh.lock); + + if (eh.isActive()) { + QPID_LOG(trace, "Handle is active"); + //We use alert mode to notify interrupts + if (pe.portev_source == PORT_SOURCE_ALERT && + handle == &impl->interruptHandle) { + QPID_LOG(trace, "Interrupt notified"); + + PollerHandle* wrappedHandle = impl->interruptHandle.getHandle(); + + if (impl->interruptHandle.queuedHandles()) { + impl->interrupt(); + eh.setActive(); + } else { + eh.setInactive(); + } + return Event(wrappedHandle, INTERRUPTED); + } + + if (pe.portev_source == PORT_SOURCE_FD) { + QPID_LOG(trace, "About to send handle: " << handle); + if (pe.portev_events & POLLHUP) { + if (eh.isHungup()) { + return Event(handle, DISCONNECTED); + } + eh.setHungup(); + } else { + eh.setInactive(); + } + QPID_LOG(trace, "Sending event (thread: " + << pthread_self() << ") for handle " << handle + << ", direction= " + << PollerPrivate::pollToDirection(pe.portev_events)); + return Event(handle, PollerPrivate::pollToDirection(pe.portev_events)); + } + } else if (eh.isDeleted()) { + //Remove the handle from the poller + int rc = ::port_dissociate(impl->portId, PORT_SOURCE_FD, + (uintptr_t) eh.fd); + if (rc == -1 && errno != EBADFD) { + QPID_POSIX_CHECK(rc); + } + } + } + + if (timeout == TIME_INFINITE) { + continue; + } + if (rc == 0 && now() > targetTimeout) { + PollerHandleDeletionManager.markAllUnusedInThisThread(); + return Event(0, TIMEOUT); + } + } while (true); +} + +// Concrete constructors +Poller::Poller() : + impl(new PollerPrivate()) +{} + +Poller::~Poller() { + delete impl; +} + +}} diff --git a/qpid/cpp/src/qpid/sys/solaris/SystemInfo.cpp b/qpid/cpp/src/qpid/sys/solaris/SystemInfo.cpp new file mode 100755 index 0000000000..765e5a7eb0 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/solaris/SystemInfo.cpp @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/SystemInfo.h" + +#define BSD_COMP +#include <sys/ioctl.h> +#include <netdb.h> +#undef BDS_COMP + + +#include <unistd.h> +#include <net/if.h> +#include <sys/types.h> +#include <sys/utsname.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <stdio.h> +#include <errno.h> +#include <limits.h> +#include <procfs.h> +#include <fcntl.h> +#include <sys/types.h> + +using namespace std; + +namespace qpid { +namespace sys { + +long SystemInfo::concurrency() { + return sysconf(_SC_NPROCESSORS_ONLN); +} + +bool SystemInfo::getLocalHostname(Address &address) { + char name[MAXHOSTNAMELEN]; + if (::gethostname(name, sizeof(name)) != 0) + return false; + address.host = name; + return true; +} + +static const string LOCALHOST("127.0.0.1"); +static const string TCP("tcp"); + +void SystemInfo::getLocalIpAddresses(uint16_t port, + std::vector<Address> &addrList) { + int s = socket(PF_INET, SOCK_STREAM, 0); + for (int i=1;;i++) { + struct lifreq ifr; + ifr.lifr_index = i; + if (::ioctl(s, SIOCGIFADDR, &ifr) < 0) { + break; + } + struct sockaddr_in *sin = (struct sockaddr_in *) &ifr.lifr_addr; + std::string addr(inet_ntoa(sin->sin_addr)); + if (addr != LOCALHOST) + addrList.push_back(Address(TCP, addr, port)); + } + if (addrList.empty()) { + addrList.push_back(Address(TCP, LOCALHOST, port)); + } + close (s); +} + +void SystemInfo::getSystemId(std::string &osName, + std::string &nodeName, + std::string &release, + std::string &version, + std::string &machine) { + struct utsname _uname; + if (uname (&_uname) == 0) { + osName = _uname.sysname; + nodeName = _uname.nodename; + release = _uname.release; + version = _uname.version; + machine = _uname.machine; + } +} + +uint32_t SystemInfo::getProcessId() +{ + return (uint32_t) ::getpid(); +} + +uint32_t SystemInfo::getParentProcessId() +{ + return (uint32_t) ::getppid(); +} + +string SystemInfo::getProcessName() +{ + psinfo processInfo; + char procfile[PATH_MAX]; + int fd; + string value; + + snprintf(procfile, PATH_MAX, "/proc/%d/psinfo", getProcessId()); + if ((fd = open(procfile, O_RDONLY)) >= 0) { + if (read(fd, (void *) &processInfo, sizeof(processInfo)) == sizeof(processInfo)) { + value = processInfo.pr_fname; + } + } + return value; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/ssl/SslHandler.cpp b/qpid/cpp/src/qpid/sys/ssl/SslHandler.cpp new file mode 100644 index 0000000000..5516d72065 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/SslHandler.cpp @@ -0,0 +1,195 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/ssl/SslHandler.h" + +#include "qpid/sys/ssl/SslIo.h" +#include "qpid/sys/ssl/SslSocket.h" +#include "qpid/framing/AMQP_HighestVersion.h" +#include "qpid/framing/ProtocolInitiation.h" +#include "qpid/log/Statement.h" + +#include <boost/bind.hpp> + +namespace qpid { +namespace sys { +namespace ssl { + + +// Buffer definition +struct Buff : public SslIO::BufferBase { + Buff() : + SslIO::BufferBase(new char[65536], 65536) + {} + ~Buff() + { delete [] bytes;} +}; + +SslHandler::SslHandler(std::string id, ConnectionCodec::Factory* f, bool _nodict) : + identifier(id), + aio(0), + factory(f), + codec(0), + readError(false), + isClient(false), + nodict(_nodict) +{} + +SslHandler::~SslHandler() { + if (codec) + codec->closed(); + delete codec; +} + +void SslHandler::init(SslIO* a, int numBuffs) { + aio = a; + + // Give connection some buffers to use + for (int i = 0; i < numBuffs; i++) { + aio->queueReadBuffer(new Buff); + } +} + +void SslHandler::write(const framing::ProtocolInitiation& data) +{ + QPID_LOG(debug, "SENT [" << identifier << "] INIT(" << data << ")"); + SslIO::BufferBase* buff = aio->getQueuedBuffer(); + if (!buff) + buff = new Buff; + framing::Buffer out(buff->bytes, buff->byteCount); + data.encode(out); + buff->dataCount = data.encodedSize(); + aio->queueWrite(buff); +} + +void SslHandler::abort() { + // TODO: can't implement currently as underlying functionality not implemented + // aio->requestCallback(boost::bind(&SslHandler::eof, this, _1)); +} +void SslHandler::activateOutput() { + aio->notifyPendingWrite(); +} + +void SslHandler::giveReadCredit(int32_t) { + // FIXME aconway 2008-12-05: not yet implemented. +} + +// Input side +void SslHandler::readbuff(SslIO& , SslIO::BufferBase* buff) { + if (readError) { + return; + } + size_t decoded = 0; + if (codec) { // Already initiated + try { + decoded = codec->decode(buff->bytes+buff->dataStart, buff->dataCount); + }catch(const std::exception& e){ + QPID_LOG(error, e.what()); + readError = true; + aio->queueWriteClose(); + } + }else{ + framing::Buffer in(buff->bytes+buff->dataStart, buff->dataCount); + framing::ProtocolInitiation protocolInit; + if (protocolInit.decode(in)) { + decoded = in.getPosition(); + QPID_LOG(debug, "RECV [" << identifier << "] INIT(" << protocolInit << ")"); + try { + codec = factory->create(protocolInit.getVersion(), *this, identifier, getSecuritySettings(aio)); + if (!codec) { + //TODO: may still want to revise this... + //send valid version header & close connection. + write(framing::ProtocolInitiation(framing::highestProtocolVersion)); + readError = true; + aio->queueWriteClose(); + } + } catch (const std::exception& e) { + QPID_LOG(error, e.what()); + readError = true; + aio->queueWriteClose(); + } + } + } + // TODO: unreading needs to go away, and when we can cope + // with multiple sub-buffers in the general buffer scheme, it will + if (decoded != size_t(buff->dataCount)) { + // Adjust buffer for used bytes and then "unread them" + buff->dataStart += decoded; + buff->dataCount -= decoded; + aio->unread(buff); + } else { + // Give whole buffer back to aio subsystem + aio->queueReadBuffer(buff); + } +} + +void SslHandler::eof(SslIO&) { + QPID_LOG(debug, "DISCONNECTED [" << identifier << "]"); + if (codec) codec->closed(); + aio->queueWriteClose(); +} + +void SslHandler::closedSocket(SslIO&, const SslSocket& s) { + // If we closed with data still to send log a warning + if (!aio->writeQueueEmpty()) { + QPID_LOG(warning, "CLOSING [" << identifier << "] unsent data (probably due to client disconnect)"); + } + delete &s; + aio->queueForDeletion(); + delete this; +} + +void SslHandler::disconnect(SslIO& a) { + // treat the same as eof + eof(a); +} + +// Notifications +void SslHandler::nobuffs(SslIO&) { +} + +void SslHandler::idle(SslIO&){ + if (isClient && codec == 0) { + codec = factory->create(*this, identifier, getSecuritySettings(aio)); + write(framing::ProtocolInitiation(codec->getVersion())); + return; + } + if (codec == 0) return; + if (codec->canEncode()) { + // Try and get a queued buffer if not then construct new one + SslIO::BufferBase* buff = aio->getQueuedBuffer(); + if (!buff) buff = new Buff; + size_t encoded=codec->encode(buff->bytes, buff->byteCount); + buff->dataCount = encoded; + aio->queueWrite(buff); + } + if (codec->isClosed()) + aio->queueWriteClose(); +} + +SecuritySettings SslHandler::getSecuritySettings(SslIO* aio) +{ + SecuritySettings settings = aio->getSecuritySettings(); + settings.nodict = nodict; + return settings; +} + + +}}} // namespace qpid::sys::ssl diff --git a/qpid/cpp/src/qpid/sys/ssl/SslHandler.h b/qpid/cpp/src/qpid/sys/ssl/SslHandler.h new file mode 100644 index 0000000000..400fa317fd --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/SslHandler.h @@ -0,0 +1,78 @@ +#ifndef QPID_SYS_SSL_SSLHANDLER_H +#define QPID_SYS_SSL_SSLHANDLER_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ConnectionCodec.h" +#include "qpid/sys/OutputControl.h" + +namespace qpid { + +namespace framing { + class ProtocolInitiation; +} + +namespace sys { +namespace ssl { + +class SslIO; +struct SslIOBufferBase; +class SslSocket; + +class SslHandler : public OutputControl { + std::string identifier; + SslIO* aio; + ConnectionCodec::Factory* factory; + ConnectionCodec* codec; + bool readError; + bool isClient; + bool nodict; + + void write(const framing::ProtocolInitiation&); + qpid::sys::SecuritySettings getSecuritySettings(SslIO* aio); + + public: + SslHandler(std::string id, ConnectionCodec::Factory* f, bool nodict); + ~SslHandler(); + void init(SslIO* a, int numBuffs); + + void setClient() { isClient = true; } + + // Output side + void abort(); + void activateOutput(); + void giveReadCredit(int32_t); + + // Input side + void readbuff(SslIO& aio, SslIOBufferBase* buff); + void eof(SslIO& aio); + void disconnect(SslIO& aio); + + // Notifications + void nobuffs(SslIO& aio); + void idle(SslIO& aio); + void closedSocket(SslIO& aio, const SslSocket& s); +}; + +}}} // namespace qpid::sys::ssl + +#endif /*!QPID_SYS_SSL_SSLHANDLER_H*/ diff --git a/qpid/cpp/src/qpid/sys/ssl/SslIo.cpp b/qpid/cpp/src/qpid/sys/ssl/SslIo.cpp new file mode 100644 index 0000000000..734ebb483a --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/SslIo.cpp @@ -0,0 +1,447 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ssl/SslIo.h" +#include "qpid/sys/ssl/SslSocket.h" +#include "qpid/sys/ssl/check.h" + +#include "qpid/sys/Time.h" +#include "qpid/sys/posix/check.h" +#include "qpid/log/Statement.h" + +// TODO The basic algorithm here is not really POSIX specific and with a bit more abstraction +// could (should) be promoted to be platform portable +#include <unistd.h> +#include <sys/socket.h> +#include <signal.h> +#include <errno.h> +#include <string.h> + +#include <boost/bind.hpp> + +using namespace qpid::sys; +using namespace qpid::sys::ssl; + +namespace { + +/* + * Make *process* not generate SIGPIPE when writing to closed + * pipe/socket (necessary as default action is to terminate process) + */ +void ignoreSigpipe() { + ::signal(SIGPIPE, SIG_IGN); +} + +/* + * We keep per thread state to avoid locking overhead. The assumption is that + * on average all the connections are serviced by all the threads so the state + * recorded in each thread is about the same. If this turns out not to be the + * case we could rebalance the info occasionally. + */ +__thread int threadReadTotal = 0; +__thread int threadMaxRead = 0; +__thread int threadReadCount = 0; +__thread int threadWriteTotal = 0; +__thread int threadWriteCount = 0; +__thread int64_t threadMaxReadTimeNs = 2 * 1000000; // start at 2ms +} + +/* + * Asynch Acceptor + */ + +SslAcceptor::SslAcceptor(const SslSocket& s, Callback callback) : + acceptedCallback(callback), + handle(s, boost::bind(&SslAcceptor::readable, this, _1), 0, 0), + socket(s) { + + s.setNonblocking(); + ignoreSigpipe(); +} + +SslAcceptor::~SslAcceptor() +{ + handle.stopWatch(); +} + +void SslAcceptor::start(Poller::shared_ptr poller) { + handle.startWatch(poller); +} + +/* + * We keep on accepting as long as there is something to accept + */ +void SslAcceptor::readable(DispatchHandle& h) { + SslSocket* s; + do { + errno = 0; + // TODO: Currently we ignore the peers address, perhaps we should + // log it or use it for connection acceptance. + try { + s = socket.accept(); + if (s) { + acceptedCallback(*s); + } else { + break; + } + } catch (const std::exception& e) { + QPID_LOG(error, "Could not accept socket: " << e.what()); + } + } while (true); + + h.rewatch(); +} + +/* + * Asynch Connector + */ + +SslConnector::SslConnector(const SslSocket& s, + Poller::shared_ptr poller, + std::string hostname, + std::string port, + ConnectedCallback connCb, + FailedCallback failCb) : + DispatchHandle(s, + 0, + boost::bind(&SslConnector::connComplete, this, _1), + boost::bind(&SslConnector::connComplete, this, _1)), + connCallback(connCb), + failCallback(failCb), + socket(s) +{ + //TODO: would be better for connect to be performed on a + //non-blocking socket, but that doesn't work at present so connect + //blocks until complete + try { + socket.connect(hostname, port); + socket.setNonblocking(); + startWatch(poller); + } catch(std::exception& e) { + failure(-1, std::string(e.what())); + } +} + +void SslConnector::connComplete(DispatchHandle& h) +{ + int errCode = socket.getError(); + + h.stopWatch(); + if (errCode == 0) { + connCallback(socket); + DispatchHandle::doDelete(); + } else { + // TODO: This need to be fixed as strerror isn't thread safe + failure(errCode, std::string(::strerror(errCode))); + } +} + +void SslConnector::failure(int errCode, std::string message) +{ + if (failCallback) + failCallback(errCode, message); + + socket.close(); + delete &socket; + + DispatchHandle::doDelete(); +} + +/* + * Asynch reader/writer + */ +SslIO::SslIO(const SslSocket& s, + ReadCallback rCb, EofCallback eofCb, DisconnectCallback disCb, + ClosedCallback cCb, BuffersEmptyCallback eCb, IdleCallback iCb) : + + DispatchHandle(s, + boost::bind(&SslIO::readable, this, _1), + boost::bind(&SslIO::writeable, this, _1), + boost::bind(&SslIO::disconnected, this, _1)), + readCallback(rCb), + eofCallback(eofCb), + disCallback(disCb), + closedCallback(cCb), + emptyCallback(eCb), + idleCallback(iCb), + socket(s), + queuedClose(false), + writePending(false) { + + s.setNonblocking(); +} + +struct deleter +{ + template <typename T> + void operator()(T *ptr){ delete ptr;} +}; + +SslIO::~SslIO() { + std::for_each( bufferQueue.begin(), bufferQueue.end(), deleter()); + std::for_each( writeQueue.begin(), writeQueue.end(), deleter()); +} + +void SslIO::queueForDeletion() { + DispatchHandle::doDelete(); +} + +void SslIO::start(Poller::shared_ptr poller) { + DispatchHandle::startWatch(poller); +} + +void SslIO::queueReadBuffer(BufferBase* buff) { + assert(buff); + buff->dataStart = 0; + buff->dataCount = 0; + bufferQueue.push_back(buff); + DispatchHandle::rewatchRead(); +} + +void SslIO::unread(BufferBase* buff) { + assert(buff); + if (buff->dataStart != 0) { + memmove(buff->bytes, buff->bytes+buff->dataStart, buff->dataCount); + buff->dataStart = 0; + } + bufferQueue.push_front(buff); + DispatchHandle::rewatchRead(); +} + +void SslIO::queueWrite(BufferBase* buff) { + assert(buff); + // If we've already closed the socket then throw the write away + if (queuedClose) { + bufferQueue.push_front(buff); + return; + } else { + writeQueue.push_front(buff); + } + writePending = false; + DispatchHandle::rewatchWrite(); +} + +void SslIO::notifyPendingWrite() { + writePending = true; + DispatchHandle::rewatchWrite(); +} + +void SslIO::queueWriteClose() { + queuedClose = true; + DispatchHandle::rewatchWrite(); +} + +/** Return a queued buffer if there are enough + * to spare + */ +SslIO::BufferBase* SslIO::getQueuedBuffer() { + // Always keep at least one buffer (it might have data that was "unread" in it) + if (bufferQueue.size()<=1) + return 0; + BufferBase* buff = bufferQueue.back(); + assert(buff); + buff->dataStart = 0; + buff->dataCount = 0; + bufferQueue.pop_back(); + return buff; +} + +/* + * We keep on reading as long as we have something to read and a buffer to put + * it in + */ +void SslIO::readable(DispatchHandle& h) { + int readTotal = 0; + AbsTime readStartTime = AbsTime::now(); + do { + // (Try to) get a buffer + if (!bufferQueue.empty()) { + // Read into buffer + BufferBase* buff = bufferQueue.front(); + assert(buff); + bufferQueue.pop_front(); + errno = 0; + int readCount = buff->byteCount-buff->dataCount; + int rc = socket.read(buff->bytes + buff->dataCount, readCount); + if (rc > 0) { + buff->dataCount += rc; + threadReadTotal += rc; + readTotal += rc; + + readCallback(*this, buff); + if (rc != readCount) { + // If we didn't fill the read buffer then time to stop reading + break; + } + + // Stop reading if we've overrun our timeslot + if (Duration(readStartTime, AbsTime::now()) > threadMaxReadTimeNs) { + break; + } + + } else { + // Put buffer back (at front so it doesn't interfere with unread buffers) + bufferQueue.push_front(buff); + assert(buff); + + // Eof or other side has gone away + if (rc == 0 || errno == ECONNRESET) { + eofCallback(*this); + h.unwatchRead(); + break; + } else if (errno == EAGAIN) { + // We have just put a buffer back so we know + // we can carry on watching for reads + break; + } else { + // Report error then just treat as a socket disconnect + QPID_LOG(error, "Error reading socket: " << getErrorString(PR_GetError())); + eofCallback(*this); + h.unwatchRead(); + break; + } + } + } else { + // Something to read but no buffer + if (emptyCallback) { + emptyCallback(*this); + } + // If we still have no buffers we can't do anything more + if (bufferQueue.empty()) { + h.unwatchRead(); + break; + } + + } + } while (true); + + ++threadReadCount; + threadMaxRead = std::max(threadMaxRead, readTotal); + return; +} + +/* + * We carry on writing whilst we have data to write and we can write + */ +void SslIO::writeable(DispatchHandle& h) { + int writeTotal = 0; + do { + // See if we've got something to write + if (!writeQueue.empty()) { + // Write buffer + BufferBase* buff = writeQueue.back(); + writeQueue.pop_back(); + errno = 0; + assert(buff->dataStart+buff->dataCount <= buff->byteCount); + int rc = socket.write(buff->bytes+buff->dataStart, buff->dataCount); + if (rc >= 0) { + threadWriteTotal += rc; + writeTotal += rc; + + // If we didn't write full buffer put rest back + if (rc != buff->dataCount) { + buff->dataStart += rc; + buff->dataCount -= rc; + writeQueue.push_back(buff); + break; + } + + // Recycle the buffer + queueReadBuffer(buff); + + // If we've already written more than the max for reading then stop + // (this is to stop writes dominating reads) + if (writeTotal > threadMaxRead) + break; + } else { + // Put buffer back + writeQueue.push_back(buff); + if (errno == ECONNRESET || errno == EPIPE) { + // Just stop watching for write here - we'll get a + // disconnect callback soon enough + h.unwatchWrite(); + break; + } else if (errno == EAGAIN) { + // We have just put a buffer back so we know + // we can carry on watching for writes + break; + } else { + QPID_LOG(error, "Error writing to socket: " << getErrorString(PR_GetError())); + h.unwatchWrite(); + break; + } + } + } else { + // If we're waiting to close the socket then can do it now as there is nothing to write + if (queuedClose) { + close(h); + break; + } + // Fd is writable, but nothing to write + if (idleCallback) { + writePending = false; + idleCallback(*this); + } + // If we still have no buffers to write we can't do anything more + if (writeQueue.empty() && !writePending && !queuedClose) { + h.unwatchWrite(); + // The following handles the case where writePending is + // set to true after the test above; in this case its + // possible that the unwatchWrite overwrites the + // desired rewatchWrite so we correct that here + if (writePending) + h.rewatchWrite(); + break; + } + } + } while (true); + + ++threadWriteCount; + return; +} + +void SslIO::disconnected(DispatchHandle& h) { + // If we've already queued close do it instead of disconnected callback + if (queuedClose) { + close(h); + } else if (disCallback) { + disCallback(*this); + h.unwatch(); + } +} + +/* + * Close the socket and callback to say we've done it + */ +void SslIO::close(DispatchHandle& h) { + h.stopWatch(); + socket.close(); + if (closedCallback) { + closedCallback(*this, socket); + } +} + +SecuritySettings SslIO::getSecuritySettings() { + SecuritySettings settings; + settings.ssf = socket.getKeyLen(); + settings.authid = socket.getClientAuthId(); + return settings; +} diff --git a/qpid/cpp/src/qpid/sys/ssl/SslIo.h b/qpid/cpp/src/qpid/sys/ssl/SslIo.h new file mode 100644 index 0000000000..8785852c24 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/SslIo.h @@ -0,0 +1,172 @@ +#ifndef _sys_ssl_SslIO +#define _sys_ssl_SslIO +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/DispatchHandle.h" +#include "qpid/sys/SecuritySettings.h" + +#include <boost/function.hpp> +#include <deque> + +namespace qpid { +namespace sys { +namespace ssl { + +class SslSocket; + +/* + * Asynchronous ssl acceptor: accepts connections then does a callback + * with the accepted fd + */ +class SslAcceptor { +public: + typedef boost::function1<void, const SslSocket&> Callback; + +private: + Callback acceptedCallback; + qpid::sys::DispatchHandle handle; + const SslSocket& socket; + +public: + SslAcceptor(const SslSocket& s, Callback callback); + ~SslAcceptor(); + void start(qpid::sys::Poller::shared_ptr poller); + +private: + void readable(qpid::sys::DispatchHandle& handle); +}; + +/* + * Asynchronous ssl connector: starts the process of initiating a + * connection and invokes a callback when completed or failed. + */ +class SslConnector : private qpid::sys::DispatchHandle { +public: + typedef boost::function1<void, const SslSocket&> ConnectedCallback; + typedef boost::function2<void, int, std::string> FailedCallback; + +private: + ConnectedCallback connCallback; + FailedCallback failCallback; + const SslSocket& socket; + +public: + SslConnector(const SslSocket& socket, + Poller::shared_ptr poller, + std::string hostname, + std::string port, + ConnectedCallback connCb, + FailedCallback failCb = 0); + +private: + void connComplete(DispatchHandle& handle); + void failure(int, std::string); +}; + +struct SslIOBufferBase { + char* const bytes; + const int32_t byteCount; + int32_t dataStart; + int32_t dataCount; + + SslIOBufferBase(char* const b, const int32_t s) : + bytes(b), + byteCount(s), + dataStart(0), + dataCount(0) + {} + + virtual ~SslIOBufferBase() + {} +}; + +/* + * Asychronous reader/writer: + * Reader accepts buffers to read into; reads into the provided buffers + * and then does a callback with the buffer and amount read. Optionally it can callback + * when there is something to read but no buffer to read it into. + * + * Writer accepts a buffer and queues it for writing; can also be given + * a callback for when writing is "idle" (ie fd is writable, but nothing to write) + * + * The class is implemented in terms of DispatchHandle to allow it to be deleted by deleting + * the contained DispatchHandle + */ +class SslIO : private qpid::sys::DispatchHandle { +public: + typedef SslIOBufferBase BufferBase; + + typedef boost::function2<void, SslIO&, BufferBase*> ReadCallback; + typedef boost::function1<void, SslIO&> EofCallback; + typedef boost::function1<void, SslIO&> DisconnectCallback; + typedef boost::function2<void, SslIO&, const SslSocket&> ClosedCallback; + typedef boost::function1<void, SslIO&> BuffersEmptyCallback; + typedef boost::function1<void, SslIO&> IdleCallback; + + +private: + ReadCallback readCallback; + EofCallback eofCallback; + DisconnectCallback disCallback; + ClosedCallback closedCallback; + BuffersEmptyCallback emptyCallback; + IdleCallback idleCallback; + const SslSocket& socket; + std::deque<BufferBase*> bufferQueue; + std::deque<BufferBase*> writeQueue; + bool queuedClose; + /** + * This flag is used to detect and handle concurrency between + * calls to notifyPendingWrite() (which can be made from any thread) and + * the execution of the writeable() method (which is always on the + * thread processing this handle. + */ + volatile bool writePending; + +public: + SslIO(const SslSocket& s, + ReadCallback rCb, EofCallback eofCb, DisconnectCallback disCb, + ClosedCallback cCb = 0, BuffersEmptyCallback eCb = 0, IdleCallback iCb = 0); + void queueForDeletion(); + + void start(qpid::sys::Poller::shared_ptr poller); + void queueReadBuffer(BufferBase* buff); + void unread(BufferBase* buff); + void queueWrite(BufferBase* buff); + void notifyPendingWrite(); + void queueWriteClose(); + bool writeQueueEmpty() { return writeQueue.empty(); } + BufferBase* getQueuedBuffer(); + + qpid::sys::SecuritySettings getSecuritySettings(); + +private: + ~SslIO(); + void readable(qpid::sys::DispatchHandle& handle); + void writeable(qpid::sys::DispatchHandle& handle); + void disconnected(qpid::sys::DispatchHandle& handle); + void close(qpid::sys::DispatchHandle& handle); +}; + +}}} + +#endif // _sys_ssl_SslIO diff --git a/qpid/cpp/src/qpid/sys/ssl/SslSocket.cpp b/qpid/cpp/src/qpid/sys/ssl/SslSocket.cpp new file mode 100644 index 0000000000..f7483a220c --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/SslSocket.cpp @@ -0,0 +1,360 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/ssl/SslSocket.h" +#include "qpid/sys/ssl/check.h" +#include "qpid/sys/ssl/util.h" +#include "qpid/Exception.h" +#include "qpid/sys/posix/check.h" +#include "qpid/sys/posix/PrivatePosix.h" + +#include <fcntl.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <sys/errno.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <netdb.h> +#include <cstdlib> +#include <string.h> +#include <iostream> + +#include <private/pprio.h> +#include <nss.h> +#include <pk11pub.h> +#include <ssl.h> +#include <key.h> + +#include <boost/format.hpp> + +namespace qpid { +namespace sys { +namespace ssl { + +namespace { +std::string getName(int fd, bool local, bool includeService = false) +{ + ::sockaddr_storage name; // big enough for any socket address + ::socklen_t namelen = sizeof(name); + + int result = -1; + if (local) { + result = ::getsockname(fd, (::sockaddr*)&name, &namelen); + } else { + result = ::getpeername(fd, (::sockaddr*)&name, &namelen); + } + + QPID_POSIX_CHECK(result); + + char servName[NI_MAXSERV]; + char dispName[NI_MAXHOST]; + if (includeService) { + if (int rc=::getnameinfo((::sockaddr*)&name, namelen, dispName, sizeof(dispName), + servName, sizeof(servName), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + throw QPID_POSIX_ERROR(rc); + return std::string(dispName) + ":" + std::string(servName); + + } else { + if (int rc=::getnameinfo((::sockaddr*)&name, namelen, dispName, sizeof(dispName), 0, 0, NI_NUMERICHOST) != 0) + throw QPID_POSIX_ERROR(rc); + return dispName; + } +} + +std::string getService(int fd, bool local) +{ + ::sockaddr_storage name; // big enough for any socket address + ::socklen_t namelen = sizeof(name); + + int result = -1; + if (local) { + result = ::getsockname(fd, (::sockaddr*)&name, &namelen); + } else { + result = ::getpeername(fd, (::sockaddr*)&name, &namelen); + } + + QPID_POSIX_CHECK(result); + + char servName[NI_MAXSERV]; + if (int rc=::getnameinfo((::sockaddr*)&name, namelen, 0, 0, + servName, sizeof(servName), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + throw QPID_POSIX_ERROR(rc); + return servName; +} + +const std::string DOMAIN_SEPARATOR("@"); +const std::string DC_SEPARATOR("."); +const std::string DC("DC"); +const std::string DN_DELIMS(" ,="); + +std::string getDomainFromSubject(std::string subject) +{ + std::string::size_type last = subject.find_first_not_of(DN_DELIMS, 0); + std::string::size_type i = subject.find_first_of(DN_DELIMS, last); + + std::string domain; + bool nextTokenIsDC = false; + while (std::string::npos != i || std::string::npos != last) + { + std::string token = subject.substr(last, i - last); + if (nextTokenIsDC) { + if (domain.size()) domain += DC_SEPARATOR; + domain += token; + nextTokenIsDC = false; + } else if (token == DC) { + nextTokenIsDC = true; + } + last = subject.find_first_not_of(DN_DELIMS, i); + i = subject.find_first_of(DN_DELIMS, last); + } + return domain; +} + +} + +SslSocket::SslSocket() : IOHandle(new IOHandlePrivate()), socket(0), prototype(0) +{ + impl->fd = ::socket (PF_INET, SOCK_STREAM, 0); + if (impl->fd < 0) throw QPID_POSIX_ERROR(errno); + socket = SSL_ImportFD(0, PR_ImportTCPSocket(impl->fd)); +} + +/** + * This form of the constructor is used with the server-side sockets + * returned from accept. Because we use posix accept rather than + * PR_Accept, we have to reset the handshake. + */ +SslSocket::SslSocket(IOHandlePrivate* ioph, PRFileDesc* model) : IOHandle(ioph), socket(0), prototype(0) +{ + socket = SSL_ImportFD(model, PR_ImportTCPSocket(impl->fd)); + NSS_CHECK(SSL_ResetHandshake(socket, true)); +} + +void SslSocket::setNonblocking() const +{ + PRSocketOptionData option; + option.option = PR_SockOpt_Nonblocking; + option.value.non_blocking = true; + PR_SetSocketOption(socket, &option); +} + +void SslSocket::connect(const std::string& host, const std::string& port) const +{ + std::stringstream namestream; + namestream << host << ":" << port; + connectname = namestream.str(); + + void* arg; + // Use the connection's cert-name if it has one; else use global cert-name + if (certname != "") { + arg = const_cast<char*>(certname.c_str()); + } else if (SslOptions::global.certName.empty()) { + arg = 0; + } else { + arg = const_cast<char*>(SslOptions::global.certName.c_str()); + } + NSS_CHECK(SSL_GetClientAuthDataHook(socket, NSS_GetClientAuthData, arg)); + NSS_CHECK(SSL_SetURL(socket, host.data())); + + char hostBuffer[PR_NETDB_BUF_SIZE]; + PRHostEnt hostEntry; + PR_CHECK(PR_GetHostByName(host.data(), hostBuffer, PR_NETDB_BUF_SIZE, &hostEntry)); + PRNetAddr address; + int value = PR_EnumerateHostEnt(0, &hostEntry, boost::lexical_cast<PRUint16>(port), &address); + if (value < 0) { + throw Exception(QPID_MSG("Error getting address for host: " << ErrorString())); + } else if (value == 0) { + throw Exception(QPID_MSG("Could not resolve address for host.")); + } + PR_CHECK(PR_Connect(socket, &address, PR_INTERVAL_NO_TIMEOUT)); + NSS_CHECK(SSL_ForceHandshake(socket)); +} + +void SslSocket::close() const +{ + if (impl->fd > 0) { + PR_Close(socket); + impl->fd = -1; + } +} + +int SslSocket::listen(uint16_t port, int backlog, const std::string& certName, bool clientAuth) const +{ + //configure prototype socket: + prototype = SSL_ImportFD(0, PR_NewTCPSocket()); + if (clientAuth) { + NSS_CHECK(SSL_OptionSet(prototype, SSL_REQUEST_CERTIFICATE, PR_TRUE)); + NSS_CHECK(SSL_OptionSet(prototype, SSL_REQUIRE_CERTIFICATE, PR_TRUE)); + } + + //get certificate and key (is this the correct way?) + CERTCertificate *cert = PK11_FindCertFromNickname(const_cast<char*>(certName.c_str()), 0); + if (!cert) throw Exception(QPID_MSG("Failed to load certificate '" << certName << "'")); + SECKEYPrivateKey *key = PK11_FindKeyByAnyCert(cert, 0); + if (!key) throw Exception(QPID_MSG("Failed to retrieve private key from certificate")); + NSS_CHECK(SSL_ConfigSecureServer(prototype, cert, key, NSS_FindCertKEAType(cert))); + SECKEY_DestroyPrivateKey(key); + CERT_DestroyCertificate(cert); + + //bind and listen + const int& socket = impl->fd; + int yes=1; + QPID_POSIX_CHECK(setsockopt(socket,SOL_SOCKET,SO_REUSEADDR,&yes,sizeof(yes))); + struct sockaddr_in name; + name.sin_family = AF_INET; + name.sin_port = htons(port); + name.sin_addr.s_addr = 0; + if (::bind(socket, (struct sockaddr*)&name, sizeof(name)) < 0) + throw Exception(QPID_MSG("Can't bind to port " << port << ": " << strError(errno))); + if (::listen(socket, backlog) < 0) + throw Exception(QPID_MSG("Can't listen on port " << port << ": " << strError(errno))); + + socklen_t namelen = sizeof(name); + if (::getsockname(socket, (struct sockaddr*)&name, &namelen) < 0) + throw QPID_POSIX_ERROR(errno); + + return ntohs(name.sin_port); +} + +SslSocket* SslSocket::accept() const +{ + int afd = ::accept(impl->fd, 0, 0); + if ( afd >= 0) { + return new SslSocket(new IOHandlePrivate(afd), prototype); + } else if (errno == EAGAIN) { + return 0; + } else { + throw QPID_POSIX_ERROR(errno); + } +} + +int SslSocket::read(void *buf, size_t count) const +{ + return PR_Read(socket, buf, count); +} + +int SslSocket::write(const void *buf, size_t count) const +{ + return PR_Write(socket, buf, count); +} + +std::string SslSocket::getSockname() const +{ + return getName(impl->fd, true); +} + +std::string SslSocket::getPeername() const +{ + return getName(impl->fd, false); +} + +std::string SslSocket::getPeerAddress() const +{ + if (!connectname.empty()) + return connectname; + return getName(impl->fd, false, true); +} + +std::string SslSocket::getLocalAddress() const +{ + return getName(impl->fd, true, true); +} + +uint16_t SslSocket::getLocalPort() const +{ + return std::atoi(getService(impl->fd, true).c_str()); +} + +uint16_t SslSocket::getRemotePort() const +{ + return atoi(getService(impl->fd, true).c_str()); +} + +int SslSocket::getError() const +{ + int result; + socklen_t rSize = sizeof (result); + + if (::getsockopt(impl->fd, SOL_SOCKET, SO_ERROR, &result, &rSize) < 0) + throw QPID_POSIX_ERROR(errno); + + return result; +} + +void SslSocket::setTcpNoDelay(bool nodelay) const +{ + if (nodelay) { + PRSocketOptionData option; + option.option = PR_SockOpt_NoDelay; + option.value.no_delay = true; + PR_SetSocketOption(socket, &option); + } +} + +void SslSocket::setCertName(const std::string& name) +{ + certname = name; +} + + +/** get the bit length of the current cipher's key */ +int SslSocket::getKeyLen() const +{ + int enabled = 0; + int keySize = 0; + SECStatus rc; + + rc = SSL_SecurityStatus( socket, + &enabled, + NULL, + NULL, + &keySize, + NULL, NULL ); + if (rc == SECSuccess && enabled) { + return keySize; + } + return 0; +} + +std::string SslSocket::getClientAuthId() const +{ + std::string authId; + CERTCertificate* cert = SSL_PeerCertificate(socket); + if (cert) { + authId = CERT_GetCommonName(&(cert->subject)); + /* + * The NSS function CERT_GetDomainComponentName only returns + * the last component of the domain name, so we have to parse + * the subject manually to extract the full domain. + */ + std::string domain = getDomainFromSubject(cert->subjectName); + if (!domain.empty()) { + authId += DOMAIN_SEPARATOR; + authId += domain; + } + CERT_DestroyCertificate(cert); + } + return authId; +} + +}}} // namespace qpid::sys::ssl diff --git a/qpid/cpp/src/qpid/sys/ssl/SslSocket.h b/qpid/cpp/src/qpid/sys/ssl/SslSocket.h new file mode 100644 index 0000000000..993859495b --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/SslSocket.h @@ -0,0 +1,132 @@ +#ifndef _sys_ssl_Socket_h +#define _sys_ssl_Socket_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IOHandle.h" +#include <nspr.h> + +#include <string> + +struct sockaddr; + +namespace qpid { +namespace sys { + +class Duration; + +namespace ssl { + +class SslSocket : public qpid::sys::IOHandle +{ +public: + /** Create a socket wrapper for descriptor. */ + SslSocket(); + + /** Set socket non blocking */ + void setNonblocking() const; + + /** Set tcp-nodelay */ + void setTcpNoDelay(bool nodelay) const; + + /** Set SSL cert-name. Allows the cert-name to be set per + * connection, overriding global cert-name settings from + * NSSInit().*/ + void setCertName(const std::string& certName); + + void connect(const std::string& host, const std::string& port) const; + + void close() const; + + /** Bind to a port and start listening. + *@param port 0 means choose an available port. + *@param backlog maximum number of pending connections. + *@param certName name of certificate to use to identify the server + *@return The bound port. + */ + int listen(uint16_t port = 0, int backlog = 10, const std::string& certName = "localhost.localdomain", bool clientAuth = false) const; + + /** + * Accept a connection from a socket that is already listening + * and has an incoming connection + */ + SslSocket* accept() const; + + // TODO The following are raw operations, maybe they need better wrapping? + int read(void *buf, size_t count) const; + int write(const void *buf, size_t count) const; + + /** Returns the "socket name" ie the address bound to + * the near end of the socket + */ + std::string getSockname() const; + + /** Returns the "peer name" ie the address bound to + * the remote end of the socket + */ + std::string getPeername() const; + + /** + * Returns an address (host and port) for the remote end of the + * socket + */ + std::string getPeerAddress() const; + /** + * Returns an address (host and port) for the local end of the + * socket + */ + std::string getLocalAddress() const; + + /** + * Returns the full address of the connection: local and remote host and port. + */ + std::string getFullAddress() const { return getLocalAddress()+"-"+getPeerAddress(); } + + uint16_t getLocalPort() const; + uint16_t getRemotePort() const; + + /** + * Returns the error code stored in the socket. This may be used + * to determine the result of a non-blocking connect. + */ + int getError() const; + + int getKeyLen() const; + std::string getClientAuthId() const; + +private: + mutable std::string connectname; + mutable PRFileDesc* socket; + std::string certname; + + /** + * 'model' socket, with configuration to use when importing + * accepted sockets for use as ssl sockets. Set on listen(), used + * in accept to pass through to newly created socket instances. + */ + mutable PRFileDesc* prototype; + + SslSocket(IOHandlePrivate* ioph, PRFileDesc* model); +}; + +}}} +#endif /*!_sys_ssl_Socket_h*/ diff --git a/qpid/cpp/src/qpid/sys/ssl/check.cpp b/qpid/cpp/src/qpid/sys/ssl/check.cpp new file mode 100644 index 0000000000..72a2e265bd --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/check.cpp @@ -0,0 +1,85 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/ssl/check.h" +#include <secerr.h> +#include <sslerr.h> +#include <boost/format.hpp> + +using boost::format; +using boost::str; + +namespace qpid { +namespace sys { +namespace ssl { + +ErrorString::ErrorString() : code(PR_GetError()), buffer(new char[PR_GetErrorTextLength()]), used(PR_GetErrorText(buffer)) {} + +ErrorString::~ErrorString() +{ + delete[] buffer; +} + +std::string ErrorString::getString() const +{ + std::string msg = std::string(buffer, used); + if (!used) { + //seems most of the NSPR/NSS errors don't have text set for + //them, add a few specific ones in here. (TODO: more complete + //list?): + return getErrorString(code); + } else { + return str(format("%1% [%2%]") % msg % code); + } +} + +std::string getErrorString(int code) +{ + std::string msg; + switch (code) { + case SSL_ERROR_EXPORT_ONLY_SERVER: msg = "Unable to communicate securely. Peer does not support high-grade encryption."; break; + case SSL_ERROR_US_ONLY_SERVER: msg = "Unable to communicate securely. Peer requires high-grade encryption which is not supported."; break; + case SSL_ERROR_NO_CYPHER_OVERLAP: msg = "Cannot communicate securely with peer: no common encryption algorithm(s)."; break; + case SSL_ERROR_NO_CERTIFICATE: msg = "Unable to find the certificate or key necessary for authentication."; break; + case SSL_ERROR_BAD_CERTIFICATE: msg = "Unable to communicate securely with peer: peers's certificate was rejected."; break; + case SSL_ERROR_UNSUPPORTED_CERTIFICATE_TYPE: msg = "Unsupported certificate type."; break; + case SSL_ERROR_WRONG_CERTIFICATE: msg = "Client authentication failed: private key in key database does not correspond to public key in certificate database."; break; + case SSL_ERROR_BAD_CERT_DOMAIN: msg = "Unable to communicate securely with peer: requested domain name does not match the server's certificate."; break; + case SSL_ERROR_BAD_CERT_ALERT: msg = "SSL peer cannot verify your certificate."; break; + case SSL_ERROR_REVOKED_CERT_ALERT: msg = "SSL peer rejected your certificate as revoked."; break; + case SSL_ERROR_EXPIRED_CERT_ALERT: msg = "SSL peer rejected your certificate as expired."; break; + + case PR_DIRECTORY_LOOKUP_ERROR: msg = "A directory lookup on a network address has failed"; break; + case PR_CONNECT_RESET_ERROR: msg = "TCP connection reset by peer"; break; + case PR_END_OF_FILE_ERROR: msg = "Encountered end of file"; break; + case SEC_ERROR_EXPIRED_CERTIFICATE: msg = "Peer's certificate has expired"; break; + default: msg = (code < -6000) ? "NSS error" : "NSPR error"; break; + } + return str(format("%1% [%2%]") % msg % code); +} + +std::ostream& operator<<(std::ostream& out, const ErrorString& err) +{ + out << err.getString(); + return out; +} + + +}}} // namespace qpid::sys::ssl diff --git a/qpid/cpp/src/qpid/sys/ssl/check.h b/qpid/cpp/src/qpid/sys/ssl/check.h new file mode 100644 index 0000000000..28d3c74ad0 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/check.h @@ -0,0 +1,57 @@ +#ifndef QPID_SYS_SSL_CHECK_H +#define QPID_SYS_SSL_CHECK_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/Msg.h" + +#include <iostream> +#include <string> +#include <nspr.h> +#include <nss.h> + +namespace qpid { +namespace sys { +namespace ssl { + +std::string getErrorString(int code); + +class ErrorString +{ + public: + ErrorString(); + ~ErrorString(); + std::string getString() const; + private: + const int code; + char* const buffer; + const size_t used; +}; + +std::ostream& operator<<(std::ostream& out, const ErrorString& err); + +}}} // namespace qpid::sys::ssl + + +#define NSS_CHECK(value) if (value != SECSuccess) { throw Exception(QPID_MSG("Failed: " << qpid::sys::ssl::ErrorString())); } +#define PR_CHECK(value) if (value != PR_SUCCESS) { throw Exception(QPID_MSG("Failed: " << qpid::sys::ssl::ErrorString())); } + +#endif /*!QPID_SYS_SSL_CHECK_H*/ diff --git a/qpid/cpp/src/qpid/sys/ssl/util.cpp b/qpid/cpp/src/qpid/sys/ssl/util.cpp new file mode 100644 index 0000000000..3078e894df --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/util.cpp @@ -0,0 +1,120 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/sys/ssl/util.h" +#include "qpid/sys/ssl/check.h" +#include "qpid/Exception.h" +#include "qpid/sys/SystemInfo.h" + +#include <unistd.h> +#include <nspr.h> +#include <nss.h> +#include <pk11pub.h> +#include <ssl.h> + +#include <iostream> +#include <fstream> +#include <boost/filesystem/operations.hpp> +#include <boost/filesystem/path.hpp> + +namespace qpid { +namespace sys { +namespace ssl { + +static const std::string LOCALHOST("127.0.0.1"); + +std::string defaultCertName() +{ + Address address; + if (SystemInfo::getLocalHostname(address)) { + return address.host; + } else { + return LOCALHOST; + } +} + +SslOptions::SslOptions() : qpid::Options("SSL Settings"), + certName(defaultCertName()), + exportPolicy(false) +{ + addOptions() + ("ssl-use-export-policy", optValue(exportPolicy), "Use NSS export policy") + ("ssl-cert-password-file", optValue(certPasswordFile, "PATH"), "File containing password to use for accessing certificate database") + ("ssl-cert-db", optValue(certDbPath, "PATH"), "Path to directory containing certificate database") + ("ssl-cert-name", optValue(certName, "NAME"), "Name of the certificate to use"); +} + +SslOptions& SslOptions::operator=(const SslOptions& o) +{ + certDbPath = o.certDbPath; + certName = o.certName; + certPasswordFile = o.certPasswordFile; + exportPolicy = o.exportPolicy; + return *this; +} + +char* promptForPassword(PK11SlotInfo*, PRBool retry, void*) +{ + if (retry) return 0; + //TODO: something else? + return PL_strdup(getpass("Please enter the password for accessing the certificate database:")); +} + +SslOptions SslOptions::global; + +char* readPasswordFromFile(PK11SlotInfo*, PRBool retry, void*) +{ + const std::string& passwordFile = SslOptions::global.certPasswordFile; + if (retry || passwordFile.empty() || !boost::filesystem::exists(passwordFile)) { + return 0; + } else { + std::ifstream file(passwordFile.c_str()); + std::string password; + file >> password; + return PL_strdup(password.c_str()); + } +} + +void initNSS(const SslOptions& options, bool server) +{ + SslOptions::global = options; + if (options.certPasswordFile.empty()) { + PK11_SetPasswordFunc(promptForPassword); + } else { + PK11_SetPasswordFunc(readPasswordFromFile); + } + NSS_CHECK(NSS_Init(options.certDbPath.c_str())); + if (options.exportPolicy) { + NSS_CHECK(NSS_SetExportPolicy()); + } else { + NSS_CHECK(NSS_SetDomesticPolicy()); + } + if (server) { + //use defaults for all args, TODO: may want to make this configurable + SSL_ConfigServerSessionIDCache(0, 0, 0, 0); + } +} + +void shutdownNSS() +{ + NSS_Shutdown(); +} + +}}} // namespace qpid::sys::ssl diff --git a/qpid/cpp/src/qpid/sys/ssl/util.h b/qpid/cpp/src/qpid/sys/ssl/util.h new file mode 100644 index 0000000000..f34adab7be --- /dev/null +++ b/qpid/cpp/src/qpid/sys/ssl/util.h @@ -0,0 +1,50 @@ +#ifndef QPID_SYS_SSL_UTIL_H +#define QPID_SYS_SSL_UTIL_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/Options.h" +#include <string> + +namespace qpid { +namespace sys { +namespace ssl { + +struct SslOptions : qpid::Options +{ + static SslOptions global; + + std::string certDbPath; + std::string certName; + std::string certPasswordFile; + bool exportPolicy; + + SslOptions(); + SslOptions& operator=(const SslOptions&); +}; + +void initNSS(const SslOptions& options, bool server = false); +void shutdownNSS(); + +}}} // namespace qpid::sys::ssl + +#endif /*!QPID_SYS_SSL_UTIL_H*/ diff --git a/qpid/cpp/src/qpid/sys/uuid.h b/qpid/cpp/src/qpid/sys/uuid.h new file mode 100644 index 0000000000..804ab34463 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/uuid.h @@ -0,0 +1,28 @@ +#ifndef _sys_uuid_h +#define _sys_uuid_h + +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifdef _WIN32 +# include "qpid/sys/windows/uuid.h" +#else +# include <uuid/uuid.h> +#endif /* _WIN32 */ + +#endif /* _sys_uuid_h */ diff --git a/qpid/cpp/src/qpid/sys/windows/AsynchIO.cpp b/qpid/cpp/src/qpid/sys/windows/AsynchIO.cpp new file mode 100644 index 0000000000..8d84fdb7b2 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/AsynchIO.cpp @@ -0,0 +1,755 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/windows/AsynchIoResult.h" +#include "qpid/sys/windows/IoHandlePrivate.h" +#include "qpid/sys/AsynchIO.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/Socket.h" +#include "qpid/sys/Poller.h" +#include "qpid/sys/Thread.h" +#include "qpid/sys/Time.h" +#include "qpid/log/Statement.h" + +#include "qpid/sys/windows/check.h" +#include "qpid/sys/windows/mingw32_compat.h" + +#include <boost/thread/once.hpp> + +#include <queue> +#include <winsock2.h> +#include <mswsock.h> +#include <windows.h> + +#include <boost/bind.hpp> + +namespace { + + typedef qpid::sys::ScopedLock<qpid::sys::Mutex> QLock; + +/* + * The function pointers for AcceptEx and ConnectEx need to be looked up + * at run time. Make sure this is done only once. + */ +boost::once_flag lookUpAcceptExOnce = BOOST_ONCE_INIT; +LPFN_ACCEPTEX fnAcceptEx = 0; +typedef void (*lookUpFunc)(const qpid::sys::Socket &); + +void lookUpAcceptEx() { + SOCKET h = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + GUID guidAcceptEx = WSAID_ACCEPTEX; + DWORD dwBytes = 0; + WSAIoctl(h, + SIO_GET_EXTENSION_FUNCTION_POINTER, + &guidAcceptEx, + sizeof(guidAcceptEx), + &fnAcceptEx, + sizeof(fnAcceptEx), + &dwBytes, + NULL, + NULL); + closesocket(h); + if (fnAcceptEx == 0) + throw qpid::Exception(QPID_MSG("Failed to look up AcceptEx")); +} + +} + +namespace qpid { +namespace sys { +namespace windows { + +/* + * Asynch Acceptor + * + */ +class AsynchAcceptor : public qpid::sys::AsynchAcceptor { + + friend class AsynchAcceptResult; + +public: + AsynchAcceptor(const Socket& s, AsynchAcceptor::Callback callback); + ~AsynchAcceptor(); + void start(Poller::shared_ptr poller); + +private: + void restart(void); + + AsynchAcceptor::Callback acceptedCallback; + const Socket& socket; +}; + +AsynchAcceptor::AsynchAcceptor(const Socket& s, Callback callback) + : acceptedCallback(callback), + socket(s) { + + s.setNonblocking(); +#if (BOOST_VERSION >= 103500) /* boost 1.35 or later reversed the args */ + boost::call_once(lookUpAcceptExOnce, lookUpAcceptEx); +#else + boost::call_once(lookUpAcceptEx, lookUpAcceptExOnce); +#endif +} + +AsynchAcceptor::~AsynchAcceptor() +{ + socket.close(); +} + +void AsynchAcceptor::start(Poller::shared_ptr poller) { + PollerHandle ph = PollerHandle(socket); + poller->monitorHandle(ph, Poller::INPUT); + restart (); +} + +void AsynchAcceptor::restart(void) { + DWORD bytesReceived = 0; // Not used, needed for AcceptEx API + AsynchAcceptResult *result = new AsynchAcceptResult(acceptedCallback, + this, + toSocketHandle(socket)); + BOOL status; + status = ::fnAcceptEx(toSocketHandle(socket), + toSocketHandle(*result->newSocket), + result->addressBuffer, + 0, + AsynchAcceptResult::SOCKADDRMAXLEN, + AsynchAcceptResult::SOCKADDRMAXLEN, + &bytesReceived, + result->overlapped()); + QPID_WINDOWS_CHECK_ASYNC_START(status); +} + + +AsynchAcceptResult::AsynchAcceptResult(AsynchAcceptor::Callback cb, + AsynchAcceptor *acceptor, + SOCKET listener) + : callback(cb), acceptor(acceptor), listener(listener) { + newSocket.reset (new Socket()); +} + +void AsynchAcceptResult::success(size_t /*bytesTransferred*/) { + ::setsockopt (toSocketHandle(*newSocket), + SOL_SOCKET, + SO_UPDATE_ACCEPT_CONTEXT, + (char*)&listener, + sizeof (listener)); + callback(*(newSocket.release())); + acceptor->restart (); + delete this; +} + +void AsynchAcceptResult::failure(int /*status*/) { + //if (status != WSA_OPERATION_ABORTED) + // Can there be anything else? ; + delete this; +} + +/* + * AsynchConnector does synchronous connects for now... to do asynch the + * IocpPoller will need some extension to register an event handle as a + * CONNECT-type "direction", the connect completion/result will need an + * event handle to associate with the connecting handle. But there's no + * time for that right now... + */ +class AsynchConnector : public qpid::sys::AsynchConnector { +private: + ConnectedCallback connCallback; + FailedCallback failCallback; + const Socket& socket; + const std::string hostname; + const std::string port; + +public: + AsynchConnector(const Socket& socket, + const std::string& hostname, + const std::string& port, + ConnectedCallback connCb, + FailedCallback failCb = 0); + void start(Poller::shared_ptr poller); +}; + +AsynchConnector::AsynchConnector(const Socket& sock, + const std::string& hname, + const std::string& p, + ConnectedCallback connCb, + FailedCallback failCb) : + connCallback(connCb), failCallback(failCb), socket(sock), + hostname(hname), port(p) +{ +} + +void AsynchConnector::start(Poller::shared_ptr) +{ + try { + socket.connect(hostname, port); + socket.setNonblocking(); + connCallback(socket); + } catch(std::exception& e) { + if (failCallback) + failCallback(socket, -1, std::string(e.what())); + socket.close(); + } +} + +} // namespace windows + +AsynchAcceptor* AsynchAcceptor::create(const Socket& s, + Callback callback) +{ + return new windows::AsynchAcceptor(s, callback); +} + +AsynchConnector* qpid::sys::AsynchConnector::create(const Socket& s, + const std::string& hostname, + const std::string& port, + ConnectedCallback connCb, + FailedCallback failCb) +{ + return new windows::AsynchConnector(s, + hostname, + port, + connCb, + failCb); +} + + +/* + * Asynch reader/writer + */ + +namespace windows { + +class AsynchIO : public qpid::sys::AsynchIO { +public: + AsynchIO(const Socket& s, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb = 0, + BuffersEmptyCallback eCb = 0, + IdleCallback iCb = 0); + ~AsynchIO(); + + // Methods inherited from qpid::sys::AsynchIO + + /** + * Notify the object is should delete itself as soon as possible. + */ + virtual void queueForDeletion(); + + /// Take any actions needed to prepare for working with the poller. + virtual void start(Poller::shared_ptr poller); + virtual void queueReadBuffer(BufferBase* buff); + virtual void unread(BufferBase* buff); + virtual void queueWrite(BufferBase* buff); + virtual void notifyPendingWrite(); + virtual void queueWriteClose(); + virtual bool writeQueueEmpty(); + virtual void startReading(); + virtual void stopReading(); + virtual void requestCallback(RequestCallback); + + /** + * getQueuedBuffer returns a buffer from the buffer queue, if one is + * available. + * + * @retval Pointer to BufferBase buffer; 0 if none is available. + */ + virtual BufferBase* getQueuedBuffer(); + +private: + ReadCallback readCallback; + EofCallback eofCallback; + DisconnectCallback disCallback; + ClosedCallback closedCallback; + BuffersEmptyCallback emptyCallback; + IdleCallback idleCallback; + const Socket& socket; + Poller::shared_ptr poller; + + std::deque<BufferBase*> bufferQueue; + std::deque<BufferBase*> writeQueue; + /* The MSVC-supplied deque is not thread-safe; keep locks to serialize + * access to the buffer queue and write queue. + */ + Mutex bufferQueueLock; + + // Number of outstanding I/O operations. + volatile LONG opsInProgress; + // Is there a write in progress? + volatile bool writeInProgress; + // Deletion requested, but there are callbacks in progress. + volatile bool queuedDelete; + // Socket close requested, but there are operations in progress. + volatile bool queuedClose; + +private: + // Dispatch events that have completed. + void notifyEof(void); + void notifyDisconnect(void); + void notifyClosed(void); + void notifyBuffersEmpty(void); + void notifyIdle(void); + + /** + * Initiate a write of the specified buffer. There's no callback for + * write completion to the AsynchIO object. + */ + void startWrite(AsynchIO::BufferBase* buff); + + void close(void); + + /** + * readComplete is called when a read request is complete. + * + * @param result Results of the operation. + */ + void readComplete(AsynchReadResult *result); + + /** + * writeComplete is called when a write request is complete. + * + * @param result Results of the operation. + */ + void writeComplete(AsynchWriteResult *result); + + /** + * Queue of completions to run. This queue enforces the requirement + * from upper layers that only one thread at a time is allowed to act + * on any given connection. Once a thread is busy processing a completion + * on this object, other threads that dispatch completions queue the + * completions here for the in-progress thread to handle when done. + * Thus, any threads can dispatch a completion from the IocpPoller, but + * this class ensures that actual processing at the connection level is + * only on one thread at a time. + */ + std::queue<AsynchIoResult *> completionQueue; + volatile bool working; + Mutex completionLock; + + /** + * Called when there's a completion to process. + */ + void completion(AsynchIoResult *result); +}; + +// This is used to encapsulate pure callbacks into a handle +class CallbackHandle : public IOHandle { +public: + CallbackHandle(AsynchIoResult::Completer completeCb, + AsynchIO::RequestCallback reqCb = 0) : + IOHandle(new IOHandlePrivate (INVALID_SOCKET, completeCb, reqCb)) + {} +}; + +AsynchIO::AsynchIO(const Socket& s, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb, + BuffersEmptyCallback eCb, + IdleCallback iCb) : + + readCallback(rCb), + eofCallback(eofCb), + disCallback(disCb), + closedCallback(cCb), + emptyCallback(eCb), + idleCallback(iCb), + socket(s), + opsInProgress(0), + writeInProgress(false), + queuedDelete(false), + queuedClose(false), + working(false) { +} + +struct deleter +{ + template <typename T> + void operator()(T *ptr){ delete ptr;} +}; + +AsynchIO::~AsynchIO() { + std::for_each( bufferQueue.begin(), bufferQueue.end(), deleter()); + std::for_each( writeQueue.begin(), writeQueue.end(), deleter()); +} + +void AsynchIO::queueForDeletion() { + queuedDelete = true; + if (opsInProgress > 0) { + QPID_LOG(info, "Delete AsynchIO queued; ops in progress"); + // AsynchIOHandler calls this then deletes itself; don't do any more + // callbacks. + readCallback = 0; + eofCallback = 0; + disCallback = 0; + closedCallback = 0; + emptyCallback = 0; + idleCallback = 0; + } + else { + delete this; + } +} + +void AsynchIO::start(Poller::shared_ptr poller0) { + PollerHandle ph = PollerHandle(socket); + poller = poller0; + poller->monitorHandle(ph, Poller::INPUT); + if (writeQueue.size() > 0) // Already have data queued for write + notifyPendingWrite(); + startReading(); +} + +void AsynchIO::queueReadBuffer(AsynchIO::BufferBase* buff) { + assert(buff); + buff->dataStart = 0; + buff->dataCount = 0; + QLock l(bufferQueueLock); + bufferQueue.push_back(buff); +} + +void AsynchIO::unread(AsynchIO::BufferBase* buff) { + assert(buff); + buff->squish(); + QLock l(bufferQueueLock); + bufferQueue.push_front(buff); +} + +void AsynchIO::queueWrite(AsynchIO::BufferBase* buff) { + assert(buff); + QLock l(bufferQueueLock); + writeQueue.push_back(buff); + if (!writeInProgress) + notifyPendingWrite(); +} + +void AsynchIO::notifyPendingWrite() { + // This method is generally called from a processing thread; transfer + // work on this to an I/O thread. Much of the upper layer code assumes + // that all I/O-related things happen in an I/O thread. + if (poller == 0) // Not really going yet... + return; + + InterlockedIncrement(&opsInProgress); + PollerHandle ph(CallbackHandle(boost::bind(&AsynchIO::completion, this, _1))); + poller->monitorHandle(ph, Poller::OUTPUT); +} + +void AsynchIO::queueWriteClose() { + queuedClose = true; + if (!writeInProgress) + notifyPendingWrite(); +} + +bool AsynchIO::writeQueueEmpty() { + QLock l(bufferQueueLock); + return writeQueue.size() == 0; +} + +/* + * Initiate a read operation. AsynchIO::readComplete() will be + * called when the read is complete and data is available. + */ +void AsynchIO::startReading() { + if (queuedDelete) + return; + + // (Try to) get a buffer; look on the front since there may be an + // "unread" one there with data remaining from last time. + AsynchIO::BufferBase *buff = 0; + { + QLock l(bufferQueueLock); + + if (!bufferQueue.empty()) { + buff = bufferQueue.front(); + assert(buff); + bufferQueue.pop_front(); + } + } + if (buff != 0) { + int readCount = buff->byteCount - buff->dataCount; + AsynchReadResult *result = + new AsynchReadResult(boost::bind(&AsynchIO::completion, this, _1), + buff, + readCount); + DWORD bytesReceived = 0, flags = 0; + InterlockedIncrement(&opsInProgress); + int status = WSARecv(toSocketHandle(socket), + const_cast<LPWSABUF>(result->getWSABUF()), 1, + &bytesReceived, + &flags, + result->overlapped(), + 0); + if (status != 0) { + int error = WSAGetLastError(); + if (error != WSA_IO_PENDING) { + result->failure(error); + result = 0; // result is invalid here + return; + } + } + // On status 0 or WSA_IO_PENDING, completion will handle the rest. + } + else { + notifyBuffersEmpty(); + } + return; +} + +// stopReading was added to prevent a race condition with read-credit on Linux. +// It may or may not be required on windows. +// +// AsynchIOHandler::readbuff() calls stopReading() inside the same +// critical section that protects startReading() in +// AsynchIOHandler::giveReadCredit(). +// +void AsynchIO::stopReading() {} + +// Queue the specified callback for invocation from an I/O thread. +void AsynchIO::requestCallback(RequestCallback callback) { + // This method is generally called from a processing thread; transfer + // work on this to an I/O thread. Much of the upper layer code assumes + // that all I/O-related things happen in an I/O thread. + if (poller == 0) // Not really going yet... + return; + + InterlockedIncrement(&opsInProgress); + PollerHandle ph(CallbackHandle( + boost::bind(&AsynchIO::completion, this, _1), + callback)); + poller->monitorHandle(ph, Poller::INPUT); +} + +/** + * Return a queued buffer if there are enough to spare. + */ +AsynchIO::BufferBase* AsynchIO::getQueuedBuffer() { + QLock l(bufferQueueLock); + // Always keep at least one buffer (it might have data that was + // "unread" in it). + if (bufferQueue.size() <= 1) + return 0; + BufferBase* buff = bufferQueue.back(); + assert(buff); + bufferQueue.pop_back(); + return buff; +} + +void AsynchIO::notifyEof(void) { + if (eofCallback) + eofCallback(*this); +} + +void AsynchIO::notifyDisconnect(void) { + if (disCallback) + disCallback(*this); +} + +void AsynchIO::notifyClosed(void) { + if (closedCallback) + closedCallback(*this, socket); +} + +void AsynchIO::notifyBuffersEmpty(void) { + if (emptyCallback) + emptyCallback(*this); +} + +void AsynchIO::notifyIdle(void) { + if (idleCallback) + idleCallback(*this); +} + +/* + * Asynch reader/writer using overlapped I/O + */ + +void AsynchIO::startWrite(AsynchIO::BufferBase* buff) { + writeInProgress = true; + InterlockedIncrement(&opsInProgress); + AsynchWriteResult *result = + new AsynchWriteResult(boost::bind(&AsynchIO::completion, this, _1), + buff, + buff->dataCount); + DWORD bytesSent = 0; + int status = WSASend(toSocketHandle(socket), + const_cast<LPWSABUF>(result->getWSABUF()), 1, + &bytesSent, + 0, + result->overlapped(), + 0); + if (status != 0) { + int error = WSAGetLastError(); + if (error != WSA_IO_PENDING) { + result->failure(error); // Also decrements in-progress count + result = 0; // result is invalid here + return; + } + } + // On status 0 or WSA_IO_PENDING, completion will handle the rest. + return; +} + +/* + * Close the socket and callback to say we've done it + */ +void AsynchIO::close(void) { + socket.close(); + notifyClosed(); +} + +void AsynchIO::readComplete(AsynchReadResult *result) { + int status = result->getStatus(); + size_t bytes = result->getTransferred(); + if (status == 0 && bytes > 0) { + bool restartRead = true; // May not if receiver doesn't want more + if (readCallback) + readCallback(*this, result->getBuff()); + if (restartRead) + startReading(); + } + else { + // No data read, so put the buffer back. It may be partially filled, + // so "unread" it back to the front of the queue. + unread(result->getBuff()); + notifyEof(); + if (status != 0) + { + notifyDisconnect(); + } + } +} + +/* + * NOTE - this completion is called for completed writes and also when + * a write is desired. The difference is in the buff - if a write is desired + * the buff is 0. + */ +void AsynchIO::writeComplete(AsynchWriteResult *result) { + int status = result->getStatus(); + size_t bytes = result->getTransferred(); + AsynchIO::BufferBase *buff = result->getBuff(); + if (buff != 0) { + writeInProgress = false; + if (status == 0 && bytes > 0) { + if (bytes < result->getRequested()) // Still more to go; resubmit + startWrite(buff); + else + queueReadBuffer(buff); // All done; back to the pool + } + else { + // An error... if it's a connection close, ignore it - it will be + // noticed and handled on a read completion any moment now. + // What to do with real error??? Save the Buffer? + } + } + + // If there are no writes outstanding, check for more writes to initiate + // (either queued or via idle). The opsInProgress count is handled in + // completion() + if (!writeInProgress) { + bool writing = false; + { + QLock l(bufferQueueLock); + if (writeQueue.size() > 0) { + buff = writeQueue.front(); + assert(buff); + writeQueue.pop_front(); + startWrite(buff); + writing = true; + } + } + if (!writing && !queuedClose) { + notifyIdle(); + } + } + return; +} + +void AsynchIO::completion(AsynchIoResult *result) { + { + ScopedLock<Mutex> l(completionLock); + if (working) { + completionQueue.push(result); + return; + } + + // First thread in with something to do; note we're working then keep + // handling completions. + working = true; + while (result != 0) { + // New scope to unlock temporarily. + { + ScopedUnlock<Mutex> ul(completionLock); + AsynchReadResult *r = dynamic_cast<AsynchReadResult*>(result); + if (r != 0) + readComplete(r); + else { + AsynchWriteResult *w = + dynamic_cast<AsynchWriteResult*>(result); + if (w != 0) + writeComplete(w); + else { + AsynchCallbackRequest *req = + dynamic_cast<AsynchCallbackRequest*>(result); + req->reqCallback(*this); + } + } + delete result; + result = 0; + InterlockedDecrement(&opsInProgress); + } + // Lock is held again. + if (completionQueue.empty()) + continue; + result = completionQueue.front(); + completionQueue.pop(); + } + working = false; + } + // Lock released; ok to close if ops are done and close requested. + // Layer above will call back to queueForDeletion() if it hasn't + // already been done. If it already has, go ahead and delete. + if (opsInProgress == 0) { + if (queuedClose) + // close() may cause a delete; don't trust 'this' on return + close(); + else if (queuedDelete) + delete this; + } +} + +} // namespace windows + +AsynchIO* qpid::sys::AsynchIO::create(const Socket& s, + AsynchIO::ReadCallback rCb, + AsynchIO::EofCallback eofCb, + AsynchIO::DisconnectCallback disCb, + AsynchIO::ClosedCallback cCb, + AsynchIO::BuffersEmptyCallback eCb, + AsynchIO::IdleCallback iCb) +{ + return new qpid::sys::windows::AsynchIO(s, rCb, eofCb, disCb, cCb, eCb, iCb); +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/windows/AsynchIoResult.h b/qpid/cpp/src/qpid/sys/windows/AsynchIoResult.h new file mode 100755 index 0000000000..b11324918b --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/AsynchIoResult.h @@ -0,0 +1,204 @@ +#ifndef _windows_asynchIoResult_h +#define _windows_asynchIoResult_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/AsynchIO.h" +#include "qpid/sys/Socket.h" +#include <memory.h> +#include <winsock2.h> +#include <ws2tcpip.h> + +namespace qpid { +namespace sys { +namespace windows { + +/* + * AsynchIoResult defines the class that receives the result of an + * asynchronous I/O operation, either send/recv or accept/connect. + * + * Operation factories should set one of these up before beginning the + * operation. Poller knows how to dispatch completion to this class. + * This class must be subclassed for needed operations; this class provides + * an interface only and cannot be instantiated. + * + * This class is tied to Windows; it inherits from OVERLAPPED so that the + * IocpPoller can cast OVERLAPPED pointers back to AsynchIoResult and call + * the completion handler. + */ +class AsynchResult : private OVERLAPPED { +public: + LPOVERLAPPED overlapped(void) { return this; } + static AsynchResult* from_overlapped(LPOVERLAPPED ol) { + return static_cast<AsynchResult*>(ol); + } + virtual void success (size_t bytesTransferred) { + bytes = bytesTransferred; + status = 0; + complete(); + } + virtual void failure (int error) { + bytes = 0; + status = error; + complete(); + } + size_t getTransferred(void) const { return bytes; } + int getStatus(void) const { return status; } + +protected: + AsynchResult() : bytes(0), status(0) + { memset(overlapped(), 0, sizeof(OVERLAPPED)); } + ~AsynchResult() {} + virtual void complete(void) = 0; + + size_t bytes; + int status; +}; + +class AsynchAcceptor; + +class AsynchAcceptResult : public AsynchResult { + + friend class AsynchAcceptor; + +public: + AsynchAcceptResult(qpid::sys::AsynchAcceptor::Callback cb, + AsynchAcceptor *acceptor, + SOCKET listener); + virtual void success (size_t bytesTransferred); + virtual void failure (int error); + +private: + virtual void complete(void) {} // No-op for this class. + + std::auto_ptr<qpid::sys::Socket> newSocket; + qpid::sys::AsynchAcceptor::Callback callback; + AsynchAcceptor *acceptor; + SOCKET listener; + + // AcceptEx needs a place to write the local and remote addresses + // when accepting the connection. Place those here; get enough for + // IPv6 addresses, even if the socket is IPv4. + enum { SOCKADDRMAXLEN = sizeof(sockaddr_in6) + 16, + SOCKADDRBUFLEN = 2 * SOCKADDRMAXLEN }; + char addressBuffer[SOCKADDRBUFLEN]; +}; + +class AsynchIoResult : public AsynchResult { +public: + typedef boost::function1<void, AsynchIoResult *> Completer; + + virtual ~AsynchIoResult() {} + qpid::sys::AsynchIO::BufferBase *getBuff(void) const { return iobuff; } + size_t getRequested(void) const { return requested; } + const WSABUF *getWSABUF(void) const { return &wsabuf; } + +protected: + void setBuff (qpid::sys::AsynchIO::BufferBase *buffer) { iobuff = buffer; } + +protected: + AsynchIoResult(Completer cb, + qpid::sys::AsynchIO::BufferBase *buff, size_t length) + : completionCallback(cb), iobuff(buff), requested(length) {} + + virtual void complete(void) = 0; + WSABUF wsabuf; + Completer completionCallback; + +private: + qpid::sys::AsynchIO::BufferBase *iobuff; + size_t requested; // Number of bytes in original I/O request +}; + +class AsynchReadResult : public AsynchIoResult { + + // complete() updates buffer then does completion callback. + virtual void complete(void) { + getBuff()->dataCount += bytes; + completionCallback(this); + } + +public: + AsynchReadResult(AsynchIoResult::Completer cb, + qpid::sys::AsynchIO::BufferBase *buff, + size_t length) + : AsynchIoResult(cb, buff, length) { + wsabuf.buf = buff->bytes + buff->dataCount; + wsabuf.len = length; + } +}; + +class AsynchWriteResult : public AsynchIoResult { + + // complete() updates buffer then does completion callback. + virtual void complete(void) { + qpid::sys::AsynchIO::BufferBase *b = getBuff(); + b->dataStart += bytes; + b->dataCount -= bytes; + completionCallback(this); + } + +public: + AsynchWriteResult(AsynchIoResult::Completer cb, + qpid::sys::AsynchIO::BufferBase *buff, + size_t length) + : AsynchIoResult(cb, buff, length) { + wsabuf.buf = buff ? buff->bytes : 0; + wsabuf.len = length; + } +}; + +class AsynchWriteWanted : public AsynchWriteResult { + + // complete() just does completion callback; no buffers used. + virtual void complete(void) { + completionCallback(this); + } + +public: + AsynchWriteWanted(AsynchIoResult::Completer cb) + : AsynchWriteResult(cb, 0, 0) { + wsabuf.buf = 0; + wsabuf.len = 0; + } +}; + +class AsynchCallbackRequest : public AsynchIoResult { + // complete() needs to simply call the completionCallback; no buffers. + virtual void complete(void) { + completionCallback(this); + } + +public: + AsynchCallbackRequest(AsynchIoResult::Completer cb, + qpid::sys::AsynchIO::RequestCallback reqCb) + : AsynchIoResult(cb, 0, 0), reqCallback(reqCb) { + wsabuf.buf = 0; + wsabuf.len = 0; + } + + qpid::sys::AsynchIO::RequestCallback reqCallback; +}; + +}}} // qpid::sys::windows + +#endif /*!_windows_asynchIoResult_h*/ diff --git a/qpid/cpp/src/qpid/sys/windows/FileSysDir.cpp b/qpid/cpp/src/qpid/sys/windows/FileSysDir.cpp new file mode 100644 index 0000000000..88f1637d48 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/FileSysDir.cpp @@ -0,0 +1,53 @@ +/* + * + * Copyright (c) 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/FileSysDir.h" +#include "qpid/sys/StrError.h" +#include "qpid/Exception.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <direct.h> +#include <errno.h> + +namespace qpid { +namespace sys { + +bool FileSysDir::exists (void) const +{ + const char *cpath = dirPath.c_str (); + struct _stat s; + if (::_stat(cpath, &s)) { + if (errno == ENOENT) { + return false; + } + throw qpid::Exception (strError(errno) + + ": Can't check directory: " + dirPath); + } + if (s.st_mode & _S_IFDIR) + return true; + throw qpid::Exception(dirPath + " is not a directory"); +} + +void FileSysDir::mkdir(void) +{ + if (::_mkdir(dirPath.c_str()) == -1) + throw Exception ("Can't create directory: " + dirPath); +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/windows/IOHandle.cpp b/qpid/cpp/src/qpid/sys/windows/IOHandle.cpp new file mode 100755 index 0000000000..250737cb99 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/IOHandle.cpp @@ -0,0 +1,42 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/windows/IoHandlePrivate.h" +#include <windows.h> + +namespace qpid { +namespace sys { + +SOCKET toFd(const IOHandlePrivate* h) +{ + return h->fd; +} + +IOHandle::IOHandle(IOHandlePrivate* h) : + impl(h) +{} + +IOHandle::~IOHandle() { + delete impl; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/windows/IoHandlePrivate.h b/qpid/cpp/src/qpid/sys/windows/IoHandlePrivate.h new file mode 100755 index 0000000000..5943db5cc7 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/IoHandlePrivate.h @@ -0,0 +1,61 @@ +#ifndef _sys_windows_IoHandlePrivate_h +#define _sys_windows_IoHandlePrivate_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/AsynchIO.h" +#include "qpid/sys/windows/AsynchIoResult.h" +#include "qpid/CommonImportExport.h" + +#include <winsock2.h> + +namespace qpid { +namespace sys { + +// Private fd related implementation details +// There should be either a valid socket handle or a completer callback. +// Handle is used to associate with poller's iocp; completer is used to +// inject a completion that will very quickly trigger a callback to the +// completer from an I/O thread. If the callback mechanism is used, there +// can be a RequestCallback set - this carries the callback object through +// from AsynchIO::requestCallback() through to the I/O completion processing. +class IOHandlePrivate { + friend QPID_COMMON_EXTERN SOCKET toSocketHandle(const Socket& s); + static IOHandlePrivate* getImpl(const IOHandle& h); + +public: + IOHandlePrivate(SOCKET f = INVALID_SOCKET, + windows::AsynchIoResult::Completer cb = 0, + AsynchIO::RequestCallback reqCallback = 0) : + fd(f), event(cb), cbRequest(reqCallback) + {} + + SOCKET fd; + windows::AsynchIoResult::Completer event; + AsynchIO::RequestCallback cbRequest; +}; + +QPID_COMMON_EXTERN SOCKET toSocketHandle(const Socket& s); + +}} + +#endif /* _sys_windows_IoHandlePrivate_h */ diff --git a/qpid/cpp/src/qpid/sys/windows/IocpPoller.cpp b/qpid/cpp/src/qpid/sys/windows/IocpPoller.cpp new file mode 100755 index 0000000000..1805dd2cd8 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/IocpPoller.cpp @@ -0,0 +1,219 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Poller.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/Dispatcher.h" + +#include "qpid/sys/windows/AsynchIoResult.h" +#include "qpid/sys/windows/IoHandlePrivate.h" +#include "qpid/sys/windows/check.h" + +#include <winsock2.h> +#include <windows.h> + +#include <assert.h> +#include <vector> +#include <exception> + +namespace qpid { +namespace sys { + +class PollerHandlePrivate { + friend class Poller; + friend class PollerHandle; + + SOCKET fd; + windows::AsynchIoResult::Completer cb; + AsynchIO::RequestCallback cbRequest; + + PollerHandlePrivate(SOCKET f, + windows::AsynchIoResult::Completer cb0 = 0, + AsynchIO::RequestCallback rcb = 0) + : fd(f), cb(cb0), cbRequest(rcb) + { + } + +}; + +PollerHandle::PollerHandle(const IOHandle& h) : + impl(new PollerHandlePrivate(toSocketHandle(static_cast<const Socket&>(h)), h.impl->event, h.impl->cbRequest)) +{} + +PollerHandle::~PollerHandle() { + delete impl; +} + +/** + * Concrete implementation of Poller to use the Windows I/O Completion + * port (IOCP) facility. + */ +class PollerPrivate { + friend class Poller; + + const HANDLE iocp; + + // The number of threads running the event loop. + volatile LONG threadsRunning; + + // Shutdown request is handled by setting isShutdown and injecting a + // well-formed completion event into the iocp. + bool isShutdown; + + PollerPrivate() : + iocp(::CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0)), + threadsRunning(0), + isShutdown(false) { + QPID_WINDOWS_CHECK_NULL(iocp); + } + + ~PollerPrivate() { + // It's probably okay to ignore any errors here as there can't be + // data loss + ::CloseHandle(iocp); + } +}; + +void Poller::shutdown() { + // Allow sloppy code to shut us down more than once. + if (impl->isShutdown) + return; + ULONG_PTR key = 1; // Tell wait() it's a shutdown, not I/O + PostQueuedCompletionStatus(impl->iocp, 0, key, 0); +} + +bool Poller::hasShutdown() +{ + return impl->isShutdown; +} + +bool Poller::interrupt(PollerHandle&) { + return false; // There's no concept of a registered handle. +} + +void Poller::run() { + do { + Poller::Event event = this->wait(); + + // Handle shutdown + switch (event.type) { + case Poller::SHUTDOWN: + return; + break; + case Poller::INVALID: // On any type of success or fail completion + break; + default: + // This should be impossible + assert(false); + } + } while (true); +} + +void Poller::monitorHandle(PollerHandle& handle, Direction dir) { + HANDLE h = (HANDLE)(handle.impl->fd); + if (h != INVALID_HANDLE_VALUE) { + HANDLE iocpHandle = ::CreateIoCompletionPort (h, impl->iocp, 0, 0); + QPID_WINDOWS_CHECK_NULL(iocpHandle); + } + else { + // INPUT is used to request a callback; OUTPUT to request a write + assert(dir == Poller::INPUT || dir == Poller::OUTPUT); + + if (dir == Poller::OUTPUT) { + windows::AsynchWriteWanted *result = + new windows::AsynchWriteWanted(handle.impl->cb); + PostQueuedCompletionStatus(impl->iocp, 0, 0, result->overlapped()); + } + else { + windows::AsynchCallbackRequest *result = + new windows::AsynchCallbackRequest(handle.impl->cb, + handle.impl->cbRequest); + PostQueuedCompletionStatus(impl->iocp, 0, 0, result->overlapped()); + } + } +} + +// All no-ops... +void Poller::unmonitorHandle(PollerHandle& /*handle*/, Direction /*dir*/) {} +void Poller::registerHandle(PollerHandle& /*handle*/) {} +void Poller::unregisterHandle(PollerHandle& /*handle*/) {} + +Poller::Event Poller::wait(Duration timeout) { + DWORD timeoutMs = 0; + DWORD numTransferred = 0; + ULONG_PTR completionKey = 0; + OVERLAPPED *overlapped = 0; + windows::AsynchResult *result = 0; + + // Wait for either an I/O operation to finish (thus signaling the + // IOCP handle) or a shutdown request to be made (thus signaling the + // shutdown event). + if (timeout == TIME_INFINITE) + timeoutMs = INFINITE; + else + timeoutMs = static_cast<DWORD>(timeout / TIME_MSEC); + + InterlockedIncrement(&impl->threadsRunning); + bool goodOp = ::GetQueuedCompletionStatus (impl->iocp, + &numTransferred, + &completionKey, + &overlapped, + timeoutMs); + LONG remainingThreads = InterlockedDecrement(&impl->threadsRunning); + if (goodOp) { + // Dequeued a successful completion. If it's a posted packet from + // shutdown() the overlapped ptr is 0 and key is 1. Else downcast + // the OVERLAPPED pointer to an AsynchIoResult and call the + // completion handler. + if (overlapped == 0 && completionKey == 1) { + // If there are other threads still running this wait, re-post + // the completion. + if (remainingThreads > 0) + PostQueuedCompletionStatus(impl->iocp, 0, completionKey, 0); + return Event(0, SHUTDOWN); + } + + result = windows::AsynchResult::from_overlapped(overlapped); + result->success (static_cast<size_t>(numTransferred)); + } + else { + if (overlapped != 0) { + // Dequeued a completion for a failed operation. Downcast back + // to the result object and inform it that the operation failed. + DWORD status = ::GetLastError(); + result = windows::AsynchResult::from_overlapped(overlapped); + result->failure (static_cast<int>(status)); + } + } + return Event(0, INVALID); // TODO - this may need to be changed. + +} + +// Concrete constructors +Poller::Poller() : + impl(new PollerPrivate()) +{} + +Poller::~Poller() { + delete impl; +} + +}} diff --git a/qpid/cpp/src/qpid/sys/windows/LockFile.cpp b/qpid/cpp/src/qpid/sys/windows/LockFile.cpp new file mode 100755 index 0000000000..048c2d5b18 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/LockFile.cpp @@ -0,0 +1,64 @@ +/* + * + * Copyright (c) 2008 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qpid/sys/LockFile.h" +#include "qpid/sys/windows/check.h" + +#include <windows.h> + +namespace qpid { +namespace sys { + +class LockFilePrivate { + friend class LockFile; + + HANDLE fd; + +public: + LockFilePrivate(HANDLE f) : fd(f) {} +}; + +LockFile::LockFile(const std::string& path_, bool create) + : path(path_), created(create) { + + HANDLE h = ::CreateFile(path.c_str(), + create ? (GENERIC_READ|GENERIC_WRITE) : GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + 0, /* Default security */ + create ? OPEN_ALWAYS : OPEN_EXISTING, + FILE_FLAG_DELETE_ON_CLOSE, /* Delete file when closed */ + NULL); + if (h == INVALID_HANDLE_VALUE) + throw qpid::Exception(path + ": " + qpid::sys::strError(GetLastError())); + + // Lock up to 4Gb + if (!::LockFile(h, 0, 0, 0xffffffff, 0)) + throw qpid::Exception(path + ": " + qpid::sys::strError(GetLastError())); + impl.reset(new LockFilePrivate(h)); +} + +LockFile::~LockFile() { + if (impl) { + if (impl->fd != INVALID_HANDLE_VALUE) { + ::UnlockFile(impl->fd, 0, 0, 0xffffffff, 0); + ::CloseHandle(impl->fd); + } + } +} + +}} /* namespace qpid::sys */ diff --git a/qpid/cpp/src/qpid/sys/windows/PipeHandle.cpp b/qpid/cpp/src/qpid/sys/windows/PipeHandle.cpp new file mode 100755 index 0000000000..062458ae5f --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/PipeHandle.cpp @@ -0,0 +1,101 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +#include "qpid/sys/PipeHandle.h" +#include "qpid/sys/windows/check.h" +#include <winsock2.h> + +namespace qpid { +namespace sys { + +PipeHandle::PipeHandle(bool nonBlocking) { + + SOCKET listener, pair[2]; + struct sockaddr_in addr; + int err; + int addrlen = sizeof(addr); + pair[0] = pair[1] = INVALID_SOCKET; + if ((listener = socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + + err = bind(listener, (const struct sockaddr*) &addr, sizeof(addr)); + if (err == SOCKET_ERROR) { + err = WSAGetLastError(); + closesocket(listener); + throw QPID_WINDOWS_ERROR(err); + } + + err = getsockname(listener, (struct sockaddr*) &addr, &addrlen); + if (err == SOCKET_ERROR) { + err = WSAGetLastError(); + closesocket(listener); + throw QPID_WINDOWS_ERROR(err); + } + + try { + if (listen(listener, 1) == SOCKET_ERROR) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + if ((pair[0] = socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + if (connect(pair[0], (const struct sockaddr*)&addr, sizeof(addr)) == SOCKET_ERROR) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + if ((pair[1] = accept(listener, NULL, NULL)) == INVALID_SOCKET) + throw QPID_WINDOWS_ERROR(WSAGetLastError()); + + closesocket(listener); + writeFd = pair[0]; + readFd = pair[1]; + } + catch (...) { + closesocket(listener); + if (pair[0] != INVALID_SOCKET) + closesocket(pair[0]); + throw; + } + + // Set the socket to non-blocking + if (nonBlocking) { + unsigned long nonblock = 1; + ioctlsocket(readFd, FIONBIO, &nonblock); + } +} + +PipeHandle::~PipeHandle() { + closesocket(readFd); + closesocket(writeFd); +} + +int PipeHandle::read(void* buf, size_t bufSize) { + return ::recv(readFd, (char *)buf, bufSize, 0); +} + +int PipeHandle::write(const void* buf, size_t bufSize) { + return ::send(writeFd, (const char *)buf, bufSize, 0); +} + +int PipeHandle::getReadHandle() { + return readFd; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/windows/PollableCondition.cpp b/qpid/cpp/src/qpid/sys/windows/PollableCondition.cpp new file mode 100644 index 0000000000..6a1d9045b4 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/PollableCondition.cpp @@ -0,0 +1,114 @@ +#ifndef QPID_SYS_WINDOWS_POLLABLECONDITION_CPP +#define QPID_SYS_WINDOWS_POLLABLECONDITION_CPP + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/PollableCondition.h" +#include "qpid/sys/IOHandle.h" +#include "qpid/sys/windows/AsynchIoResult.h" +#include "qpid/sys/windows/IoHandlePrivate.h" + +#include <boost/bind.hpp> +#include <windows.h> + +namespace qpid { +namespace sys { + +// PollableConditionPrivate will reuse the IocpPoller's ability to queue +// a completion to the IOCP and have it dispatched to the completer callback +// noted in the IOHandlePrivate when the request is queued. The +// AsynchCallbackRequest object is not really used - we already have the +// desired callback for the user of PollableCondition. +class PollableConditionPrivate : private IOHandle { + friend class PollableCondition; + +private: + PollableConditionPrivate(const sys::PollableCondition::Callback& cb, + sys::PollableCondition& parent, + const boost::shared_ptr<sys::Poller>& poller); + ~PollableConditionPrivate(); + + void poke(); + void dispatch(windows::AsynchIoResult *result); + +private: + PollableCondition::Callback cb; + PollableCondition& parent; + boost::shared_ptr<sys::Poller> poller; + LONG isSet; +}; + +PollableConditionPrivate::PollableConditionPrivate(const sys::PollableCondition::Callback& cb, + sys::PollableCondition& parent, + const boost::shared_ptr<sys::Poller>& poller) + : IOHandle(new sys::IOHandlePrivate(INVALID_SOCKET, + boost::bind(&PollableConditionPrivate::dispatch, this, _1))), + cb(cb), parent(parent), poller(poller), isSet(0) +{ +} + +PollableConditionPrivate::~PollableConditionPrivate() +{ +} + +void PollableConditionPrivate::poke() +{ + // monitorHandle will queue a completion for the IOCP; when it's handled, a + // poller thread will call back to dispatch() below. + PollerHandle ph(*this); + poller->monitorHandle(ph, Poller::INPUT); +} + +void PollableConditionPrivate::dispatch(windows::AsynchIoResult *result) +{ + delete result; // Poller::monitorHandle() allocates this + cb(parent); + if (isSet) + poke(); +} + + /* PollableCondition */ + +PollableCondition::PollableCondition(const Callback& cb, + const boost::shared_ptr<sys::Poller>& poller) + : impl(new PollableConditionPrivate(cb, *this, poller)) +{ +} + +PollableCondition::~PollableCondition() +{ + delete impl; +} + +void PollableCondition::set() { + // Add one to the set count and poke it to provoke a callback + ::InterlockedIncrement(&impl->isSet); + impl->poke(); +} + +void PollableCondition::clear() { + ::InterlockedExchange(&impl->isSet, 0); +} + +}} // namespace qpid::sys + +#endif /*!QPID_SYS_WINDOWS_POLLABLECONDITION_CPP*/ diff --git a/qpid/cpp/src/qpid/sys/windows/Shlib.cpp b/qpid/cpp/src/qpid/sys/windows/Shlib.cpp new file mode 100644 index 0000000000..ba18747eb4 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/Shlib.cpp @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Shlib.h" +#include "qpid/Exception.h" +#include "qpid/sys/windows/check.h" +#include <windows.h> + +namespace qpid { +namespace sys { + +void Shlib::load(const char* name) { + HMODULE h = LoadLibrary(name); + if (h == NULL) { + throw QPID_WINDOWS_ERROR(GetLastError()); + } + handle = static_cast<void*>(h); +} + +void Shlib::unload() { + if (handle) { + if (FreeLibrary(static_cast<HMODULE>(handle)) == 0) { + throw QPID_WINDOWS_ERROR(GetLastError()); + } + handle = 0; + } +} + +void* Shlib::getSymbol(const char* name) { + // Double cast avoids warning about casting function pointer to object + void *sym = reinterpret_cast<void*>(reinterpret_cast<intptr_t>(GetProcAddress(static_cast<HMODULE>(handle), name))); + if (sym == NULL) + throw QPID_WINDOWS_ERROR(GetLastError()); + return sym; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/windows/Socket.cpp b/qpid/cpp/src/qpid/sys/windows/Socket.cpp new file mode 100755 index 0000000000..baa80f04e0 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/Socket.cpp @@ -0,0 +1,289 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +// Ensure we get all of winsock2.h +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0501 +#endif + +#include "qpid/sys/Socket.h" +#include "qpid/sys/SocketAddress.h" +#include "qpid/sys/windows/IoHandlePrivate.h" +#include "qpid/sys/windows/check.h" +#include "qpid/sys/Time.h" + +#include <cstdlib> +#include <string.h> + +#include <winsock2.h> + +#include <boost/format.hpp> +#include <boost/lexical_cast.hpp> + +// Need to initialize WinSock. Ideally, this would be a singleton or embedded +// in some one-time initialization function. I tried boost singleton and could +// not get it to compile (and others located in google had the same problem). +// So, this simple static with an interlocked increment will do for known +// use cases at this time. Since this will only shut down winsock at process +// termination, there may be some problems with client programs that also +// expect to load and unload winsock, but we'll see... +// If someone does get an easy-to-use singleton sometime, converting to it +// may be preferable. + +namespace { + +static LONG volatile initialized = 0; + +class WinSockSetup { + // : public boost::details::pool::singleton_default<WinSockSetup> { + +public: + WinSockSetup() { + LONG timesEntered = InterlockedIncrement(&initialized); + if (timesEntered > 1) + return; + err = 0; + WORD wVersionRequested; + WSADATA wsaData; + + /* Request WinSock 2.2 */ + wVersionRequested = MAKEWORD(2, 2); + err = WSAStartup(wVersionRequested, &wsaData); + } + + ~WinSockSetup() { + WSACleanup(); + } + +public: + int error(void) const { return err; } + +protected: + DWORD err; +}; + +static WinSockSetup setup; + +} /* namespace */ + +namespace qpid { +namespace sys { + +namespace { + +std::string getName(SOCKET fd, bool local) +{ + sockaddr_in name; // big enough for any socket address + socklen_t namelen = sizeof(name); + if (local) { + QPID_WINSOCK_CHECK(::getsockname(fd, (sockaddr*)&name, &namelen)); + } else { + QPID_WINSOCK_CHECK(::getpeername(fd, (sockaddr*)&name, &namelen)); + } + + char servName[NI_MAXSERV]; + char dispName[NI_MAXHOST]; + if (int rc = ::getnameinfo((sockaddr*)&name, namelen, + dispName, sizeof(dispName), + servName, sizeof(servName), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + throw qpid::Exception(QPID_MSG(gai_strerror(rc))); + return std::string(dispName) + ":" + std::string(servName); +} +} // namespace + +Socket::Socket() : + IOHandle(new IOHandlePrivate), + nonblocking(false), + nodelay(false) +{ + SOCKET& socket = impl->fd; + if (socket != INVALID_SOCKET) Socket::close(); + SOCKET s = ::socket (PF_INET, SOCK_STREAM, 0); + if (s == INVALID_SOCKET) throw QPID_WINDOWS_ERROR(WSAGetLastError()); + socket = s; +} + +Socket::Socket(IOHandlePrivate* h) : + IOHandle(h), + nonblocking(false), + nodelay(false) +{} + +void +Socket::createSocket(const SocketAddress& sa) const +{ + SOCKET& socket = impl->fd; + if (socket != INVALID_SOCKET) Socket::close(); + + SOCKET s = ::socket (getAddrInfo(sa).ai_family, + getAddrInfo(sa).ai_socktype, + 0); + if (s == INVALID_SOCKET) throw QPID_WINDOWS_ERROR(WSAGetLastError()); + socket = s; + + try { + if (nonblocking) setNonblocking(); + if (nodelay) setTcpNoDelay(); + } catch (std::exception&) { + closesocket(s); + socket = INVALID_SOCKET; + throw; + } +} + +void Socket::setNonblocking() const { + u_long nonblock = 1; + QPID_WINSOCK_CHECK(ioctlsocket(impl->fd, FIONBIO, &nonblock)); +} + +void Socket::connect(const std::string& host, const std::string& port) const +{ + SocketAddress sa(host, port); + connect(sa); +} + +void +Socket::connect(const SocketAddress& addr) const +{ + peername = addr.asString(false); + + const SOCKET& socket = impl->fd; + const addrinfo *addrs = &(getAddrInfo(addr)); + int error = 0; + WSASetLastError(0); + while (addrs != 0) { + if ((::connect(socket, addrs->ai_addr, addrs->ai_addrlen) == 0) || + (WSAGetLastError() == WSAEWOULDBLOCK)) + break; + // Error... save this error code and see if there are other address + // to try before throwing the exception. + error = WSAGetLastError(); + addrs = addrs->ai_next; + } + if (error) + throw qpid::Exception(QPID_MSG(strError(error) << ": " << peername)); +} + +void +Socket::close() const +{ + SOCKET& socket = impl->fd; + if (socket == INVALID_SOCKET) return; + QPID_WINSOCK_CHECK(closesocket(socket)); + socket = INVALID_SOCKET; +} + + +int Socket::write(const void *buf, size_t count) const +{ + const SOCKET& socket = impl->fd; + int sent = ::send(socket, (const char *)buf, count, 0); + if (sent == SOCKET_ERROR) + return -1; + return sent; +} + +int Socket::read(void *buf, size_t count) const +{ + const SOCKET& socket = impl->fd; + int received = ::recv(socket, (char *)buf, count, 0); + if (received == SOCKET_ERROR) + return -1; + return received; +} + +int Socket::listen(const std::string&, const std::string& port, int backlog) const +{ + const SOCKET& socket = impl->fd; + BOOL yes=1; + QPID_WINSOCK_CHECK(setsockopt(socket, SOL_SOCKET, SO_REUSEADDR, (char *)&yes, sizeof(yes))); + struct sockaddr_in name; + memset(&name, 0, sizeof(name)); + name.sin_family = AF_INET; + name.sin_port = htons(boost::lexical_cast<uint16_t>(port)); + name.sin_addr.s_addr = 0; + if (::bind(socket, (struct sockaddr*)&name, sizeof(name)) == SOCKET_ERROR) + throw Exception(QPID_MSG("Can't bind to port " << port << ": " << strError(WSAGetLastError()))); + if (::listen(socket, backlog) == SOCKET_ERROR) + throw Exception(QPID_MSG("Can't listen on port " << port << ": " << strError(WSAGetLastError()))); + + socklen_t namelen = sizeof(name); + QPID_WINSOCK_CHECK(::getsockname(socket, (struct sockaddr*)&name, &namelen)); + return ntohs(name.sin_port); +} + +Socket* Socket::accept() const +{ + SOCKET afd = ::accept(impl->fd, 0, 0); + if (afd != INVALID_SOCKET) + return new Socket(new IOHandlePrivate(afd)); + else if (WSAGetLastError() == EAGAIN) + return 0; + else throw QPID_WINDOWS_ERROR(WSAGetLastError()); +} + +std::string Socket::getPeerAddress() const +{ + if (peername.empty()) + peername = getName(impl->fd, false); + return peername; +} + +std::string Socket::getLocalAddress() const +{ + if (localname.empty()) + localname = getName(impl->fd, true); + return localname; +} + +int Socket::getError() const +{ + int result; + socklen_t rSize = sizeof (result); + + QPID_WINSOCK_CHECK(::getsockopt(impl->fd, SOL_SOCKET, SO_ERROR, (char *)&result, &rSize)); + return result; +} + +void Socket::setTcpNoDelay() const +{ + int flag = 1; + int result = setsockopt(impl->fd, + IPPROTO_TCP, + TCP_NODELAY, + (char *)&flag, + sizeof(flag)); + QPID_WINSOCK_CHECK(result); + nodelay = true; +} + +inline IOHandlePrivate* IOHandlePrivate::getImpl(const qpid::sys::IOHandle &h) +{ + return h.impl; +} + +SOCKET toSocketHandle(const Socket& s) +{ + return IOHandlePrivate::getImpl(s)->fd; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/windows/SocketAddress.cpp b/qpid/cpp/src/qpid/sys/windows/SocketAddress.cpp new file mode 100644 index 0000000000..ac43cd2d23 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/SocketAddress.cpp @@ -0,0 +1,76 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +// Ensure we get all of winsock2.h +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0501 +#endif + +#include "qpid/sys/SocketAddress.h" + +#include "qpid/sys/windows/check.h" + +#include <winsock2.h> +#include <ws2tcpip.h> +#include <string.h> + +namespace qpid { +namespace sys { + +SocketAddress::SocketAddress(const std::string& host0, const std::string& port0) : + host(host0), + port(port0), + addrInfo(0) +{ + ::addrinfo hints; + ::memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; // In order to allow AF_INET6 we'd have to change createTcp() as well + hints.ai_socktype = SOCK_STREAM; + + const char* node = 0; + if (host.empty()) { + hints.ai_flags |= AI_PASSIVE; + } else { + node = host.c_str(); + } + const char* service = port.empty() ? "0" : port.c_str(); + + int n = ::getaddrinfo(node, service, &hints, &addrInfo); + if (n != 0) + throw Exception(QPID_MSG("Cannot resolve " << host << ": " << ::gai_strerror(n))); +} + +SocketAddress::~SocketAddress() +{ + ::freeaddrinfo(addrInfo); +} + +std::string SocketAddress::asString(bool) const +{ + return host + ":" + port; +} + +const ::addrinfo& getAddrInfo(const SocketAddress& sa) +{ + return *sa.addrInfo; +} + +}} diff --git a/qpid/cpp/src/qpid/sys/windows/SslAsynchIO.cpp b/qpid/cpp/src/qpid/sys/windows/SslAsynchIO.cpp new file mode 100644 index 0000000000..11a3389e45 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/SslAsynchIO.cpp @@ -0,0 +1,661 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "SslAsynchIO.h" +#include "qpid/sys/Mutex.h" +#include "qpid/sys/Socket.h" +#include "qpid/sys/Poller.h" +#include "qpid/sys/Thread.h" +#include "qpid/sys/Time.h" +#include "qpid/log/Statement.h" + +#include "qpid/sys/windows/check.h" + +// security.h needs to see this to distinguish from kernel use. +#define SECURITY_WIN32 +#include <security.h> +#include <Schnlsp.h> +#undef SECURITY_WIN32 + +#include <queue> +#include <boost/bind.hpp> + +namespace { + + /* + * To make the SSL encryption more efficient, set up a new BufferBase + * that leaves room for the SSL header to be prepended and the SSL + * trailer to be appended. + * + * This works by accepting a properly formed BufferBase, remembering it, + * and resetting the members of this struct to reflect the reserved + * header and trailer areas. It's only needed for giving buffers up to + * the frame layer for writing into. + */ + struct SslIoBuff : public qpid::sys::AsynchIO::BufferBase { + std::auto_ptr<qpid::sys::AsynchIO::BufferBase> aioBuff; + + SslIoBuff (qpid::sys::AsynchIO::BufferBase *base, + const SecPkgContext_StreamSizes &sizes) + : qpid::sys::AsynchIO::BufferBase(&base->bytes[sizes.cbHeader], + std::min(base->byteCount - sizes.cbHeader - sizes.cbTrailer, + sizes.cbMaximumMessage)), + aioBuff(base) + {} + + ~SslIoBuff() {} + qpid::sys::AsynchIO::BufferBase* release() { return aioBuff.release(); } + }; +} + +namespace qpid { +namespace sys { +namespace windows { + +SslAsynchIO::SslAsynchIO(const qpid::sys::Socket& s, + CredHandle hCred, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb, + BuffersEmptyCallback eCb, + IdleCallback iCb, + NegotiateDoneCallback nCb) : + credHandle(hCred), + aio(0), + state(Negotiating), + readCallback(rCb), + idleCallback(iCb), + negotiateDoneCallback(nCb), + callbacksInProgress(0), + queuedDelete(false), + leftoverPlaintext(0) +{ + SecInvalidateHandle(&ctxtHandle); + peerAddress = s.getPeerAddress(); + aio = qpid::sys::AsynchIO::create(s, + boost::bind(&SslAsynchIO::sslDataIn, this, _1, _2), + eofCb, + disCb, + cCb, + eCb, + boost::bind(&SslAsynchIO::idle, this, _1)); +} + +SslAsynchIO::~SslAsynchIO() { + if (leftoverPlaintext) { + delete leftoverPlaintext; + leftoverPlaintext = 0; + } +} + +void SslAsynchIO::queueForDeletion() { + // This method effectively disconnects the layer above; pass it on the + // AsynchIO and delete. + aio->queueForDeletion(); + queuedDelete = true; + if (!callbacksInProgress) + delete this; +} + +void SslAsynchIO::start(qpid::sys::Poller::shared_ptr poller) { + aio->start(poller); + startNegotiate(); +} + +void SslAsynchIO::queueReadBuffer(AsynchIO::BufferBase* buff) { + aio->queueReadBuffer(buff); +} + +void SslAsynchIO::unread(AsynchIO::BufferBase* buff) { + // This is plaintext data being given back for more. Since it's already + // decrypted, don't give it back to the aio layer; keep it to append + // any new data for the upper layer. + assert(buff); + buff->squish(); + assert(leftoverPlaintext == 0); + leftoverPlaintext = buff; +} + +void SslAsynchIO::queueWrite(AsynchIO::BufferBase* buff) { + // @@TODO: Need to delay the write if the session is renegotiating. + + // Should not have gotten here without an SslIoBuff. This assert is + // primarily to catch any stray cases where write is called with a buffer + // not obtained via getQueuedBuffer. + SslIoBuff *sslBuff = dynamic_cast<SslIoBuff*>(buff); + assert(sslBuff != 0); + + // Encrypt and hand off to the io layer. Remember that the upper layer's + // encoding was working on, and adjusting counts for, the SslIoBuff. + // Update the count of the original BufferBase before handing off to + // the I/O layer. + buff = sslBuff->release(); + SecBuffer buffs[4]; + buffs[0].cbBuffer = schSizes.cbHeader; + buffs[0].BufferType = SECBUFFER_STREAM_HEADER; + buffs[0].pvBuffer = buff->bytes; // This space was left by SslIoBuff + buffs[1].cbBuffer = sslBuff->dataCount; + buffs[1].BufferType = SECBUFFER_DATA; + buffs[1].pvBuffer = sslBuff->bytes; + buffs[2].cbBuffer = schSizes.cbTrailer; + buffs[2].BufferType = SECBUFFER_STREAM_TRAILER; + buffs[2].pvBuffer = &sslBuff->bytes[sslBuff->dataCount]; + buffs[3].cbBuffer = 0; + buffs[3].BufferType = SECBUFFER_EMPTY; + buffs[3].pvBuffer = 0; + SecBufferDesc buffDesc; + buffDesc.ulVersion = SECBUFFER_VERSION; + buffDesc.cBuffers = 4; + buffDesc.pBuffers = buffs; + SECURITY_STATUS status = ::EncryptMessage(&ctxtHandle, 0, &buffDesc, 0); + + // EncryptMessage encrypts the data in place. The header and trailer + // areas were left previously and must now be included in the updated + // count of bytes to write to the peer. + delete sslBuff; + buff->dataCount = buffs[0].cbBuffer + buffs[1].cbBuffer + buffs[2].cbBuffer; + aio->queueWrite(buff); +} + +void SslAsynchIO::notifyPendingWrite() { + aio->notifyPendingWrite(); +} + +void SslAsynchIO::queueWriteClose() { + if (state == Negotiating) { + // Never got going, so don't bother trying to close SSL down orderly. + state = ShuttingDown; + aio->queueWriteClose(); + return; + } + + state = ShuttingDown; + + DWORD shutdown = SCHANNEL_SHUTDOWN; + SecBuffer shutBuff; + shutBuff.cbBuffer = sizeof(DWORD); + shutBuff.BufferType = SECBUFFER_TOKEN; + shutBuff.pvBuffer = &shutdown; + SecBufferDesc desc; + desc.ulVersion = SECBUFFER_VERSION; + desc.cBuffers = 1; + desc.pBuffers = &shutBuff; + ::ApplyControlToken(&ctxtHandle, &desc); + negotiateStep(0); + // When the shutdown sequence is done, negotiateDone() will handle + // shutting down aio. +} + +bool SslAsynchIO::writeQueueEmpty() { + return aio->writeQueueEmpty(); +} + +/* + * Initiate a read operation. AsynchIO::readComplete() will be + * called when the read is complete and data is available. + */ +void SslAsynchIO::startReading() { + aio->startReading(); +} + +void SslAsynchIO::stopReading() { + aio->stopReading(); +} + +// Queue the specified callback for invocation from an I/O thread. +void SslAsynchIO::requestCallback(RequestCallback callback) { + aio->requestCallback(callback); +} + +/** + * Return a queued buffer read to put new data in for writing. + * This method ALWAYS returns a SslIoBuff reflecting a BufferBase from + * the aio layer that has header and trailer space reserved. + */ +AsynchIO::BufferBase* SslAsynchIO::getQueuedBuffer() { + SslIoBuff *sslBuff = 0; + BufferBase* buff = aio->getQueuedBuffer(); + if (buff == 0) + return 0; + + sslBuff = new SslIoBuff(buff, schSizes); + return sslBuff; +} + +unsigned int SslAsynchIO::getSslKeySize() { + SecPkgContext_KeyInfo info; + memset(&info, 0, sizeof(info)); + ::QueryContextAttributes(&ctxtHandle, SECPKG_ATTR_KEY_INFO, &info); + return info.KeySize; +} + +void SslAsynchIO::negotiationDone() { + switch(state) { + case Negotiating: + ::QueryContextAttributes(&ctxtHandle, + SECPKG_ATTR_STREAM_SIZES, + &schSizes); + state = Running; + if (negotiateDoneCallback) + negotiateDoneCallback(SEC_E_OK); + break; + case Redo: + state = Running; + break; + case ShuttingDown: + aio->queueWriteClose(); + break; + default: + assert(0); + } +} + +void SslAsynchIO::negotiationFailed(SECURITY_STATUS status) { + QPID_LOG(notice, "SSL negotiation failed to " << peerAddress << ": " << + qpid::sys::strError(status)); + if (negotiateDoneCallback) + negotiateDoneCallback(status); + else + queueWriteClose(); +} + +void SslAsynchIO::sslDataIn(qpid::sys::AsynchIO& a, BufferBase *buff) { + if (state != Running) { + negotiateStep(buff); + return; + } + + // Decrypt the buffer; if there's legit data, pass it on through. + // However, it's also possible that the peer hasn't supplied enough + // data yet, or the session needs to be renegotiated, or the session + // is ending. + SecBuffer recvBuffs[4]; + recvBuffs[0].cbBuffer = buff->dataCount; + recvBuffs[0].BufferType = SECBUFFER_DATA; + recvBuffs[0].pvBuffer = &buff->bytes[buff->dataStart]; + recvBuffs[1].BufferType = SECBUFFER_EMPTY; + recvBuffs[2].BufferType = SECBUFFER_EMPTY; + recvBuffs[3].BufferType = SECBUFFER_EMPTY; + SecBufferDesc buffDesc; + buffDesc.ulVersion = SECBUFFER_VERSION; + buffDesc.cBuffers = 4; + buffDesc.pBuffers = recvBuffs; + SECURITY_STATUS status = ::DecryptMessage(&ctxtHandle, &buffDesc, 0, NULL); + if (status != SEC_E_OK) { + if (status == SEC_E_INCOMPLETE_MESSAGE) { + // Give the partially filled buffer back and get more data + a.unread(buff); + } + else { + // Don't need this any more... + a.queueReadBuffer(buff); + + if (status == SEC_I_RENEGOTIATE) { + state = Redo; + negotiateStep(0); + } + else if (status == SEC_I_CONTEXT_EXPIRED) { + queueWriteClose(); + } + else { + throw QPID_WINDOWS_ERROR(status); + } + } + return; + } + + // All decrypted and verified... continue with AMQP. The recvBuffs have + // been set up by DecryptMessage to demarcate the SSL header, data, and + // trailer, as well as any extra data left over. Walk through and find + // that info, adjusting the buff data accordingly to reflect only the + // actual decrypted data. + // If there's extra data, copy that out to a new buffer and run through + // this method again. + BufferBase *extraBuff = 0; + for (int i = 0; i < 4; i++) { + switch (recvBuffs[i].BufferType) { + case SECBUFFER_STREAM_HEADER: + buff->dataStart += recvBuffs[i].cbBuffer; + // Fall through - also don't count these bytes as data + case SECBUFFER_STREAM_TRAILER: + buff->dataCount -= recvBuffs[i].cbBuffer; + break; + case SECBUFFER_EXTRA: + // Very important to get this buffer from the downstream aio. + // The ones constructed from the local getQueuedBuffer() are + // restricted size for encrypting. However, data coming up from + // TCP may have a bunch of SSL segments coalesced and be much + // larger than the maximum single SSL segment. + extraBuff = a.getQueuedBuffer(); + if (0 == extraBuff) + throw QPID_WINDOWS_ERROR(WSAENOBUFS); + memmove(extraBuff->bytes, + recvBuffs[i].pvBuffer, + recvBuffs[i].cbBuffer); + extraBuff->dataCount = recvBuffs[i].cbBuffer; + break; + default: + break; + } + } + + // Since we've already taken (possibly) all the available bytes from the + // aio layer, need to be sure that everything that's processable is + // processed before returning back to aio. It could be that any + // leftoverPlaintext data plus new buff data won't fit in one buffer, so + // need to keep going around the input processing loop until either + // all the bytes are gone, or there's less than a full frame remaining + // (so we can count on more bytes being on the way via aio). + do { + BufferBase *temp = 0; + // Now that buff reflects only decrypted data, see if there was any + // partial data left over from last time. If so, append this new + // data to that and release the current buff back to aio. Assume that + // leftoverPlaintext was squished so the data starts at 0. + if (leftoverPlaintext != 0) { + // There is leftover data; append all the new data that will fit. + int32_t count = buff->dataCount; + if (leftoverPlaintext->dataCount + count > leftoverPlaintext->byteCount) + count = (leftoverPlaintext->byteCount - leftoverPlaintext->dataCount); + ::memmove(&leftoverPlaintext->bytes[leftoverPlaintext->dataCount], + &buff->bytes[buff->dataStart], + count); + leftoverPlaintext->dataCount += count; + buff->dataCount -= count; + buff->dataStart += count; + if (buff->dataCount == 0) { + a.queueReadBuffer(buff); + buff = 0; + } + // Prepare to pass the buffer up. Beware that the read callback + // may do an unread(), so move the leftoverPlaintext pointer + // out of the way. It also may release the buffer back to aio, + // so in either event, the pointer passed to the callback is not + // valid on return. + temp = leftoverPlaintext; + leftoverPlaintext = 0; + } + else { + temp = buff; + buff = 0; + } + if (readCallback) { + // The callback guard here is to prevent an upcall from deleting + // this out from under us via queueForDeletion(). + ++callbacksInProgress; + readCallback(*this, temp); + --callbacksInProgress; + } + else + a.queueReadBuffer(temp); // What else can we do with this??? + } while (buff != 0); + + // Ok, the current decrypted data is done. If there was any extra data, + // go back and handle that. + if (extraBuff != 0) + sslDataIn(a, extraBuff); + + // If the upper layer queued for delete, do that now that all the + // callbacks are done. + if (queuedDelete && callbacksInProgress == 0) + delete this; +} + +void SslAsynchIO::idle(qpid::sys::AsynchIO&) { + // Don't relay idle indication to layer above until SSL session is up. + if (state == Running) { + state = Running; + if (idleCallback) + idleCallback(*this); + } +} + + /**************************************************/ + +ClientSslAsynchIO::ClientSslAsynchIO(const std::string& brokerHost, + const qpid::sys::Socket& s, + CredHandle hCred, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb, + BuffersEmptyCallback eCb, + IdleCallback iCb, + NegotiateDoneCallback nCb) : + SslAsynchIO(s, hCred, rCb, eofCb, disCb, cCb, eCb, iCb, nCb), + serverHost(brokerHost) +{ +} + +void ClientSslAsynchIO::startNegotiate() { + // SEC_CHAR is non-const, so do all the typing here. + SEC_CHAR *host = const_cast<SEC_CHAR *>(serverHost.c_str()); + + // Need a buffer to receive the token to send to the server. + BufferBase *buff = aio->getQueuedBuffer(); + ULONG ctxtRequested = ISC_REQ_STREAM; + ULONG ctxtAttrs; + // sendBuffs gets information to forward to the peer. + SecBuffer sendBuffs[2]; + sendBuffs[0].cbBuffer = buff->byteCount; + sendBuffs[0].BufferType = SECBUFFER_TOKEN; + sendBuffs[0].pvBuffer = buff->bytes; + sendBuffs[1].cbBuffer = 0; + sendBuffs[1].BufferType = SECBUFFER_EMPTY; + sendBuffs[1].pvBuffer = 0; + SecBufferDesc sendBuffDesc; + sendBuffDesc.ulVersion = SECBUFFER_VERSION; + sendBuffDesc.cBuffers = 2; + sendBuffDesc.pBuffers = sendBuffs; + SECURITY_STATUS status = ::InitializeSecurityContext(&credHandle, + NULL, + host, + ctxtRequested, + 0, + 0, + NULL, + 0, + &ctxtHandle, + &sendBuffDesc, + &ctxtAttrs, + NULL); + if (status == SEC_I_CONTINUE_NEEDED) { + buff->dataCount = sendBuffs[0].cbBuffer; + aio->queueWrite(buff); + } +} + +void ClientSslAsynchIO::negotiateStep(BufferBase* buff) { + // SEC_CHAR is non-const, so do all the typing here. + SEC_CHAR *host = const_cast<SEC_CHAR *>(serverHost.c_str()); + ULONG ctxtRequested = ISC_REQ_STREAM; + ULONG ctxtAttrs; + + // tokenBuffs describe the buffer that's coming in. It should have + // a token from the SSL server. + SecBuffer tokenBuffs[2]; + tokenBuffs[0].cbBuffer = buff ? buff->dataCount : 0; + tokenBuffs[0].BufferType = SECBUFFER_TOKEN; + tokenBuffs[0].pvBuffer = buff ? buff->bytes : 0; + tokenBuffs[1].cbBuffer = 0; + tokenBuffs[1].BufferType = SECBUFFER_EMPTY; + tokenBuffs[1].pvBuffer = 0; + SecBufferDesc tokenBuffDesc; + tokenBuffDesc.ulVersion = SECBUFFER_VERSION; + tokenBuffDesc.cBuffers = 2; + tokenBuffDesc.pBuffers = tokenBuffs; + + // Need a buffer to receive any token to send back to the server. + BufferBase *sendbuff = aio->getQueuedBuffer(); + // sendBuffs gets information to forward to the peer. + SecBuffer sendBuffs[2]; + sendBuffs[0].cbBuffer = sendbuff->byteCount; + sendBuffs[0].BufferType = SECBUFFER_TOKEN; + sendBuffs[0].pvBuffer = sendbuff->bytes; + sendBuffs[1].cbBuffer = 0; + sendBuffs[1].BufferType = SECBUFFER_EMPTY; + sendBuffs[1].pvBuffer = 0; + SecBufferDesc sendBuffDesc; + sendBuffDesc.ulVersion = SECBUFFER_VERSION; + sendBuffDesc.cBuffers = 2; + sendBuffDesc.pBuffers = sendBuffs; + + SECURITY_STATUS status = ::InitializeSecurityContext(&credHandle, + &ctxtHandle, + host, + ctxtRequested, + 0, + 0, + &tokenBuffDesc, + 0, + NULL, + &sendBuffDesc, + &ctxtAttrs, + NULL); + + if (status == SEC_E_INCOMPLETE_MESSAGE) { + // Not enough - get more data from the server then try again. + aio->unread(buff); + aio->queueReadBuffer(sendbuff); // Don't need this one for now... + return; + } + // Done with the buffer that came in... + if (buff) + aio->queueReadBuffer(buff); + if (status == SEC_I_CONTINUE_NEEDED) { + sendbuff->dataCount = sendBuffs[0].cbBuffer; + aio->queueWrite(sendbuff); + return; + } + // Nothing to send back to the server... + aio->queueReadBuffer(sendbuff); + // SEC_I_CONTEXT_EXPIRED means session stop complete; SEC_E_OK can be + // either session stop or negotiation done (session up). + if (status == SEC_E_OK || status == SEC_I_CONTEXT_EXPIRED) + negotiationDone(); + else + negotiationFailed(status); +} + +/*************************************************/ + +ServerSslAsynchIO::ServerSslAsynchIO(bool clientMustAuthenticate, + const qpid::sys::Socket& s, + CredHandle hCred, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb, + BuffersEmptyCallback eCb, + IdleCallback iCb, + NegotiateDoneCallback nCb) : + SslAsynchIO(s, hCred, rCb, eofCb, disCb, cCb, eCb, iCb, nCb), + clientAuth(clientMustAuthenticate) +{ +} + +void ServerSslAsynchIO::startNegotiate() { + // Nothing... need the client to send a token first. +} + +void ServerSslAsynchIO::negotiateStep(BufferBase* buff) { + ULONG ctxtRequested = ASC_REQ_STREAM; + if (clientAuth) + ctxtRequested |= ASC_REQ_MUTUAL_AUTH; + ULONG ctxtAttrs; + + // tokenBuffs describe the buffer that's coming in. It should have + // a token from the SSL server except if shutting down or renegotiating. + SecBuffer tokenBuffs[2]; + tokenBuffs[0].cbBuffer = buff ? buff->dataCount : 0; + tokenBuffs[0].BufferType = SECBUFFER_TOKEN; + tokenBuffs[0].pvBuffer = buff ? buff->bytes : 0; + tokenBuffs[1].cbBuffer = 0; + tokenBuffs[1].BufferType = SECBUFFER_EMPTY; + tokenBuffs[1].pvBuffer = 0; + SecBufferDesc tokenBuffDesc; + tokenBuffDesc.ulVersion = SECBUFFER_VERSION; + tokenBuffDesc.cBuffers = 2; + tokenBuffDesc.pBuffers = tokenBuffs; + + // Need a buffer to receive any token to send back to the server. + BufferBase *sendbuff = aio->getQueuedBuffer(); + // sendBuffs gets information to forward to the peer. + SecBuffer sendBuffs[2]; + sendBuffs[0].cbBuffer = sendbuff->byteCount; + sendBuffs[0].BufferType = SECBUFFER_TOKEN; + sendBuffs[0].pvBuffer = sendbuff->bytes; + sendBuffs[1].cbBuffer = 0; + sendBuffs[1].BufferType = SECBUFFER_EMPTY; + sendBuffs[1].pvBuffer = 0; + SecBufferDesc sendBuffDesc; + sendBuffDesc.ulVersion = SECBUFFER_VERSION; + sendBuffDesc.cBuffers = 2; + sendBuffDesc.pBuffers = sendBuffs; + PCtxtHandle ctxtHandlePtr = (SecIsValidHandle(&ctxtHandle)) ? &ctxtHandle : 0; + SECURITY_STATUS status = ::AcceptSecurityContext(&credHandle, + ctxtHandlePtr, + &tokenBuffDesc, + ctxtRequested, + 0, + &ctxtHandle, + &sendBuffDesc, + &ctxtAttrs, + NULL); + if (status == SEC_E_INCOMPLETE_MESSAGE) { + // Not enough - get more data from the server then try again. + if (buff) + aio->unread(buff); + aio->queueReadBuffer(sendbuff); // Don't need this one for now... + return; + } + // Done with the buffer that came in... + if (buff) + aio->queueReadBuffer(buff); + if (status == SEC_I_CONTINUE_NEEDED) { + sendbuff->dataCount = sendBuffs[0].cbBuffer; + aio->queueWrite(sendbuff); + return; + } + // There may have been a token generated; if so, send it to the client. + if (sendBuffs[0].cbBuffer > 0) { + sendbuff->dataCount = sendBuffs[0].cbBuffer; + aio->queueWrite(sendbuff); + } + else + // Nothing to send back to the server... + aio->queueReadBuffer(sendbuff); + + // SEC_I_CONTEXT_EXPIRED means session stop complete; SEC_E_OK can be + // either session stop or negotiation done (session up). + if (status == SEC_E_OK || status == SEC_I_CONTEXT_EXPIRED) { + if (clientAuth) + QPID_LOG(warning, "DID WE CHECK FOR CLIENT AUTH???"); + + negotiationDone(); + } + else { + negotiationFailed(status); + } +} + +}}} // namespace qpid::sys::windows diff --git a/qpid/cpp/src/qpid/sys/windows/SslAsynchIO.h b/qpid/cpp/src/qpid/sys/windows/SslAsynchIO.h new file mode 100644 index 0000000000..3cdf2c8f08 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/SslAsynchIO.h @@ -0,0 +1,191 @@ +#ifndef _sys_windows_SslAsynchIO +#define _sys_windows_SslAsynchIO + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/AsynchIO.h" +#include "qpid/sys/IntegerTypes.h" +#include "qpid/sys/Poller.h" +#include "qpid/CommonImportExport.h" +#include <boost/function.hpp> +#include <boost/shared_ptr.hpp> +#include <windows.h> +// security.h needs to see this to distinguish from kernel use. +#define SECURITY_WIN32 +#include <security.h> +#include <Schnlsp.h> +#undef SECURITY_WIN32 + +namespace qpid { +namespace sys { +namespace windows { + +class Socket; +class Poller; + +/* + * SSL/Schannel shim between the frame-handling and AsynchIO layers. + * SslAsynchIO creates a regular AsynchIO object to handle I/O and this class + * gets involved for SSL negotiations and encrypt/decrypt. The details of + * how this all works are invisible to the layers on either side. The only + * change from normal AsynchIO usage is that there's an extra callback + * from SslAsynchIO to indicate that the initial session negotiation is + * complete. + * + * The details of session negotiation are different for client and server + * SSL roles. These differences are handled by deriving separate client + * and server role classes. + */ +class SslAsynchIO : public qpid::sys::AsynchIO { +public: + typedef boost::function1<void, SECURITY_STATUS> NegotiateDoneCallback; + + SslAsynchIO(const qpid::sys::Socket& s, + CredHandle hCred, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb = 0, + BuffersEmptyCallback eCb = 0, + IdleCallback iCb = 0, + NegotiateDoneCallback nCb = 0); + ~SslAsynchIO(); + + virtual void queueForDeletion(); + + virtual void start(qpid::sys::Poller::shared_ptr poller); + virtual void queueReadBuffer(BufferBase* buff); + virtual void unread(BufferBase* buff); + virtual void queueWrite(BufferBase* buff); + virtual void notifyPendingWrite(); + virtual void queueWriteClose(); + virtual bool writeQueueEmpty(); + virtual void startReading(); + virtual void stopReading(); + virtual void requestCallback(RequestCallback); + virtual BufferBase* getQueuedBuffer(); + + QPID_COMMON_EXTERN unsigned int getSslKeySize(); + +protected: + CredHandle credHandle; + + // AsynchIO layer below that's actually doing the I/O + qpid::sys::AsynchIO *aio; + + // Track what the state of the SSL session is. Have to know when it's + // time to notify the upper layer that the session is up, and also to + // know when it's not legit to pass data through to either side. + enum { Negotiating, Running, Redo, ShuttingDown } state; + bool sessionUp; + CtxtHandle ctxtHandle; + TimeStamp credExpiry; + + // Client- and server-side SSL subclasses implement these to do the + // proper negotiation steps. negotiateStep() is called with a buffer + // just received from the peer. + virtual void startNegotiate() = 0; + virtual void negotiateStep(BufferBase *buff) = 0; + + // The negotiating steps call one of these when it's finalized: + void negotiationDone(); + void negotiationFailed(SECURITY_STATUS status); + +private: + // These are callbacks from AsynchIO to here. + void sslDataIn(qpid::sys::AsynchIO& a, BufferBase *buff); + void idle(qpid::sys::AsynchIO&); + + // These callbacks are to the layer above. + ReadCallback readCallback; + IdleCallback idleCallback; + NegotiateDoneCallback negotiateDoneCallback; + volatile unsigned int callbacksInProgress; // >0 if w/in callbacks + volatile bool queuedDelete; + + // Address of peer, in case it's needed for logging. + std::string peerAddress; + + // Partial buffer of decrypted plaintext given back by the layer above. + AsynchIO::BufferBase *leftoverPlaintext; + + SecPkgContext_StreamSizes schSizes; +}; + +/* + * SSL/Schannel client-side shim between the frame-handling and AsynchIO + * layers. + */ +class ClientSslAsynchIO : public SslAsynchIO { +public: + // Args same as for SslIoShim, with the addition of brokerHost which is + // the expected SSL name of the server. + QPID_COMMON_EXTERN ClientSslAsynchIO(const std::string& brokerHost, + const qpid::sys::Socket& s, + CredHandle hCred, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb = 0, + BuffersEmptyCallback eCb = 0, + IdleCallback iCb = 0, + NegotiateDoneCallback nCb = 0); + +private: + std::string serverHost; + + // Client- and server-side SSL subclasses implement these to do the + // proper negotiation steps. negotiateStep() is called with a buffer + // just received from the peer. + void startNegotiate(); + void negotiateStep(BufferBase *buff); +}; +/* + * SSL/Schannel server-side shim between the frame-handling and AsynchIO + * layers. + */ +class ServerSslAsynchIO : public SslAsynchIO { +public: + QPID_COMMON_EXTERN ServerSslAsynchIO(bool clientMustAuthenticate, + const qpid::sys::Socket& s, + CredHandle hCred, + ReadCallback rCb, + EofCallback eofCb, + DisconnectCallback disCb, + ClosedCallback cCb = 0, + BuffersEmptyCallback eCb = 0, + IdleCallback iCb = 0, + NegotiateDoneCallback nCb = 0); + +private: + bool clientAuth; + + // Client- and server-side SSL subclasses implement these to do the + // proper negotiation steps. negotiateStep() is called with a buffer + // just received from the peer. + void startNegotiate(); + void negotiateStep(BufferBase *buff); +}; + +}}} // namespace qpid::sys::windows + +#endif // _sys_windows_SslAsynchIO diff --git a/qpid/cpp/src/qpid/sys/windows/StrError.cpp b/qpid/cpp/src/qpid/sys/windows/StrError.cpp new file mode 100755 index 0000000000..546d399d16 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/StrError.cpp @@ -0,0 +1,52 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/StrError.h" +#include <string> +#include <string.h> +#include <windows.h> + +namespace qpid { +namespace sys { + +std::string strError(int err) { + const size_t bufsize = 512; + char buf[bufsize]; + buf[0] = 0; + if (0 == FormatMessage (FORMAT_MESSAGE_MAX_WIDTH_MASK + | FORMAT_MESSAGE_FROM_SYSTEM, + 0, + err, + 0, // Default language + buf, + bufsize, + 0)) + { +#ifdef _MSC_VER + strerror_s(buf, bufsize, err); +#else + return std::string(strerror(err)); +#endif + } + return std::string(buf); +} + +}} diff --git a/qpid/cpp/src/qpid/sys/windows/SystemInfo.cpp b/qpid/cpp/src/qpid/sys/windows/SystemInfo.cpp new file mode 100755 index 0000000000..4da440bdd4 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/SystemInfo.cpp @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +/* GetNativeSystemInfo call requires _WIN32_WINNT 0x0501 or higher */ +#ifndef _WIN32_WINNT +# define _WIN32_WINNT 0x0501 +#endif + +#include "qpid/sys/IntegerTypes.h" +#include "qpid/sys/SystemInfo.h" + +#include <winsock2.h> +#include <ws2tcpip.h> +#include <windows.h> +#include <tlhelp32.h> + +#ifndef HOST_NAME_MAX +# define HOST_NAME_MAX 256 +#endif + +namespace qpid { +namespace sys { + +long SystemInfo::concurrency() { + SYSTEM_INFO sys_info; + ::GetSystemInfo (&sys_info); + long activeProcessors = 0; + DWORD_PTR mask = sys_info.dwActiveProcessorMask; + while (mask != 0) { + if (mask & 1) + ++activeProcessors; + mask >>= 1; + } + return activeProcessors; +} + +bool SystemInfo::getLocalHostname (Address &address) { + char name[HOST_NAME_MAX]; + if (::gethostname(name, sizeof(name)) != 0) { + errno = WSAGetLastError(); + return false; + } + address.host = name; + return true; +} + +static const std::string LOCALHOST("127.0.0.1"); +static const std::string TCP("tcp"); + +void SystemInfo::getLocalIpAddresses (uint16_t port, + std::vector<Address> &addrList) { + enum { MAX_URL_INTERFACES = 100 }; + + SOCKET s = socket (PF_INET, SOCK_STREAM, 0); + if (s != INVALID_SOCKET) { + INTERFACE_INFO interfaces[MAX_URL_INTERFACES]; + DWORD filledBytes = 0; + WSAIoctl (s, + SIO_GET_INTERFACE_LIST, + 0, + 0, + interfaces, + sizeof (interfaces), + &filledBytes, + 0, + 0); + unsigned int interfaceCount = filledBytes / sizeof (INTERFACE_INFO); + for (unsigned int i = 0; i < interfaceCount; ++i) { + if (interfaces[i].iiFlags & IFF_UP) { + std::string addr(inet_ntoa(interfaces[i].iiAddress.AddressIn.sin_addr)); + if (addr != LOCALHOST) + addrList.push_back(Address(TCP, addr, port)); + } + } + closesocket (s); + } +} + +void SystemInfo::getSystemId (std::string &osName, + std::string &nodeName, + std::string &release, + std::string &version, + std::string &machine) +{ + osName = "Microsoft Windows"; + + char node[MAX_COMPUTERNAME_LENGTH + 1]; + DWORD nodelen = MAX_COMPUTERNAME_LENGTH + 1; + GetComputerName (node, &nodelen); + nodeName = node; + + OSVERSIONINFOEX vinfo; + vinfo.dwOSVersionInfoSize = sizeof(vinfo); + GetVersionEx ((OSVERSIONINFO *)&vinfo); + + SYSTEM_INFO sinfo; + GetNativeSystemInfo(&sinfo); + + switch(vinfo.dwMajorVersion) { + case 5: + switch(vinfo.dwMinorVersion) { + case 0: + release ="2000"; + break; + case 1: + release = "XP"; + break; + case 2: + if (sinfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_AMD64 || + sinfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_IA64) + release = "XP-64"; + else + release = "Server 2003"; + break; + default: + release = "Windows"; + } + break; + case 6: + if (vinfo.wProductType == VER_NT_SERVER) + release = "Server 2008"; + else + release = "Vista"; + break; + default: + release = "Microsoft Windows"; + } + version = vinfo.szCSDVersion; + + switch(sinfo.wProcessorArchitecture) { + case PROCESSOR_ARCHITECTURE_AMD64: + machine = "x86-64"; + break; + case PROCESSOR_ARCHITECTURE_IA64: + machine = "IA64"; + break; + case PROCESSOR_ARCHITECTURE_INTEL: + machine = "x86"; + break; + default: + machine = "unknown"; + break; + } +} + +uint32_t SystemInfo::getProcessId() +{ + return static_cast<uint32_t>(::GetCurrentProcessId()); +} + +uint32_t SystemInfo::getParentProcessId() +{ + // Only want info for the current process, so ask for something specific. + // The module info won't be used here but it keeps the snapshot limited to + // the current process so a search through all processes is not needed. + HANDLE snap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, 0); + if (snap == INVALID_HANDLE_VALUE) + return 0; + PROCESSENTRY32 entry; + entry.dwSize = sizeof(entry); + if (!Process32First(snap, &entry)) + entry.th32ParentProcessID = 0; + CloseHandle(snap); + return static_cast<uint32_t>(entry.th32ParentProcessID); +} + +std::string SystemInfo::getProcessName() +{ + std::string name; + + // Only want info for the current process, so ask for something specific. + // The module info won't be used here but it keeps the snapshot limited to + // the current process so a search through all processes is not needed. + HANDLE snap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, 0); + if (snap == INVALID_HANDLE_VALUE) + return name; + PROCESSENTRY32 entry; + entry.dwSize = sizeof(entry); + if (!Process32First(snap, &entry)) + entry.szExeFile[0] = '\0'; + CloseHandle(snap); + name = entry.szExeFile; + return name; +} + +}} // namespace qpid::sys diff --git a/qpid/cpp/src/qpid/sys/windows/Thread.cpp b/qpid/cpp/src/qpid/sys/windows/Thread.cpp new file mode 100755 index 0000000000..583a9613a3 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/Thread.cpp @@ -0,0 +1,100 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Thread.h" +#include "qpid/sys/Runnable.h" +#include "qpid/sys/windows/check.h" + +#include <process.h> +#include <windows.h> + +namespace { +unsigned __stdcall runRunnable(void* p) +{ + static_cast<qpid::sys::Runnable*>(p)->run(); + _endthreadex(0); + return 0; +} +} + +namespace qpid { +namespace sys { + +class ThreadPrivate { + friend class Thread; + + HANDLE threadHandle; + unsigned threadId; + + ThreadPrivate(Runnable* runnable) { + uintptr_t h = _beginthreadex(0, + 0, + runRunnable, + runnable, + 0, + &threadId); + QPID_WINDOWS_CHECK_CRT_NZ(h); + threadHandle = reinterpret_cast<HANDLE>(h); + } + + ThreadPrivate() + : threadHandle(GetCurrentThread()), threadId(GetCurrentThreadId()) {} +}; + +Thread::Thread() {} + +Thread::Thread(Runnable* runnable) : impl(new ThreadPrivate(runnable)) {} + +Thread::Thread(Runnable& runnable) : impl(new ThreadPrivate(&runnable)) {} + +Thread::operator bool() { + return impl; +} + +bool Thread::operator==(const Thread& t) const { + return impl->threadId == t.impl->threadId; +} + +bool Thread::operator!=(const Thread& t) const { + return !(*this==t); +} + +void Thread::join() { + if (impl) { + DWORD status = WaitForSingleObject (impl->threadHandle, INFINITE); + QPID_WINDOWS_CHECK_NOT(status, WAIT_FAILED); + CloseHandle (impl->threadHandle); + impl->threadHandle = 0; + } +} + +unsigned long Thread::logId() { + return GetCurrentThreadId(); +} + +/* static */ +Thread Thread::current() { + Thread t; + t.impl.reset(new ThreadPrivate()); + return t; +} + +}} /* qpid::sys */ diff --git a/qpid/cpp/src/qpid/sys/windows/Time.cpp b/qpid/cpp/src/qpid/sys/windows/Time.cpp new file mode 100644 index 0000000000..25c50819cd --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/Time.cpp @@ -0,0 +1,136 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/sys/Time.h" +#include <ostream> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread_time.hpp> +#include <windows.h> + +using namespace boost::posix_time; + +namespace { + +// High-res timing support. This will display times since program start, +// more or less. Keep track of the start value and the conversion factor to +// seconds. +bool timeInitialized = false; +LARGE_INTEGER start; +double freq = 1.0; + +} + +namespace qpid { +namespace sys { + +AbsTime::AbsTime(const AbsTime& t, const Duration& d) { + if (d == Duration::max()) { + timepoint = ptime(max_date_time); + } + else { + time_duration td = microseconds(d.nanosecs / 1000); + timepoint = t.timepoint + td; + } +} + +AbsTime AbsTime::FarFuture() { + AbsTime ff; + ptime maxd(max_date_time); + ff.timepoint = maxd; + return ff; +} + +AbsTime AbsTime::Epoch() { + AbsTime time_epoch; + time_epoch.timepoint = boost::posix_time::from_time_t(0); + return time_epoch; +} + +AbsTime AbsTime::now() { + AbsTime time_now; + time_now.timepoint = boost::get_system_time(); + return time_now; +} + +Duration::Duration(const AbsTime& start, const AbsTime& finish) { + time_duration d = finish.timepoint - start.timepoint; + nanosecs = d.total_nanoseconds(); +} + +std::ostream& operator<<(std::ostream& o, const Duration& d) { + return o << int64_t(d) << "ns"; +} + +std::ostream& operator<<(std::ostream& o, const AbsTime& t) { + std::string time_string = to_simple_string(t.timepoint); + return o << time_string; +} + + +void sleep(int secs) { + ::Sleep(secs * 1000); +} + +void usleep(uint64_t usecs) { + DWORD msecs = usecs / 1000; + if (msecs == 0) + msecs = 1; + ::Sleep(msecs); +} + +void outputFormattedNow(std::ostream& o) { + ::time_t rawtime; + ::tm timeinfo; + char time_string[100]; + + ::time( &rawtime ); +#ifdef _MSC_VER + ::localtime_s(&timeinfo, &rawtime); +#else + timeinfo = *(::localtime(&rawtime)); +#endif + ::strftime(time_string, 100, + "%Y-%m-%d %H:%M:%S", + &timeinfo); + o << time_string << " "; +} + +void outputHiresNow(std::ostream& o) { + if (!timeInitialized) { + start.QuadPart = 0; + LARGE_INTEGER iFreq; + iFreq.QuadPart = 1; + QueryPerformanceCounter(&start); + QueryPerformanceFrequency(&iFreq); + freq = static_cast<double>(iFreq.QuadPart); + timeInitialized = true; + } + LARGE_INTEGER iNow; + iNow.QuadPart = 0; + QueryPerformanceCounter(&iNow); + iNow.QuadPart -= start.QuadPart; + if (iNow.QuadPart < 0) + iNow.QuadPart = 0; + double now = static_cast<double>(iNow.QuadPart); + now /= freq; // now is seconds after this + o << std::fixed << std::setprecision(8) << std::setw(16) << std::setfill('0') << now << "s "; +} +}} diff --git a/qpid/cpp/src/qpid/sys/windows/mingw32_compat.h b/qpid/cpp/src/qpid/sys/windows/mingw32_compat.h new file mode 100644 index 0000000000..51f613cc25 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/mingw32_compat.h @@ -0,0 +1,39 @@ +#ifndef _sys_windows_mingw32_compat +#define _sys_windows_mingw32_compat +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#ifdef WIN32 +#ifndef _MSC_VER + +// +// The following definitions for extension function GUIDs and signatures are taken from +// MswSock.h in the Windows32 SDK. These rightfully belong in the mingw32 version of +// mswsock.h, but are not included presently. +// + +#define WSAID_ACCEPTEX {0xb5367df1,0xcbac,0x11cf,{0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92}} +typedef BOOL (PASCAL *LPFN_ACCEPTEX)(SOCKET,SOCKET,PVOID,DWORD,DWORD,DWORD,LPDWORD,LPOVERLAPPED); + +#endif +#endif + +#endif diff --git a/qpid/cpp/src/qpid/sys/windows/uuid.cpp b/qpid/cpp/src/qpid/sys/windows/uuid.cpp new file mode 100644 index 0000000000..3316ecbc00 --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/uuid.cpp @@ -0,0 +1,67 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include <rpc.h> +#ifdef uuid_t /* Done in rpcdce.h */ +# undef uuid_t +#endif + +#include "qpid/sys/windows/uuid.h" + +#include <string.h> + +void uuid_clear (uuid_t uu) { + UuidCreateNil (reinterpret_cast<UUID*>(uu)); +} + +void uuid_copy (uuid_t dst, const uuid_t src) { + memcpy (dst, src, qpid::sys::UuidSize); +} + +void uuid_generate (uuid_t out) { + UuidCreate (reinterpret_cast<UUID*>(out)); +} + +int uuid_is_null (const uuid_t uu) { + RPC_STATUS unused; + return UuidIsNil ((UUID*)uu, &unused); +} + +int uuid_parse (const char *in, uuid_t uu) { + return UuidFromString ((unsigned char*)in, (UUID*)uu) == RPC_S_OK ? 0 : -1; +} + +void uuid_unparse (const uuid_t uu, char *out) { + unsigned char *formatted; + if (UuidToString((UUID*)uu, &formatted) == RPC_S_OK) { +#ifdef _MSC_VER + strncpy_s (out, 36+1, (char*)formatted, _TRUNCATE); +#else + strncpy (out, (char*)formatted, 36+1); +#endif + RpcStringFree(&formatted); + } +} + +int uuid_compare (const uuid_t a, const uuid_t b) { + RPC_STATUS unused; + return !UuidEqual((UUID*)a, (UUID*)b, &unused); +} diff --git a/qpid/cpp/src/qpid/sys/windows/uuid.h b/qpid/cpp/src/qpid/sys/windows/uuid.h new file mode 100644 index 0000000000..8ab132e9ce --- /dev/null +++ b/qpid/cpp/src/qpid/sys/windows/uuid.h @@ -0,0 +1,39 @@ +#ifndef _sys_windows_uuid_h +#define _sys_windows_uuid_h + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "qpid/types/ImportExport.h" +#include <qpid/sys/IntegerTypes.h> + +namespace qpid { namespace sys { const size_t UuidSize = 16; }} +typedef uint8_t uuid_t[qpid::sys::UuidSize]; + +QPID_TYPES_EXTERN void uuid_clear (uuid_t uu); +QPID_TYPES_EXTERN void uuid_copy (uuid_t dst, const uuid_t src); +QPID_TYPES_EXTERN void uuid_generate (uuid_t out); +QPID_TYPES_EXTERN int uuid_is_null (const uuid_t uu); // Returns 1 if null, else 0 +QPID_TYPES_EXTERN int uuid_parse (const char *in, uuid_t uu); // Returns 0 on success, else -1 +QPID_TYPES_EXTERN void uuid_unparse (const uuid_t uu, char *out); +QPID_TYPES_EXTERN int uuid_compare (const uuid_t a, const uuid_t b); + +#endif /*!_sys_windows_uuid_h*/ |