diff options
Diffstat (limited to 'qpid/java/broker/src/main/java/org')
74 files changed, 4136 insertions, 989 deletions
diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/AMQBrokerManagerMBean.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/AMQBrokerManagerMBean.java index 7216841a94..22bdae8267 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/AMQBrokerManagerMBean.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/AMQBrokerManagerMBean.java @@ -77,7 +77,7 @@ public class AMQBrokerManagerMBean extends AMQManagedObject implements ManagedBr @MBeanConstructor("Creates the Broker Manager MBean") public AMQBrokerManagerMBean(VirtualHost.VirtualHostMBean virtualHostMBean) throws JMException { - super(ManagedBroker.class, ManagedBroker.TYPE); + super(ManagedBroker.class, ManagedBroker.TYPE, ManagedBroker.VERSION); _virtualHostMBean = virtualHostMBean; VirtualHost virtualHost = virtualHostMBean.getVirtualHost(); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/AMQChannel.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/AMQChannel.java index 5a01888ccf..72a2780c32 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/AMQChannel.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/AMQChannel.java @@ -251,7 +251,6 @@ public class AMQChannel } catch (NoRouteException e) { - //_currentMessage.takeReference(); _returnMessages.add(e); } } @@ -432,7 +431,7 @@ public class AMQChannel { if (_log.isDebugEnabled()) { - _log.debug(debugIdentity() + " Adding unacked message(" + entry.getMessage().toString() + " DT:" + deliveryTag + _log.debug(debugIdentity() + " Adding unacked message(" + entry.toString() + " DT:" + deliveryTag + ") with a queue(" + entry.getQueue() + ") for " + subscription); } } @@ -552,7 +551,7 @@ public class AMQChannel } else { - _log.warn(System.identityHashCode(this) + " Requested requeue of message(" + unacked.getMessage().debugIdentity() + _log.warn(System.identityHashCode(this) + " Requested requeue of message(" + unacked.debugIdentity() + "):" + deliveryTag + " but no queue defined and no DeadLetter queue so DROPPING message."); unacked.dequeueAndDelete(_storeContext); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/Main.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/Main.java index 780a17940e..49619ac5b0 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/Main.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/Main.java @@ -26,6 +26,8 @@ import java.net.BindException; import java.net.InetAddress; import java.net.InetSocketAddress; +import javax.management.NotCompliantMBeanException; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; @@ -42,10 +44,13 @@ import org.apache.mina.common.IoAcceptor; import org.apache.mina.transport.socket.nio.SocketAcceptorConfig; import org.apache.mina.transport.socket.nio.SocketSessionConfig; import org.apache.mina.util.NewThreadExecutor; +import org.apache.qpid.AMQException; import org.apache.qpid.common.QpidProperties; import org.apache.qpid.framing.ProtocolVersion; import org.apache.qpid.pool.ReadWriteThreadModel; import org.apache.qpid.server.configuration.ServerConfiguration; +import org.apache.qpid.server.configuration.management.ConfigurationManagementMBean; +import org.apache.qpid.server.logging.management.LoggingManagementMBean; import org.apache.qpid.server.protocol.AMQPFastProtocolHandler; import org.apache.qpid.server.protocol.AMQPProtocolProvider; import org.apache.qpid.server.registry.ApplicationRegistry; @@ -232,16 +237,29 @@ public class Main String logConfig = commandLine.getOptionValue("l"); String logWatchConfig = commandLine.getOptionValue("w", "0"); + + int logWatchTime = 0; + try + { + logWatchTime = Integer.parseInt(logWatchConfig); + } + catch (NumberFormatException e) + { + System.err.println("Log watch configuration value of " + logWatchConfig + " is invalid. Must be " + + "a non-negative integer. Using default of zero (no watching configured"); + } + + File logConfigFile; if (logConfig != null) { - File logConfigFile = new File(logConfig); - configureLogging(logConfigFile, logWatchConfig); + logConfigFile = new File(logConfig); + configureLogging(logConfigFile, logWatchTime); } else { File configFileDirectory = configFile.getParentFile(); - File logConfigFile = new File(configFileDirectory, DEFAULT_LOG_CONFIG_FILENAME); - configureLogging(logConfigFile, logWatchConfig); + logConfigFile = new File(configFileDirectory, DEFAULT_LOG_CONFIG_FILENAME); + configureLogging(logConfigFile, logWatchTime); } ConfigurationFileApplicationRegistry config = new ConfigurationFileApplicationRegistry(configFile); @@ -249,7 +267,12 @@ public class Main updateManagementPort(serverConfig, commandLine.getOptionValue("m")); ApplicationRegistry.initialise(config); + + configureLoggingManagementMBean(logConfigFile, logWatchTime); + ConfigurationManagementMBean configMBean = new ConfigurationManagementMBean(); + configMBean.register(); + //fixme .. use QpidProperties.getVersionString when we have fixed the classpath issues // that are causing the broker build to pick up the wrong properties file and hence say // Starting Qpid Client @@ -445,19 +468,8 @@ public class Main return ip; } - private void configureLogging(File logConfigFile, String logWatchConfig) + private void configureLogging(File logConfigFile, int logWatchTime) { - int logWatchTime = 0; - try - { - logWatchTime = Integer.parseInt(logWatchConfig); - } - catch (NumberFormatException e) - { - System.err.println("Log watch configuration value of " + logWatchConfig + " is invalid. Must be " - + "a non-negative integer. Using default of zero (no watching configured"); - } - if (logConfigFile.exists() && logConfigFile.canRead()) { System.out.println("Configuring logger using configuration file " + logConfigFile.getAbsolutePath()); @@ -481,4 +493,17 @@ public class Main } } + private void configureLoggingManagementMBean(File logConfigFile, int logWatchTime) throws Exception + { + LoggingManagementMBean blm = new LoggingManagementMBean(logConfigFile.getPath(),logWatchTime); + + try + { + blm.register(); + } + catch (AMQException e) + { + throw new InitException("Unable to initialise the Logging Management MBean: ", e); + } + } } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/ack/TxAck.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/ack/TxAck.java index 918fcd8407..95de0dc8c3 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/ack/TxAck.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/ack/TxAck.java @@ -102,7 +102,7 @@ public class TxAck implements TxnOp //buffer must be marked as persistent: for (QueueEntry msg : _unacked.values()) { - if (msg.getMessage().isPersistent()) + if (msg.isPersistent()) { return true; } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/ack/UnacknowledgedMessageMapImpl.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/ack/UnacknowledgedMessageMapImpl.java index ac3b0b5e49..5c38185696 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/ack/UnacknowledgedMessageMapImpl.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/ack/UnacknowledgedMessageMapImpl.java @@ -89,7 +89,7 @@ public class UnacknowledgedMessageMapImpl implements UnacknowledgedMessageMap QueueEntry message = _map.remove(deliveryTag); if(message != null) { - _unackedSize -= message.getMessage().getSize(); + _unackedSize -= message.getSize(); } @@ -115,7 +115,7 @@ public class UnacknowledgedMessageMapImpl implements UnacknowledgedMessageMap synchronized (_lock) { _map.put(deliveryTag, message); - _unackedSize += message.getMessage().getSize(); + _unackedSize += message.getSize(); _lastDeliveryTag = deliveryTag; } } @@ -181,7 +181,7 @@ public class UnacknowledgedMessageMapImpl implements UnacknowledgedMessageMap it.remove(); - _unackedSize -= unacked.getValue().getMessage().getSize(); + _unackedSize -= unacked.getValue().getSize(); if (unacked.getKey() == deliveryTag) diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/QueueConfiguration.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/QueueConfiguration.java index 90d6caec99..e6c5dee90d 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/QueueConfiguration.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/QueueConfiguration.java @@ -21,8 +21,10 @@ package org.apache.qpid.server.configuration; import java.util.List; +import java.io.File; import org.apache.commons.configuration.Configuration; +import org.apache.qpid.server.registry.ApplicationRegistry; public class QueueConfiguration { @@ -31,13 +33,20 @@ public class QueueConfiguration private Configuration _config; private String _name; + private VirtualHostConfiguration _virtualHostConfiguration; - public QueueConfiguration(String name, Configuration config) + public QueueConfiguration(String name, Configuration config, VirtualHostConfiguration virtualHostConfiguration) { + _virtualHostConfiguration = virtualHostConfiguration; _config = config; _name = name; } + public VirtualHostConfiguration getVirtualHostConfiguration() + { + return _virtualHostConfiguration; + } + public boolean getDurable() { return _config.getBoolean("durable" ,false); @@ -103,4 +112,13 @@ public class QueueConfiguration return _config.getLong("minimumAlertRepeatGap", 0); } + public long getMemoryUsageMaximum() + { + return _config.getLong("maximumMemoryUsage", 100 * 1024 * 1024); //100Meg + } + + public long getMemoryUsageMinimum() + { + return _config.getLong("minimumMemoryUsage", 0); + } } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/ServerConfiguration.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/ServerConfiguration.java index b3c08a2a95..c0fe42c5c2 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/ServerConfiguration.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/ServerConfiguration.java @@ -33,8 +33,18 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationFactory; import org.apache.commons.configuration.SystemConfiguration; import org.apache.commons.configuration.XMLConfiguration; - -public class ServerConfiguration +import org.apache.qpid.server.configuration.management.ConfigurationManagementMBean; +import org.apache.qpid.server.registry.ApplicationRegistry; +import org.apache.qpid.server.virtualhost.VirtualHost; +import org.apache.qpid.server.virtualhost.VirtualHostRegistry; +import org.apache.qpid.tools.messagestore.MessageStoreTool; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import sun.misc.Signal; +import sun.misc.SignalHandler; + +public class ServerConfiguration implements SignalHandler { private static Configuration _config; @@ -52,6 +62,13 @@ public class ServerConfiguration private Map<String, VirtualHostConfiguration> _virtualHosts = new HashMap<String, VirtualHostConfiguration>(); private SecurityConfiguration _securityConfiguration = null; + private File _configFile; + + private Logger _log = LoggerFactory.getLogger(this.getClass()); + + private ConfigurationManagementMBean _mbean; + + // Map of environment variables to config items private static final Map<String, String> envVarMap = new HashMap<String, String>(); @@ -82,6 +99,8 @@ public class ServerConfiguration public ServerConfiguration(File configurationURL) throws ConfigurationException { this(parseConfig(configurationURL)); + _configFile = configurationURL; + sun.misc.Signal.handle(new sun.misc.Signal("HUP"), this); } public ServerConfiguration(Configuration conf) throws ConfigurationException @@ -94,8 +113,9 @@ public class ServerConfiguration _securityConfiguration = new SecurityConfiguration(conf.subset("security")); setupVirtualHosts(conf); + } - + private void setupVirtualHosts(Configuration conf) throws ConfigurationException { List vhosts = conf.getList("virtualhosts"); @@ -113,7 +133,7 @@ public class ServerConfiguration CompositeConfiguration mungedConf = new CompositeConfiguration(); mungedConf.addConfiguration(conf.subset("virtualhosts.virtualhost."+name)); mungedConf.addConfiguration(vhostConfiguration.subset("virtualhost." + name)); - VirtualHostConfiguration vhostConfig = new VirtualHostConfiguration(name, mungedConf); + VirtualHostConfiguration vhostConfig = new VirtualHostConfiguration(name, mungedConf, this); _virtualHosts.put(vhostConfig.getName(), vhostConfig); } } @@ -181,6 +201,42 @@ public class ServerConfiguration return conf; } + @Override + public void handle(Signal arg0) + { + try + { + reparseConfigFile(); + } + catch (ConfigurationException e) + { + _log.error("Could not reload configuration file", e); + } + } + + public void reparseConfigFile() throws ConfigurationException + { + if (_configFile != null) + { + Configuration newConfig = parseConfig(_configFile); + _securityConfiguration = new SecurityConfiguration(newConfig.subset("security")); + ApplicationRegistry.getInstance().getAccessManager().configurePlugins(_securityConfiguration); + + VirtualHostRegistry vhostRegistry = ApplicationRegistry.getInstance().getVirtualHostRegistry(); + for (String hostname : _virtualHosts.keySet()) + { + VirtualHost vhost = vhostRegistry.getVirtualHost(hostname); + SecurityConfiguration hostSecurityConfig = new SecurityConfiguration(newConfig.subset("virtualhosts.virtualhost."+hostname+".security")); + vhost.getAccessManager().configureHostPlugins(hostSecurityConfig); + } + } + } + + public String getQpidWork() + { + return System.getProperty("QPID_WORK", System.getProperty("java.io.tmpdir")); + } + public void setJMXManagementPort(int mport) { _jmxPort = mport; @@ -248,11 +304,6 @@ public class ServerConfiguration return _config.getInt("advanced.framesize", DEFAULT_FRAME_SIZE); } - public boolean getManagementSecurityEnabled() - { - return _config.getBoolean("management.security-enabled", false); - } - public boolean getProtectIOEnabled() { return _config.getBoolean("broker.connector.protectio.enabled", false); @@ -454,8 +505,10 @@ public class ServerConfiguration _config.setProperty("housekeeping.expiredMessageCheckPeriod", value); } - public long getHousekeepingExpiredMessageCheckPeriod() + public long getHousekeepingCheckPeriod() { - return _config.getLong("housekeeping.expiredMessageCheckPeriod", DEFAULT_HOUSEKEEPING_PERIOD); + return _config.getLong("housekeeping.checkPeriod", + _config.getLong("housekeeping.expiredMessageCheckPeriod", + DEFAULT_HOUSEKEEPING_PERIOD)); } } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/VirtualHostConfiguration.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/VirtualHostConfiguration.java index 91d0b8d8da..343abe4b5a 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/VirtualHostConfiguration.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/VirtualHostConfiguration.java @@ -28,6 +28,7 @@ import java.util.Map; import org.apache.commons.configuration.CompositeConfiguration; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.qpid.server.registry.ApplicationRegistry; import org.apache.qpid.server.store.MemoryMessageStore; @@ -37,10 +38,12 @@ public class VirtualHostConfiguration private String _name; private Map<String, QueueConfiguration> _queues = new HashMap<String, QueueConfiguration>(); private Map<String, ExchangeConfiguration> _exchanges = new HashMap<String, ExchangeConfiguration>(); + private ServerConfiguration _serverConfiguration; - - public VirtualHostConfiguration(String name, Configuration config) throws ConfigurationException + public VirtualHostConfiguration(String name, Configuration config, + ServerConfiguration serverConfiguration) throws ConfigurationException { + _serverConfiguration = serverConfiguration; _config = config; _name = name; @@ -52,7 +55,7 @@ public class VirtualHostConfiguration CompositeConfiguration mungedConf = new CompositeConfiguration(); mungedConf.addConfiguration(_config.subset("queues.queue." + queueName)); mungedConf.addConfiguration(_config.subset("queues")); - _queues.put(queueName, new QueueConfiguration(queueName, mungedConf)); + _queues.put(queueName, new QueueConfiguration(queueName, mungedConf, this)); } i = _config.getList("exchanges.exchange.name").iterator(); @@ -67,6 +70,21 @@ public class VirtualHostConfiguration } } + /** + * All future usages should use the constructor that takes the ServerConfiguration. + * + * This can be removed after QPID-1696 has been resolved. + * + * @param name + * @param mungedConf + * @throws ConfigurationException + */ + @Deprecated + public VirtualHostConfiguration(String name, Configuration mungedConf) throws ConfigurationException + { + this(name,mungedConf, ApplicationRegistry.getInstance().getConfiguration()); + } + public String getName() { return _name; @@ -74,7 +92,7 @@ public class VirtualHostConfiguration public long getHousekeepingExpiredMessageCheckPeriod() { - return _config.getLong("housekeeping.expiredMessageCheckPeriod", ApplicationRegistry.getInstance().getConfiguration().getHousekeepingExpiredMessageCheckPeriod()); + return _config.getLong("housekeeping.expiredMessageCheckPeriod", _serverConfiguration.getHousekeepingCheckPeriod()); } public String getAuthenticationDatabase() @@ -127,4 +145,25 @@ public class VirtualHostConfiguration return _queues.get(queueName); } + public long getMemoryUsageMaximum() + { + return _config.getLong("queues.maximumMemoryUsage", 0); + } + + public long getMemoryUsageMinimum() + { + return _config.getLong("queues.minimumMemoryUsage", 0); + } + + public ServerConfiguration getServerConfiguration() + { + return _serverConfiguration; + } + + public static final String FLOW_TO_DISK_PATH = "flowToDiskPath"; + public String getFlowToDiskLocation() + { + return _config.getString(FLOW_TO_DISK_PATH, getServerConfiguration().getQpidWork()); + } + } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/management/ConfigurationManagement.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/management/ConfigurationManagement.java new file mode 100644 index 0000000000..8e4bf01c6a --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/management/ConfigurationManagement.java @@ -0,0 +1,43 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.configuration.management; + +import javax.management.MBeanOperationInfo; + +import org.apache.commons.configuration.ConfigurationException; +import org.apache.qpid.server.management.MBeanOperation; + +public interface ConfigurationManagement +{ + + String TYPE = "ConfigurationManagement"; + int VERSION = 1; + + /** + * Reload the + * @throws ConfigurationException + */ + @MBeanOperation(name="reloadSecurityConfiguration", + description = "Force a reload of the security configuration sections", + impact = MBeanOperationInfo.ACTION) + void reloadSecurityConfiguration() throws ConfigurationException; + +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/management/ConfigurationManagementMBean.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/management/ConfigurationManagementMBean.java new file mode 100644 index 0000000000..ead6053d70 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/configuration/management/ConfigurationManagementMBean.java @@ -0,0 +1,49 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.configuration.management; + +import javax.management.NotCompliantMBeanException; + +import org.apache.commons.configuration.ConfigurationException; +import org.apache.qpid.server.management.AMQManagedObject; +import org.apache.qpid.server.registry.ApplicationRegistry; + +public class ConfigurationManagementMBean extends AMQManagedObject implements ConfigurationManagement +{ + + public ConfigurationManagementMBean() throws NotCompliantMBeanException + { + super(ConfigurationManagement.class, ConfigurationManagement.TYPE, ConfigurationManagement.VERSION); + } + + @Override + public String getObjectInstanceName() + { + return ConfigurationManagement.TYPE; + } + + @Override + public void reloadSecurityConfiguration() throws ConfigurationException + { + ApplicationRegistry.getInstance().getConfiguration().reparseConfigFile(); + } + +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/exchange/AbstractExchange.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/exchange/AbstractExchange.java index 8d24626b73..b6c741bbec 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/exchange/AbstractExchange.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/exchange/AbstractExchange.java @@ -38,13 +38,9 @@ import org.apache.qpid.server.management.Managable; import org.apache.qpid.server.management.ManagedObject; import org.apache.qpid.server.management.ManagedObjectRegistry; import org.apache.qpid.server.queue.QueueRegistry; -import org.apache.qpid.server.queue.AMQQueue; import org.apache.qpid.server.registry.ApplicationRegistry; import org.apache.qpid.server.virtualhost.VirtualHost; -import java.util.List; -import java.util.Map; - public abstract class AbstractExchange implements Exchange, Managable { private AMQShortString _name; @@ -81,7 +77,7 @@ public abstract class AbstractExchange implements Exchange, Managable public ExchangeMBean() throws NotCompliantMBeanException { - super(ManagedExchange.class, ManagedExchange.TYPE); + super(ManagedExchange.class, ManagedExchange.TYPE, ManagedExchange.VERSION); } protected void init() throws OpenDataException diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/exchange/ManagedExchange.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/exchange/ManagedExchange.java index 5d6d68b3c8..317ff385ab 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/exchange/ManagedExchange.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/exchange/ManagedExchange.java @@ -40,6 +40,7 @@ import org.apache.qpid.server.queue.ManagedQueue; public interface ManagedExchange { static final String TYPE = "Exchange"; + static final int VERSION = 1; /** * Returns the name of the managed exchange. diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/BytesOnlyCreditManager.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/BytesOnlyCreditManager.java index 0743e4bb8d..0bb428cd8f 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/BytesOnlyCreditManager.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/BytesOnlyCreditManager.java @@ -1,6 +1,6 @@ package org.apache.qpid.server.flow; -import org.apache.qpid.server.queue.AMQMessage; +import org.apache.qpid.server.queue.QueueEntry; import java.util.concurrent.atomic.AtomicLong; @@ -49,9 +49,9 @@ public class BytesOnlyCreditManager extends AbstractFlowCreditManager return _bytesCredit.get() > 0L; } - public boolean useCreditForMessage(AMQMessage msg) + public boolean useCreditForMessage(QueueEntry queueEntry) { - final long msgSize = msg.getSize(); + final long msgSize = queueEntry.getSize(); if(hasCredit()) { if(_bytesCredit.addAndGet(-msgSize) >= 0) diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/FlowCreditManager.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/FlowCreditManager.java index a249a6e63a..297e5a4826 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/FlowCreditManager.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/FlowCreditManager.java @@ -1,6 +1,7 @@ package org.apache.qpid.server.flow; import org.apache.qpid.server.queue.AMQMessage; +import org.apache.qpid.server.queue.QueueEntry; /* * @@ -40,5 +41,5 @@ public interface FlowCreditManager public boolean hasCredit(); - public boolean useCreditForMessage(AMQMessage msg); + public boolean useCreditForMessage(QueueEntry queueEntry); } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/LimitlessCreditManager.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/LimitlessCreditManager.java index d63431c3eb..437b7b0469 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/LimitlessCreditManager.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/LimitlessCreditManager.java @@ -1,6 +1,6 @@ package org.apache.qpid.server.flow; -import org.apache.qpid.server.queue.AMQMessage; +import org.apache.qpid.server.queue.QueueEntry; /* * @@ -37,7 +37,7 @@ public class LimitlessCreditManager extends AbstractFlowCreditManager implements return true; } - public boolean useCreditForMessage(AMQMessage msg) + public boolean useCreditForMessage(QueueEntry queueEntry) { return true; } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/MessageAndBytesCreditManager.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/MessageAndBytesCreditManager.java index 5f0acec03f..15ecb5f292 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/MessageAndBytesCreditManager.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/MessageAndBytesCreditManager.java @@ -1,6 +1,6 @@ package org.apache.qpid.server.flow; -import org.apache.qpid.server.queue.AMQMessage; +import org.apache.qpid.server.queue.QueueEntry; /* * @@ -52,7 +52,7 @@ public class MessageAndBytesCreditManager extends AbstractFlowCreditManager impl return (_messageCredit > 0L) && ( _bytesCredit > 0L ); } - public synchronized boolean useCreditForMessage(AMQMessage msg) + public synchronized boolean useCreditForMessage(QueueEntry queueEntry) { if(_messageCredit == 0L) { @@ -61,7 +61,7 @@ public class MessageAndBytesCreditManager extends AbstractFlowCreditManager impl } else { - final long msgSize = msg.getSize(); + final long msgSize = queueEntry.getSize(); if(msgSize > _bytesCredit) { setSuspended(true); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/MessageOnlyCreditManager.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/MessageOnlyCreditManager.java index c1b3a09006..3e28d779b1 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/MessageOnlyCreditManager.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/MessageOnlyCreditManager.java @@ -1,6 +1,6 @@ package org.apache.qpid.server.flow; -import org.apache.qpid.server.queue.AMQMessage; +import org.apache.qpid.server.queue.QueueEntry; import java.util.concurrent.atomic.AtomicLong; @@ -50,7 +50,7 @@ public class MessageOnlyCreditManager extends AbstractFlowCreditManager implemen return _messageCredit.get() > 0L; } - public boolean useCreditForMessage(AMQMessage msg) + public boolean useCreditForMessage(QueueEntry queueEntry) { if(hasCredit()) { diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/Pre0_10CreditManager.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/Pre0_10CreditManager.java index be0300f2c1..5cdd3a0328 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/Pre0_10CreditManager.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/flow/Pre0_10CreditManager.java @@ -20,7 +20,7 @@ */ package org.apache.qpid.server.flow; -import org.apache.qpid.server.queue.AMQMessage; +import org.apache.qpid.server.queue.QueueEntry; public class Pre0_10CreditManager extends AbstractFlowCreditManager implements FlowCreditManager { @@ -123,7 +123,7 @@ public class Pre0_10CreditManager extends AbstractFlowCreditManager implements F && (_messageCreditLimit == 0L || _messageCredit > 0); } - public synchronized boolean useCreditForMessage(final AMQMessage msg) + public synchronized boolean useCreditForMessage(final QueueEntry queueEntry) { if(_messageCreditLimit != 0L) { @@ -137,10 +137,10 @@ public class Pre0_10CreditManager extends AbstractFlowCreditManager implements F } else { - if((_bytesCredit >= msg.getSize()) || (_bytesCredit == _bytesCreditLimit)) + if((_bytesCredit >= queueEntry.getSize()) || (_bytesCredit == _bytesCreditLimit)) { _messageCredit--; - _bytesCredit -= msg.getSize(); + _bytesCredit -= queueEntry.getSize(); return true; } @@ -166,9 +166,9 @@ public class Pre0_10CreditManager extends AbstractFlowCreditManager implements F } else { - if((_bytesCredit >= msg.getSize()) || (_bytesCredit == _bytesCreditLimit)) + if((_bytesCredit >= queueEntry.getSize()) || (_bytesCredit == _bytesCreditLimit)) { - _bytesCredit -= msg.getSize(); + _bytesCredit -= queueEntry.getSize(); return true; } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/handler/BasicGetMethodHandler.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/handler/BasicGetMethodHandler.java index 0f492a21bb..a626114792 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/handler/BasicGetMethodHandler.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/handler/BasicGetMethodHandler.java @@ -129,7 +129,7 @@ public class BasicGetMethodHandler implements StateAwareMethodListener<BasicGetB public void deliverToClient(final Subscription sub, final QueueEntry entry, final long deliveryTag)
throws AMQException
{
- singleMessageCredit.useCreditForMessage(entry.getMessage());
+ singleMessageCredit.useCreditForMessage(entry);
session.getProtocolOutputConverter().writeGetOk(entry, channel.getChannelId(),
deliveryTag, queue.getMessageCount());
@@ -181,9 +181,9 @@ public class BasicGetMethodHandler implements StateAwareMethodListener<BasicGetB super(channel, protocolSession, consumerTag, filters, noLocal, creditManager, deliveryMethod, recordMethod);
}
- public boolean wouldSuspend(QueueEntry msg)
+ public boolean wouldSuspend(QueueEntry queueEntry)
{
- return !getCreditManager().useCreditForMessage(msg.getMessage());
+ return !getCreditManager().useCreditForMessage(queueEntry);
}
}
diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/handler/BasicRejectMethodHandler.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/handler/BasicRejectMethodHandler.java index bd70cd7776..8b04315d33 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/handler/BasicRejectMethodHandler.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/handler/BasicRejectMethodHandler.java @@ -96,7 +96,7 @@ public class BasicRejectMethodHandler implements StateAwareMethodListener<BasicR if (_logger.isDebugEnabled()) { - _logger.debug("Rejecting: DT:" + deliveryTag + "-" + queueEntry.getMessage().debugIdentity() + + _logger.debug("Rejecting: DT:" + deliveryTag + "-" + queueEntry.debugIdentity() + ": Requeue:" + body.getRequeue() + //": Resend:" + evt.getMethod().resend + " on channel:" + channel.debugIdentity()); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/logging/management/LoggingManagement.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/logging/management/LoggingManagement.java new file mode 100644 index 0000000000..79d60a6df0 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/logging/management/LoggingManagement.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * + */ +package org.apache.qpid.server.logging.management; + +import java.io.IOException; + +import org.apache.qpid.server.management.MBeanAttribute; +import org.apache.qpid.server.management.MBeanOperation; +import org.apache.qpid.server.management.MBeanOperationParameter; + +import javax.management.MBeanOperationInfo; +import javax.management.openmbean.TabularData; + +public interface LoggingManagement +{ + String TYPE = "LoggingManagement"; + int VERSION = 1; + + //TabularType and contained CompositeType key/description information + String[] COMPOSITE_ITEM_NAMES = {"LoggerName", "Level"}; + String[] COMPOSITE_ITEM_DESCRIPTIONS = {"Name of the logger", "Level of the logger"}; + String[] TABULAR_UNIQUE_INDEX = {COMPOSITE_ITEM_NAMES[0]}; + + /** + * Attribute to represent the log4j xml configuration file's LogWatch interval. + * @return The logwatch interval in seconds. + */ + @MBeanAttribute(name="Log4jLogWatchInterval", + description = "The log4j xml configuration file LogWatch interval (in seconds). 0 indicates not being checked.") + Integer getLog4jLogWatchInterval(); + + + //****** log4j runtime operations ****** // + + /** + * Sets the level of an active Log4J logger + * @param logger The name of the logger + * @param level The level to set the logger to + * @return True if successful, false if unsuccessful (eg if an invalid level is specified) + */ + @MBeanOperation(name = "setRuntimeLoggerLevel", description = "Set the runtime logging level for an active log4j logger.", + impact = MBeanOperationInfo.ACTION) + boolean setRuntimeLoggerLevel(@MBeanOperationParameter(name = "logger", description = "Logger name")String logger, + @MBeanOperationParameter(name = "level", description = "Logger level")String level); + + /** + * Retrieves a TabularData set of the active log4j loggers and their levels + * @return TabularData set of CompositeData rows with logger name and level, or null if there is a problem with the TabularData type + */ + @MBeanOperation(name = "viewEffectiveRuntimeLoggerLevels", description = "View the effective runtime logging level " + + "for active log4j logger's.", impact = MBeanOperationInfo.INFO) + TabularData viewEffectiveRuntimeLoggerLevels(); + + /** + * Sets the level of the active Log4J RootLogger + * @param level The level to set the RootLogger to + * @return True if successful, false if unsuccessful (eg if an invalid level is specified) + */ + @MBeanOperation(name = "setRuntimeRootLoggerLevel", description = "Set the runtime logging level for the active log4j Root Logger.", + impact = MBeanOperationInfo.ACTION) + boolean setRuntimeRootLoggerLevel(@MBeanOperationParameter(name = "level", description = "Logger level")String level); + + /** + * Attribute to represent the level of the active Log4J RootLogger + * @return The level of the RootLogger. + */ + @MBeanAttribute(name = "getRuntimeRootLoggerLevel", description = "Get the runtime logging level for the active log4j Root Logger.") + String getRuntimeRootLoggerLevel(); + + + //****** log4j XML configuration file operations ****** // + + /** + * Updates the level of an existing Log4J logger within the xml configuration file + * @param logger The name of the logger + * @param level The level to set the logger to + * @return True if successful, false if unsuccessful (eg if an invalid logger or level is specified) + * @throws IOException if there is an error parsing the configuration file. + */ + @MBeanOperation(name = "setConfigFileLoggerLevel", description = "Set the logging level for an existing logger " + + "in the log4j xml configuration file", impact = MBeanOperationInfo.ACTION) + boolean setConfigFileLoggerLevel(@MBeanOperationParameter(name = "logger", description = "logger name")String logger, + @MBeanOperationParameter(name = "level", description = "Logger level")String level) throws IOException; + + /** + * Retrieves a TabularData set of the existing Log4J loggers within the xml configuration file + * @return TabularData set of CompositeData rows with logger name and level, or null if there is a problem with the TabularData type + * @throws IOException if there is an error parsing the configuration file. + */ + @MBeanOperation(name = "viewConfigFileLoggerLevels", description = "Get the logging level defined for the logger's " + + "in the log4j xml configuration file.", impact = MBeanOperationInfo.INFO) + TabularData viewConfigFileLoggerLevels() throws IOException; + + /** + * Updates the level of the Log4J RootLogger within the xml configuration file if it is present + * @param level The level to set the logger to + * @return True if successful, false if not (eg an invalid level is specified, or root logger level isnt already defined) + * @throws IOException if there is an error parsing the configuration file. + */ + @MBeanOperation(name = "setConfigFileRootLoggerLevel", description = "Set the logging level for the Root Logger " + + "in the log4j xml configuration file.", impact = MBeanOperationInfo.ACTION) + boolean setConfigFileRootLoggerLevel(@MBeanOperationParameter(name = "level", description = "Logger level")String level) throws IOException; + + /** + * Attribute to represent the level of the Log4J RootLogger within the xml configuration file + * @return The level of the RootLogger, or null if it is not present + */ + @MBeanAttribute(name = "getConfigFileRootLoggerLevel", description = "Get the logging level for the Root Logger " + + "in the log4j xml configuration file.") + String getConfigFileRootLoggerLevel() throws IOException; +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/logging/management/LoggingManagementMBean.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/logging/management/LoggingManagementMBean.java new file mode 100644 index 0000000000..f84cbbd786 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/logging/management/LoggingManagementMBean.java @@ -0,0 +1,667 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * + */ +package org.apache.qpid.server.logging.management; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; + +import org.apache.qpid.server.management.MBeanDescription; +import org.apache.qpid.server.management.AMQManagedObject; + +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.log4j.xml.Log4jEntityResolver; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.ErrorHandler; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; + +import javax.management.JMException; +import javax.management.openmbean.CompositeData; +import javax.management.openmbean.CompositeDataSupport; +import javax.management.openmbean.CompositeType; +import javax.management.openmbean.OpenDataException; +import javax.management.openmbean.OpenType; +import javax.management.openmbean.SimpleType; +import javax.management.openmbean.TabularData; +import javax.management.openmbean.TabularDataSupport; +import javax.management.openmbean.TabularType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + + +/** MBean class for BrokerLoggingManagerMBean. It implements all the management features exposed for managing logging. */ +@MBeanDescription("Logging Management Interface") +public class LoggingManagementMBean extends AMQManagedObject implements LoggingManagement +{ + + private static final Logger _logger = Logger.getLogger(LoggingManagementMBean.class); + private String _log4jConfigFileName; + private int _log4jLogWatchInterval; + + static TabularType _loggerLevelTabularType; + static CompositeType _loggerLevelCompositeType; + + static + { + try + { + OpenType[] loggerLevelItemTypes = new OpenType[]{SimpleType.STRING, SimpleType.STRING}; + + _loggerLevelCompositeType = new CompositeType("LoggerLevelList", "Logger Level Data", + COMPOSITE_ITEM_NAMES, COMPOSITE_ITEM_DESCRIPTIONS, loggerLevelItemTypes); + + _loggerLevelTabularType = new TabularType("LoggerLevel", "List of loggers with levels", + _loggerLevelCompositeType, TABULAR_UNIQUE_INDEX); + } + catch (OpenDataException e) + { + _logger.error("Tabular data setup for viewing logger levels was incorrect."); + _loggerLevelTabularType = null; + } + } + + public LoggingManagementMBean(String log4jConfigFileName, int log4jLogWatchInterval) throws JMException + { + super(LoggingManagement.class, LoggingManagement.TYPE, LoggingManagement.VERSION); + _log4jConfigFileName = log4jConfigFileName; + _log4jLogWatchInterval = log4jLogWatchInterval; + } + + public String getObjectInstanceName() + { + return LoggingManagement.TYPE; + } + + public Integer getLog4jLogWatchInterval() + { + return _log4jLogWatchInterval; + } + + @SuppressWarnings("unchecked") + public synchronized boolean setRuntimeLoggerLevel(String logger, String level) + { + //check specified level is valid + Level newLevel; + try + { + newLevel = getLevel(level); + } + catch (Exception e) + { + return false; + } + + //check specified logger exists + Enumeration loggers = LogManager.getCurrentLoggers(); + Boolean loggerExists = false; + + while(loggers.hasMoreElements()) + { + Logger log = (Logger) loggers.nextElement(); + if (log.getName().equals(logger)) + { + loggerExists = true; + break; + } + } + + if(!loggerExists) + { + return false; + } + + //set the logger to the new level + _logger.info("Setting level to " + level + " for logger: " + logger); + + Logger log = Logger.getLogger(logger); + log.setLevel(newLevel); + + return true; + } + + @SuppressWarnings("unchecked") + public synchronized TabularData viewEffectiveRuntimeLoggerLevels() + { + if (_loggerLevelTabularType == null) + { + _logger.warn("TabluarData type not set up correctly"); + return null; + } + + _logger.info("Getting levels for currently active log4j loggers"); + + Enumeration loggers = LogManager.getCurrentLoggers(); + + TabularData loggerLevelList = new TabularDataSupport(_loggerLevelTabularType); + + Logger logger; + String loggerName; + String level; + + try + { + while(loggers.hasMoreElements()){ + logger = (Logger) loggers.nextElement(); + + loggerName = logger.getName(); + level = logger.getEffectiveLevel().toString(); + + Object[] itemData = {loggerName, level}; + CompositeData loggerData = new CompositeDataSupport(_loggerLevelCompositeType, COMPOSITE_ITEM_NAMES, itemData); + loggerLevelList.put(loggerData); + } + } + catch (OpenDataException e) + { + _logger.warn("Unable to create logger level list due to :" + e); + return null; + } + + return loggerLevelList; + + } + + public synchronized String getRuntimeRootLoggerLevel() + { + Logger rootLogger = Logger.getRootLogger(); + + return rootLogger.getLevel().toString(); + } + + public synchronized boolean setRuntimeRootLoggerLevel(String level) + { + Level newLevel; + try + { + newLevel = getLevel(level); + } + catch (Exception e) + { + return false; + } + + _logger.info("Setting RootLogger level to " + level); + + Logger log = Logger.getRootLogger(); + log.setLevel(newLevel); + + return true; + } + + //method to convert from a string to a log4j Level, throws exception if the given value is invalid + private Level getLevel(String level) throws Exception + { + Level newLevel = Level.toLevel(level); + + //above Level.toLevel call returns a DEBUG Level if the request fails. Check the result. + if (newLevel.equals(Level.DEBUG) && !(level.equalsIgnoreCase("debug"))) + { + //received DEBUG but we did not ask for it, the Level request failed. + throw new Exception("Invalid level name"); + } + + return newLevel; + } + + //handler to catch errors signalled by the JAXP parser and throw an appropriate exception + private class SaxErrorHandler implements ErrorHandler + { + + public void error(SAXParseException e) throws SAXException + { + throw new SAXException("Error parsing XML file: " + e.getMessage()); + } + + public void fatalError(SAXParseException e) throws SAXException + { + throw new SAXException("Fatal error parsing XML file: " + e.getMessage()); + } + + public void warning(SAXParseException e) throws SAXException + { + throw new SAXException("Warning parsing XML file: " + e.getMessage()); + } + } + + //method to parse the XML configuration file, validating it in the process, and returning a DOM Document of the content. + private synchronized Document parseConfigFile(String fileName) throws IOException + { + //check file was specified, exists, and is readable + if(fileName == null) + { + _logger.warn("No log4j XML configuration file has been set"); + throw new IOException("No log4j XML configuration file has been set"); + } + + File configFile = new File(fileName); + + if (!configFile.exists()) + { + _logger.warn("Specified log4j XML configuration file does not exist: " + fileName); + throw new IOException("Specified log4j XML configuration file does not exist"); + } + else if (!configFile.canRead()) + { + _logger.warn("Specified log4j XML configuration file is not readable: " + fileName); + throw new IOException("Specified log4j XML configuration file is not readable"); + } + + //parse it + DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder docBuilder; + Document doc; + + ErrorHandler errHandler = new SaxErrorHandler(); + try + { + docFactory.setValidating(true); + docBuilder = docFactory.newDocumentBuilder(); + docBuilder.setErrorHandler(errHandler); + docBuilder.setEntityResolver(new Log4jEntityResolver()); + doc = docBuilder.parse(fileName); + } + catch (ParserConfigurationException e) + { + _logger.warn("Unable to parse the log4j XML file due to possible configuration error: " + e); + //recommended that MBeans should use java.* and javax.* exceptions only + throw new IOException("Unable to parse the log4j XML file due to possible configuration error: " + e.getMessage()); + } + catch (SAXException e) + { + _logger.warn("The specified log4j XML file is invalid: " + e); + //recommended that MBeans should use standard java.* and javax.* exceptions only + throw new IOException("The specified log4j XML file is invalid: " + e.getMessage()); + } + catch (IOException e) + { + _logger.warn("Unable to parse the specified log4j XML file" + e); + throw new IOException("Unable to parse the specified log4j XML file", e); + } + + return doc; + } + + + private synchronized boolean writeUpdatedConfigFile(String log4jConfigFileName, Document doc) throws IOException + { + File log4jConfigFile = new File(log4jConfigFileName); + + if (!log4jConfigFile.canWrite()) + { + _logger.warn("Specified log4j XML configuration file is not writable: " + log4jConfigFile); + throw new IOException("Specified log4j XML configuration file is not writable"); + } + + Transformer transformer = null; + try + { + transformer = TransformerFactory.newInstance().newTransformer(); + } + catch (Exception e) + { + _logger.warn("Could not create an XML transformer: " +e); + return false; + } + + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty(OutputKeys.DOCTYPE_SYSTEM, "log4j.dtd"); + DOMSource source = new DOMSource(doc); + + File tmp; + try + { + tmp = File.createTempFile("LogManMBeanTemp", ".tmp"); + tmp.deleteOnExit(); + StreamResult result = new StreamResult(tmp); + transformer.transform(source, result); + } + catch (TransformerException e) + { + _logger.warn("Could not transform the XML into new file: " +e); + return false; + } + catch (IOException e) + { + _logger.warn("Could not create the new file: " +e); + return false; + } + + // Swap temp file in to replace existing configuration file. + File old = new File(log4jConfigFile.getAbsoluteFile() + ".old"); + if (old.exists()) + { + old.delete(); + } + log4jConfigFile.renameTo(old); + return tmp.renameTo(log4jConfigFile); + } + + + /* The log4j XML configuration file DTD defines three possible element + * combinations for specifying optional logger+level settings. + * Must account for the following: + * + * <category name="x"> <priority value="y"/> </category> OR + * <category name="x"> <level value="y"/> </category> OR + * <logger name="x"> <level value="y"/> </logger> + * + * Noting also that the level/priority child element is optional too, + * and not the only possible child element. + */ + + + public synchronized TabularData viewConfigFileLoggerLevels() throws IOException + { + if (_loggerLevelTabularType == null) + { + _logger.warn("TabluarData type not set up correctly"); + return null; + } + + _logger.info("Getting logger levels from log4j configuration file"); + + Document doc = parseConfigFile(_log4jConfigFileName); + + TabularData loggerLevelList = new TabularDataSupport(_loggerLevelTabularType); + + //retrieve the 'category' element nodes + NodeList categoryElements = doc.getElementsByTagName("category"); + + String categoryName; + String priority = null; + + for (int i = 0; i < categoryElements.getLength(); i++) + { + Element categoryElement = (Element) categoryElements.item(i); + categoryName = categoryElement.getAttribute("name"); + + //retrieve the category's mandatory 'priority' or 'level' element's value. + //It may not be the only child node, so request by tag name. + NodeList priorityElements = categoryElement.getElementsByTagName("priority"); + NodeList levelElements = categoryElement.getElementsByTagName("level"); + + if (priorityElements.getLength() != 0) + { + Element priorityElement = (Element) priorityElements.item(0); + priority = priorityElement.getAttribute("value").toUpperCase(); + } + else if (levelElements.getLength() != 0) + { + Element levelElement = (Element) levelElements.item(0); + priority = levelElement.getAttribute("value").toUpperCase(); + } + else + { + //there is no exiting priority or level to view, move onto next category/logger + continue; + } + + try + { + Object[] itemData = {categoryName, priority}; + CompositeData loggerData = new CompositeDataSupport(_loggerLevelCompositeType, COMPOSITE_ITEM_NAMES, itemData); + loggerLevelList.put(loggerData); + } + catch (OpenDataException e) + { + _logger.warn("Unable to create logger level list due to :" + e); + return null; + } + } + + //retrieve the 'logger' element nodes + NodeList loggerElements = doc.getElementsByTagName("logger"); + + String loggerName; + String level; + + for (int i = 0; i < loggerElements.getLength(); i++) + { + Element loggerElement = (Element) loggerElements.item(i); + loggerName = loggerElement.getAttribute("name"); + + //retrieve the logger's mandatory 'level' element's value + //It may not be the only child node, so request by tag name. + NodeList levelElements = loggerElement.getElementsByTagName("level"); + + Element levelElement = (Element) levelElements.item(0); + level = levelElement.getAttribute("value").toUpperCase(); + + try + { + Object[] itemData = {loggerName, level}; + CompositeData loggerData = new CompositeDataSupport(_loggerLevelCompositeType, COMPOSITE_ITEM_NAMES, itemData); + loggerLevelList.put(loggerData); + } + catch (OpenDataException e) + { + _logger.warn("Unable to create logger level list due to :" + e); + return null; + } + } + + return loggerLevelList; + } + + public synchronized boolean setConfigFileLoggerLevel(String logger, String level) throws IOException + { + //check that the specified level is a valid log4j Level + try + { + getLevel(level); + } + catch (Exception e) + { + //it isnt a valid level + return false; + } + + _logger.info("Setting level to " + level + " for logger '" + logger + + "' in log4j xml configuration file: " + _log4jConfigFileName); + + Document doc = parseConfigFile(_log4jConfigFileName); + + //retrieve the 'category' and 'logger' element nodes + NodeList categoryElements = doc.getElementsByTagName("category"); + NodeList loggerElements = doc.getElementsByTagName("logger"); + + //collect them into a single elements list + List<Element> logElements = new ArrayList<Element>(); + + for (int i = 0; i < categoryElements.getLength(); i++) + { + logElements.add((Element) categoryElements.item(i)); + } + for (int i = 0; i < loggerElements.getLength(); i++) + { + logElements.add((Element) loggerElements.item(i)); + } + + //try to locate the specified logger/category in the elements retrieved + Element logElement = null; + for (Element e : logElements) + { + if (e.getAttribute("name").equals(logger)) + { + logElement = e; + break; + } + } + + if (logElement == null) + { + //no loggers/categories with given name found, does not exist to update + _logger.warn("Specified logger does not exist in the configuration file: " +logger); + return false; + } + + //retrieve the optional 'priority' or 'level' sub-element value. + //It may not be the only child node, so request by tag name. + NodeList priorityElements = logElement.getElementsByTagName("priority"); + NodeList levelElements = logElement.getElementsByTagName("level"); + + Element levelElement = null; + if (priorityElements.getLength() != 0) + { + levelElement = (Element) priorityElements.item(0); + } + else if (levelElements.getLength() != 0) + { + levelElement = (Element) levelElements.item(0); + } + else + { + //there is no exiting priority or level element to update + return false; + } + + //update the element with the new level/priority + levelElement.setAttribute("value", level); + + //output the new file + return writeUpdatedConfigFile(_log4jConfigFileName, doc); + } + + + /* The log4j XML configuration file DTD defines 2 possible element + * combinations for specifying the optional root logger level settings + * Must account for the following: + * + * <root> <priority value="y"/> </root> OR + * <root> <level value="y"/> </root> + * + * Noting also that the level/priority child element is optional too, + * and not the only possible child element. + */ + + public synchronized String getConfigFileRootLoggerLevel() throws IOException + { + _logger.info("Getting root logger level from log4j configuration file"); + + Document doc = parseConfigFile(_log4jConfigFileName); + + //retrieve the optional 'root' element node + NodeList rootElements = doc.getElementsByTagName("root"); + + if (rootElements.getLength() == 0) + { + //there is not root logger definition + return null; + } + + Element rootElement = (Element) rootElements.item(0); + + //retrieve the optional 'priority' or 'level' element value. + //It may not be the only child node, so request by tag name. + NodeList priorityElements = rootElement.getElementsByTagName("priority"); + NodeList levelElements = rootElement.getElementsByTagName("level"); + String priority = null; + + if (priorityElements.getLength() != 0) + { + Element priorityElement = (Element) priorityElements.item(0); + priority = priorityElement.getAttribute("value"); + } + else if(levelElements.getLength() != 0) + { + Element levelElement = (Element) levelElements.item(0); + priority = levelElement.getAttribute("value"); + } + + if(priority != null) + { + return priority.toUpperCase(); + } + else + { + return null; + } + } + + public synchronized boolean setConfigFileRootLoggerLevel(String level) throws IOException + { + //check that the specified level is a valid log4j Level + try + { + getLevel(level); + } + catch (Exception e) + { + //it isnt a valid level + return false; + } + + _logger.info("Setting level to " + level + " for the Root logger in " + + "log4j xml configuration file: " + _log4jConfigFileName); + + Document doc = parseConfigFile(_log4jConfigFileName); + + //retrieve the optional 'root' element node + NodeList rootElements = doc.getElementsByTagName("root"); + + if (rootElements.getLength() == 0) + { + return false; + } + + Element rootElement = (Element) rootElements.item(0); + + //retrieve the optional 'priority' or 'level' sub-element value. + //It may not be the only child node, so request by tag name. + NodeList priorityElements = rootElement.getElementsByTagName("priority"); + NodeList levelElements = rootElement.getElementsByTagName("level"); + + Element levelElement = null; + if (priorityElements.getLength() != 0) + { + levelElement = (Element) priorityElements.item(0); + } + else if (levelElements.getLength() != 0) + { + levelElement = (Element) levelElements.item(0); + } + else + { + //there is no exiting priority/level to update + return false; + } + + //update the element with the new level/priority + levelElement.setAttribute("value", level); + + //output the new file + return writeUpdatedConfigFile(_log4jConfigFileName, doc); + } +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/AMQManagedObject.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/AMQManagedObject.java index a2c2bd62a2..c6e07f6f48 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/AMQManagedObject.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/AMQManagedObject.java @@ -50,10 +50,10 @@ public abstract class AMQManagedObject extends DefaultManagedObject protected MBeanInfo _mbeanInfo; - protected AMQManagedObject(Class<?> managementInterface, String typeName) + protected AMQManagedObject(Class<?> managementInterface, String typeName, int version) throws NotCompliantMBeanException { - super(managementInterface, typeName); + super(managementInterface, typeName, version); buildMBeanInfo(); } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/DefaultManagedObject.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/DefaultManagedObject.java index 84526dbc11..67aee90ba4 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/DefaultManagedObject.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/DefaultManagedObject.java @@ -39,13 +39,15 @@ public abstract class DefaultManagedObject extends StandardMBean implements Mana private Class<?> _managementInterface; private String _typeName; + private int _version; - protected DefaultManagedObject(Class<?> managementInterface, String typeName) + protected DefaultManagedObject(Class<?> managementInterface, String typeName, int version) throws NotCompliantMBeanException { super(managementInterface); _managementInterface = managementInterface; _typeName = typeName; + _version = version; } public String getType() @@ -115,6 +117,10 @@ public abstract class DefaultManagedObject extends StandardMBean implements Mana objectName.append(getHierarchicalName(this)); objectName.append("name=").append(name); + objectName.append(","); + objectName.append("version=").append(_version); + + return new ObjectName(objectName.toString()); } @@ -132,6 +138,9 @@ public abstract class DefaultManagedObject extends StandardMBean implements Mana objectName.append(hierarchyName.substring(0, hierarchyName.lastIndexOf(","))); } + objectName.append(","); + objectName.append("version=").append(_version); + return new ObjectName(objectName.toString()); } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/JMXManagedObjectRegistry.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/JMXManagedObjectRegistry.java index aea2f9d872..f02e858250 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/JMXManagedObjectRegistry.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/JMXManagedObjectRegistry.java @@ -107,8 +107,6 @@ public class JMXManagedObjectRegistry implements ManagedObjectRegistry } IApplicationRegistry appRegistry = ApplicationRegistry.getInstance(); - - boolean jmxmpSecurity = appRegistry.getConfiguration().getManagementSecurityEnabled(); int port = appRegistry.getConfiguration().getJMXManagementPort(); //retrieve the Principal Database assigned to JMX authentication duties @@ -119,184 +117,152 @@ public class JMXManagedObjectRegistry implements ManagedObjectRegistry final JMXConnectorServer cs; HashMap<String,Object> env = new HashMap<String,Object>(); - if (jmxmpSecurity) + //Socket factories for the RMIConnectorServer, either default or SLL depending on configuration + RMIClientSocketFactory csf; + RMIServerSocketFactory ssf; + + //check ssl enabled option in config, default to true if option is not set + boolean sslEnabled = appRegistry.getConfiguration().getManagementSSLEnabled(); + + if (sslEnabled) { - // For SASL using JMXMP - JMXServiceURL jmxURL = new JMXServiceURL("jmxmp", null, port); + //set the SSL related system properties used by the SSL RMI socket factories to the values + //given in the configuration file, unless command line settings have already been specified + String keyStorePath; - String saslType = null; - if (db instanceof Base64MD5PasswordFilePrincipalDatabase) + if(System.getProperty("javax.net.ssl.keyStore") != null) { - saslType = "SASL/CRAM-MD5"; - env.put("jmx.remote.profiles", "SASL/CRAM-MD5"); - CRAMMD5HashedInitialiser initialiser = new CRAMMD5HashedInitialiser(); - initialiser.initialise(db); - env.put("jmx.remote.sasl.callback.handler", initialiser.getCallbackHandler()); + keyStorePath = System.getProperty("javax.net.ssl.keyStore"); } - else if (db instanceof PlainPasswordFilePrincipalDatabase) + else { - saslType = "SASL/PLAIN"; - PlainInitialiser initialiser = new PlainInitialiser(); - initialiser.initialise(db); - env.put("jmx.remote.sasl.callback.handler", initialiser.getCallbackHandler()); - env.put("jmx.remote.profiles", "SASL/PLAIN"); + keyStorePath = appRegistry.getConfiguration().getManagementKeyStorePath(); } - //workaround NPE generated from env map classloader issue when using Eclipse 3.4 to launch - env.put("jmx.remote.profile.provider.class.loader", this.getClass().getClassLoader()); - - _log.warn("Starting JMXMP based JMX ConnectorServer on port '" + port + "' with " + saslType); - _startupLog.warn("Starting JMXMP based JMX ConnectorServer on port '" + port + "' with " + saslType); - - cs = JMXConnectorServerFactory.newJMXConnectorServer(jmxURL, env, _mbeanServer); - } - else - { - //Socket factories for the RMIConnectorServer, either default or SLL depending on configuration - RMIClientSocketFactory csf; - RMIServerSocketFactory ssf; - - //check ssl enabled option in config, default to true if option is not set - boolean sslEnabled = appRegistry.getConfiguration().getManagementSSLEnabled(); - - if (sslEnabled) + //check the keystore path value is valid + if (keyStorePath == null) { - //set the SSL related system properties used by the SSL RMI socket factories to the values - //given in the configuration file, unless command line settings have already been specified - String keyStorePath; - - if(System.getProperty("javax.net.ssl.keyStore") != null) + throw new ConfigurationException("JMX management SSL keystore path not defined, " + + "unable to start SSL protected JMX ConnectorServer"); + } + else + { + //ensure the system property is set + System.setProperty("javax.net.ssl.keyStore", keyStorePath); + + //check the file is usable + File ksf = new File(keyStorePath); + + if (!ksf.exists()) { - keyStorePath = System.getProperty("javax.net.ssl.keyStore"); + throw new FileNotFoundException("Cannot find JMX management SSL keystore file " + ksf); } - else{ - keyStorePath = appRegistry.getConfiguration().getManagementKeyStorePath(); + if (!ksf.canRead()) + { + throw new FileNotFoundException("Cannot read JMX management SSL keystore file: " + + ksf + ". Check permissions."); } - //check the keystore path value is valid - if (keyStorePath == null) + _log.info("JMX ConnectorServer using SSL keystore file " + ksf.getAbsolutePath()); + _startupLog.info("JMX ConnectorServer using SSL keystore file " + ksf.getAbsolutePath()); + } + + //check the key store password is set + if (System.getProperty("javax.net.ssl.keyStorePassword") == null) + { + + if (appRegistry.getConfiguration().getManagementKeyStorePassword() == null) { - throw new ConfigurationException("JMX management SSL keystore path not defined, " + - "unable to start SSL protected JMX ConnectorServer"); + throw new ConfigurationException("JMX management SSL keystore password not defined, " + + "unable to start requested SSL protected JMX server"); } else { - //ensure the system property is set - System.setProperty("javax.net.ssl.keyStore", keyStorePath); - - //check the file is usable - File ksf = new File(keyStorePath); - - if (!ksf.exists()) - { - throw new FileNotFoundException("Cannot find JMX management SSL keystore file " + ksf); - } - if (!ksf.canRead()) - { - throw new FileNotFoundException("Cannot read JMX management SSL keystore file: " - + ksf + ". Check permissions."); - } - - _log.info("JMX ConnectorServer using SSL keystore file " + ksf.getAbsolutePath()); - _startupLog.info("JMX ConnectorServer using SSL keystore file " + ksf.getAbsolutePath()); + System.setProperty("javax.net.ssl.keyStorePassword", + appRegistry.getConfiguration().getManagementKeyStorePassword()); } + } - //check the key store password is set - if (System.getProperty("javax.net.ssl.keyStorePassword") == null) - { - - if (appRegistry.getConfiguration().getManagementKeyStorePassword() == null) - { - throw new ConfigurationException("JMX management SSL keystore password not defined, " + - "unable to start requested SSL protected JMX server"); - } - else - { - System.setProperty("javax.net.ssl.keyStorePassword", - appRegistry.getConfiguration().getManagementKeyStorePassword()); - } - } + //create the SSL RMI socket factories + csf = new SslRMIClientSocketFactory(); + ssf = new SslRMIServerSocketFactory(); - //create the SSL RMI socket factories - csf = new SslRMIClientSocketFactory(); - ssf = new SslRMIServerSocketFactory(); + _log.warn("Starting JMX ConnectorServer on port '"+ port + "' (+" + + (port +PORT_EXPORT_OFFSET) + ") with SSL"); + _startupLog.warn("Starting JMX ConnectorServer on port '"+ port + "' (+" + + (port +PORT_EXPORT_OFFSET) + ") with SSL"); + } + else + { + //Do not specify any specific RMI socket factories, resulting in use of the defaults. + csf = null; + ssf = null; - _log.warn("Starting JMX ConnectorServer on port '"+ port + "' (+" + - (port +PORT_EXPORT_OFFSET) + ") with SSL"); - _startupLog.warn("Starting JMX ConnectorServer on port '"+ port + "' (+" + - (port +PORT_EXPORT_OFFSET) + ") with SSL"); - } - else - { - //Do not specify any specific RMI socket factories, resulting in use of the defaults. - csf = null; - ssf = null; - - _log.warn("Starting JMX ConnectorServer on port '" + port + "' (+" + (port +PORT_EXPORT_OFFSET) + ")"); - _startupLog.warn("Starting JMX ConnectorServer on port '" + port + "' (+" + (port +PORT_EXPORT_OFFSET) + ")"); - } - - //add a JMXAuthenticator implementation the env map to authenticate the RMI based JMX connector server - RMIPasswordAuthenticator rmipa = new RMIPasswordAuthenticator(); - rmipa.setPrincipalDatabase(db); - env.put(JMXConnectorServer.AUTHENTICATOR, rmipa); - - /* - * Start a RMI registry on the management port, to hold the JMX RMI ConnectorServer stub. - * Using custom socket factory to prevent anyone (including us unfortunately) binding to the registry using RMI. - * As a result, only binds made using the object reference will succeed, thus securing it from external change. - */ - System.setProperty("java.rmi.server.randomIDs", "true"); - _rmiRegistry = LocateRegistry.createRegistry(port, null, new CustomRMIServerSocketFactory()); - - /* - * We must now create the RMI ConnectorServer manually, as the JMX Factory methods use RMI calls - * to bind the ConnectorServer to the registry, which will now fail as for security we have - * locked it from any RMI based modifications, including our own. Instead, we will manually bind - * the RMIConnectorServer stub to the registry using its object reference, which will still succeed. - * - * The registry is exported on the defined management port 'port'. We will export the RMIConnectorServer - * on 'port +1'. Use of these two well-defined ports will ease any navigation through firewall's. - */ - final RMIServerImpl rmiConnectorServerStub = new RMIJRMPServerImpl(port+PORT_EXPORT_OFFSET, csf, ssf, env); - final String hostname = InetAddress.getLocalHost().getHostName(); - final JMXServiceURL externalUrl = new JMXServiceURL( - "service:jmx:rmi://"+hostname+":"+(port+PORT_EXPORT_OFFSET)+"/jndi/rmi://"+hostname+":"+port+"/jmxrmi"); - - final JMXServiceURL internalUrl = new JMXServiceURL("rmi", hostname, port+PORT_EXPORT_OFFSET); - cs = new RMIConnectorServer(internalUrl, env, rmiConnectorServerStub, _mbeanServer) + _log.warn("Starting JMX ConnectorServer on port '" + port + "' (+" + (port +PORT_EXPORT_OFFSET) + ")"); + _startupLog.warn("Starting JMX ConnectorServer on port '" + port + "' (+" + (port +PORT_EXPORT_OFFSET) + ")"); + } + + //add a JMXAuthenticator implementation the env map to authenticate the RMI based JMX connector server + RMIPasswordAuthenticator rmipa = new RMIPasswordAuthenticator(); + rmipa.setPrincipalDatabase(db); + env.put(JMXConnectorServer.AUTHENTICATOR, rmipa); + + /* + * Start a RMI registry on the management port, to hold the JMX RMI ConnectorServer stub. + * Using custom socket factory to prevent anyone (including us unfortunately) binding to the registry using RMI. + * As a result, only binds made using the object reference will succeed, thus securing it from external change. + */ + System.setProperty("java.rmi.server.randomIDs", "true"); + _rmiRegistry = LocateRegistry.createRegistry(port, null, new CustomRMIServerSocketFactory()); + + /* + * We must now create the RMI ConnectorServer manually, as the JMX Factory methods use RMI calls + * to bind the ConnectorServer to the registry, which will now fail as for security we have + * locked it from any RMI based modifications, including our own. Instead, we will manually bind + * the RMIConnectorServer stub to the registry using its object reference, which will still succeed. + * + * The registry is exported on the defined management port 'port'. We will export the RMIConnectorServer + * on 'port +1'. Use of these two well-defined ports will ease any navigation through firewall's. + */ + final RMIServerImpl rmiConnectorServerStub = new RMIJRMPServerImpl(port+PORT_EXPORT_OFFSET, csf, ssf, env); + final String hostname = InetAddress.getLocalHost().getHostName(); + final JMXServiceURL externalUrl = new JMXServiceURL( + "service:jmx:rmi://"+hostname+":"+(port+PORT_EXPORT_OFFSET)+"/jndi/rmi://"+hostname+":"+port+"/jmxrmi"); + + final JMXServiceURL internalUrl = new JMXServiceURL("rmi", hostname, port+PORT_EXPORT_OFFSET); + cs = new RMIConnectorServer(internalUrl, env, rmiConnectorServerStub, _mbeanServer) + { + @Override + public synchronized void start() throws IOException { - @Override - public synchronized void start() throws IOException + try { - try - { - //manually bind the connector server to the registry at key 'jmxrmi', like the out-of-the-box agent - _rmiRegistry.bind("jmxrmi", rmiConnectorServerStub); - } - catch (AlreadyBoundException abe) - { - //key was already in use. shouldnt happen here as its a new registry, unbindable by normal means. - - //IOExceptions are the only checked type throwable by the method, wrap and rethrow - IOException ioe = new IOException(abe.getMessage()); - ioe.initCause(abe); - throw ioe; - } - - //now do the normal tasks - super.start(); - } - - @Override - public JMXServiceURL getAddress() - { - //must return our pre-crafted url that includes the full details, inc JNDI details - return externalUrl; - } + //manually bind the connector server to the registry at key 'jmxrmi', like the out-of-the-box agent + _rmiRegistry.bind("jmxrmi", rmiConnectorServerStub); + } + catch (AlreadyBoundException abe) + { + //key was already in use. shouldnt happen here as its a new registry, unbindable by normal means. - }; - } + //IOExceptions are the only checked type throwable by the method, wrap and rethrow + IOException ioe = new IOException(abe.getMessage()); + ioe.initCause(abe); + throw ioe; + } + + //now do the normal tasks + super.start(); + } + + @Override + public JMXServiceURL getAddress() + { + //must return our pre-crafted url that includes the full details, inc JNDI details + return externalUrl; + } + + }; + //Add the custom invoker as an MBeanServerForwarder, and start the RMIConnectorServer. MBeanServerForwarder mbsf = MBeanInvocationHandlerImpl.newProxyInstance(); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/MBeanInvocationHandlerImpl.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/MBeanInvocationHandlerImpl.java index a0ecc2bd85..e9b4d85e66 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/MBeanInvocationHandlerImpl.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/MBeanInvocationHandlerImpl.java @@ -20,6 +20,8 @@ */ package org.apache.qpid.server.management; +import org.apache.qpid.server.configuration.management.ConfigurationManagement; +import org.apache.qpid.server.logging.management.LoggingManagement; import org.apache.qpid.server.security.access.management.UserManagement; import org.apache.log4j.Logger; @@ -37,6 +39,8 @@ import java.lang.reflect.Method; import java.security.AccessController; import java.security.Principal; import java.security.AccessControlContext; +import java.util.ArrayList; +import java.util.HashSet; import java.util.Set; import java.util.Properties; @@ -53,9 +57,16 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler public final static String READWRITE = "readwrite"; public final static String READONLY = "readonly"; private final static String DELEGATE = "JMImplementation:type=MBeanServerDelegate"; - private MBeanServer mbs; + private MBeanServer _mbs; private static Properties _userRoles = new Properties(); + private static HashSet<String> _adminOnlyMethods = new HashSet<String>(); + { + _adminOnlyMethods.add(UserManagement.TYPE); + _adminOnlyMethods.add(LoggingManagement.TYPE); + _adminOnlyMethods.add(ConfigurationManagement.TYPE); + } + public static MBeanServerForwarder newProxyInstance() { final InvocationHandler handler = new MBeanInvocationHandlerImpl(); @@ -71,7 +82,7 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler if (methodName.equals("getMBeanServer")) { - return mbs; + return _mbs; } if (methodName.equals("setMBeanServer")) @@ -80,11 +91,11 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler { throw new IllegalArgumentException("Null MBeanServer"); } - if (mbs != null) + if (_mbs != null) { throw new IllegalArgumentException("MBeanServer object already initialized"); } - mbs = (MBeanServer) args[0]; + _mbs = (MBeanServer) args[0]; return null; } @@ -95,12 +106,12 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler // Allow operations performed locally on behalf of the connector server itself if (subject == null) { - return method.invoke(mbs, args); + return method.invoke(_mbs, args); } if (args == null || DELEGATE.equals(args[0])) { - return method.invoke(mbs, args); + return method.invoke(_mbs, args); } // Restrict access to "createMBean" and "unregisterMBean" to any user @@ -124,7 +135,7 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler { if (isAdmin(identity)) { - return method.invoke(mbs, args); + return method.invoke(_mbs, args); } else { @@ -135,14 +146,14 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler // Following users can perform any operation other than "createMBean" and "unregisterMBean" if (isAllowedToModify(identity)) { - return method.invoke(mbs, args); + return method.invoke(_mbs, args); } // These users can only call "getAttribute" on the MBeanServerDelegate MBean // Here we can add other fine grained permissions like specific method for a particular mbean if (isReadOnlyUser(identity) && isReadOnlyMethod(method, args)) { - return method.invoke(mbs, args); + return method.invoke(_mbs, args); } throw new SecurityException("Access denied"); @@ -153,9 +164,9 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler if (args[0] instanceof ObjectName) { ObjectName object = (ObjectName) args[0]; - return UserManagement.TYPE.equals(object.getKeyProperty("type")); + + return _adminOnlyMethods.contains(object.getKeyProperty("type")); } - return false; } @@ -196,7 +207,10 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler private boolean isReadOnlyMethod(Method method, Object[] args) { String methodName = method.getName(); - if (methodName.startsWith("query") || methodName.startsWith("get")) + + //handle standard get/set/query and select 'is' methods from MBeanServer + if (methodName.startsWith("query") || methodName.startsWith("get") + ||methodName.startsWith("isInstanceOf") || methodName.startsWith("isRegistered")) { return true; } @@ -205,8 +219,11 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler return false; } + //handle invocation of other methods on mbeans if ((args[0] instanceof ObjectName) && (methodName.equals("invoke"))) { + + //get invoked method name String mbeanMethod = (args.length > 1) ? (String) args[1] : null; if (mbeanMethod == null) { @@ -215,7 +232,8 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler try { - MBeanInfo mbeanInfo = mbs.getMBeanInfo((ObjectName) args[0]); + //check if the given method is tagged with an INFO impact attribute + MBeanInfo mbeanInfo = _mbs.getMBeanInfo((ObjectName) args[0]); if (mbeanInfo != null) { MBeanOperationInfo[] opInfos = mbeanInfo.getOperations(); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/ManagedBroker.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/ManagedBroker.java index 45e2e91ed7..c18417fc43 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/management/ManagedBroker.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/management/ManagedBroker.java @@ -40,12 +40,13 @@ public interface ManagedBroker { static final String TYPE = "VirtualHostManager"; + static final int VERSION = 1 ; + /** * Creates a new Exchange. * @param name * @param type * @param durable - * @param passive * @throws IOException * @throws JMException */ @@ -73,7 +74,6 @@ public interface ManagedBroker * @param queueName * @param durable * @param owner - * @param autoDelete * @throws IOException * @throws JMException */ diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/plugins/PluginManager.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/plugins/PluginManager.java index 1b7919e8b7..5d8fa3e9d7 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/plugins/PluginManager.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/plugins/PluginManager.java @@ -35,6 +35,7 @@ import org.apache.qpid.server.security.access.ACLPluginFactory; import org.apache.qpid.server.security.access.plugins.AllowAll; import org.apache.qpid.server.security.access.plugins.DenyAll; import org.apache.qpid.server.security.access.plugins.SimpleXML; +import org.apache.qpid.server.security.access.plugins.network.FirewallPlugin; import org.osgi.framework.BundleActivator; import org.osgi.framework.BundleException; import org.osgi.util.tracker.ServiceTracker; @@ -165,6 +166,7 @@ public class PluginManager _securityPlugins.put(SimpleXML.class.getName(), SimpleXML.FACTORY); _securityPlugins.put(AllowAll.class.getName(), AllowAll.FACTORY); _securityPlugins.put(DenyAll.class.getName(), DenyAll.FACTORY); + _securityPlugins.put(FirewallPlugin.class.getName(), FirewallPlugin.FACTORY); } return _securityPlugins; } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/protocol/AMQProtocolSessionMBean.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/protocol/AMQProtocolSessionMBean.java index bd072985c4..5dd3cc075a 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/protocol/AMQProtocolSessionMBean.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/protocol/AMQProtocolSessionMBean.java @@ -37,7 +37,6 @@ */ package org.apache.qpid.server.protocol; -import java.security.Principal; import java.util.Date; import java.util.List; @@ -58,7 +57,6 @@ import javax.management.openmbean.TabularDataSupport; import javax.management.openmbean.TabularType; import org.apache.qpid.AMQException; -import org.apache.qpid.framing.AMQFrame; import org.apache.qpid.framing.AMQShortString; import org.apache.qpid.framing.ConnectionCloseBody; import org.apache.qpid.framing.MethodRegistry; @@ -93,7 +91,7 @@ public class AMQProtocolSessionMBean extends AMQManagedObject implements Managed @MBeanConstructor("Creates an MBean exposing an AMQ Broker Connection") public AMQProtocolSessionMBean(AMQMinaProtocolSession session) throws NotCompliantMBeanException, OpenDataException { - super(ManagedConnection.class, ManagedConnection.TYPE); + super(ManagedConnection.class, ManagedConnection.TYPE, ManagedConnection.VERSION); _session = session; String remote = getRemoteAddress(); remote = "anonymous".equals(remote) ? (remote + hashCode()) : remote; diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/protocol/ManagedConnection.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/protocol/ManagedConnection.java index e6e713ac6d..e75b09a0cb 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/protocol/ManagedConnection.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/protocol/ManagedConnection.java @@ -41,6 +41,7 @@ import org.apache.qpid.server.management.MBeanOperationParameter; public interface ManagedConnection { static final String TYPE = "Connection"; + static final int VERSION = 1; @MBeanAttribute(name = "ClientId", description = "Client Id") String getClientId(); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQMessage.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQMessage.java index e96d2ba874..8dac12fe24 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQMessage.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQMessage.java @@ -60,30 +60,6 @@ public interface AMQMessage //Check the status of this message - /** - * Called selectors to determin if the message has already been sent - * - * @return _deliveredToConsumer - */ - boolean getDeliveredToConsumer(); - - /** - * Called to enforce the 'immediate' flag. - * - * @returns true if the message is marked for immediate delivery but has not been marked as delivered - * to a consumer - */ - boolean immediateAndNotDelivered(); - - /** - * Checks to see if the message has expired. If it has the message is dequeued. - * - * @return true if the message has expire - * - * @throws org.apache.qpid.AMQException - */ - boolean expired() throws AMQException; - /** Is this a persistent message * * @return true if the message is persistent @@ -91,13 +67,8 @@ public interface AMQMessage boolean isPersistent(); - /** - * Called when this message is delivered to a consumer. (used to implement the 'immediate' flag functionality). - * And for selector efficiency. - */ - void setDeliveredToConsumer(); + boolean isImmediate(); - void setExpiration(long expiration); void setClientIdentifier(AMQProtocolSession.ProtocolSessionIdentifier sessionIdentifier); @@ -113,9 +84,16 @@ public interface AMQMessage void addContentBodyFrame(StoreContext storeContext, ContentChunk contentChunk, boolean isLastContentBody) throws AMQException; + void recoverFromMessageMetaData(MessageMetaData mmd); + + void recoverContentBodyFrame(ContentChunk contentChunk, boolean isLastContentBody) throws AMQException; String toString(); String debugIdentity(); + + void setExpiration(long expiration); + + long getExpiration(); } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQMessageHandle.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQMessageHandle.java deleted file mode 100644 index 93ac21fc7c..0000000000 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQMessageHandle.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.qpid.server.queue; - -import org.apache.qpid.AMQException; -import org.apache.qpid.framing.ContentHeaderBody; -import org.apache.qpid.server.store.StoreContext; -import org.apache.qpid.framing.abstraction.ContentChunk; -import org.apache.qpid.framing.abstraction.MessagePublishInfo; - -/** - * A pluggable way of getting message data. Implementations can provide intelligent caching for example or - * even no caching at all to minimise the broker memory footprint. - */ -public interface AMQMessageHandle -{ - ContentHeaderBody getContentHeaderBody(StoreContext context) throws AMQException; - - /** - * - * @return the messageId for the message associated with this handle - */ - Long getMessageId(); - - - /** - * @return the number of body frames associated with this message - */ - int getBodyCount(StoreContext context) throws AMQException; - - /** - * @return the size of the body - */ - long getBodySize(StoreContext context) throws AMQException; - - /** - * Get a particular content body - * @param index the index of the body to retrieve, must be between 0 and getBodyCount() - 1 - * @return a content body - * @throws IllegalArgumentException if the index is invalid - */ - ContentChunk getContentChunk(StoreContext context, int index) throws IllegalArgumentException, AMQException; - - void addContentBodyFrame(StoreContext storeContext, ContentChunk contentBody, boolean isLastContentBody) throws AMQException; - - MessagePublishInfo getMessagePublishInfo(StoreContext context) throws AMQException; - - boolean isPersistent(); - - void setPublishAndContentHeaderBody(StoreContext storeContext, MessagePublishInfo messagePublishInfo, - ContentHeaderBody contentHeaderBody) - throws AMQException; - - void removeMessage(StoreContext storeContext) throws AMQException; - - long getArrivalTime(); -} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQPriorityQueue.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQPriorityQueue.java index 34a70c6969..ade780d0bb 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQPriorityQueue.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQPriorityQueue.java @@ -36,12 +36,12 @@ public class AMQPriorityQueue extends SimpleAMQQueue int priorities) throws AMQException { - super(name, durable, owner, autoDelete, virtualHost, new PriorityQueueList.Factory(priorities)); + super(name, durable, owner, autoDelete, virtualHost, new PriorityQueueEntryList.Factory(priorities)); } public int getPriorities() { - return ((PriorityQueueList) _entries).getPriorities(); + return ((PriorityQueueEntryList) _entries).getPriorities(); } @Override diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueue.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueue.java index 9fadbb0cdc..43ec6c4d15 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueue.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueue.java @@ -68,6 +68,8 @@ public interface AMQQueue extends Managable, Comparable<AMQQueue> boolean isEmpty(); + boolean isFlowed(); + int getMessageCount(); int getUndeliveredMessageCount(); @@ -111,7 +113,15 @@ public interface AMQQueue extends Managable, Comparable<AMQQueue> void removeMessagesFromQueue(long fromMessageId, long toMessageId, StoreContext storeContext); + long getMemoryUsageMaximum(); + + void setMemoryUsageMaximum(long maximumMemoryUsage); + + long getMemoryUsageMinimum(); + + void setMemoryUsageMinimum(long minimumMemoryUsage); + long getMemoryUsageCurrent(); long getMaximumMessageSize(); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueueFactory.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueueFactory.java index eb0a011e93..6ba22321f1 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueueFactory.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueueFactory.java @@ -23,13 +23,17 @@ package org.apache.qpid.server.queue; import org.apache.qpid.AMQException; import org.apache.qpid.framing.AMQShortString; import org.apache.qpid.framing.FieldTable; +import org.apache.qpid.protocol.AMQConstant; import org.apache.qpid.server.configuration.QueueConfiguration; import org.apache.qpid.server.virtualhost.VirtualHost; - public class AMQQueueFactory { public static final AMQShortString X_QPID_PRIORITIES = new AMQShortString("x-qpid-priorities"); + public static final AMQShortString QPID_MAX_COUNT = new AMQShortString("qpid.max_count"); + public static final AMQShortString QPID_MAX_SIZE = new AMQShortString("qpid.max_size"); + public static final AMQShortString QPID_POLICY_TYPE = new AMQShortString("qpid.policy_type"); + public static final String QPID_FLOW_TO_DISK = "flow_to_disk"; public static AMQQueue createAMQQueueImpl(AMQShortString name, boolean durable, @@ -39,10 +43,26 @@ public class AMQQueueFactory throws AMQException { - final int priorities = arguments == null ? 1 : arguments.containsKey(X_QPID_PRIORITIES) ? arguments.getInteger(X_QPID_PRIORITIES) : 1; + int priorities = 1; + + if (arguments != null && arguments.containsKey(X_QPID_PRIORITIES)) + { + Integer priority = arguments.getInteger(X_QPID_PRIORITIES); + + if (priority != null) + { + priorities = priority.intValue(); + } + else + { + throw new AMQException(AMQConstant.INVALID_ARGUMENT, + "Queue create request with non integer value for :" + X_QPID_PRIORITIES + "=" + arguments.get(X_QPID_PRIORITIES), null); + } + + } AMQQueue q = null; - if(priorities > 1) + if (priorities > 1) { q = new AMQPriorityQueue(name, durable, owner, autoDelete, virtualHost, priorities); } @@ -51,6 +71,40 @@ public class AMQQueueFactory q = new SimpleAMQQueue(name, durable, owner, autoDelete, virtualHost); } + final String queuePolicyType = arguments == null ? null : + arguments.containsKey(QPID_POLICY_TYPE) ? arguments.getString(QPID_POLICY_TYPE) : null; + + if (queuePolicyType != null) + { + if (queuePolicyType.equals(QPID_FLOW_TO_DISK)) + { + if (arguments.containsKey(QPID_MAX_SIZE)) + { + + final long queueSize = arguments.getInteger(QPID_MAX_SIZE); + + if (queueSize < 0) + { + throw new AMQException(AMQConstant.INVALID_ARGUMENT, + "Queue create request with negative size:" + queueSize, null); + } + + q.setMemoryUsageMaximum(queueSize); + } + else + { + throw new AMQException(AMQConstant.INVALID_ARGUMENT, + "Queue create request with no qpid.max_size value,", null); + } + } + else + { + throw new AMQException(AMQConstant.NOT_IMPLEMENTED, + "Queue create request with unknown Policy Type:" + queuePolicyType, null); + } + + } + //Register the new queue virtualHost.getQueueRegistry().registerQueue(q); return q; @@ -66,9 +120,9 @@ public class AMQQueueFactory FieldTable arguments = null; boolean priority = config.getPriority(); int priorities = config.getPriorities(); - if(priority || priorities > 0) + if (priority || priorities > 0) { - if(arguments == null) + if (arguments == null) { arguments = new FieldTable(); } @@ -85,6 +139,8 @@ public class AMQQueueFactory q.setMaximumMessageSize(config.getMaximumMessageSize()); q.setMaximumMessageCount(config.getMaximumMessageCount()); q.setMinimumAlertRepeatGap(config.getMinimumAlertRepeatGap()); + q.setMemoryUsageMaximum(config.getMemoryUsageMaximum()); + q.setMemoryUsageMinimum(config.getMemoryUsageMinimum()); return q; } } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueueMBean.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueueMBean.java index a08719875d..2ff54fb748 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueueMBean.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/AMQQueueMBean.java @@ -100,7 +100,7 @@ public class AMQQueueMBean extends AMQManagedObject implements ManagedQueue, Que @MBeanConstructor("Creates an MBean exposing an AMQQueue") public AMQQueueMBean(AMQQueue queue) throws JMException { - super(ManagedQueue.class, ManagedQueue.TYPE); + super(ManagedQueue.class, ManagedQueue.TYPE, ManagedQueue.VERSION); _queue = queue; _queueName = jmxEncode(new StringBuffer(queue.getName()), 0).toString(); } @@ -221,11 +221,12 @@ public class AMQQueueMBean extends AMQManagedObject implements ManagedQueue, Que _queue.setMaximumMessageCount(value); } + /** + * returns the maximum total size of messages(bytes) in the queue. + */ public Long getMaximumQueueDepth() { - long queueDepthInBytes = _queue.getMaximumQueueDepth(); - - return queueDepthInBytes >> 10; + return _queue.getMaximumQueueDepth(); } public void setMaximumQueueDepth(Long value) @@ -233,20 +234,49 @@ public class AMQQueueMBean extends AMQManagedObject implements ManagedQueue, Que _queue.setMaximumQueueDepth(value); } + public Long getMemoryUsageMaximum() + { + return _queue.getMemoryUsageMaximum(); + } + + public void setMemoryUsageMaximum(Long maximumMemoryUsage) + { + _queue.setMemoryUsageMaximum(maximumMemoryUsage); + } + + public Long getMemoryUsageMinimum() + { + return _queue.getMemoryUsageMinimum(); + } + + public void setMemoryUsageMinimum(Long minimumMemoryUsage) + { + _queue.setMemoryUsageMinimum(minimumMemoryUsage); + } + + public Long getMemoryUsageCurrent() + { + return _queue.getMemoryUsageCurrent(); + } + + public boolean isFlowed() + { + return _queue.isFlowed(); + } + /** - * returns the size of messages(KB) in the queue. + * returns the total size of messages(bytes) in the queue. */ public Long getQueueDepth() throws JMException { - long queueBytesSize = _queue.getQueueDepth(); - - return queueBytesSize >> 10; + return _queue.getQueueDepth(); } /** * Checks if there is any notification to be send to the listeners + * @param queueEntry */ - public void checkForNotification(AMQMessage msg) throws AMQException + public void checkForNotification(QueueEntry queueEntry) throws AMQException { final Set<NotificationCheck> notificationChecks = _queue.getNotificationChecks(); @@ -260,7 +290,7 @@ public class AMQQueueMBean extends AMQManagedObject implements ManagedQueue, Que { if (check.isMessageSpecific() || (_lastNotificationTimes[check.ordinal()] < thresholdTime)) { - if (check.notifyIfNecessary(msg, _queue, this)) + if (check.notifyIfNecessary(queueEntry, _queue, this)) { _lastNotificationTimes[check.ordinal()] = currentTime; } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/FileQueueBackingStore.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/FileQueueBackingStore.java new file mode 100644 index 0000000000..0e5a4efba6 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/FileQueueBackingStore.java @@ -0,0 +1,358 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.queue; + +import org.apache.log4j.Logger; +import org.apache.mina.common.ByteBuffer; +import org.apache.qpid.AMQException; +import org.apache.qpid.framing.AMQShortString; +import org.apache.qpid.framing.BasicContentHeaderProperties; +import org.apache.qpid.framing.ContentHeaderBody; +import org.apache.qpid.framing.abstraction.ContentChunk; +import org.apache.qpid.framing.abstraction.MessagePublishInfo; +import org.apache.qpid.util.FileUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +public class FileQueueBackingStore implements QueueBackingStore +{ + private static final Logger _log = Logger.getLogger(FileQueueBackingStore.class); + + private String _flowToDiskLocation; + + public FileQueueBackingStore(String location) + { + _flowToDiskLocation = location; + } + + public AMQMessage load(Long messageId) + { + _log.info("Loading Message (ID:" + messageId + ")"); + + MessageMetaData mmd; + + File handle = getFileHandle(messageId); + + ObjectInputStream input = null; + + Exception error = null; + try + { + input = new ObjectInputStream(new FileInputStream(handle)); + + long arrivaltime = input.readLong(); + + final AMQShortString exchange = new AMQShortString(input.readUTF()); + final AMQShortString routingKey = new AMQShortString(input.readUTF()); + final boolean mandatory = input.readBoolean(); + final boolean immediate = input.readBoolean(); + + int bodySize = input.readInt(); + byte[] underlying = new byte[bodySize]; + + input.readFully(underlying, 0, bodySize); + + ByteBuffer buf = ByteBuffer.wrap(underlying); + + ContentHeaderBody chb = ContentHeaderBody.createFromBuffer(buf, bodySize); + + int chunkCount = input.readInt(); + + // There are WAY to many annonymous MPIs in the code this should be made concrete. + MessagePublishInfo info = new MessagePublishInfo() + { + + public AMQShortString getExchange() + { + return exchange; + } + + public void setExchange(AMQShortString exchange) + { + + } + + public boolean isImmediate() + { + return immediate; + } + + public boolean isMandatory() + { + return mandatory; + } + + public AMQShortString getRoutingKey() + { + return routingKey; + } + }; + + mmd = new MessageMetaData(info, chb, chunkCount); + mmd.setArrivalTime(arrivaltime); + + AMQMessage message; + if (((BasicContentHeaderProperties) chb.properties).getDeliveryMode() == 2) + { + message = new PersistentAMQMessage(messageId, null); + } + else + { + message = new TransientAMQMessage(messageId); + } + + message.recoverFromMessageMetaData(mmd); + + for (int chunk = 0; chunk < chunkCount; chunk++) + { + int length = input.readInt(); + + byte[] data = new byte[length]; + + input.readFully(data, 0, length); + + try + { + message.recoverContentBodyFrame(new RecoverDataBuffer(length, data), (chunk + 1 == chunkCount)); + } + catch (AMQException e) + { + //ignore as this will not occur. + // It is thrown by the _transactionLog method in load on PersistentAMQMessage + // but we have created the message with a null log and will never call that method. + } + } + + return message; + } + catch (Exception e) + { + error = e; + } + finally + { + try + { + input.close(); + // We can purge the message here then reflow it if required but I believe it to be cleaner to leave it + // on disk until it has been deleted from the queue at that point we can be sure we won't need the data + //handle.delete(); + } + catch (IOException e) + { + _log.info("Unable to close input on message(" + messageId + ") recovery due to:" + e.getMessage()); + } + } + + throw new UnableToRecoverMessageException(error); + } + + public void unload(AMQMessage message) throws UnableToFlowMessageException + { + long messageId = message.getMessageId(); + + File handle = getFileHandle(messageId); + + //If we have written the data once then we don't need to do it again. + if (handle.exists()) + { + if (_log.isDebugEnabled()) + { + _log.debug("Message(ID:" + messageId + ") already unloaded."); + } + return; + } + + if (_log.isInfoEnabled()) + { + _log.info("Unloading Message (ID:" + messageId + ")"); + } + + ObjectOutputStream writer = null; + Exception error = null; + + try + { + writer = new ObjectOutputStream(new FileOutputStream(handle)); + + writer.writeLong(message.getArrivalTime()); + + MessagePublishInfo mpi = message.getMessagePublishInfo(); + writer.writeUTF(String.valueOf(mpi.getExchange())); + writer.writeUTF(String.valueOf(mpi.getRoutingKey())); + writer.writeBoolean(mpi.isMandatory()); + writer.writeBoolean(mpi.isImmediate()); + ContentHeaderBody chb = message.getContentHeaderBody(); + + // write out the content header body + final int bodySize = chb.getSize(); + byte[] underlying = new byte[bodySize]; + ByteBuffer buf = ByteBuffer.wrap(underlying); + chb.writePayload(buf); + + writer.writeInt(bodySize); + writer.write(underlying, 0, bodySize); + + int bodyCount = message.getBodyCount(); + writer.writeInt(bodyCount); + + //WriteContentBody + for (int index = 0; index < bodyCount; index++) + { + ContentChunk chunk = message.getContentChunk(index); + int length = chunk.getSize(); + + byte[] chunk_underlying = new byte[length]; + + ByteBuffer chunk_buf = chunk.getData(); + + chunk_buf.duplicate().rewind().get(chunk_underlying); + + writer.writeInt(length); + writer.write(chunk_underlying, 0, length); + } + } + catch (FileNotFoundException e) + { + error = e; + } + catch (IOException e) + { + error = e; + } + finally + { + // In a FileNotFound situation writer will be null. + if (writer != null) + { + try + { + writer.flush(); + writer.close(); + } + catch (IOException e) + { + error = e; + } + } + } + + if (error != null) + { + _log.error("Unable to unload message(" + messageId + ") to disk, restoring state."); + handle.delete(); + throw new UnableToFlowMessageException(messageId, error); + } + } + + /** + * Use the messageId to calculate the file path on disk. + * + * Current implementation will give us 256 bins. + * Therefore the maximum messages that can be flowed before error/platform is: + * ext3 : 256 bins * 32000 = 8192000 + * FAT32 : 256 bins * 65534 = 16776704 + * Other FS have much greater limits than we need to worry about. + * + * @param messageId the Message we need a file Handle for. + * + * @return the File handle + */ + private File getFileHandle(long messageId) + { + // grab the 8 LSB to give us 256 bins + long bin = messageId & 0xFFL; + + String bin_path = _flowToDiskLocation + File.separator + bin; + File bin_dir = new File(bin_path); + + if (!bin_dir.exists()) + { + bin_dir.mkdirs(); + } + + String id = bin_path + File.separator + messageId; + + return new File(id); + } + + public void delete(Long messageId) + { + File handle = getFileHandle(messageId); + + if (handle.exists()) + { + if (_log.isInfoEnabled()) + { + _log.info("Message(" + messageId + ") delete flowToDisk."); + } + if (!handle.delete()) + { + throw new RuntimeException("Unable to delete flowToDisk data"); + } + } + } + + public void close() + { + _log.info("Closing Backing store at:" + _flowToDiskLocation); + if (!FileUtils.delete(new File(_flowToDiskLocation), true)) + { + _log.error("Unable to fully delete backing store location"); + } + } + + private class RecoverDataBuffer implements ContentChunk + { + private int _length; + private ByteBuffer _dataBuffer; + + public RecoverDataBuffer(int length, byte[] data) + { + _length = length; + _dataBuffer = ByteBuffer.wrap(data); + } + + public int getSize() + { + return _length; + } + + public ByteBuffer getData() + { + return _dataBuffer; + } + + public void reduceToFit() + { + + } + + } + +} + diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/FileQueueBackingStoreFactory.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/FileQueueBackingStoreFactory.java new file mode 100644 index 0000000000..21073c22ae --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/FileQueueBackingStoreFactory.java @@ -0,0 +1,166 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.queue; + +import org.apache.commons.configuration.ConfigurationException; +import org.apache.log4j.Logger; +import org.apache.qpid.server.configuration.VirtualHostConfiguration; +import org.apache.qpid.server.virtualhost.VirtualHost; +import org.apache.qpid.util.FileUtils; + +import java.io.File; + +public class FileQueueBackingStoreFactory implements QueueBackingStoreFactory +{ + private static final Logger _log = Logger.getLogger(FileQueueBackingStoreFactory.class); + + private String _flowToDiskLocation; + private static final String QUEUE_BACKING_DIR = "queueBacking"; + + public void configure(VirtualHost virtualHost, VirtualHostConfiguration config) throws ConfigurationException + { + setFlowToDisk(virtualHost.getName(), config.getFlowToDiskLocation()); + } + + private void setFlowToDisk(String vHostName, String location) throws ConfigurationException + { + if (vHostName == null) + { + throw new ConfigurationException("Unable to setup to Flow to Disk as Virtualhost name was not specified"); + } + + if (location == null) + { + throw new ConfigurationException("Unable to setup to Flow to Disk as location was not specified."); + } + + _flowToDiskLocation = location; + + _flowToDiskLocation += File.separator + QUEUE_BACKING_DIR + File.separator + vHostName; + + //Check the location we will create QUEUE_BACKING_DIR in. + File root = new File(location); + if (!root.exists()) + { + throw new ConfigurationException("Specified Flow to Disk root does not exist:" + root.getAbsolutePath()); + } + else + { + + if (root.isFile()) + { + throw new ConfigurationException("Unable to create Temporary Flow to Disk store as specified root is a file:" + + root.getAbsolutePath()); + } + + if (!root.canWrite()) + { + throw new ConfigurationException("Unable to create Temporary Flow to Disk store. Unable to write to specified root:" + + root.getAbsolutePath()); + } + + } + + // if we don't mark QUEUE_BAKCING_DIR as a deleteOnExit it will remain. + File backingDir = new File(location + File.separator + QUEUE_BACKING_DIR); + if (backingDir.exists()) + { + if (!FileUtils.delete(backingDir, true)) + { + throw new ConfigurationException("Unable to delete existing Flow to Disk root at:" + + backingDir.getAbsolutePath()); + } + + if (backingDir.isFile()) + { + throw new ConfigurationException("Unable to create Temporary Flow to Disk root as specified location is a file:" + + backingDir.getAbsolutePath()); + } + } + + backingDir.deleteOnExit(); + if (!backingDir.mkdirs()) + { + throw new ConfigurationException("Unable to create Temporary Flow to Disk root:" + location + File.separator + QUEUE_BACKING_DIR); + } + + + File store = new File(_flowToDiskLocation); + if (store.exists()) + { + if (!FileUtils.delete(store, true)) + { + throw new ConfigurationException("Unable to delete existing Flow to Disk store at:" + + store.getAbsolutePath()); + } + + if (store.isFile()) + { + throw new ConfigurationException("Unable to create Temporary Flow to Disk store as specified location is a file:" + + store.getAbsolutePath()); + } + + } + + _log.info("Creating Flow to Disk Store : " + store.getAbsolutePath()); + store.deleteOnExit(); + if (!store.mkdir()) + { + throw new ConfigurationException("Unable to create Temporary Flow to Disk store:" + store.getAbsolutePath()); + } + } + + public QueueBackingStore createBacking(AMQQueue queue) + { + return new FileQueueBackingStore(createStore(queue.getName().toString())); + } + + private String createStore(String name) + { + return createStore(name, 0); + } + + private String createStore(String name, int index) + { + + String store = _flowToDiskLocation + File.separator + name; + if (index > 0) + { + store += "-" + index; + } + + //TODO ensure store is safe for the OS + + File storeFile = new File(store); + + if (storeFile.exists()) + { + return createStore(name, index + 1); + } + + storeFile.mkdirs(); + + storeFile.deleteOnExit(); + + return store; + } + +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/FlowableBaseQueueEntryList.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/FlowableBaseQueueEntryList.java new file mode 100644 index 0000000000..0c4b8a0b42 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/FlowableBaseQueueEntryList.java @@ -0,0 +1,485 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.queue; + +import org.apache.log4j.Logger; +import org.apache.qpid.pool.ReferenceCountingExecutorService; +import org.apache.qpid.server.virtualhost.VirtualHost; + +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +/** This is an abstract base class to handle */ +public abstract class FlowableBaseQueueEntryList implements QueueEntryList +{ + protected static final Logger _log = Logger.getLogger(FlowableBaseQueueEntryList.class); + + private final AtomicInteger _atomicQueueCount = new AtomicInteger(0); + private final AtomicLong _atomicQueueSize = new AtomicLong(0L); + protected final AtomicLong _atomicQueueInMemory = new AtomicLong(0L); + /** The maximum amount of memory that is allocated to this queue. Beyond this the queue will flow to disk. */ + + protected long _memoryUsageMaximum = -1L; + + /** The minimum amount of memory that is allocated to this queue. If the queueDepth hits this level then more flowed data can be read in. */ + protected long _memoryUsageMinimum = 0; + private volatile AtomicBoolean _flowed; + private QueueBackingStore _backingStore; + protected AMQQueue _queue; + private Executor _inhaler; + private Executor _purger; + private AtomicBoolean _stopped; + private AtomicReference<MessageInhaler> _asynchronousInhaler = new AtomicReference(null); + protected boolean _disabled; + private AtomicReference<MessagePurger> _asynchronousPurger = new AtomicReference(null); + private static final int BATCH_PROCESS_COUNT = 100; + + FlowableBaseQueueEntryList(AMQQueue queue) + { + _queue = queue; + _flowed = new AtomicBoolean(false); + VirtualHost vhost = queue.getVirtualHost(); + if (vhost != null) + { + _backingStore = vhost.getQueueBackingStoreFactory().createBacking(queue); + } + + _stopped = new AtomicBoolean(false); + _inhaler = ReferenceCountingExecutorService.getInstance().acquireExecutorService(); + _purger = ReferenceCountingExecutorService.getInstance().acquireExecutorService(); + _disabled = true; + } + + public void setFlowed(boolean flowed) + { + if (_flowed.get() != flowed) + { + _log.warn("Marking Queue(" + _queue.getName() + ") as flowed (" + flowed + ")"); + _flowed.set(flowed); + } + } + + protected void showUsage() + { + showUsage(""); + } + + protected void showUsage(String prefix) + { + if (_log.isDebugEnabled()) + { + _log.debug(prefix + " Queue(" + _queue + ":" + _queue.getName() + ") usage:" + memoryUsed() + + "/" + getMemoryUsageMinimum() + "<>" + getMemoryUsageMaximum() + + "/" + dataSize()); + } + } + + public boolean isFlowed() + { + return _flowed.get(); + } + + public int size() + { + return _atomicQueueCount.get(); + } + + public long dataSize() + { + return _atomicQueueSize.get(); + } + + public long memoryUsed() + { + return _atomicQueueInMemory.get(); + } + + public void setMemoryUsageMaximum(long maximumMemoryUsage) + { + _memoryUsageMaximum = maximumMemoryUsage; + + if (maximumMemoryUsage >= 0) + { + _disabled = false; + } + + // Don't attempt to start the inhaler/purger unless we have a minimum value specified. + if (_memoryUsageMaximum >= 0) + { + setMemoryUsageMinimum(_memoryUsageMaximum / 2); + + // if we have now have to much memory in use we need to purge. + if (_memoryUsageMaximum < _atomicQueueInMemory.get()) + { + setFlowed(true); + startPurger(); + } + } + else + { + if (_log.isInfoEnabled()) + { + _log.info("Disabling Flow to Disk for queue:" + _queue.getName()); + } + _disabled = true; + } + } + + public long getMemoryUsageMaximum() + { + return _memoryUsageMaximum; + } + + public void setMemoryUsageMinimum(long minimumMemoryUsage) + { + _memoryUsageMinimum = minimumMemoryUsage; + + // Don't attempt to start the inhaler unless we have a minimum value specified. + if (_memoryUsageMinimum > 0) + { + checkAndStartInhaler(); + } + } + + private void checkAndStartInhaler() + { + // If we've increased the minimum memory above what we have in memory then + // we need to inhale more if there is more + if (_atomicQueueInMemory.get() < _memoryUsageMinimum && _atomicQueueSize.get() > 0) + { + startInhaler(); + } + } + + private void startInhaler() + { + MessageInhaler inhaler = new MessageInhaler(); + + if (_asynchronousInhaler.compareAndSet(null, inhaler)) + { + _inhaler.execute(inhaler); + } + } + + private void startPurger() + { + MessagePurger purger = new MessagePurger(); + + if (_asynchronousPurger.compareAndSet(null, purger)) + { + _purger.execute(purger); + } + } + + public long getMemoryUsageMinimum() + { + return _memoryUsageMinimum; + } + + /** + * Only to be called by the QueueEntry + * + * @param queueEntry the entry to unload + */ + public void entryUnloadedUpdateMemory(QueueEntry queueEntry) + { + if (_atomicQueueInMemory.addAndGet(-queueEntry.getSize()) < 0) + { + _log.error("InMemory Count just went below 0:" + queueEntry.debugIdentity()); + } + + checkAndStartInhaler(); + } + + /** + * Only to be called from the QueueEntry + * + * @param queueEntry the entry to load + */ + public void entryLoadedUpdateMemory(QueueEntry queueEntry) + { + if (_atomicQueueInMemory.addAndGet(queueEntry.getSize()) > _memoryUsageMaximum) + { + _log.error("Loaded to much data!:" + _atomicQueueInMemory.get() + "/" + _memoryUsageMaximum); + setFlowed(true); + startPurger(); + } + } + + public void stop() + { + if (!_stopped.getAndSet(true)) + { + // The SimpleAMQQueue keeps running when stopped so we should just release the services + // rather than actively shutdown our threads. + //Shutdown thread for inhaler. + ReferenceCountingExecutorService.getInstance().releaseExecutorService(); + ReferenceCountingExecutorService.getInstance().releaseExecutorService(); + + _backingStore.close(); + } + } + + protected void incrementCounters(final QueueEntryImpl queueEntry) + { + _atomicQueueCount.incrementAndGet(); + _atomicQueueSize.addAndGet(queueEntry.getSize()); + long inUseMemory = _atomicQueueInMemory.addAndGet(queueEntry.getSize()); + + if (!_disabled && inUseMemory > _memoryUsageMaximum) + { + setFlowed(true); + queueEntry.unload(); + } + } + + protected void dequeued(QueueEntryImpl queueEntry) + { + _atomicQueueCount.decrementAndGet(); + _atomicQueueSize.addAndGet(-queueEntry.getSize()); + if (!queueEntry.isFlowed()) + { + if (_atomicQueueInMemory.addAndGet(-queueEntry.getSize()) < 0) + { + _log.error("InMemory Count just went below 0 on dequeue."); + } + } + } + + public QueueBackingStore getBackingStore() + { + return _backingStore; + } + + private class MessageInhaler implements Runnable + { + public void run() + { + String threadName = Thread.currentThread().getName(); + Thread.currentThread().setName("Inhaler-" + _queue.getVirtualHost().getName() + "-" + _queue.getName()); + try + { + inhaleList(this); + } + finally + { + Thread.currentThread().setName(threadName); + } + } + } + + private void inhaleList(MessageInhaler messageInhaler) + { + if (_log.isInfoEnabled()) + { + _log.info("Inhaler Running:" + _queue.getName()); + showUsage("Inhaler Running:" + _queue.getName()); + } + // If in memory count is at or over max then we can't inhale + if (_atomicQueueInMemory.get() >= _memoryUsageMaximum) + { + if (_log.isDebugEnabled()) + { + _log.debug("Unable to start inhaling as we are already over quota:" + + _atomicQueueInMemory.get() + ">=" + _memoryUsageMaximum); + } + return; + } + + _asynchronousInhaler.compareAndSet(messageInhaler, null); + int inhaled = 1; + + while ((_atomicQueueInMemory.get() < _memoryUsageMaximum) // we havn't filled our max memory + && (_atomicQueueInMemory.get() < _atomicQueueSize.get()) // we haven't loaded all that is available + && (inhaled < BATCH_PROCESS_COUNT) // limit the number of runs we do + && (inhaled > 0) // ensure we could inhale something + && _asynchronousInhaler.compareAndSet(null, messageInhaler)) // Ensure we are the running inhaler + { + inhaled = 0; + QueueEntryIterator iterator = iterator(); + + // If the inhaler is running and delivery rate picks up ensure that we just don't chase the delivery thread. + while ((_atomicQueueInMemory.get() < _memoryUsageMaximum) + && !iterator.getNode().isAvailable() && iterator.advance()) + { + //Find first AVAILABLE node + } + + // Because the above loop checks then moves on to the next entry a check for atTail will return true but + // we won't have checked the last entry to see if we can load it. So create atEndofList and update it based + // on the return from advance() which returns true if it can advance. + boolean atEndofList = false; + while ((_atomicQueueInMemory.get() < _memoryUsageMaximum) // we havn't filled our max memory + && (inhaled < BATCH_PROCESS_COUNT) // limit the number of runs we do + && !atEndofList) // We have reached end of list QueueEntries + { + QueueEntry entry = iterator.getNode(); + + if (entry.isAvailable() && entry.isFlowed()) + { + if (_atomicQueueInMemory.get() + entry.getSize() > _memoryUsageMaximum) + { + // We don't have space for this message so we need to stop inhaling. + if (_log.isDebugEnabled()) + { + _log.debug("Entry won't fit in memory stopping inhaler:" + entry.debugIdentity()); + } + inhaled = BATCH_PROCESS_COUNT; + } + else + { + entry.load(); + inhaled++; + } + } + + atEndofList = !iterator.advance(); + } + + if (iterator.atTail()) + { + setFlowed(false); + } + + _asynchronousInhaler.set(null); + } + + if (_log.isInfoEnabled()) + { + _log.info("Inhaler Stopping:" + _queue.getName()); + showUsage("Inhaler Stopping:" + _queue.getName()); + } + + //If we have become flowed or have more capacity since we stopped then schedule the thread to run again. + if (_flowed.get() && _atomicQueueInMemory.get() < _memoryUsageMaximum) + { + if (_log.isInfoEnabled()) + { + _log.info("Rescheduling Inhaler:" + _queue.getName()); + } + _inhaler.execute(messageInhaler); + } + + } + + private class MessagePurger implements Runnable + { + public void run() + { + String threadName = Thread.currentThread().getName(); + Thread.currentThread().setName("Purger-" + _queue.getVirtualHost().getName() + "-" + _queue.getName()); + try + { + purgeList(this); + } + finally + { + Thread.currentThread().setName(threadName); + } + } + } + + private void purgeList(MessagePurger messagePurger) + { + // If in memory count is at or over max then we can't inhale + if (_atomicQueueInMemory.get() <= _memoryUsageMinimum) + { + if (_log.isDebugEnabled()) + { + _log.debug("Unable to start purging as we are already below our minimum cache level:" + + _atomicQueueInMemory.get() + "<=" + _memoryUsageMinimum); + } + return; + } + + if (_log.isInfoEnabled()) + { + _log.info("Purger Running:" + _queue.getName()); + showUsage("Purger Running:" + _queue.getName()); + } + + _asynchronousPurger.compareAndSet(messagePurger, null); + int purged = 0; + + while ((_atomicQueueInMemory.get() > _memoryUsageMinimum) + && purged < BATCH_PROCESS_COUNT + && _asynchronousPurger.compareAndSet(null, messagePurger)) + { + QueueEntryIterator iterator = iterator(); + + //There are potentially AQUIRED messages that can be purged but we can't purge the last AQUIRED message + // as it may have just become AQUIRED and not yet delivered. + + //To be safe only purge available messages. This should be fine as long as we have a small prefetch. + while (!iterator.getNode().isAvailable() && iterator.advance()) + { + //Find first AVAILABLE node + } + + // Count up the memory usage to find our minimum point + long memoryUsage = 0; + boolean atTail = false; + while ((memoryUsage < _memoryUsageMaximum) && !atTail) + { + QueueEntry entry = iterator.getNode(); + + if (entry.isAvailable() && !entry.isFlowed()) + { + memoryUsage += entry.getSize(); + } + + atTail = !iterator.advance(); + } + + //Purge remainging mesages on queue + while (!atTail && (purged < BATCH_PROCESS_COUNT)) + { + QueueEntry entry = iterator.getNode(); + + if (entry.isAvailable() && !entry.isFlowed()) + { + entry.unload(); + purged++; + } + + atTail = !iterator.advance(); + } + + _asynchronousPurger.set(null); + } + + if (_log.isInfoEnabled()) + { + _log.info("Purger Stopping:" + _queue.getName()); + showUsage("Purger Stopping:" + _queue.getName()); + } + + //If we are still flowed and are over the minimum value then schedule to run again. + if (_flowed.get() && _atomicQueueInMemory.get() > _memoryUsageMinimum) + { + _log.info("Rescheduling Purger:" + _queue.getName()); + _purger.execute(messagePurger); + } + } +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/ManagedQueue.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/ManagedQueue.java index 2bc94995e9..d2cf90b42d 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/ManagedQueue.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/ManagedQueue.java @@ -41,6 +41,7 @@ import org.apache.qpid.server.management.MBeanOperationParameter; public interface ManagedQueue { static final String TYPE = "Queue"; + static final int VERSION = 2; /** * Returns the Name of the ManagedQueue. @@ -71,7 +72,7 @@ public interface ManagedQueue * @return * @throws IOException */ - @MBeanAttribute(name="QueueDepth", description="Size of messages(KB) in the queue") + @MBeanAttribute(name="QueueDepth", description="The total size(Bytes) of messages in the queue") Long getQueueDepth() throws IOException, JMException; /** @@ -180,9 +181,66 @@ public interface ManagedQueue * @param value * @throws IOException */ - @MBeanAttribute(name="MaximumQueueDepth", description="The threshold high value(KB) for Queue Depth") + @MBeanAttribute(name="MaximumQueueDepth", description="The threshold high value(Bytes) for Queue Depth") void setMaximumQueueDepth(Long value) throws IOException; + //TODO change descriptions + /** + * View the limit on the memory that this queue will utilise. + * + * Used by Flow to Disk. + * + * @return The maximum memory(B) that the queue will occuy. + */ + public Long getMemoryUsageMaximum(); + + /** + * Place a limit on the memory that this queue will utilise. + * + * Used by Flow to Disk + * + * @param maximumMemoryUsage The new maximum memory(B) to be used by this queue + */ + @MBeanAttribute(name="MemoryUsageMaximum", description="The maximum memory(B) that the queue will occupy.") + public void setMemoryUsageMaximum(Long maximumMemoryUsage); + + /** + * View the minimum amount of memory that has been defined for this queue. + * + * Used by Flow to Disk + * + * @return The minimum amount of queue data(B) that the queue will attempt to keep in memory + */ + public Long getMemoryUsageMinimum(); + + /** + * Set the minimum amount of memory that has been defined for this queue. + * + * Used by Flow to Disk + * + * @param minimumMemoryUsage The new minimum memory(B) level to be used by this queue + */ + @MBeanAttribute(name="MemoryUsageMinimum", description="The minimum memory(B) that the queue will occupy.") + public void setMemoryUsageMinimum(Long minimumMemoryUsage); + + /** + * View the amount of memory(B) that this queue is using. + * + * @return The current memory(B) usage of this queue. + */ + @MBeanAttribute(name="MemoryUsageCurrent", description="The current amount of memory(B) used by this queue.") + public Long getMemoryUsageCurrent(); + + /** + * When a queue exceeds its MemoryUsageMaximum value then the Queue will start flowing to disk. + * + * This boolean is used to show that change in state. + * + * @return true if the Queue is currently flowing to disk + */ + @MBeanAttribute(name="isFlowed", description="true if the queue is currently flowing to disk.") + public boolean isFlowed(); + //********** Operations *****************// diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/NotificationCheck.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/NotificationCheck.java index e33b0c83c7..a83d661de2 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/NotificationCheck.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/NotificationCheck.java @@ -20,14 +20,12 @@ */
package org.apache.qpid.server.queue;
-import org.apache.qpid.AMQException;
-
public enum NotificationCheck
{
MESSAGE_COUNT_ALERT
{
- boolean notifyIfNecessary(AMQMessage msg, AMQQueue queue, QueueNotificationListener listener)
+ boolean notifyIfNecessary(QueueEntry queueEntry, AMQQueue queue, QueueNotificationListener listener)
{
int msgCount;
final long maximumMessageCount = queue.getMaximumMessageCount();
@@ -41,19 +39,19 @@ public enum NotificationCheck },
MESSAGE_SIZE_ALERT(true)
{
- boolean notifyIfNecessary(AMQMessage msg, AMQQueue queue, QueueNotificationListener listener)
+ boolean notifyIfNecessary(QueueEntry queueEntry, AMQQueue queue, QueueNotificationListener listener)
{
final long maximumMessageSize = queue.getMaximumMessageSize();
if(maximumMessageSize != 0)
{
// Check for threshold message size
- long messageSize = (msg == null) ? 0 : msg.getContentHeaderBody().bodySize;
+ long messageSize = (queueEntry == null) ? 0 : queueEntry.getSize();
if (messageSize >= maximumMessageSize)
{
listener.notifyClients(this, queue, messageSize + "b : Maximum message size threshold (" +
maximumMessageSize + ") breached. [Message ID=" +
- (msg == null ? "null" : msg.getMessageId()) + "]");
+ (queueEntry == null ? "null" : queueEntry.getMessageId()) + "]");
return true;
}
}
@@ -63,7 +61,7 @@ public enum NotificationCheck },
QUEUE_DEPTH_ALERT
{
- boolean notifyIfNecessary(AMQMessage msg, AMQQueue queue, QueueNotificationListener listener)
+ boolean notifyIfNecessary(QueueEntry queueEntry, AMQQueue queue, QueueNotificationListener listener)
{
// Check for threshold queue depth in bytes
final long maximumQueueDepth = queue.getMaximumQueueDepth();
@@ -84,7 +82,7 @@ public enum NotificationCheck },
MESSAGE_AGE_ALERT
{
- boolean notifyIfNecessary(AMQMessage msg, AMQQueue queue, QueueNotificationListener listener)
+ boolean notifyIfNecessary(QueueEntry queueEntry, AMQQueue queue, QueueNotificationListener listener)
{
final long maxMessageAge = queue.getMaximumMessageAge();
@@ -126,6 +124,6 @@ public enum NotificationCheck return _messageSpecific;
}
- abstract boolean notifyIfNecessary(AMQMessage msg, AMQQueue queue, QueueNotificationListener listener);
+ abstract boolean notifyIfNecessary(QueueEntry queueEntry, AMQQueue queue, QueueNotificationListener listener);
}
diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PersistentAMQMessage.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PersistentAMQMessage.java index 92c10b0347..9c644cc010 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PersistentAMQMessage.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PersistentAMQMessage.java @@ -52,7 +52,8 @@ public class PersistentAMQMessage extends TransientAMQMessage throws AMQException { super.setPublishAndContentHeaderBody(storeContext, messagePublishInfo, contentHeaderBody); - MessageMetaData mmd = new MessageMetaData(messagePublishInfo, contentHeaderBody, _contentBodies == null ? 0 : _contentBodies.size(), _arrivalTime); + MessageMetaData mmd = new MessageMetaData(messagePublishInfo, contentHeaderBody, + _contentBodies == null ? 0 : _contentBodies.size(), _arrivalTime); _transactionLog.storeMessageMetaData(storeContext, _messageId, mmd); } @@ -63,13 +64,7 @@ public class PersistentAMQMessage extends TransientAMQMessage return true; } - public void recoverFromMessageMetaData(MessageMetaData mmd) - { - _arrivalTime = mmd.getArrivalTime(); - _contentHeaderBody = mmd.getContentHeaderBody(); - _messagePublishInfo = mmd.getMessagePublishInfo(); - } - + @Override public void recoverContentBodyFrame(ContentChunk contentChunk, boolean isLastContentBody) throws AMQException { super.addContentBodyFrame(null, contentChunk, isLastContentBody); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PriorityQueueEntryList.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PriorityQueueEntryList.java new file mode 100644 index 0000000000..5dd76a5299 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PriorityQueueEntryList.java @@ -0,0 +1,466 @@ +/* +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*/ +package org.apache.qpid.server.queue; + +import org.apache.qpid.framing.CommonContentHeaderProperties; + +public class PriorityQueueEntryList extends FlowableBaseQueueEntryList implements QueueEntryList +{ + private final AMQQueue _queue; + private final QueueEntryList[] _priorityLists; + private final int _priorities; + private final int _priorityOffset; + + public PriorityQueueEntryList(AMQQueue queue, int priorities) + { + super(queue); + _queue = queue; + _priorityLists = new QueueEntryList[priorities]; + _priorities = priorities; + _priorityOffset = 5 - ((priorities + 1) / 2); + for (int i = 0; i < priorities; i++) + { + _priorityLists[i] = new SimpleQueueEntryList(queue); + } + + showUsage("Created:" + _queue.getName()); + } + + public int getPriorities() + { + return _priorities; + } + + public AMQQueue getQueue() + { + return _queue; + } + + public QueueEntry add(AMQMessage message) + { + int index = ((CommonContentHeaderProperties) ((message.getContentHeaderBody().properties))).getPriority() - _priorityOffset; + if (index >= _priorities) + { + index = _priorities - 1; + } + else if (index < 0) + { + index = 0; + } + + long requriedSize = message.getSize(); + // Check and see if list would flow on adding message + if (!_disabled && !isFlowed() && _priorityLists[index].memoryUsed() + requriedSize > _priorityLists[index].getMemoryUsageMaximum()) + { + if (_log.isDebugEnabled()) + { + _log.debug("Message(" + message.debugIdentity() + ") Add of size (" + + requriedSize + ") will cause flow. Searching for space"); + } + + long reclaimed = 0; + + //work down the priorities looking for memory + + //First: Don't take all the memory. So look for a queue that has more than 50% free + long currentMax; + int scavangeIndex = 0; + + if (scavangeIndex == index) + { + scavangeIndex++; + } + + while (scavangeIndex < _priorities && reclaimed <= requriedSize) + { + currentMax = _priorityLists[scavangeIndex].getMemoryUsageMaximum(); + long used = _priorityLists[scavangeIndex].memoryUsed(); + + if (used < currentMax / 2) + { + long newMax = currentMax / 2; + + _priorityLists[scavangeIndex].setMemoryUsageMaximum(newMax); + + reclaimed += currentMax - newMax; + if (_log.isDebugEnabled()) + { + _log.debug("Reclaiming(1) :" + (currentMax - newMax) + "(" + reclaimed + "/" + requriedSize + ") from queue:" + scavangeIndex); + } + break; + } + else + { + scavangeIndex++; + if (scavangeIndex == index) + { + scavangeIndex++; + } + } + } + + //Second: Just take the free memory we need + if (scavangeIndex == _priorities) + { + scavangeIndex = 0; + if (scavangeIndex == index) + { + scavangeIndex++; + } + + while (scavangeIndex < _priorities && reclaimed <= requriedSize) + { + currentMax = _priorityLists[scavangeIndex].getMemoryUsageMaximum(); + long used = _priorityLists[scavangeIndex].memoryUsed(); + + if (used < currentMax) + { + long newMax = currentMax - used; + + // if there are no messages at this priority just take it all + if (newMax == currentMax) + { + newMax = 0; + } + + _priorityLists[scavangeIndex].setMemoryUsageMaximum(newMax); + + reclaimed += currentMax - newMax; + if (_log.isDebugEnabled()) + { + _log.debug("Reclaiming(2) :" + (currentMax - newMax) + "(" + reclaimed + "/" + requriedSize + ") from queue:" + scavangeIndex); + } + break; + } + else + { + scavangeIndex++; + if (scavangeIndex == index) + { + scavangeIndex++; + } + } + } + + //Third: Take required memory + if (scavangeIndex == _priorities) + { + scavangeIndex = 0; + if (scavangeIndex == index) + { + scavangeIndex++; + } + while (scavangeIndex < _priorities && reclaimed <= requriedSize) + { + currentMax = _priorityLists[scavangeIndex].getMemoryUsageMaximum(); + + if (currentMax > 0 ) + { + long newMax = currentMax; + // Just take the amount of space required for this message. + if (newMax > requriedSize) + { + newMax = requriedSize; + } + _priorityLists[scavangeIndex].setMemoryUsageMaximum(newMax); + + reclaimed += currentMax - newMax; + if (_log.isDebugEnabled()) + { + _log.debug("Reclaiming(3) :" + (currentMax - newMax) + "(" + reclaimed + "/" + requriedSize + ") from queue:" + scavangeIndex); + } + break; + } + else + { + scavangeIndex++; + if (scavangeIndex == index) + { + scavangeIndex++; + } + } + } + } + } + + //Increment Maximum + if (reclaimed > 0) + { + if (_log.isDebugEnabled()) + { + _log.debug("Increasing queue(" + index + ") maximum by " + reclaimed + + " to " + (_priorityLists[index].getMemoryUsageMaximum() + reclaimed)); + } + _priorityLists[index].setMemoryUsageMaximum(_priorityLists[index].getMemoryUsageMaximum() + reclaimed); + } + else + { + _log.debug("No space found."); + } + + if (_log.isTraceEnabled()) + { + showUsage("Add"); + } + } + + return _priorityLists[index].add(message); + } + + @Override + protected void showUsage(String prefix) + { + if (_log.isDebugEnabled()) + { + if (prefix.length() != 0) + { + _log.debug(prefix); + } + for (int index = 0; index < _priorities; index++) + { + QueueEntryList queueEntryList = _priorityLists[index]; + _log.debug("Queue (" + _queue + ":" + _queue.getName() + ")[" + index + "] usage:" + queueEntryList.memoryUsed() + + "/" + queueEntryList.getMemoryUsageMaximum() + + "/" + queueEntryList.dataSize()); + } + } + } + + public QueueEntry next(QueueEntry node) + { + QueueEntryImpl nodeImpl = (QueueEntryImpl) node; + QueueEntry next = nodeImpl.getNext(); + + if (next == null) + { + QueueEntryList nodeEntryList = nodeImpl.getQueueEntryList(); + int index; + for (index = _priorityLists.length - 1; _priorityLists[index] != nodeEntryList; index--) + { + ; + } + + while (next == null && index != 0) + { + index--; + next = ((QueueEntryImpl) _priorityLists[index].getHead()).getNext(); + } + + } + return next; + } + + private final class PriorityQueueEntryListIterator implements QueueEntryIterator + { + private final QueueEntryIterator[] _iterators = new QueueEntryIterator[_priorityLists.length]; + private QueueEntry _lastNode; + + PriorityQueueEntryListIterator() + { + for (int i = 0; i < _priorityLists.length; i++) + { + _iterators[i] = _priorityLists[i].iterator(); + } + _lastNode = _iterators[_iterators.length - 1].getNode(); + } + + public boolean atTail() + { + for (int i = 0; i < _iterators.length; i++) + { + if (!_iterators[i].atTail()) + { + return false; + } + } + return true; + } + + public QueueEntry getNode() + { + return _lastNode; + } + + public boolean advance() + { + for (int i = _iterators.length - 1; i >= 0; i--) + { + if (_iterators[i].advance()) + { + _lastNode = _iterators[i].getNode(); + return true; + } + } + return false; + } + } + + public QueueEntryIterator iterator() + { + return new PriorityQueueEntryListIterator(); + } + + public QueueEntry getHead() + { + return _priorityLists[_priorities - 1].getHead(); + } + + static class Factory implements QueueEntryListFactory + { + private final int _priorities; + + Factory(int priorities) + { + _priorities = priorities; + } + + public QueueEntryList createQueueEntryList(AMQQueue queue) + { + return new PriorityQueueEntryList(queue, _priorities); + } + } + + @Override + public boolean isFlowed() + { + boolean flowed = false; + boolean full = true; + + if (_log.isTraceEnabled()) + { + showUsage("isFlowed"); + } + + for (QueueEntryList queueEntryList : _priorityLists) + { + //full = full && queueEntryList.getMemoryUsageMaximum() == queueEntryList.memoryUsed(); + full = full && queueEntryList.getMemoryUsageMaximum() <= queueEntryList.dataSize(); + flowed = flowed || (queueEntryList.isFlowed()); + } + return flowed && full; + } + + @Override + public int size() + { + int size = 0; + for (QueueEntryList queueEntryList : _priorityLists) + { + size += queueEntryList.size(); + } + + return size; + } + + @Override + public long dataSize() + { + int dataSize = 0; + for (QueueEntryList queueEntryList : _priorityLists) + { + dataSize += queueEntryList.dataSize(); + } + + return dataSize; + } + + @Override + public long memoryUsed() + { + int memoryUsed = 0; + for (QueueEntryList queueEntryList : _priorityLists) + { + memoryUsed += queueEntryList.memoryUsed(); + } + + return memoryUsed; + } + + @Override + public void setMemoryUsageMaximum(long maximumMemoryUsage) + { + _memoryUsageMaximum = maximumMemoryUsage; + + if (maximumMemoryUsage >= 0) + { + _disabled = false; + } + + long share = maximumMemoryUsage / _priorities; + + //Apply a share of the maximum To each prioirty quue + for (QueueEntryList queueEntryList : _priorityLists) + { + queueEntryList.setMemoryUsageMaximum(share); + } + + if (maximumMemoryUsage < 0) + { + if (_log.isInfoEnabled()) + { + _log.info("Disabling Flow to Disk for queue:" + _queue.getName()); + } + _disabled = true; + return; + } + + //ensure we use the full allocation of memory + long remainder = maximumMemoryUsage - (share * _priorities); + if (remainder > 0) + { + _priorityLists[_priorities - 1].setMemoryUsageMaximum(share + remainder); + } + } + + @Override + public long getMemoryUsageMaximum() + { + return _memoryUsageMaximum; + } + + @Override + public void setMemoryUsageMinimum(long minimumMemoryUsage) + { + _memoryUsageMinimum = minimumMemoryUsage; + + //Apply a share of the minimum To each prioirty quue + for (QueueEntryList queueEntryList : _priorityLists) + { + queueEntryList.setMemoryUsageMaximum(minimumMemoryUsage / _priorities); + } + } + + @Override + public long getMemoryUsageMinimum() + { + return _memoryUsageMinimum; + } + + @Override + public void stop() + { + super.stop(); + for (QueueEntryList queueEntryList : _priorityLists) + { + queueEntryList.stop(); + } + } +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PriorityQueueList.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PriorityQueueList.java deleted file mode 100644 index 7be2827e0f..0000000000 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/PriorityQueueList.java +++ /dev/null @@ -1,160 +0,0 @@ -/* -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*/ -package org.apache.qpid.server.queue; - -import org.apache.qpid.framing.CommonContentHeaderProperties; -import org.apache.qpid.AMQException; - -public class PriorityQueueList implements QueueEntryList -{ - private final AMQQueue _queue; - private final QueueEntryList[] _priorityLists; - private final int _priorities; - private final int _priorityOffset; - - public PriorityQueueList(AMQQueue queue, int priorities) - { - _queue = queue; - _priorityLists = new QueueEntryList[priorities]; - _priorities = priorities; - _priorityOffset = 5-((priorities + 1)/2); - for(int i = 0; i < priorities; i++) - { - _priorityLists[i] = new SimpleQueueEntryList(queue); - } - } - - public int getPriorities() - { - return _priorities; - } - - public AMQQueue getQueue() - { - return _queue; - } - - public QueueEntry add(AMQMessage message) - { - int index = ((CommonContentHeaderProperties)((message.getContentHeaderBody().properties))).getPriority() - _priorityOffset; - if(index >= _priorities) - { - index = _priorities-1; - } - else if(index < 0) - { - index = 0; - } - return _priorityLists[index].add(message); - } - - public QueueEntry next(QueueEntry node) - { - QueueEntryImpl nodeImpl = (QueueEntryImpl)node; - QueueEntry next = nodeImpl.getNext(); - - if(next == null) - { - QueueEntryList nodeEntryList = nodeImpl.getQueueEntryList(); - int index; - for(index = _priorityLists.length-1; _priorityLists[index] != nodeEntryList; index--); - - while(next == null && index != 0) - { - index--; - next = ((QueueEntryImpl)_priorityLists[index].getHead()).getNext(); - } - - } - return next; - } - - private final class PriorityQueueEntryListIterator implements QueueEntryIterator - { - private final QueueEntryIterator[] _iterators = new QueueEntryIterator[ _priorityLists.length ]; - private QueueEntry _lastNode; - - PriorityQueueEntryListIterator() - { - for(int i = 0; i < _priorityLists.length; i++) - { - _iterators[i] = _priorityLists[i].iterator(); - } - _lastNode = _iterators[_iterators.length - 1].getNode(); - } - - - public boolean atTail() - { - for(int i = 0; i < _iterators.length; i++) - { - if(!_iterators[i].atTail()) - { - return false; - } - } - return true; - } - - public QueueEntry getNode() - { - return _lastNode; - } - - public boolean advance() - { - for(int i = _iterators.length-1; i >= 0; i--) - { - if(_iterators[i].advance()) - { - _lastNode = _iterators[i].getNode(); - return true; - } - } - return false; - } - } - - public QueueEntryIterator iterator() - { - return new PriorityQueueEntryListIterator(); - } - - public QueueEntry getHead() - { - return _priorityLists[_priorities-1].getHead(); - } - - static class Factory implements QueueEntryListFactory - { - private final int _priorities; - - Factory(int priorities) - { - _priorities = priorities; - } - - public QueueEntryList createQueueEntryList(AMQQueue queue) - { - return new PriorityQueueList(queue, _priorities); - } - } -} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueBackingStore.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueBackingStore.java new file mode 100644 index 0000000000..5efb95d0c0 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueBackingStore.java @@ -0,0 +1,36 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.queue; + +import org.apache.qpid.server.configuration.VirtualHostConfiguration; +import org.apache.qpid.server.virtualhost.VirtualHost; +import org.apache.commons.configuration.ConfigurationException; + +public interface QueueBackingStore +{ + AMQMessage load(Long messageId); + + void unload(AMQMessage message) throws UnableToFlowMessageException; + + void delete(Long messageId); + + void close(); +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueBackingStoreFactory.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueBackingStoreFactory.java new file mode 100644 index 0000000000..3dd23a2f40 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueBackingStoreFactory.java @@ -0,0 +1,32 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.queue; + +import org.apache.qpid.server.virtualhost.VirtualHost; +import org.apache.qpid.server.configuration.VirtualHostConfiguration; +import org.apache.commons.configuration.ConfigurationException; + +public interface QueueBackingStoreFactory +{ + void configure(VirtualHost virtualHost, VirtualHostConfiguration config) throws ConfigurationException; + + public QueueBackingStore createBacking(AMQQueue queue); +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntry.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntry.java index 09600b9d28..7fc5df4e9e 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntry.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntry.java @@ -119,18 +119,48 @@ public interface QueueEntry extends Comparable<QueueEntry>, Filterable<AMQExcept final static EntryState EXPIRED_STATE = new ExpiredState(); final static EntryState NON_SUBSCRIPTION_ACQUIRED_STATE = new NonSubscriptionAcquiredState(); + /** Flag to indicate that this message requires 'immediate' delivery. */ + + final static byte IMMEDIATE = 0x01; + + /** + * Flag to indicate whether this message has been delivered to a consumer. Used in implementing return functionality + * for messages published with the 'immediate' flag. + */ + + final static byte DELIVERED_TO_CONSUMER = 0x02; + + AMQQueue getQueue(); AMQMessage getMessage(); + Long getMessageId(); + long getSize(); + /** + * Called selectors to determin if the message has already been sent + * + * @return _deliveredToConsumer + */ boolean getDeliveredToConsumer(); + /** + * Checks to see if the message has expired. If it has the message is dequeued. + * + * @return true if the message has expire + * + * @throws org.apache.qpid.AMQException + */ boolean expired() throws AMQException; + public void setExpiration(final long expiration); + boolean isAcquired(); + boolean isAvailable(); + boolean acquire(); boolean acquire(Subscription sub); @@ -143,10 +173,22 @@ public interface QueueEntry extends Comparable<QueueEntry>, Filterable<AMQExcept void setDeliveredToSubscription(); + /** + * Called when this message is delivered to a consumer. (used to implement the 'immediate' flag functionality). + * And for selector efficiency. + */ + public void setDeliveredToConsumer(); + void release(); String debugIdentity(); + /** + * Called to enforce the 'immediate' flag. + * + * @returns true if the message is marked for immediate delivery but has not been marked as delivered + * to a consumer + */ boolean immediateAndNotDelivered(); void setRedelivered(boolean b); @@ -180,4 +222,11 @@ public interface QueueEntry extends Comparable<QueueEntry>, Filterable<AMQExcept void addStateChangeListener(StateChangeListener listener); boolean removeStateChangeListener(StateChangeListener listener); -} + + void unload(); + + void load(); + + boolean isFlowed(); + +}
\ No newline at end of file diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntryImpl.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntryImpl.java index 911ed8321b..fceaf75a9e 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntryImpl.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntryImpl.java @@ -20,25 +20,23 @@ */ package org.apache.qpid.server.queue; +import org.apache.log4j.Logger; import org.apache.qpid.AMQException; import org.apache.qpid.framing.ContentHeaderBody; import org.apache.qpid.server.store.StoreContext; import org.apache.qpid.server.subscription.Subscription; -import org.apache.log4j.Logger; -import java.util.Set; import java.util.HashSet; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; - +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; public class QueueEntryImpl implements QueueEntry { - /** - * Used for debugging purposes. - */ + /** Used for debugging purposes. */ private static final Logger _log = Logger.getLogger(QueueEntryImpl.class); private final SimpleQueueEntryList _queueEntryList; @@ -52,44 +50,50 @@ public class QueueEntryImpl implements QueueEntry private volatile EntryState _state = AVAILABLE_STATE; private static final - AtomicReferenceFieldUpdater<QueueEntryImpl, EntryState> + AtomicReferenceFieldUpdater<QueueEntryImpl, EntryState> _stateUpdater = - AtomicReferenceFieldUpdater.newUpdater - (QueueEntryImpl.class, EntryState.class, "_state"); - + AtomicReferenceFieldUpdater.newUpdater + (QueueEntryImpl.class, EntryState.class, "_state"); private volatile Set<StateChangeListener> _stateChangeListeners; private static final - AtomicReferenceFieldUpdater<QueueEntryImpl, Set> - _listenersUpdater = - AtomicReferenceFieldUpdater.newUpdater - (QueueEntryImpl.class, Set.class, "_stateChangeListeners"); - + AtomicReferenceFieldUpdater<QueueEntryImpl, Set> + _listenersUpdater = + AtomicReferenceFieldUpdater.newUpdater + (QueueEntryImpl.class, Set.class, "_stateChangeListeners"); private static final - AtomicLongFieldUpdater<QueueEntryImpl> + AtomicLongFieldUpdater<QueueEntryImpl> _entryIdUpdater = - AtomicLongFieldUpdater.newUpdater - (QueueEntryImpl.class, "_entryId"); - + AtomicLongFieldUpdater.newUpdater + (QueueEntryImpl.class, "_entryId"); private volatile long _entryId; volatile QueueEntryImpl _next; + private long _messageSize; + private QueueBackingStore _backingStore; + private AtomicBoolean _flowed; + private Long _messageId; + + private byte _flags = 0; + + private long _expiration; + + private static final byte IMMEDIATE_AND_DELIVERED = (byte) (IMMEDIATE | DELIVERED_TO_CONSUMER); + private boolean _persistent; QueueEntryImpl(SimpleQueueEntryList queueEntryList) { - this(queueEntryList,null,Long.MIN_VALUE); + this(queueEntryList, null, Long.MIN_VALUE); _state = DELETED_STATE; } - public QueueEntryImpl(SimpleQueueEntryList queueEntryList, AMQMessage message, final long entryId) { - _queueEntryList = queueEntryList; - _message = message; + this(queueEntryList, message); _entryIdUpdater.set(this, entryId); } @@ -98,6 +102,20 @@ public class QueueEntryImpl implements QueueEntry { _queueEntryList = queueEntryList; _message = message; + if (message != null) + { + _messageId = message.getMessageId(); + _messageSize = message.getSize(); + + if (message.isImmediate()) + { + _flags |= IMMEDIATE; + } + _expiration = message.getExpiration(); + _persistent = message.isPersistent(); + } + _backingStore = queueEntryList.getBackingStore(); + _flowed = new AtomicBoolean(false); } protected void setEntryId(long entryId) @@ -117,22 +135,54 @@ public class QueueEntryImpl implements QueueEntry public AMQMessage getMessage() { + if (_message == null) + { + return _backingStore.load(_messageId); + } return _message; } + public Long getMessageId() + { + return _messageId; + } + public long getSize() { - return getMessage().getSize(); + return _messageSize; } public boolean getDeliveredToConsumer() { - return getMessage().getDeliveredToConsumer(); + return (_flags & DELIVERED_TO_CONSUMER) != 0; + } + + public void setDeliveredToConsumer() + { + _flags |= DELIVERED_TO_CONSUMER; + + // We have delivered this message so we can unload it if we are flowed. + if (_queueEntryList.isFlowed()) + { + unload(); + } } public boolean expired() throws AMQException { - return getMessage().expired(); + if (_expiration != 0L) + { + long now = System.currentTimeMillis(); + + return (now > _expiration); + } + + return false; + } + + public void setExpiration(final long expiration) + { + _expiration = expiration; } public boolean isAcquired() @@ -140,6 +190,11 @@ public class QueueEntryImpl implements QueueEntry return _state.getState() == State.ACQUIRED; } + public boolean isAvailable() + { + return _state.getState() == State.AVAILABLE; + } + public boolean acquire() { return acquire(NON_SUBSCRIPTION_ACQUIRED_STATE); @@ -147,8 +202,8 @@ public class QueueEntryImpl implements QueueEntry private boolean acquire(final EntryState state) { - boolean acquired = _stateUpdater.compareAndSet(this,AVAILABLE_STATE, state); - if(acquired && _stateChangeListeners != null) + boolean acquired = _stateUpdater.compareAndSet(this, AVAILABLE_STATE, state); + if (acquired && _stateChangeListeners != null) { notifyStateChange(State.AVAILABLE, State.ACQUIRED); } @@ -169,33 +224,40 @@ public class QueueEntryImpl implements QueueEntry public void setDeliveredToSubscription() { - getMessage().setDeliveredToConsumer(); + _flags |= DELIVERED_TO_CONSUMER; } public void release() { - _stateUpdater.set(this,AVAILABLE_STATE); + _stateUpdater.set(this, AVAILABLE_STATE); } public String debugIdentity() { - return getMessage().debugIdentity(); + String entry = "[State:" + _state.getState().name() + "]"; + if (_message == null) + { + return entry + "(Message Unloaded ID:" + _messageId + ")"; + } + else + { + return entry + _message.debugIdentity(); + } } - - public boolean immediateAndNotDelivered() + public boolean immediateAndNotDelivered() { - return _message.immediateAndNotDelivered(); + return (_flags & IMMEDIATE_AND_DELIVERED) == IMMEDIATE; } public ContentHeaderBody getContentHeaderBody() throws AMQException { - return _message.getContentHeaderBody(); + return getMessage().getContentHeaderBody(); } public boolean isPersistent() throws AMQException { - return _message.isPersistent(); + return _persistent; } public boolean isRedelivered() @@ -206,21 +268,21 @@ public class QueueEntryImpl implements QueueEntry public void setRedelivered(boolean redelivered) { _redelivered = redelivered; - // todo - here we could mark this message as redelivered so we don't have to mark - // all messages on recover as redelivered. + // todo - here we could record this message as redelivered on this queue in the transactionLog + // so we don't have to mark all messages on recover as redelivered. } public Subscription getDeliveredSubscription() { - EntryState state = _state; - if (state instanceof SubscriptionAcquiredState) - { - return ((SubscriptionAcquiredState) state).getSubscription(); - } - else - { - return null; - } + EntryState state = _state; + if (state instanceof SubscriptionAcquiredState) + { + return ((SubscriptionAcquiredState) state).getSubscription(); + } + else + { + return null; + } } @@ -247,7 +309,7 @@ public class QueueEntryImpl implements QueueEntry } public boolean isRejectedBy(Subscription subscription) - { + { if (_rejectedBy != null) // We have subscriptions that rejected this message { @@ -259,11 +321,10 @@ public class QueueEntryImpl implements QueueEntry } } - public void requeue(final StoreContext storeContext) throws AMQException { getQueue().requeue(storeContext, this); - if(_stateChangeListeners != null) + if (_stateChangeListeners != null) { notifyStateChange(QueueEntry.State.ACQUIRED, QueueEntry.State.AVAILABLE); } @@ -273,7 +334,7 @@ public class QueueEntryImpl implements QueueEntry { EntryState state = _state; - if((state.getState() == State.ACQUIRED) &&_stateUpdater.compareAndSet(this, state, DEQUEUED_STATE)) + if ((state.getState() == State.ACQUIRED) && _stateUpdater.compareAndSet(this, state, DEQUEUED_STATE)) { if (state instanceof SubscriptionAcquiredState) { @@ -281,6 +342,8 @@ public class QueueEntryImpl implements QueueEntry s.restoreCredit(this); } + _queueEntryList.dequeued(this); + getQueue().dequeue(storeContext, this); if (_stateChangeListeners != null) @@ -292,7 +355,7 @@ public class QueueEntryImpl implements QueueEntry private void notifyStateChange(final State oldState, final State newState) { - for(StateChangeListener l : _stateChangeListeners) + for (StateChangeListener l : _stateChangeListeners) { l.stateChanged(this, oldState, newState); } @@ -317,7 +380,7 @@ public class QueueEntryImpl implements QueueEntry public void addStateChangeListener(StateChangeListener listener) { Set<StateChangeListener> listeners = _stateChangeListeners; - if(listeners == null) + if (listeners == null) { _listenersUpdater.compareAndSet(this, null, new CopyOnWriteArraySet<StateChangeListener>()); listeners = _stateChangeListeners; @@ -329,7 +392,7 @@ public class QueueEntryImpl implements QueueEntry public boolean removeStateChangeListener(StateChangeListener listener) { Set<StateChangeListener> listeners = _stateChangeListeners; - if(listeners != null) + if (listeners != null) { return listeners.remove(listener); } @@ -337,10 +400,66 @@ public class QueueEntryImpl implements QueueEntry return false; } + public void unload() + { + if (_message != null && _backingStore != null) + { + + try + { + _backingStore.unload(_message); + + if (_log.isDebugEnabled()) + { + _log.debug("Unloaded:" + debugIdentity()); + } + _message = null; + + //Update the memoryState if this load call resulted in the message being purged from memory + if (!_flowed.getAndSet(true)) + { + _queueEntryList.entryUnloadedUpdateMemory(this); + } + + } + catch (UnableToFlowMessageException utfme) + { + // There is no recovery needed as the memory states remain unchanged. + if (_log.isDebugEnabled()) + { + _log.debug("Unable to Flow message:" + debugIdentity() + ", due to:" + utfme.getMessage()); + } + } + } + } + + public void load() + { + if (_messageId != null && _backingStore != null) + { + _message = _backingStore.load(_messageId); + + if (_log.isDebugEnabled()) + { + _log.debug("Loaded:" + debugIdentity()); + } + + //Update the memoryState if this load call resulted in the message comming in to memory + if (_flowed.getAndSet(false)) + { + _queueEntryList.entryLoadedUpdateMemory(this); + } + } + } + + public boolean isFlowed() + { + return _flowed.get(); + } public int compareTo(final QueueEntry o) { - QueueEntryImpl other = (QueueEntryImpl)o; + QueueEntryImpl other = (QueueEntryImpl) o; return getEntryId() > other.getEntryId() ? 1 : getEntryId() < other.getEntryId() ? -1 : 0; } @@ -348,13 +467,13 @@ public class QueueEntryImpl implements QueueEntry { QueueEntryImpl next = nextNode(); - while(next != null && next.isDeleted()) + while (next != null && next.isDeleted()) { final QueueEntryImpl newNext = next.nextNode(); - if(newNext != null) + if (newNext != null) { - SimpleQueueEntryList._nextUpdater.compareAndSet(this,next, newNext); + SimpleQueueEntryList._nextUpdater.compareAndSet(this, next, newNext); next = nextNode(); } else @@ -380,9 +499,13 @@ public class QueueEntryImpl implements QueueEntry { EntryState state = _state; - if(state != DELETED_STATE && _stateUpdater.compareAndSet(this,state,DELETED_STATE)) + if (state != DELETED_STATE && _stateUpdater.compareAndSet(this, state, DELETED_STATE)) { - _queueEntryList.advanceHead(); + _queueEntryList.advanceHead(); + if (_backingStore != null) + { + _backingStore.delete(_messageId); + } return true; } else @@ -395,4 +518,5 @@ public class QueueEntryImpl implements QueueEntry { return _queueEntryList; } + } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntryList.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntryList.java index 313e076f61..a58c6eaf7d 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntryList.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/QueueEntryList.java @@ -1,23 +1,23 @@ /* -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*/ + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ package org.apache.qpid.server.queue; public interface QueueEntryList @@ -31,4 +31,36 @@ public interface QueueEntryList QueueEntryIterator iterator(); QueueEntry getHead(); + + void setFlowed(boolean flowed); + + boolean isFlowed(); + + int size(); + + long dataSize(); + + long memoryUsed(); + + void setMemoryUsageMaximum(long maximumMemoryUsage); + + long getMemoryUsageMaximum(); + + void setMemoryUsageMinimum(long minimumMemoryUsage); + + long getMemoryUsageMinimum(); + + /** + * Immediately update memory usage based on the unload of this queueEntry, potentially start inhaler. + * @param queueEntry the entry that has been unloaded + */ + void entryUnloadedUpdateMemory(QueueEntry queueEntry); + + /** + * Immediately update memory usage based on the load of this queueEntry + * @param queueEntry the entry that has been loaded + */ + void entryLoadedUpdateMemory(QueueEntry queueEntry); + + void stop(); } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/SimpleAMQQueue.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/SimpleAMQQueue.java index 501e90b4d7..63ec56c1af 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/SimpleAMQQueue.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/SimpleAMQQueue.java @@ -19,7 +19,6 @@ import org.apache.qpid.framing.AMQShortString; import org.apache.qpid.framing.FieldTable; import org.apache.qpid.pool.ReadWriteRunnable; import org.apache.qpid.pool.ReferenceCountingExecutorService; -import org.apache.qpid.server.configuration.QueueConfiguration; import org.apache.qpid.server.exchange.Exchange; import org.apache.qpid.server.management.ManagedObject; import org.apache.qpid.server.output.ProtocolOutputConverter; @@ -73,10 +72,6 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener private final List<Task> _deleteTaskList = new CopyOnWriteArrayList<Task>(); - private final AtomicInteger _atomicQueueCount = new AtomicInteger(0); - - private final AtomicLong _atomicQueueSize = new AtomicLong(0L); - private final AtomicInteger _activeSubscriberCount = new AtomicInteger(); protected final SubscriptionList _subscriptionList = new SubscriptionList(this); @@ -105,6 +100,7 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener /** the minimum interval between sending out consecutive alerts of the same type */ public long _minimumAlertRepeatGap = ApplicationRegistry.getInstance().getConfiguration().getMinimumAlertRepeatGap(); + private static final int MAX_ASYNC_DELIVERIES = 10; private final Set<NotificationCheck> _notificationChecks = EnumSet.noneOf(NotificationCheck.class); @@ -159,7 +155,6 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener } resetNotifications(); - } public void resetNotifications() @@ -188,6 +183,11 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener return _autoDelete; } + public boolean isFlowed() + { + return _entries.isFlowed(); + } + public AMQShortString getOwner() { return _owner; @@ -321,10 +321,6 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener public QueueEntry enqueue(StoreContext storeContext, AMQMessage message) throws AMQException { - - incrementQueueCount(); - incrementQueueSize(message); - _totalMessagesReceived.incrementAndGet(); QueueEntry entry; @@ -421,7 +417,7 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener deliverAsync(); } - _managedObject.checkForNotification(entry.getMessage()); + _managedObject.checkForNotification(entry); return entry; } @@ -465,20 +461,11 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener // Simple Queues don't :-) } - private void incrementQueueSize(final AMQMessage message) - { - getAtomicQueueSize().addAndGet(message.getSize()); - } - - private void incrementQueueCount() - { - getAtomicQueueCount().incrementAndGet(); - } - private void deliverMessage(final Subscription sub, final QueueEntry entry) throws AMQException { _deliveredMessages.incrementAndGet(); + sub.send(entry); } @@ -573,8 +560,6 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener */ public void dequeue(StoreContext storeContext, QueueEntry entry) throws FailedDequeueException { - decrementQueueCount(); - decrementQueueSize(entry); if (entry.acquiredBySubscription()) { _deliveredMessages.decrementAndGet(); @@ -582,10 +567,9 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener try { - AMQMessage msg = entry.getMessage(); - if (msg.isPersistent()) + if (entry.isPersistent()) { - _virtualHost.getTransactionLog().dequeueMessage(storeContext, this, msg.getMessageId()); + _virtualHost.getTransactionLog().dequeueMessage(storeContext, this, entry.getMessageId()); } } @@ -604,15 +588,6 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener } - private void decrementQueueSize(final QueueEntry entry) - { - getAtomicQueueSize().addAndGet(-entry.getMessage().getSize()); - } - - void decrementQueueCount() - { - getAtomicQueueCount().decrementAndGet(); - } public boolean resend(final QueueEntry entry, final Subscription subscription) throws AMQException { @@ -658,14 +633,19 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener return getMessageCount() == 0; } + public long getMemoryUsageCurrent() + { + return getQueueInMemory(); + } + public int getMessageCount() { - return getAtomicQueueCount().get(); + return getQueueCount(); } public long getQueueDepth() { - return getAtomicQueueSize().get(); + return getQueueSize(); } public int getUndeliveredMessageCount() @@ -741,14 +721,19 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener return _name.compareTo(o.getName()); } - public AtomicInteger getAtomicQueueCount() + public int getQueueCount() + { + return _entries.size(); + } + + public long getQueueSize() { - return _atomicQueueCount; + return _entries.dataSize(); } - public AtomicLong getAtomicQueueSize() + public long getQueueInMemory() { - return _atomicQueueSize; + return _entries.memoryUsed(); } private boolean isExclusiveSubscriber() @@ -775,7 +760,7 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener public boolean accept(QueueEntry entry) { - final long messageId = entry.getMessage().getMessageId(); + final long messageId = entry.getMessageId(); return messageId >= fromMessageId && messageId <= toMessageId; } @@ -794,7 +779,7 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener public boolean accept(QueueEntry entry) { - _complete = entry.getMessage().getMessageId() == messageId; + _complete = entry.getMessageId() == messageId; return _complete; } @@ -843,7 +828,7 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener public boolean accept(QueueEntry entry) { - final long messageId = entry.getMessage().getMessageId(); + final long messageId = entry.getMessageId(); return (messageId >= fromMessageId) && (messageId <= toMessageId) && entry.acquire(); @@ -862,11 +847,9 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener // Move the messages in the transaction log. for (QueueEntry entry : entries) { - AMQMessage message = entry.getMessage(); - - if (message.isPersistent() && toQueue.isDurable()) + if (entry.isPersistent() && toQueue.isDurable()) { - transactionLog.enqueueMessage(storeContext, toQueue, message.getMessageId()); + transactionLog.enqueueMessage(storeContext, toQueue, entry.getMessageId()); } // dequeue will remove the messages from the queue entry.dequeue(storeContext); @@ -901,6 +884,9 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener for (QueueEntry entry : entries) { toQueue.enqueue(storeContext, entry.getMessage()); + // As we only did a dequeue above now that we have moved the message we should perform a delete. + // We cannot do this earlier as the message will be lost if flowed. + //entry.delete(); } } catch (MessageCleanupException e) @@ -927,7 +913,7 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener public boolean accept(QueueEntry entry) { - final long messageId = entry.getMessage().getMessageId(); + final long messageId = entry.getMessageId(); if ((messageId >= fromMessageId) && (messageId <= toMessageId)) { @@ -953,11 +939,9 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener // Move the messages in on the transaction log. for (QueueEntry entry : entries) { - AMQMessage message = entry.getMessage(); - - if (!entry.isDeleted() && message.isPersistent() && toQueue.isDurable()) + if (!entry.isDeleted() && entry.isPersistent() && toQueue.isDurable()) { - transactionLog.enqueueMessage(storeContext, toQueue, message.getMessageId()); + transactionLog.enqueueMessage(storeContext, toQueue, entry.getMessageId()); } } @@ -1016,7 +1000,7 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener { QueueEntry node = queueListIterator.getNode(); - final long messageId = node.getMessage().getMessageId(); + final long messageId = node.getMessageId(); if ((messageId >= fromMessageId) && (messageId <= toMessageId) @@ -1116,6 +1100,7 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener if (!_stopped.getAndSet(true)) { ReferenceCountingExecutorService.getInstance().releaseExecutorService(); + _entries.stop(); } } @@ -1451,12 +1436,33 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener } else { - _managedObject.checkForNotification(node.getMessage()); + _managedObject.checkForNotification(node); } } } + + public long getMemoryUsageMaximum() + { + return _entries.getMemoryUsageMaximum(); + } + + public void setMemoryUsageMaximum(long maximumMemoryUsage) + { + _entries.setMemoryUsageMaximum(maximumMemoryUsage); + } + + public long getMemoryUsageMinimum() + { + return _entries.getMemoryUsageMinimum(); + } + + public void setMemoryUsageMinimum(long minimumMemoryUsage) + { + _entries.setMemoryUsageMinimum(minimumMemoryUsage); + } + public long getMinimumAlertRepeatGap() { return _minimumAlertRepeatGap; @@ -1597,7 +1603,7 @@ public class SimpleAMQQueue implements AMQQueue, Subscription.StateListener for (int i = 0; i < num && !it.atTail(); i++) { it.advance(); - ids.add(it.getNode().getMessage().getMessageId()); + ids.add(it.getNode().getMessageId()); } return ids; } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/SimpleQueueEntryList.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/SimpleQueueEntryList.java index a46c5ae2e8..a10e332ef5 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/SimpleQueueEntryList.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/SimpleQueueEntryList.java @@ -22,8 +22,9 @@ import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; * under the License. * */ -public class SimpleQueueEntryList implements QueueEntryList +public class SimpleQueueEntryList extends FlowableBaseQueueEntryList { + private final QueueEntryImpl _head; private volatile QueueEntryImpl _tail; @@ -41,12 +42,9 @@ public class SimpleQueueEntryList implements QueueEntryList AtomicReferenceFieldUpdater.newUpdater (QueueEntryImpl.class, QueueEntryImpl.class, "_next"); - - - - public SimpleQueueEntryList(AMQQueue queue) { + super(queue); _queue = queue; _head = new QueueEntryImpl(this); _tail = _head; @@ -77,6 +75,9 @@ public class SimpleQueueEntryList implements QueueEntryList public QueueEntry add(AMQMessage message) { QueueEntryImpl node = new QueueEntryImpl(this, message); + + incrementCounters(node); + for (;;) { QueueEntryImpl tail = _tail; @@ -101,12 +102,12 @@ public class SimpleQueueEntryList implements QueueEntryList } } + public QueueEntry next(QueueEntry node) { return ((QueueEntryImpl)node).getNext(); } - public class QueueEntryIteratorImpl implements QueueEntryIterator { @@ -172,7 +173,9 @@ public class SimpleQueueEntryList implements QueueEntryList { return new SimpleQueueEntryList(queue); } + } - + + } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/TransientAMQMessage.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/TransientAMQMessage.java index fa4e85a043..4c9fe81439 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/TransientAMQMessage.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/TransientAMQMessage.java @@ -44,8 +44,6 @@ public class TransientAMQMessage implements AMQMessage /** Used for debugging purposes. */ protected static final Logger _log = Logger.getLogger(AMQMessage.class); - private final AtomicInteger _referenceCount = new AtomicInteger(1); - protected ContentHeaderBody _contentHeaderBody; protected MessagePublishInfo _messagePublishInfo; @@ -56,23 +54,11 @@ public class TransientAMQMessage implements AMQMessage protected final Long _messageId; - /** Flag to indicate that this message requires 'immediate' delivery. */ - - private static final byte IMMEDIATE = 0x01; - - /** - * Flag to indicate whether this message has been delivered to a consumer. Used in implementing return functionality - * for messages published with the 'immediate' flag. - */ - - private static final byte DELIVERED_TO_CONSUMER = 0x02; private byte _flags = 0; - private long _expiration; - private AMQProtocolSession.ProtocolSessionIdentifier _sessionIdentifier; - private static final byte IMMEDIATE_AND_DELIVERED = (byte) (IMMEDIATE | DELIVERED_TO_CONSUMER); + private long _expiration; /** * Used to iterate through all the body frames associated with this message. Will not keep all the data in memory @@ -164,14 +150,19 @@ public class TransientAMQMessage implements AMQMessage public String debugIdentity() { - return "(HC:" + System.identityHashCode(this) + " ID:" + getMessageId() + " Ref:" + _referenceCount.get() + ")"; + return "(HC:" + System.identityHashCode(this) + " ID:" + getMessageId() +")"; } - public void setExpiration(final long expiration) + public void setExpiration(long expiration) { _expiration = expiration; } + public long getExpiration() + { + return _expiration; + } + public Iterator<AMQDataBlock> getBodyFrameIterator(AMQProtocolSession protocolSession, int channel) { return new BodyFrameIterator(protocolSession, channel); @@ -192,57 +183,6 @@ public class TransientAMQMessage implements AMQMessage return _messageId; } - /** - * Called selectors to determin if the message has already been sent - * - * @return _deliveredToConsumer - */ - public boolean getDeliveredToConsumer() - { - return (_flags & DELIVERED_TO_CONSUMER) != 0; - } - - /** - * Called to enforce the 'immediate' flag. - * - * @returns true if the message is marked for immediate delivery but has not been marked as delivered - * to a consumer - */ - public boolean immediateAndNotDelivered() - { - - return (_flags & IMMEDIATE_AND_DELIVERED) == IMMEDIATE; - - } - - /** - * Checks to see if the message has expired. If it has the message is dequeued. - * - * @return true if the message has expire - * - * @throws AMQException - */ - public boolean expired() throws AMQException - { - - if (_expiration != 0L) - { - long now = System.currentTimeMillis(); - - return (now > _expiration); - } - - return false; - } - - /** - * Called when this message is delivered to a consumer. (used to implement the 'immediate' flag functionality). - * And for selector efficiency. - */ - public void setDeliveredToConsumer() - { - _flags |= DELIVERED_TO_CONSUMER; - } public long getSize() { @@ -317,6 +257,11 @@ public class TransientAMQMessage implements AMQMessage return false; } + public boolean isImmediate() + { + return _messagePublishInfo.isImmediate(); + } + /** * This is called when all the content has been received. * @@ -339,33 +284,49 @@ public class TransientAMQMessage implements AMQMessage throw new NullPointerException("PublishInfo cannot be null"); } - _messagePublishInfo = messagePublishInfo; + _arrivalTime = System.currentTimeMillis(); + + _contentHeaderBody = contentHeaderBody; + _messagePublishInfo = messagePublishInfo; - if (contentHeaderBody.bodySize == 0) - { - _contentBodies = Collections.EMPTY_LIST; - } + updateHeaderAndFlags(); + } - _arrivalTime = System.currentTimeMillis(); + public long getArrivalTime() + { + return _arrivalTime; + } - if (messagePublishInfo.isImmediate()) + public void recoverFromMessageMetaData(MessageMetaData mmd) + { + _arrivalTime = mmd.getArrivalTime(); + _contentHeaderBody = mmd.getContentHeaderBody(); + _messagePublishInfo = mmd.getMessagePublishInfo(); + + updateHeaderAndFlags(); + } + + private void updateHeaderAndFlags() + { + if (_contentHeaderBody.bodySize == 0) { - _flags |= IMMEDIATE; + _contentBodies = Collections.EMPTY_LIST; } } - public long getArrivalTime() + public void recoverContentBodyFrame(ContentChunk contentChunk, boolean isLastContentBody) throws AMQException { - return _arrivalTime; + addContentBodyFrame(null, contentChunk, isLastContentBody); } + public String toString() { // return "Message[" + debugIdentity() + "]: " + _messageId + "; ref count: " + _referenceCount + "; taken : " + // _taken + " by :" + _takenBySubcription; - return "Message[" + debugIdentity() + "]: " + getMessageId() + "; ref count: " + _referenceCount; + return "Message[" + debugIdentity() + "]: " + getMessageId() ; } } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/UnableToFlowMessageException.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/UnableToFlowMessageException.java new file mode 100644 index 0000000000..03cfed8533 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/UnableToFlowMessageException.java @@ -0,0 +1,29 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.queue; + +public class UnableToFlowMessageException extends Exception +{ + public UnableToFlowMessageException(long messageId, Exception error) + { + super("Unable to Flow Message:"+messageId, error); + } +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/UnableToRecoverMessageException.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/UnableToRecoverMessageException.java new file mode 100644 index 0000000000..cae5bc6327 --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/queue/UnableToRecoverMessageException.java @@ -0,0 +1,29 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.queue; + +public class UnableToRecoverMessageException extends RuntimeException +{ + public UnableToRecoverMessageException(Exception error) + { + super(error); + } +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/registry/ApplicationRegistry.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/registry/ApplicationRegistry.java index 477beeadcb..22b4623ae1 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/registry/ApplicationRegistry.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/registry/ApplicationRegistry.java @@ -24,6 +24,7 @@ import java.net.InetSocketAddress; import java.util.HashMap; import java.util.Map; +import org.apache.commons.configuration.ConfigurationException; import org.apache.log4j.Logger; import org.apache.mina.common.IoAcceptor; import org.apache.qpid.server.configuration.ServerConfiguration; @@ -261,7 +262,7 @@ public abstract class ApplicationRegistry implements IApplicationRegistry return _virtualHostRegistry; } - public ACLManager getAccessManager() + public ACLManager getAccessManager() throws ConfigurationException { return new ACLManager(_configuration.getSecurityConfiguration(), _pluginManager); } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/registry/IApplicationRegistry.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/registry/IApplicationRegistry.java index a1f30c6eed..bbfda3addc 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/registry/IApplicationRegistry.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/registry/IApplicationRegistry.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.net.InetSocketAddress; import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; import org.apache.qpid.server.configuration.ServerConfiguration; import org.apache.qpid.server.management.ManagedObjectRegistry; import org.apache.qpid.server.plugins.PluginManager; @@ -64,7 +65,7 @@ public interface IApplicationRegistry VirtualHostRegistry getVirtualHostRegistry(); - ACLManager getAccessManager(); + ACLManager getAccessManager() throws ConfigurationException; PluginManager getPluginManager(); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLManager.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLManager.java index 57c6098874..6f7f66fad2 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLManager.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLManager.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; import org.apache.log4j.Logger; import org.apache.qpid.framing.AMQShortString; import org.apache.qpid.server.configuration.SecurityConfiguration; @@ -49,12 +50,12 @@ public class ACLManager private Map<String, ACLPlugin> _globalPlugins = new HashMap<String, ACLPlugin>(); private Map<String, ACLPlugin> _hostPlugins = new HashMap<String, ACLPlugin>(); - public ACLManager(SecurityConfiguration configuration, PluginManager manager) + public ACLManager(SecurityConfiguration configuration, PluginManager manager) throws ConfigurationException { this(configuration, manager, null); } - public ACLManager(SecurityConfiguration configuration, PluginManager manager, ACLPluginFactory securityPlugin) + public ACLManager(SecurityConfiguration configuration, PluginManager manager, ACLPluginFactory securityPlugin) throws ConfigurationException { _pluginManager = manager; @@ -73,12 +74,12 @@ public class ACLManager } - public void configureHostPlugins(SecurityConfiguration hostConfig) + public void configureHostPlugins(SecurityConfiguration hostConfig) throws ConfigurationException { _hostPlugins = configurePlugins(hostConfig); } - public Map<String, ACLPlugin> configurePlugins(SecurityConfiguration hostConfig) + public Map<String, ACLPlugin> configurePlugins(SecurityConfiguration hostConfig) throws ConfigurationException { Configuration securityConfig = hostConfig.getConfiguration(); Map<String, ACLPlugin> plugins = new HashMap<String, ACLPlugin>(); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLPlugin.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLPlugin.java index ca760f3360..032184ec39 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLPlugin.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLPlugin.java @@ -21,6 +21,7 @@ package org.apache.qpid.server.security.access; import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; import org.apache.qpid.framing.AMQShortString; import org.apache.qpid.server.exchange.Exchange; import org.apache.qpid.server.protocol.AMQProtocolSession; @@ -36,7 +37,7 @@ public interface ACLPlugin ABSTAIN } - void setConfiguration(Configuration config); + void setConfiguration(Configuration config) throws ConfigurationException; // These return true if the plugin thinks the action should be allowed, and false if not. diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLPluginFactory.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLPluginFactory.java index aee6af93d0..256f093477 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLPluginFactory.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/ACLPluginFactory.java @@ -21,12 +21,13 @@ package org.apache.qpid.server.security.access; import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; public interface ACLPluginFactory { public boolean supportsTag(String name); - public ACLPlugin newInstance(Configuration config); + public ACLPlugin newInstance(Configuration config) throws ConfigurationException; } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/management/AMQUserManagementMBean.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/management/AMQUserManagementMBean.java index f04aecd0a5..121f571abe 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/management/AMQUserManagementMBean.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/management/AMQUserManagementMBean.java @@ -64,9 +64,9 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana private static final Logger _logger = Logger.getLogger(AMQUserManagementMBean.class); private PrincipalDatabase _principalDatabase; - private String _accessFileName; private Properties _accessRights; - // private File _accessFile; + private File _accessFile; + private ReentrantLock _accessRightsUpdate = new ReentrantLock(); // Setup for the TabularType @@ -104,7 +104,7 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana public AMQUserManagementMBean() throws JMException { - super(UserManagement.class, UserManagement.TYPE); + super(UserManagement.class, UserManagement.TYPE, UserManagement.VERSION); } public String getObjectInstanceName() @@ -129,9 +129,10 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana public boolean setRights(String username, boolean read, boolean write, boolean admin) { - if (_accessRights.get(username) == null) + Object oldRights = null; + if ((oldRights =_accessRights.get(username)) == null) { - // If the user doesn't exist in the user rights file check that they at least have an account. + // If the user doesn't exist in the access rights file check that they at least have an account. if (_principalDatabase.getUser(username) == null) { return false; @@ -140,7 +141,6 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana try { - _accessRightsUpdate.lock(); // Update the access rights @@ -166,8 +166,29 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana _accessRights.remove(username); } } + + //save the rights file + try + { + saveAccessFile(); + } + catch (IOException e) + { + _logger.warn("Problem occured saving '" + _accessFile + "', the access right changes will not be preserved: " + e); - saveAccessFile(); + //the rights file was not successfully saved, restore user rights to previous value + _logger.warn("Reverting attempted rights update for user'" + username + "'"); + if (oldRights != null) + { + _accessRights.put(username, oldRights); + } + else + { + _accessRights.remove(username); + } + + return false; + } } finally { @@ -184,9 +205,23 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana { if (_principalDatabase.createPrincipal(new UsernamePrincipal(username), password)) { - _accessRights.put(username, ""); - - return setRights(username, read, write, admin); + if (!setRights(username, read, write, admin)) + { + //unable to set rights for user, remove account + try + { + _principalDatabase.deletePrincipal(new UsernamePrincipal(username)); + } + catch (AccountNotFoundException e) + { + //ignore + } + return false; + } + else + { + return true; + } } return false; @@ -194,7 +229,6 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana public boolean deleteUser(String username) { - try { if (_principalDatabase.deletePrincipal(new UsernamePrincipal(username))) @@ -204,7 +238,16 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana _accessRightsUpdate.lock(); _accessRights.remove(username); - saveAccessFile(); + + try + { + saveAccessFile(); + } + catch (IOException e) + { + _logger.warn("Problem occured saving '" + _accessFile + "', the access right changes will not be preserved: " + e); + return false; + } } finally { @@ -213,15 +256,15 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana _accessRightsUpdate.unlock(); } } - return true; } } catch (AccountNotFoundException e) { _logger.warn("Attempt to delete user (" + username + ") that doesn't exist"); + return false; } - return false; + return true; } public boolean reloadData() @@ -233,12 +276,12 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana } catch (ConfigurationException e) { - _logger.info("Reload failed due to:" + e); + _logger.warn("Reload failed due to:" + e); return false; } catch (IOException e) { - _logger.info("Reload failed due to:" + e); + _logger.warn("Reload failed due to:" + e); return false; } // Reload successful @@ -320,10 +363,24 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana */ public void setAccessFile(String accessFile) throws IOException, ConfigurationException { - _accessFileName = accessFile; - - if (_accessFileName != null) + if (accessFile != null) { + _accessFile = new File(accessFile); + if (!_accessFile.exists()) + { + throw new ConfigurationException("'" + _accessFile + "' does not exist"); + } + + if (!_accessFile.canRead()) + { + throw new ConfigurationException("Cannot read '" + _accessFile + "'."); + } + + if (!_accessFile.canWrite()) + { + _logger.warn("Unable to write to access rights file '" + _accessFile + "', changes will not be preserved."); + } + loadAccessFile(); } else @@ -334,39 +391,34 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana private void loadAccessFile() throws IOException, ConfigurationException { - try + if(_accessFile == null) { - _accessRightsUpdate.lock(); - - Properties accessRights = new Properties(); - - File accessFile = new File(_accessFileName); - - if (!accessFile.exists()) + _logger.error("No jmx access rights file has been specified."); + return; + } + + if(_accessFile.exists()) + { + try { - throw new ConfigurationException("'" + _accessFileName + "' does not exist"); - } + _accessRightsUpdate.lock(); - if (!accessFile.canRead()) - { - throw new ConfigurationException("Cannot read '" + _accessFileName + "'."); + Properties accessRights = new Properties(); + accessRights.load(new FileInputStream(_accessFile)); + checkAccessRights(accessRights); + setAccessRights(accessRights); } - - if (!accessFile.canWrite()) + finally { - _logger.warn("Unable to write to access file '" + _accessFileName + "' changes will not be preserved."); + if (_accessRightsUpdate.isHeldByCurrentThread()) + { + _accessRightsUpdate.unlock(); + } } - - accessRights.load(new FileInputStream(accessFile)); - checkAccessRights(accessRights); - setAccessRights(accessRights); } - finally + else { - if (_accessRightsUpdate.isHeldByCurrentThread()) - { - _accessRightsUpdate.unlock(); - } + _logger.error("Specified jmxaccess rights file '" + _accessFile + "' does not exist."); } } @@ -385,33 +437,24 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana } } - private void saveAccessFile() + private void saveAccessFile() throws IOException { try { _accessRightsUpdate.lock(); - try - { - // Create temporary file - File tmp = File.createTempFile(_accessFileName, ".tmp"); - // Rename current file - File rights = new File(_accessFileName); + // Create temporary file + File tmp = File.createTempFile(_accessFile.getName(), ".tmp"); - FileOutputStream output = new FileOutputStream(tmp); - _accessRights.store(output, "Generated by AMQUserManagementMBean Console : Last edited by user:" + getCurrentJMXUser()); - output.close(); + FileOutputStream output = new FileOutputStream(tmp); + _accessRights.store(output, "Generated by AMQUserManagementMBean Console : Last edited by user:" + getCurrentJMXUser()); + output.close(); - // Rename new file to main file - tmp.renameTo(rights); + // Rename new file to main file + tmp.renameTo(_accessFile); - // delete tmp - tmp.delete(); - } - catch (IOException e) - { - _logger.warn("Problem occured saving '" + _accessFileName + "' changes may not be preserved. :" + e); - } + // delete tmp + tmp.delete(); } finally { @@ -420,6 +463,7 @@ public class AMQUserManagementMBean extends AMQManagedObject implements UserMana _accessRightsUpdate.unlock(); } } + } private String getCurrentJMXUser() diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/management/UserManagement.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/management/UserManagement.java index 658d7ebbd3..9fcdd4cd17 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/management/UserManagement.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/management/UserManagement.java @@ -33,7 +33,9 @@ import java.io.IOException; public interface UserManagement { + String TYPE = "UserManagement"; + int VERSION = 2; //********** Operations *****************// /** @@ -115,4 +117,5 @@ public interface UserManagement impact = MBeanOperationInfo.INFO) TabularData viewUsers(); + } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/plugins/network/FirewallFactory.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/plugins/network/FirewallFactory.java index 7fcf4a0494..a1a399e5bf 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/plugins/network/FirewallFactory.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/plugins/network/FirewallFactory.java @@ -21,6 +21,7 @@ package org.apache.qpid.server.security.access.plugins.network; import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; import org.apache.qpid.server.security.access.ACLPlugin; import org.apache.qpid.server.security.access.ACLPluginFactory; @@ -28,7 +29,7 @@ public class FirewallFactory implements ACLPluginFactory { @Override - public ACLPlugin newInstance(Configuration config) + public ACLPlugin newInstance(Configuration config) throws ConfigurationException { FirewallPlugin plugin = new FirewallPlugin(); plugin.setConfiguration(config); diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/plugins/network/FirewallPlugin.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/plugins/network/FirewallPlugin.java index cb8b6f6fed..39397966f0 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/plugins/network/FirewallPlugin.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/access/plugins/network/FirewallPlugin.java @@ -23,12 +23,18 @@ package org.apache.qpid.server.security.access.plugins.network; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.util.Iterator; import java.util.List; import java.util.regex.Pattern; +import org.apache.commons.configuration.CompositeConfiguration; import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.XMLConfiguration; import org.apache.qpid.server.protocol.AMQMinaProtocolSession; import org.apache.qpid.server.protocol.AMQProtocolSession; +import org.apache.qpid.server.security.access.ACLPlugin; +import org.apache.qpid.server.security.access.ACLPluginFactory; import org.apache.qpid.server.security.access.plugins.AbstractACLPlugin; import org.apache.qpid.server.virtualhost.VirtualHost; import org.apache.qpid.util.NetMatcher; @@ -36,6 +42,21 @@ import org.apache.qpid.util.NetMatcher; public class FirewallPlugin extends AbstractACLPlugin { + public static final ACLPluginFactory FACTORY = new ACLPluginFactory() + { + public boolean supportsTag(String name) + { + return name.startsWith("firewall"); + } + + public ACLPlugin newInstance(Configuration config) throws ConfigurationException + { + FirewallPlugin plugin = new FirewallPlugin(); + plugin.setConfiguration(config); + return plugin; + } + }; + public class FirewallRule { @@ -149,7 +170,7 @@ public class FirewallPlugin extends AbstractACLPlugin } @Override - public void setConfiguration(Configuration config) + public void setConfiguration(Configuration config) throws ConfigurationException { // Get default action String defaultAction = config.getString("[@default-action]"); @@ -165,15 +186,21 @@ public class FirewallPlugin extends AbstractACLPlugin { _default = AuthzResult.DENIED; } + CompositeConfiguration finalConfig = new CompositeConfiguration(config); + + List subFiles = config.getList("firewall.xml[@fileName]"); + for (Object subFile : subFiles) + { + finalConfig.addConfiguration(new XMLConfiguration((String) subFile)); + } - int numRules = config.getList("rule[@access]").size(); // all rules must - // have an access - // attribute + // all rules must have an access attribute + int numRules = finalConfig.getList("rule[@access]").size(); _rules = new FirewallRule[numRules]; for (int i = 0; i < numRules; i++) { - FirewallRule rule = new FirewallRule(config.getString("rule(" + i + ")[@access]"), config.getList("rule(" - + i + ")[@network]"), config.getList("rule(" + i + ")[@hostname]")); + FirewallRule rule = new FirewallRule(finalConfig.getString("rule(" + i + ")[@access]"), finalConfig.getList("rule(" + + i + ")[@network]"), finalConfig.getList("rule(" + i + ")[@hostname]")); _rules[i] = rule; } } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/Base64MD5PasswordFilePrincipalDatabase.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/Base64MD5PasswordFilePrincipalDatabase.java index 69ad9014db..3c211746e3 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/Base64MD5PasswordFilePrincipalDatabase.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/Base64MD5PasswordFilePrincipalDatabase.java @@ -152,8 +152,39 @@ public class Base64MD5PasswordFilePrincipalDatabase implements PrincipalDatabase public boolean verifyPassword(String principal, char[] password) throws AccountNotFoundException { char[] pwd = lookupPassword(principal); + + if (pwd == null) + { + throw new AccountNotFoundException("Unable to lookup the specfied users password"); + } + + byte[] byteArray = new byte[password.length]; + int index = 0; + for (char c : password) + { + byteArray[index++] = (byte) c; + } + + byte[] MD5byteArray; + try + { + MD5byteArray = HashedUser.getMD5(byteArray); + } + catch (Exception e1) + { + _logger.warn("Unable to hash password for user '" + principal + "' for comparison"); + return false; + } + + char[] hashedPassword = new char[MD5byteArray.length]; - return compareCharArray(pwd, password); + index = 0; + for (byte c : MD5byteArray) + { + hashedPassword[index++] = (char) c; + } + + return compareCharArray(pwd, hashedPassword); } private boolean compareCharArray(char[] a, char[] b) @@ -193,7 +224,7 @@ public class Base64MD5PasswordFilePrincipalDatabase implements PrincipalDatabase { _userUpdate.lock(); char[] orig = user.getPassword(); - user.setPassword(password); + user.setPassword(password,false); try { @@ -204,7 +235,7 @@ public class Base64MD5PasswordFilePrincipalDatabase implements PrincipalDatabase _logger.error("Unable to save password file, password change for user'" + principal + "' will revert at restart"); //revert the password change - user.setPassword(orig); + user.setPassword(orig,true); return false; } return true; @@ -230,7 +261,17 @@ public class Base64MD5PasswordFilePrincipalDatabase implements PrincipalDatabase return false; } - HashedUser user = new HashedUser(principal.getName(), password); + HashedUser user; + try + { + user = new HashedUser(principal.getName(), password); + } + catch (Exception e1) + { + _logger.warn("Unable to create new user '" + principal.getName() + "'"); + return false; + } + try { diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/HashedUser.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/HashedUser.java index 4d92e3fb4c..3690e7f92a 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/HashedUser.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/HashedUser.java @@ -25,6 +25,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; import java.io.UnsupportedEncodingException; +import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.Principal; @@ -63,10 +64,22 @@ public class HashedUser implements Principal } } - public HashedUser(String name, char[] password) + public HashedUser(String name, char[] password) throws UnsupportedEncodingException, NoSuchAlgorithmException { _name = name; - setPassword(password); + setPassword(password,false); + } + + public static byte[] getMD5(byte[] data) throws NoSuchAlgorithmException, UnsupportedEncodingException + { + MessageDigest md = MessageDigest.getInstance("MD5"); + + for (byte b : data) + { + md.update(b); + } + + return md.digest(); } public String getName() @@ -84,9 +97,31 @@ public class HashedUser implements Principal return _password; } - void setPassword(char[] password) + void setPassword(char[] password, boolean alreadyHashed) throws UnsupportedEncodingException, NoSuchAlgorithmException { - _password = password; + if(alreadyHashed){ + _password = password; + } + else + { + byte[] byteArray = new byte[password.length]; + int index = 0; + for (char c : password) + { + byteArray[index++] = (byte) c; + } + + byte[] MD5byteArray = getMD5(byteArray); + + _password = new char[MD5byteArray.length]; + + index = 0; + for (byte c : MD5byteArray) + { + _password[index++] = (char) c; + } + } + _modified = true; _encodedPassword = null; } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/PlainPasswordFilePrincipalDatabase.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/PlainPasswordFilePrincipalDatabase.java index 9da954d74f..5e4678a63b 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/PlainPasswordFilePrincipalDatabase.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/PlainPasswordFilePrincipalDatabase.java @@ -34,11 +34,13 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.io.PrintStream; import java.security.Principal; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; /** @@ -50,13 +52,18 @@ import java.util.regex.Pattern; */ public class PlainPasswordFilePrincipalDatabase implements PrincipalDatabase { + public static final String DEFAULT_ENCODING = "utf-8"; + private static final Logger _logger = Logger.getLogger(PlainPasswordFilePrincipalDatabase.class); - protected File _passwordFile; + private File _passwordFile; - protected Pattern _regexp = Pattern.compile(":"); + private Pattern _regexp = Pattern.compile(":"); - protected Map<String, AuthenticationProviderInitialiser> _saslServers; + private Map<String, AuthenticationProviderInitialiser> _saslServers; + + private Map<String, PlainUser> _users = new HashMap<String, PlainUser>(); + private ReentrantLock _userUpdate = new ReentrantLock(); public PlainPasswordFilePrincipalDatabase() { @@ -83,7 +90,7 @@ public class PlainPasswordFilePrincipalDatabase implements PrincipalDatabase _saslServers.put(cram.getMechanismName(), cram); } - public void setPasswordFile(String passwordFile) throws FileNotFoundException + public void setPasswordFile(String passwordFile) throws IOException { File f = new File(passwordFile); _logger.info("PlainPasswordFile using file " + f.getAbsolutePath()); @@ -97,10 +104,20 @@ public class PlainPasswordFilePrincipalDatabase implements PrincipalDatabase throw new FileNotFoundException("Cannot read password file " + f + ". Check permissions."); } + + loadPasswordFile(); } - public void setPassword(Principal principal, PasswordCallback callback) throws IOException, - AccountNotFoundException + /** + * SASL Callback Mechanism - sets the Password in the PasswordCallback based on the value in the PasswordFile + * If you want to change the password for a user, use updatePassword instead. + * + * @param principal The Principal to set the password for + * @param callback The PasswordCallback to call setPassword on + * + * @throws AccountNotFoundException If the Principal cannot be found in this Database + */ + public void setPassword(Principal principal, PasswordCallback callback) throws AccountNotFoundException { if (_passwordFile == null) { @@ -111,6 +128,7 @@ public class PlainPasswordFilePrincipalDatabase implements PrincipalDatabase throw new IllegalArgumentException("principal must not be null"); } char[] pwd = lookupPassword(principal.getName()); + if (pwd != null) { callback.setPassword(pwd); @@ -121,33 +139,151 @@ public class PlainPasswordFilePrincipalDatabase implements PrincipalDatabase } } + /** + * Used to verify that the presented Password is correct. Currently only used by Management Console + * + * @param principal The principal to authenticate + * @param password The plaintext password to check + * + * @return true if password is correct + * + * @throws AccountNotFoundException if the principal cannot be found + */ public boolean verifyPassword(String principal, char[] password) throws AccountNotFoundException { - try - { - char[] pwd = lookupPassword(principal); - return compareCharArray(pwd, password); - } - catch (IOException e) + char[] pwd = lookupPassword(principal); + + if (pwd == null) { - return false; + throw new AccountNotFoundException("Unable to lookup the specfied users password"); } + + return compareCharArray(pwd, password); + } + /** + * Changes the password for the specified user + * + * @param principal to change the password for + * @param password plaintext password to set the password too + */ public boolean updatePassword(Principal principal, char[] password) throws AccountNotFoundException { - return false; // updates denied + PlainUser user = _users.get(principal.getName()); + + if (user == null) + { + throw new AccountNotFoundException(principal.getName()); + } + + try + { + try + { + _userUpdate.lock(); + char[] orig = user.getPassword(); + user.setPassword(password); + + try + { + savePasswordFile(); + } + catch (IOException e) + { + _logger.error("Unable to save password file, password change for user '" + principal + "' discarded"); + //revert the password change + user.setPassword(orig); + return false; + } + return true; + } + finally + { + if (_userUpdate.isHeldByCurrentThread()) + { + _userUpdate.unlock(); + } + } + } + catch (Exception e) + { + return false; + } } public boolean createPrincipal(Principal principal, char[] password) { - return false; // updates denied + if (_users.get(principal.getName()) != null) + { + return false; + } + + PlainUser user = new PlainUser(principal.getName(), password); + + try + { + _userUpdate.lock(); + _users.put(user.getName(), user); + + try + { + savePasswordFile(); + return true; + } + catch (IOException e) + { + //remove the use on failure. + _users.remove(user.getName()); + _logger.warn("Unable to create user '" + user.getName()); + return false; + } + } + finally + { + if (_userUpdate.isHeldByCurrentThread()) + { + _userUpdate.unlock(); + } + } } public boolean deletePrincipal(Principal principal) throws AccountNotFoundException { - return false; // updates denied + PlainUser user = _users.get(principal.getName()); + + if (user == null) + { + throw new AccountNotFoundException(principal.getName()); + } + + try + { + _userUpdate.lock(); + user.delete(); + + try + { + savePasswordFile(); + } + catch (IOException e) + { + _logger.error("Unable to remove user '" + user.getName() + "' from password file."); + return false; + } + + _users.remove(user.getName()); + } + finally + { + if (_userUpdate.isHeldByCurrentThread()) + { + _userUpdate.unlock(); + } + } + + return true; } public Map<String, AuthenticationProviderInitialiser> getMechanisms() @@ -157,21 +293,14 @@ public class PlainPasswordFilePrincipalDatabase implements PrincipalDatabase public List<Principal> getUsers() { - return new LinkedList<Principal>(); //todo + return new LinkedList<Principal>(_users.values()); } public Principal getUser(String username) { - try - { - if (lookupPassword(username) != null) - { - return new UsernamePrincipal(username); - } - } - catch (IOException e) + if (_users.containsKey(username)) { - //fall through to null return + return new UsernamePrincipal(username); } return null; } @@ -197,49 +326,166 @@ public class PlainPasswordFilePrincipalDatabase implements PrincipalDatabase * Looks up the password for a specified user in the password file. Note this code is <b>not</b> secure since it * creates strings of passwords. It should be modified to create only char arrays which get nulled out. * - * @param name the name of the principal to lookup - * - * @return char[] of the password + * @param name The principal name to lookup * - * @throws java.io.IOException whilst accessing the file + * @return a char[] for use in SASL. */ - private char[] lookupPassword(String name) throws IOException + private char[] lookupPassword(String name) + { + PlainUser user = _users.get(name); + if (user == null) + { + return null; + } + else + { + return user.getPassword(); + } + } + + private void loadPasswordFile() throws IOException + { + try + { + _userUpdate.lock(); + _users.clear(); + + BufferedReader reader = null; + try + { + reader = new BufferedReader(new FileReader(_passwordFile)); + String line; + + while ((line = reader.readLine()) != null) + { + String[] result = _regexp.split(line); + if (result == null || result.length < 2 || result[0].startsWith("#")) + { + continue; + } + + PlainUser user = new PlainUser(result); + _logger.info("Created user:" + user); + _users.put(user.getName(), user); + } + } + finally + { + if (reader != null) + { + reader.close(); + } + } + } + finally + { + if (_userUpdate.isHeldByCurrentThread()) + { + _userUpdate.unlock(); + } + } + } + + private void savePasswordFile() throws IOException { - BufferedReader reader = null; try { - reader = new BufferedReader(new FileReader(_passwordFile)); - String line; + _userUpdate.lock(); + + BufferedReader reader = null; + PrintStream writer = null; + File tmp = File.createTempFile(_passwordFile.getName(), ".tmp"); - while ((line = reader.readLine()) != null) + try { - if (!line.startsWith("#")) + writer = new PrintStream(tmp); + reader = new BufferedReader(new FileReader(_passwordFile)); + String line; + + while ((line = reader.readLine()) != null) { String[] result = _regexp.split(line); - if (result == null || result.length < 2) + if (result == null || result.length < 2 || result[0].startsWith("#")) { + writer.write(line.getBytes(DEFAULT_ENCODING)); + writer.println(); continue; } - if (name.equals(result[0])) + PlainUser user = _users.get(result[0]); + + if (user == null) { - return result[1].toCharArray(); + writer.write(line.getBytes(DEFAULT_ENCODING)); + writer.println(); + } + else if (!user.isDeleted()) + { + if (!user.isModified()) + { + writer.write(line.getBytes(DEFAULT_ENCODING)); + writer.println(); + } + else + { + byte[] password = user.getPasswordBytes(); + + writer.write((user.getName() + ":").getBytes(DEFAULT_ENCODING)); + writer.write(password); + writer.println(); + + user.saved(); + } + } + } + + for (PlainUser user : _users.values()) + { + if (user.isModified()) + { + byte[] password; + password = user.getPasswordBytes(); + writer.write((user.getName() + ":").getBytes(DEFAULT_ENCODING)); + writer.write(password); + writer.println(); + user.saved(); } } } - return null; + finally + { + if (reader != null) + { + reader.close(); + } + + if (writer != null) + { + writer.close(); + } + + // Swap temp file to main password file. + File old = new File(_passwordFile.getAbsoluteFile() + ".old"); + if (old.exists()) + { + old.delete(); + } + _passwordFile.renameTo(old); + tmp.renameTo(_passwordFile); + tmp.delete(); + } } finally { - if (reader != null) + if (_userUpdate.isHeldByCurrentThread()) { - reader.close(); + _userUpdate.unlock(); } } } - + public void reload() throws IOException { - //This PD is not cached, so do nothing. + loadPasswordFile(); } } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/PlainUser.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/PlainUser.java new file mode 100644 index 0000000000..46a78a55aa --- /dev/null +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/database/PlainUser.java @@ -0,0 +1,106 @@ +/* +* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.security.auth.database; + +import org.apache.log4j.Logger; + +import java.security.Principal; + +public class PlainUser implements Principal +{ + private String _name; + private char[] _password; + private boolean _modified = false; + private boolean _deleted = false; + + PlainUser(String[] data) + { + if (data.length != 2) + { + throw new IllegalArgumentException("User Data should be length 2, username, password"); + } + + _name = data[0]; + + _password = data[1].toCharArray(); + + } + + public PlainUser(String name, char[] password) + { + _name = name; + _password = password; + _modified = true; + } + + public String getName() + { + return _name; + } + + public String toString() + { + return _name; + } + + char[] getPassword() + { + return _password; + } + + byte[] getPasswordBytes() + { + byte[] byteArray = new byte[_password.length]; + int index = 0; + for (char c : _password) + { + byteArray[index++] = (byte) c; + } + return byteArray; + } + + void setPassword(char[] password) + { + _password = password; + _modified = true; + } + + public boolean isModified() + { + return _modified; + } + + public boolean isDeleted() + { + return _deleted; + } + + public void delete() + { + _deleted = true; + } + + public void saved() + { + _modified = false; + } + +} diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/rmi/RMIPasswordAuthenticator.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/rmi/RMIPasswordAuthenticator.java index 378b17e733..77040e896c 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/rmi/RMIPasswordAuthenticator.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/security/auth/rmi/RMIPasswordAuthenticator.java @@ -20,23 +20,14 @@ */ package org.apache.qpid.server.security.auth.rmi; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; import java.util.Collections; import javax.management.remote.JMXAuthenticator; import javax.management.remote.JMXPrincipal; import javax.security.auth.Subject; -import javax.security.auth.callback.PasswordCallback; import javax.security.auth.login.AccountNotFoundException; -import org.apache.qpid.server.security.auth.database.Base64MD5PasswordFilePrincipalDatabase; -import org.apache.qpid.server.security.auth.database.PlainPasswordFilePrincipalDatabase; import org.apache.qpid.server.security.auth.database.PrincipalDatabase; -import org.apache.qpid.server.security.auth.sasl.UsernamePrincipal; public class RMIPasswordAuthenticator implements JMXAuthenticator { @@ -48,7 +39,6 @@ public class RMIPasswordAuthenticator implements JMXAuthenticator static final String CREDENTIALS_REQUIRED = "User details are required. " + "Please ensure you are using an up to date management console to connect."; - public static final String DEFAULT_ENCODING = "utf-8"; private PrincipalDatabase _db = null; public RMIPasswordAuthenticator() @@ -91,56 +81,26 @@ public class RMIPasswordAuthenticator implements JMXAuthenticator throw new SecurityException(SHOULD_BE_NON_NULL); } + // Verify that a PD has been set. + if (_db == null) + { + throw new SecurityException(UNABLE_TO_LOOKUP); + } + boolean authenticated = false; // Perform authentication try { - PasswordCallback pwCallback = new PasswordCallback("prompt",false); - UsernamePrincipal uname = new UsernamePrincipal(username); - - if (_db instanceof Base64MD5PasswordFilePrincipalDatabase) - { - //retrieve the stored password for the given user - _db.setPassword(uname, pwCallback); - - //compare the MD5Hash of the given password with the stored value - if (Arrays.equals(getMD5Hash(password), pwCallback.getPassword())) - { - authenticated = true; - } - } - else if (_db instanceof PlainPasswordFilePrincipalDatabase) - { - //retrieve the users stored password and compare with given value - _db.setPassword(uname, pwCallback); - - if (password.equals(new String(pwCallback.getPassword()))) - { - authenticated = true; - } - } - else - { - throw new SecurityException(UNABLE_TO_LOOKUP); + if (_db.verifyPassword(username, password.toCharArray())) + { + authenticated = true; } } catch (AccountNotFoundException e) { throw new SecurityException(INVALID_CREDENTIALS); } - catch (UnsupportedEncodingException e) - { - throw new SecurityException(UNABLE_TO_LOOKUP); - } - catch (NoSuchAlgorithmException e) - { - throw new SecurityException(UNABLE_TO_LOOKUP); - } - catch (IOException e) - { - throw new SecurityException(UNABLE_TO_LOOKUP); - } if (authenticated) { @@ -155,28 +115,5 @@ public class RMIPasswordAuthenticator implements JMXAuthenticator throw new SecurityException(INVALID_CREDENTIALS); } } - - public static char[] getMD5Hash(String text) throws NoSuchAlgorithmException, UnsupportedEncodingException - { - byte[] data = text.getBytes(DEFAULT_ENCODING); - MessageDigest md = MessageDigest.getInstance("MD5"); - - for (byte b : data) - { - md.update(b); - } - - byte[] digest = md.digest(); - - char[] hash = new char[digest.length ]; - - int index = 0; - for (byte b : digest) - { - hash[index++] = (char) b; - } - - return hash; - } }
\ No newline at end of file diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/subscription/SubscriptionImpl.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/subscription/SubscriptionImpl.java index 119a4b1692..bc1f56fee1 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/subscription/SubscriptionImpl.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/subscription/SubscriptionImpl.java @@ -387,6 +387,7 @@ public abstract class SubscriptionImpl implements Subscription, FlowCreditManage //todo - client id should be recoreded and this test removed but handled below if (_noLocal) { + //todo getPublisherClientInstance should be moved to QueueEntryImpl final Object publisherId = entry.getMessage().getPublisherClientInstance(); // We don't want local messages so check to see if message is one we sent @@ -407,6 +408,7 @@ public abstract class SubscriptionImpl implements Subscription, FlowCreditManage //todo - client id should be recoreded and this test removed but handled here + //todo getPublisherIdentifier should be moved to QueueEntryImpl if (localInstance != null && localInstance.equals(entry.getMessage().getPublisherIdentifier())) { return false; @@ -498,9 +500,9 @@ public abstract class SubscriptionImpl implements Subscription, FlowCreditManage } - public boolean wouldSuspend(QueueEntry msg) + public boolean wouldSuspend(QueueEntry queueEntry) { - return !_creditManager.useCreditForMessage(msg.getMessage());//_channel.wouldSuspend(msg.getMessage()); + return !_creditManager.useCreditForMessage(queueEntry); } public void getSendLock() diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/txn/NonTransactionalContext.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/txn/NonTransactionalContext.java index 0ca8135f71..2f27e1405a 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/txn/NonTransactionalContext.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/txn/NonTransactionalContext.java @@ -127,9 +127,9 @@ public class NonTransactionalContext implements TransactionalContext { if (debug) { - _log.debug("Discarding message: " + queueEntry.getMessage().getMessageId()); + _log.debug("Discarding message: " + queueEntry.getMessageId()); } - if(queueEntry.getMessage().isPersistent()) + if(queueEntry.isPersistent()) { beginTranIfNecessary(); } @@ -175,9 +175,9 @@ public class NonTransactionalContext implements TransactionalContext if (debug) { - _log.debug("Discarding message: " + queueEntry.getMessage().getMessageId()); + _log.debug("Discarding message: " + queueEntry.getMessageId()); } - if(queueEntry.getMessage().isPersistent()) + if(queueEntry.isPersistent()) { beginTranIfNecessary(); } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/virtualhost/ManagedVirtualHost.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/virtualhost/ManagedVirtualHost.java index 85d804457e..f4c81fbbb8 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/virtualhost/ManagedVirtualHost.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/virtualhost/ManagedVirtualHost.java @@ -31,6 +31,7 @@ import org.apache.qpid.server.management.MBeanAttribute; public interface ManagedVirtualHost
{
static final String TYPE = "VirtualHost";
+ static final int VERSION = 1;
/**
* Returns the name of the managed virtualHost.
diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/server/virtualhost/VirtualHost.java b/qpid/java/broker/src/main/java/org/apache/qpid/server/virtualhost/VirtualHost.java index e1b770b1d3..5d2a31b80d 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/server/virtualhost/VirtualHost.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/server/virtualhost/VirtualHost.java @@ -48,6 +48,9 @@ import org.apache.qpid.server.queue.AMQQueue; import org.apache.qpid.server.queue.AMQQueueFactory; import org.apache.qpid.server.queue.DefaultQueueRegistry; import org.apache.qpid.server.queue.QueueRegistry; +import org.apache.qpid.server.queue.QueueBackingStore; +import org.apache.qpid.server.queue.FileQueueBackingStoreFactory; +import org.apache.qpid.server.queue.QueueBackingStoreFactory; import org.apache.qpid.server.registry.ApplicationRegistry; import org.apache.qpid.server.routing.RoutingTable; import org.apache.qpid.server.security.access.ACLManager; @@ -84,7 +87,10 @@ public class VirtualHost implements Accessable private ACLManager _accessManager; private final Timer _houseKeepingTimer; - + + private VirtualHostConfiguration _configuration; + private QueueBackingStoreFactory _queueBackingStoreFactory; + public void setAccessableName(String name) { _logger.warn("Setting Accessable Name for VirualHost is not allowed. (" @@ -106,6 +112,16 @@ public class VirtualHost implements Accessable return _routingTable; } + public VirtualHostConfiguration getConfiguration() + { + return _configuration ; + } + + public QueueBackingStoreFactory getQueueBackingStoreFactory() + { + return _queueBackingStoreFactory; + } + /** * Abstract MBean class. This has some of the methods implemented from management intrerface for exchanges. Any * implementaion of an Exchange MBean should extend this class. @@ -114,7 +130,7 @@ public class VirtualHost implements Accessable { public VirtualHostMBean() throws NotCompliantMBeanException { - super(ManagedVirtualHost.class, "VirtualHost"); + super(ManagedVirtualHost.class, ManagedVirtualHost.TYPE, ManagedVirtualHost.VERSION); } public String getObjectInstanceName() @@ -137,7 +153,6 @@ public class VirtualHost implements Accessable /** * Normal Constructor - * @param name * @param hostConfig * @throws Exception */ @@ -148,6 +163,7 @@ public class VirtualHost implements Accessable public VirtualHost(VirtualHostConfiguration hostConfig, TransactionLog transactionLog) throws Exception { + _configuration = hostConfig; _name = hostConfig.getName(); if (_name == null || _name.length() == 0) @@ -179,6 +195,9 @@ public class VirtualHost implements Accessable initialiseRoutingTable(hostConfig); } + _queueBackingStoreFactory = new FileQueueBackingStoreFactory(); + _queueBackingStoreFactory.configure(this, hostConfig); + _exchangeFactory.initialise(hostConfig); _exchangeRegistry.initialise(); @@ -403,6 +422,12 @@ public class VirtualHost implements Accessable //Stop Connections _connectionRegistry.close(); + //Stop Housekeeping + if (_houseKeepingTimer != null) + { + _houseKeepingTimer.cancel(); + } + //Stop the Queues processing if (_queueRegistry != null) { @@ -410,13 +435,7 @@ public class VirtualHost implements Accessable { queue.stop(); } - } - - //Stop Housekeeping - if (_houseKeepingTimer != null) - { - _houseKeepingTimer.cancel(); - } + } //Close TransactionLog if (_transactionLog != null) diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/tools/messagestore/commands/Move.java b/qpid/java/broker/src/main/java/org/apache/qpid/tools/messagestore/commands/Move.java index a8dd58ca83..7fe16062fc 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/tools/messagestore/commands/Move.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/tools/messagestore/commands/Move.java @@ -172,7 +172,7 @@ public class Move extends AbstractCommand { for (QueueEntry msg : messages) { - ids.add(msg.getMessage().getMessageId()); + ids.add(msg.getMessageId()); } } } diff --git a/qpid/java/broker/src/main/java/org/apache/qpid/tools/messagestore/commands/Show.java b/qpid/java/broker/src/main/java/org/apache/qpid/tools/messagestore/commands/Show.java index d46ba85069..49afcb1340 100644 --- a/qpid/java/broker/src/main/java/org/apache/qpid/tools/messagestore/commands/Show.java +++ b/qpid/java/broker/src/main/java/org/apache/qpid/tools/messagestore/commands/Show.java @@ -352,7 +352,7 @@ public class Show extends AbstractCommand isredelivered.add(entry.isRedelivered() ? "true" : "false"); - isdelivered.add(msg.getDeliveredToConsumer() ? "true" : "false"); + isdelivered.add(entry.getDeliveredToConsumer() ? "true" : "false"); BasicContentHeaderProperties headers = null; |