diff options
| author | Keith Wall <kwall@apache.org> | 2014-03-25 17:54:10 +0000 |
|---|---|---|
| committer | Keith Wall <kwall@apache.org> | 2014-03-25 17:54:10 +0000 |
| commit | cd6130384dc5f27ad494eabf8a2b15ca79280aa1 (patch) | |
| tree | 77d7b1f0ced2cea6b031327fcb5c8143d763cf9d | |
| parent | fcc3f654b60b7dd2180afe73e8809545725b41af (diff) | |
| parent | 809061e0024b74f89afdeff8ba83d6514589f417 (diff) | |
| download | qpid-python-cd6130384dc5f27ad494eabf8a2b15ca79280aa1.tar.gz | |
NO-JIRA: Merge changes from trunk.
Command was:
svn merge https://svn.apache.org/repos/asf/qpid/trunk
git-svn-id: https://svn.apache.org/repos/asf/qpid/branches/java-broker-bdb-ha2@1581428 13f79535-47bb-0310-9956-ffa450edef68
148 files changed, 6789 insertions, 4696 deletions
diff --git a/qpid/QPID_VERSION.txt b/qpid/QPID_VERSION.txt index 5a9e6bda25..eec15f9025 100644 --- a/qpid/QPID_VERSION.txt +++ b/qpid/QPID_VERSION.txt @@ -1 +1 @@ -0.27 +0.29 diff --git a/qpid/cpp/INSTALL b/qpid/cpp/INSTALL index 6295a23cc9..06c6fbfe45 100644 --- a/qpid/cpp/INSTALL +++ b/qpid/cpp/INSTALL @@ -159,8 +159,7 @@ all of the above plus: NOTE: make sure to install the related '-devel' packages also!!!! -NOTE: Python 3.x is know to NOT work - please use 2.7 or earlier. -Ruby should be prior to version 2. +NOTE: Python 3.x is known to NOT work - please use 2.7 or earlier. To build the QMF (Qpid Management Framework) bindings for Ruby and Python, the following must also be installed: diff --git a/qpid/cpp/QPID_VERSION.txt b/qpid/cpp/QPID_VERSION.txt index 5a9e6bda25..eec15f9025 100644 --- a/qpid/cpp/QPID_VERSION.txt +++ b/qpid/cpp/QPID_VERSION.txt @@ -1 +1 @@ -0.27 +0.29 diff --git a/qpid/cpp/bindings/qpid/dotnet/configure-windows.ps1 b/qpid/cpp/bindings/qpid/dotnet/configure-windows.ps1 index 50d9741e95..e37ad31129 100644 --- a/qpid/cpp/bindings/qpid/dotnet/configure-windows.ps1 +++ b/qpid/cpp/bindings/qpid/dotnet/configure-windows.ps1 @@ -17,51 +17,49 @@ # under the License.
#
-#
-# configure-windows.ps1
-# =====================
+Set-PSDebug -Trace 0
+Set-PSDebug -strict
+$ErrorActionPreference='Stop'
+
+$global:usageText = @("#
+# configure-windows.ps1 [studio-and-architecture [boost-root-directory]]
#
# This script configures a qpid\cpp developer build environment under Windows
# to enable working with cpp\bindings\qpid\dotnet binding source code.
#
-# * Supports multiple versions of Visual Studio (VS2008, VS2010, VS2012)
-# as CMake generator.
-#
-# * Supports 32-bit and/or 64-bit development platforms.
-#
-# * User chooses in-source or out-of-source build directories.
-#
-# - 'In-source' builds happen when CMake is run from directory qpid\cpp.
-# Hundreds of CMake-generated output files are placed in qpid\cpp\src.
-# These files go right on top of files that are part of the source tree
-# in qpid\cpp\src.
-# In-source builds support only one platform.
-# Choose only a 32-bit or a 64-bit platform but not both.
-#
-# - Out-of-source builds happen when the user chooses another directory
-# under qpid in which to run CMake. Out-of-source builds are required
-# in order to build both x86 and x64 targets using the same source tree.
-# For each build platform (32-bit x86 or Win32, or 64-bit x64) the user
-# specifies a build directory and a specific version of Boost.
-# Many platform/Boost-version directories may reside side by side.
-#
-# * User chooses to run CMake or not.
-#
-# - When a new build directory is created then the user is given the
-# option of running CMake in that directory. Running CMake is a
-# necessary step as CMake creates important source, solution, and
-# project files.
-#
-# - If a directory "looks like" is has already had CMake run in it
-# then this script skips running CMake again.
-#
-# * User chooses to include Proton or not.
-#
-# - Proton is included by having variable PROTON_ROOT reference the
-# directory where proton was installed.
+# ARG PROCESSING
+# ==============
+# All arguments may be specified on the command line. If not then this
+# script will prompt the user for choices.
+#
+# * studio-and-architecture
+# User chooses a version of Visual Studio and a target platform:
+# 2012-x86
+# 2012-x64
+# 2010-x86
+# 2010-x64
+# 2008-x86
+# 2008-x64
+#
+# * boost-root-directory
+# The path to the version of boost to be used in this build
+#
+# CONFIGURATION OUTPUT
+# ====================
+# This script uses these build and install directories for the build:
+# build_2008_x86 install_2008_x86
+# build_2010_x86 install_2010_x86
+# build_2008_x64 install_2008_x64
+# build_2010_x64 install_2010_x64
+#
+#
+# PROTON AMQP 1.0
+# ===============
+# Proton is included automatically by having previously executed proton's
+# ""make install"" to the install_xx_xxxx directory that qpid is about to use.
#
# Prerequisites
-#
+# =============
# 1. Powershell must be installed.
# 2. 32-bit and/or 64-bit Boost libraries must be installed in separate
# directories. A user system may have any number of Boost library
@@ -70,7 +68,7 @@ # 3. CMake 2.8 (or later) must be installed. The cmake\bin directory
# must be in the user's path.
# 4. Boost library specifications may or may not be in the user's path.
-# The script author recommeds not to have Boost in the path and only
+# The script author recommends not to have Boost in the path and only
# allow the Boost path to be specified by generated command procedures.
# 5. Visual Studio build environment must be installed.
#
@@ -78,88 +76,73 @@ # Use case: Create a new build environment
# ========================================
#
-# Required Boost:
+# Required VS2010 Boost:
# 32-bit library - C:\Boost
# 64-bit library - D:\Boost_64
#
# Required Qpid checkout tree
# C:\svn\qpid\...
#
-# Run this script. It will ask for four directories:
-#
-# Needed info User clicks on or adds new
-# ---------------- --------------------------
-# 32-bit Boost C:\boost
-# 32-bit build dir C:\svn\qpid\build32
-# 64-bit Boost D:\Boost_64
-# 64-bit build dir C:\svn\qpid\build64
-#
-# In this example the build dirs are new. The script will prompt
-# asking if CMake is to run in the build directories. User chooses Yes.
-#
-# Now this script runs CMake twice, once each with the 32-bit and 64-bit
-# generators.
-# * This step creates qpid-cpp.sln and related project files.
-# C:\svn\qpid\build32\qpid-cpp.sln
-# C:\svn\qpid\build64\qpid-cpp.sln
-#
-# This script generates other scripts as follows:
-#
-# C:\svn\qpid\build32\start-devenv-messaging-x86-32bit.ps1
-# C:\svn\qpid\build32\start-devenv-messaging-x86-32bit.bat
-# C:\svn\qpid\build32\setenv-messaging-x86-32bit.bat
+# Run this script, select VS2010 compiler, x86 platform:
#
-# C:\svn\qpid\build64\start-devenv-messaging-x64-64bit.ps1
-# C:\svn\qpid\build64\start-devenv-messaging-x64-64bit.bat
-# C:\svn\qpid\build64\setenv-messaging-x64-64bit.bat
+# configure-windows.ps1 2010-x86 C:\Boost
#
-# Next the user compiles solution qpid\build32\qpid-cpp.sln.
+# This script will automatically create build directory
+# C:\svn\qpid\build_2010_x86
#
-# Using the generated scripts:
+# Next this script runs CMake.
#
-# Case 1. Run an executable in 32-bit mode.
-# 1. Open a command prompt.
-# 2. > CD c:\svn\qpid\build32
-# 3. > CALL setenv-messaging-x86-32bit.bat
-# 4. > CD src\debug
-# 5. > run the chosen program
-#
-# Note: Step #3 adds Boost to the user's path. This script selects the
-# version of Boost to match the executables in src\debug.
-#
-# Case 2. Launch Visual Studio org.apache.qpid.messaging.sln in 64-bit mode.
-# 1. > CD c:\svn\qpid\build64
-# 2. > powershell start-devenv-messaging-x64-64bit.ps1
-# or
-# 1. Double-click c:\svn\qpid\build64\start-devenv-messaging-x64-64bit.bat
+# * This step creates qpid-cpp.sln and related project files.
+# C:\svn\qpid\build_2010_x86\qpid-cpp.sln
#
-# Note: In this case the scripts set QPID_BUILD_ROOT to point to the out-of-
-# source directory used by qpid-cpp.sln. Also the scripts put Boost in
-# the path so that executables may run directly from Visual Studio.
+# This script generates several other helper scripts:
#
-
-Set-PSDebug -Trace 0
-Set-PSDebug -strict
-$ErrorActionPreference='Stop'
+# C:\svn\qpid\build_2010_x86\start-devenv-messaging-x86-32bit.ps1
+# C:\svn\qpid\build_2010_x86\start-devenv-messaging-x86-32bit.bat
+# C:\svn\qpid\build_2010_x86\setenv-messaging-x86-32bit.bat
+# C:\svn\qpid\build_2010_x86\run-cmake.bat
+# C:\svn\qpid\build_2010_x86\run-qpid-devenv-debug.bat
+")
#############################
# global strings to be written to script files
#
$global:txtPath = '$env:PATH'
$global:txtQR = '$env:QPID_BUILD_ROOT'
-$global:txtPR = '$env:PROTON_ROOT'
$global:txtWH = 'Write-Host'
#############################
# Visual Studio version selection dialog items and choice
#
-[array]$global:VsVersionCmakeChoiceList = "Visual Studio 2012", "Visual Studio 2010", "Visual Studio 2008"
-$global:vsVersion = ''
-$global:cmakeGenerator = ''
-$global:vsSubdir = ''
-$global:cmakeCompiler = ''
-$global:cmakeCommandLine32 = ''
-$global:cmakeCommandLine64 = ''
+[array]$global:VsVersionCmakeChoiceList = `
+ "Visual Studio 2012 - x86", `
+ "Visual Studio 2012 - x64", `
+ "Visual Studio 2010 - x86", `
+ "Visual Studio 2010 - x64", `
+ "Visual Studio 2008 - x86", `
+ "Visual Studio 2008 - x64"
+$global:vsSelectedOption = ''
+$global:vsVersion = '' # "Visual Studio 2010"
+$global:vsShortName = '' # "2010"
+$global:cmakeGenerator = '' # "Visual Studio 10"
+$global:vsSubdir = '' # "msvc10"
+$global:cmakeCompiler = '' # "-vc100"
+$global:build32or64 = '' # "32" or "64"
+$global:buildPathSizeId = '' # "x86" or "x64"
+$global:vsEnvironment = '' # ""%VS100COMNTOOLS%..\..\vcvarsall.bat" x86"
+$global:vsBuildTarget = '' # "Debug|Win32"
+
+$global:cmakeCommandLine = ''
+
+$global:boostRootPath = ''
+
+#############################
+# Usage
+#
+function Usage
+{
+ Write-Host $global:usageText
+}
#############################
# Select-Folder
@@ -198,7 +181,7 @@ function AskYesOrNo ($Question="No question?", $Title="No Title?") #
function SanityCheckBoostPath ($path=0)
{
- $result = $true
+ $result = 1
$displayPath = ""
if ($path -ne $null) {
@@ -208,11 +191,11 @@ function SanityCheckBoostPath ($path=0) foreach ($pattern in $toTest) {
$target = Join-Path $path $pattern
if (!(Test-Path -path $target)) {
- $result = $false
+ $result = 0
}
}
} else {
- $result = $false
+ $result = 0
}
if (! $result) {
@@ -223,44 +206,13 @@ function SanityCheckBoostPath ($path=0) #############################
-# SanityCheckProtonInstallPath
-# A path is a "proton install path" if it contains
-# both bin and include subdirectories.
-#
-function SanityCheckProtonInstallPath ($path=0)
-{
- $result = $true
- $displayPath = ""
-
- if ($path -ne $null) {
- $displayPath = $path
-
- $toTest = ('include', 'bin')
- foreach ($pattern in $toTest) {
- $target = Join-Path $path $pattern
- if (!(Test-Path -path $target)) {
- $result = $false
- }
- }
- } else {
- $result = $false
- }
-
- if (! $result) {
- Write-Host "The path ""$displayPath"" does not appear to be a Proton install root path."
- }
- $result
-}
-
-
-#############################
# SanityCheckBuildPath
# A path is a "build path" if it contains
# various subdirectories.
#
function SanityCheckBuildPath ($path=0)
{
- $result = $true
+ $result = 1
$displayPath = ""
if ($path -ne $null) {
$displayPath = $path
@@ -270,11 +222,11 @@ function SanityCheckBuildPath ($path=0) foreach ($pattern in $toTest) {
$target = Join-Path $path $pattern
if (!(Test-Path -path $target)) {
- $result = $false
+ $result = 0
}
}
} else {
- $result = $false
+ $result = 0
}
if (! $result) {
Write-Host "The path ""$displayPath"" does not appear to be a Qpid C++ build root path."
@@ -300,16 +252,14 @@ function WriteDotnetBindingSlnLauncherPs1 [string] $nBits,
[string] $outfileName,
[string] $studioVersion,
- [string] $studioSubdir,
- [string] $protonRoot
+ [string] $studioSubdir
)
$out = @("#
# Launch $slnName in $studioVersion $vsPlatform ($nBits-bit) environment
#
-$global:txtPath = ""$protonRoot\bin;$boostRoot\lib;$global:txtPath""
+$global:txtPath = ""$boostRoot\lib;$global:txtPath""
$global:txtQR = ""$buildRoot""
-$global:txtPR = ""$protonRoot""
$global:txtWH ""Launch $slnName in $studioVersion $vsPlatform ($nBits-bit) environment.""
$cppDir\bindings\qpid\dotnet\$vsSubdir\$slnName
")
@@ -366,8 +316,7 @@ function WriteDotnetBindingEnvSetupBat [string] $outfileName,
[string] $studioVersion,
[string] $studioSubdir,
- [string] $cmakeLine,
- [string] $protonRoot
+ [string] $cmakeLine
)
$out = @("@ECHO OFF
@@ -383,9 +332,8 @@ REM The solution was generated with cmake command line: REM $cmakeLine
ECHO %PATH% | FINDSTR /I boost > NUL
IF %ERRORLEVEL% EQU 0 ECHO WARNING: Boost is defined in your path multiple times!
-SET PATH=$protonRoot\bin;$boostRoot\lib;%PATH%
+SET PATH=$boostRoot\lib;%PATH%
SET QPID_BUILD_ROOT=$buildRoot
-SET PROTON_ROOT=$protonRoot
ECHO Environment set for $slnName $studioVersion $vsPlatform $nBits-bit development.
")
Write-Host " $buildRoot\$outfileName"
@@ -393,32 +341,132 @@ ECHO Environment set for $slnName $studioVersion $vsPlatform $nBits-bit developm }
#############################
-# Return the SelectedItem from the dropdown list and close the form.
+# WriteCmakeRerunnerBat
+# Write a batch file that runs cmake again
#
-function Return-DropDown {
+function WriteCmakeRerunnerBat
+{
+ param
+ (
+ [string] $slnName,
+ [string] $boostRoot,
+ [string] $buildRoot,
+ [string] $vsPlatform,
+ [string] $nBits,
+ [string] $outfileName,
+ [string] $studioVersion,
+ [string] $studioSubdir,
+ [string] $cmakeLine
+ )
+
+ $out = @("@ECHO OFF
+REM
+REM Call this command procedure from a command prompt to rerun cmake
+REM $studioVersion $vsPlatform ($nBits-bit)
+REM
+$cmakeLine
+")
+ Write-Host " $buildRoot\$outfileName"
+ $out | Out-File "$buildRoot\$outfileName" -encoding ASCII
+}
+
+#############################
+# WriteMakeInstallBat
+# Write a batch file that runs "make install" for debug build
+#
+function WriteMakeInstallBat
+{
+ param
+ (
+ [string] $buildRoot,
+ [string] $outfileName,
+ [string] $varfileName,
+ [string] $vsEnvironment,
+ [string] $vsBuildTarget
+ )
+
+ $out = @("@ECHO OFF
+REM
+REM Call this command procedure from a command prompt to run 'make install'
+REM
+setlocal
+call $varfileName
+call $vsEnvironment
+devenv qpid-cpp.sln /build $vsBuildTarget /project INSTALL
+endlocal
+")
+ Write-Host " $buildRoot\$outfileName"
+ $out | Out-File "$buildRoot\$outfileName" -encoding ASCII
+}
+
+#############################
+# Given a visual studio selection from command line or selection list
+# Return the visual studio and architecture settings or exit
+#
+function ParseStudioSelection
+{
+ param
+ (
+ [string] $vsSelection
+ )
+ Write-Host "Checking studio version: $vsSelection"
+ if ($vsSelection.Contains("2012")) {
+ $global:vsVersion = "Visual Studio 2012"
+ $global:cmakeGenerator = "Visual Studio 11"
+ $global:vsSubdir = "msvc11"
+ $global:cmakeCompiler = "-vc110"
+ $global:vsShortName = "2012"
+ $global:vsEnvironment = """%VS110COMNTOOLS%..\..\VC\vcvarsall.bat"""
+ } elseif ($vsSelection.Contains("2010")) {
+ $global:vsVersion = "Visual Studio 2010"
+ $global:cmakeGenerator = "Visual Studio 10"
+ $global:vsSubdir = "msvc10"
+ $global:cmakeCompiler = "-vc100"
+ $global:vsShortName = "2010"
+ $global:vsEnvironment = """%VS100COMNTOOLS%..\..\VC\vcvarsall.bat"""
+ } elseif ($vsSelection.Contains("2008")) {
+ $global:vsVersion = "Visual Studio 2008"
+ $global:cmakeGenerator = "Visual Studio 9 2008"
+ $global:vsSubdir = "msvc9"
+ $global:cmakeCompiler = "-vc90"
+ $global:vsShortName = "2008"
+ $global:vsEnvironment = """%VS90COMNTOOLS%..\..\VC\vcvarsall.bat"""
+ } else {
+ Write-Host "Visual Studio must be 2008, 2010, or 2012"
+ exit
+ }
+ $global:vsSelectedOption = $vsSelection
+
+ if ($vsSelection.Contains("x86")) {
+ $global:buildPathSizeId = "x86"
+ $global:build32or64 = "32"
+ $global:vsEnvironment += " x86"
+ $global:vsBuildTarget = """Debug|Win32"""
+ } elseif ($vsSelection.Contains("x64")) {
+ $global:buildPathSizeId = "x64"
+ $global:build32or64 = "64"
+ $global:vsEnvironment += " amd64"
+ $global:vsBuildTarget = """Debug|x64"""
+ # Promote CMAKE generator to 64 bit variant
+ $global:cmakeGenerator += " Win64"
+ } else {
+ Write-Host "Studio selection must contain x86 or x64"
+ exit
+ }
+}
+
+#############################
+# When the user presses 'select' then this function handles it.
+# Return the visual studio and architecture.
+# Close the form.
+#
+function SelectVisualStudio {
if ($DropDown.SelectedItem -ne $null) {
- $global:vsVersion = $DropDown.SelectedItem.ToString()
- if ($global:vsVersion -eq 'Visual Studio 2012') {
- $global:cmakeGenerator = "Visual Studio 11"
- $global:vsSubdir = "msvc11"
- $global:cmakeCompiler = "-vc110"
- } else {
- if ($global:vsVersion -eq 'Visual Studio 2010') {
- $global:cmakeGenerator = "Visual Studio 10"
- $global:vsSubdir = "msvc10"
- $global:cmakeCompiler = "-vc100"
- } else {
- if ($global:vsVersion -eq 'Visual Studio 2008') {
- $global:cmakeGenerator = "Visual Studio 9 2008"
- $global:vsSubdir = "msvc9"
- $global:cmakeCompiler = "-vc90"
- } else {
- Write-Host "Visual Studio must be 2008, 2010, or 2012"
- exit
- }
- }
- }
- $Form.Close()
+ $vsVersion = $DropDown.SelectedItem.ToString()
+
+ ParseStudioSelection $vsVersion
+
+ $Form.Close() 2> $null
Write-Host "Selected generator: $global:cmakeGenerator"
}
}
@@ -432,29 +480,24 @@ function SelectVisualStudioVersion { $Form.width = 350
$Form.height = 150
- $Form.Text = "Select Visual Studio Version"
+ $Form.Text = "Select Visual Studio Version and platform"
$DropDown = new-object System.Windows.Forms.ComboBox
$DropDown.Location = new-object System.Drawing.Size(120,10)
$DropDown.Size = new-object System.Drawing.Size(150,30)
ForEach ($Item in $global:VsVersionCmakeChoiceList) {
- $DropDown.Items.Add($Item)
+ [void] $DropDown.Items.Add($Item)
}
$DropDown.SelectedIndex = 0
$Form.Controls.Add($DropDown)
-# $DropDownLabel.Location = new-object System.Drawing.Size(10,10)
-# $DropDownLabel.size = new-object System.Drawing.Size(100,20)
-# $DropDownLabel.Text = ""
-# $Form.Controls.Add($DropDownLabel)
-
$Button = new-object System.Windows.Forms.Button
$Button.Location = new-object System.Drawing.Size(120,50)
$Button.Size = new-object System.Drawing.Size(120,20)
$Button.Text = "Select"
- $Button.Add_Click({Return-DropDown})
+ $Button.Add_Click({SelectVisualStudio})
$form.Controls.Add($Button)
$Form.Add_Shown({$Form.Activate()})
@@ -476,232 +519,148 @@ function SelectVisualStudioVersion { [System.Reflection.Assembly]::LoadWithPartialName("System.Drawing") | Out-Null
#############################
-# User dialog to select a version of Visual Studio as CMake generator
-#
-SelectVisualStudioVersion
-
-#############################
-# User dialog to get optional 32-bit boost, proton, and build paths
+# Get command line args
#
-$boost32 = Select-Folder -message "Select 32-bit BOOST_ROOT folder for $global:vsVersion build. Press CANCEL to skip 32-bit processing."
-
-$defined32 = ($boost32 -ne $null) -and ($boost32 -ne '')
-if ($defined32) {
- $found = SanityCheckBoostPath $boost32
- if (! $found) {
+if ($args.Length -ge 1) {
+ if (($args[0].Contains("help")) -or ($args[0].Contains("-h"))) {
+ Usage
exit
}
+
+ $vsVer = $args[0]
+ ParseStudioSelection $vsVer
}
-$make32 = $false
-if ($defined32) {
-
- $proton32folder = Select-Folder -message "Select 32-bit Proton install folder for $global:vsVersion build."
-
- $found = ($proton32folder -ne $null) -and ($proton32folder -ne '')
- if ($found) {
- $found = SanityCheckProtonInstallPath $proton32folder
- }
- if ($found) {
- $proton32cmake = """-DPROTON_ROOT=$proton32folder"""
- } else {
- $proton32cmake = ""
- }
-
- $build32 = Select-Folder -message "Select 32-bit QPID_BUILD_ROOT folder for $global:vsVersion build." -path $projRoot
-
- $found = ($build32 -ne $null) -and ($build32 -ne '')
- if (! $found) {
- Write-Host "You must select a build root folder for 32-bit builds"
- exit
- }
- $found = SanityCheckBuildPath $build32
- if ($found) {
- Write-Host "Directory ""$build32"" is already set by CMake. CMake will not be re-run."
- } else {
- $make32 = AskYesOrNo "Run CMake in $build32 ?" "32-Bit Builds - Choose CMake run or not"
- }
+if ($args.Length -ge 2) {
+ $global:boostRootPath = $args[1]
}
#############################
-# User dialog to get optional 64-bit boost, proton, and build paths
+# User dialog to select a version of Visual Studio as CMake generator
#
-$boost64 = Select-Folder -message "Select 64-bit BOOST_ROOT folder for $global:vsVersion build. Press CANCEL to skip 64-bit processing."
-
-$defined64 = ($boost64 -ne $null) -and ($boost64 -ne '')
-if ($defined64) {
- $found = SanityCheckBoostPath $boost64
- if (! $found) {
- exit
- }
+if ($global:vsVersion -eq '') {
+ SelectVisualStudioVersion
}
-$make64 = $false
-if ($defined64) {
- $proton64folder = Select-Folder -message "Select 64-bit Proton install folder for $global:vsVersion build."
-
- $found = ($proton64folder -ne $null) -and ($proton64folder -ne '')
- if ($found) {
- $found = SanityCheckProtonInstallPath $proton64folder
- }
- if ($found) {
- $proton64cmake = """-DPROTON_ROOT=$proton64folder"""
- } else {
- $proton64cmake = ""
- }
-
- $build64 = Select-Folder -message "Select 64-bit QPID_BUILD_ROOT folder for $global:vsVersion build." -path $projRoot
+#############################
+# User dialog to get boost paths
+#
+if ($global:boostRootPath -eq '') {
+ $global:boostRootPath = Select-Folder -message "Select BOOST_ROOT folder for $global:vsSelectedOption build. Press CANCEL to quit"
+}
- $found = ($build64 -ne $null) -and ($build64 -ne '')
+#############################
+# Decide to run cmake or not.
+# If the build directory is absent the run cmake
+# If the build directory already exists then it's the user's choice
+#
+$make = 0
+$defined = ($global:boostRootPath -ne $null) -and ($global:boostRootPath -ne '')
+if ($defined) {
+ $found = SanityCheckBoostPath $global:boostRootPath
if (! $found) {
- Write-Host "You must select a build root folder for 64-bit builds"
exit
}
- $found = SanityCheckBuildPath $build64
+
+ $build = Join-Path $projRoot "build_${global:vsShortName}_${global:buildPathSizeId}"
+ $install = Join-Path $projRoot "install_${global:vsShortName}_${global:buildPathSizeId}"
+
+ $found = SanityCheckBuildPath $build
if ($found) {
- Write-Host "Directory ""$build64"" is already set by CMake. CMake will not be re-run."
+ $make = AskYesOrNo "build directory ""$build"" appears to have run cmake already. Run cmake again?"
} else {
- $make64 = AskYesOrNo "Run CMake in $build64 ?" "64-Bit Builds - Choose CMake run or not"
+ $make = 1
}
}
-#############################
-# Conditionally run CMake
-#
-# 32-bit X86
-#
-if ($make32) {
- cd "$build32"
- $global:cmakeCommandLine32 = "CMake -G ""$global:cmakeGenerator"" ""-DBUILD_DOCS=No"" ""-DCMAKE_INSTALL_PREFIX=$build32/install"" ""-DBoost_COMPILER=$global:cmakeCompiler"" ""-DBOOST_ROOT=$boost32"" $proton32cmake $cppDir"
- Write-Host "Running 32-bit CMake in $build32 : $global:cmakeCommandLine32"
- CMake -G "$global:cmakeGenerator" "-DBUILD_DOCS=No" "-DCMAKE_INSTALL_PREFIX=$build32/install" "-DBoost_COMPILER=$global:cmakeCompiler" "-DBOOST_ROOT=$boost32" $proton32cmake $cppDir
-} else {
- Write-Host "Skipped 32-bit CMake."
+if (! $make) {
+ exit
}
+#############################
+# run CMake
#
-# 64-bit X64
-#
-if ($make64) {
- cd "$build64"
- Write-Host "Running 64-bit CMake in $build64"
- $global:cmakeCommandLine64 = "CMake -G ""$global:cmakeGenerator Win64"" ""-DBUILD_DOCS=No"" ""-DCMAKE_INSTALL_PREFIX=$build64/install"" ""-DBoost_COMPILER=$global:cmakeCompiler"" ""-DBOOST_ROOT=$boost64"" $proton64cmake $cppDir"
- Write-Host $global:cmakeCommandLine64
- Write-Host ""
- CMake -G "$global:cmakeGenerator Win64" "-DBUILD_DOCS=No" "-DCMAKE_INSTALL_PREFIX=$build64/install" "-DBoost_COMPILER=$global:cmakeCompiler" "-DBOOST_ROOT=$boost64" $proton64cmake $cppDir
-} else {
- Write-Host "Skipped 64-bit CMake."
+if(!(Test-Path -Path $build)) {
+ New-Item -ItemType directory -Path $build
}
+cd "$build"
+$global:cmakeCommandLine = "CMake -G ""$global:cmakeGenerator"" "
+$global:cmakeCommandLine += """-DBUILD_DOCS=No"" "
+$global:cmakeCommandLine += """-DCMAKE_INSTALL_PREFIX=$install"" "
+$global:cmakeCommandLine += """-DBoost_COMPILER=$global:cmakeCompiler"" "
+$global:cmakeCommandLine += """-DBOOST_ROOT=$global:boostRootPath"" "
+$global:cmakeCommandLine += """-DINSTALL_QMFGEN=No"" "
+$global:cmakeCommandLine += $cppDir
+Write-Host "Running CMake in $build : $global:cmakeCommandLine"
+& cmd /c "$global:cmakeCommandLine 2>&1"
+
#############################
# Emit scripts
#
-# 32-bit scripts
-#
-if ($defined32) {
-
- Write-Host "Writing 32-bit scripts..."
-
- ###########
- # Powershell script to launch org.apache.qpid.messaging.sln
- #
- WriteDotnetBindingSlnLauncherPs1 -slnName "org.apache.qpid.messaging.sln" `
- -boostRoot "$boost32" `
- -buildRoot "$build32" `
- -cppDir "$cppDir" `
- -vsPlatform "x86" `
- -nBits "32" `
- -outfileName "start-devenv-messaging-$global:vsSubdir-x86-32bit.ps1" `
- -studioVersion "$global:vsVersion" `
- -studioSubdir "$global:vsSubdir" `
- -protonRoot "$proton32folder"
-
-
- ###########
- # Batch script (that you doubleclick) to launch powershell script
- # that launches org.apache.qpid.messaging.sln.
- #
- WriteDotnetBindingSlnLauncherBat -slnName "org.apache.qpid.messaging.sln" `
- -buildRoot "$build32" `
- -vsPlatform "x86" `
- -nBits "32" `
- -psScriptName "start-devenv-messaging-$global:vsSubdir-x86-32bit.ps1" `
- -outfileName "start-devenv-messaging-$global:vsSubdir-x86-32bit.bat" `
- -studioVersion "$global:vsVersion" `
- -studioSubdir "$global:vsSubdir"
-
- ###########
- # Batch script (that you CALL from a command prompt)
- # to establish the org.apache.qpid.messaging.sln build environment.
- #
- WriteDotnetBindingEnvSetupBat -slnName "org.apache.qpid.messaging.sln" `
- -boostRoot "$boost32" `
- -buildRoot "$build32" `
- -vsPlatform "x86" `
- -nBits "32" `
- -outfileName "setenv-messaging-$global:vsSubdir-x86-32bit.bat" `
- -studioVersion "$global:vsVersion" `
- -studioSubdir "$global:vsSubdir" `
- -cmakeLine "$global:cmakeCommandLine32" `
- -protonRoot "$proton32folder"
-
-} else {
- Write-Host "Skipped writing 32-bit scripts."
-}
-
-#############################
-# 64-bit scripts
-#
-if ($defined64) {
-
- Write-Host "Writing 64-bit scripts..."
-
- ###########
- # Powershell script to launch org.apache.qpid.messaging.sln
- #
- WriteDotnetBindingSlnLauncherPs1 -slnName "org.apache.qpid.messaging.sln" `
- -boostRoot "$boost64" `
- -buildRoot "$build64" `
- -cppDir "$cppDir" `
- -vsPlatform "x64" `
- -nBits "64" `
- -outfileName "start-devenv-messaging-$global:vsSubdir-x64-64bit.ps1" `
- -studioVersion "$global:vsVersion" `
- -studioSubdir "$global:vsSubdir"
-
-
- ###########
- # Batch script (that you doubleclick) to launch powershell script
- # that launches org.apache.qpid.messaging.sln.
- #
- WriteDotnetBindingSlnLauncherBat -slnName "org.apache.qpid.messaging.sln" `
- -buildRoot "$build64" `
- -vsPlatform "x64" `
- -nBits "64" `
- -psScriptName "start-devenv-messaging-$global:vsSubdir-x64-64bit.ps1" `
- -outfileName "start-devenv-messaging-$global:vsSubdir-x64-64bit.bat" `
- -studioVersion "$global:vsVersion" `
- -studioSubdir "$global:vsSubdir" `
- -protonRoot "$proton64folder"
-
- ###########
- # Batch script (that you CALL from a command prompt)
- # to establish the org.apache.qpid.messaging.sln build environment.
- #
- WriteDotnetBindingEnvSetupBat -slnName "org.apache.qpid.messaging.sln" `
- -boostRoot "$boost64" `
- -buildRoot "$build64" `
- -vsPlatform "x64" `
- -nBits "64" `
- -outfileName "setenv-messaging-$global:vsSubdir-x64-64bit.bat" `
+Write-Host "Writing helper scripts..."
+
+###########
+# Powershell script to launch org.apache.qpid.messaging.sln
+#
+WriteDotnetBindingSlnLauncherPs1 -slnName "org.apache.qpid.messaging.sln" `
+ -boostRoot "$global:boostRootPath" `
+ -buildRoot "$build" `
+ -cppDir "$cppDir" `
+ -vsPlatform $global:buildPathSizeId `
+ -nBits $global:build32or64 `
+ -outfileName "start-devenv-messaging-$global:vsSubdir-$global:buildPathSizeId-$global:build32or64-bit.ps1" `
+ -studioVersion "$global:vsVersion" `
+ -studioSubdir "$global:vsSubdir"
+
+###########
+# Batch script (that you doubleclick) to launch powershell script
+# that launches org.apache.qpid.messaging.sln.
+#
+WriteDotnetBindingSlnLauncherBat -slnName "org.apache.qpid.messaging.sln" `
+ -buildRoot "$build" `
+ -vsPlatform $global:buildPathSizeId `
+ -nBits $global:build32or64 `
+ -psScriptName "start-devenv-messaging-$global:vsSubdir-$global:buildPathSizeId-$global:build32or64-bit.ps1" `
+ -outfileName "start-devenv-messaging-$global:vsSubdir-$global:buildPathSizeId-$global:build32or64-bit.bat" `
-studioVersion "$global:vsVersion" `
- -studioSubdir "$global:vsSubdir" `
- -cmakeLine "$global:cmakeCommandLine64" `
- -protonRoot "$proton64folder"
-
-} else {
- Write-Host "Skipped writing 64-bit scripts."
-}
+ -studioSubdir "$global:vsSubdir"
+
+###########
+# Batch script (that you CALL from a command prompt)
+# to establish the org.apache.qpid.messaging.sln build environment.
+#
+WriteDotnetBindingEnvSetupBat -slnName "org.apache.qpid.messaging.sln" `
+ -boostRoot "$global:boostRootPath" `
+ -buildRoot "$build" `
+ -vsPlatform $global:buildPathSizeId `
+ -nBits $global:build32or64 `
+ -outfileName "setenv-messaging-$global:vsSubdir-$global:buildPathSizeId-$global:build32or64-bit.bat" `
+ -studioVersion "$global:vsVersion" `
+ -studioSubdir "$global:vsSubdir" `
+ -cmakeLine "$global:cmakeCommandLine"
+
+###########
+# Batch script to re-run cmake
+#
+WriteCmakeRerunnerBat -slnName "org.apache.qpid.messaging.sln" `
+ -boostRoot ""$global:boostRootPath"" `
+ -buildRoot "$build" `
+ -vsPlatform $global:buildPathSizeId `
+ -nBits $global:build32or64 `
+ -outfileName "run-cmake.bat" `
+ -studioVersion "$global:vsVersion" `
+ -studioSubdir "$global:vsSubdir" `
+ -cmakeLine "$global:cmakeCommandLine"
+
+###########
+# Batch script to do command line "make install"
+#
+WriteMakeInstallBat -buildRoot "$build" `
+ -outfileName "make-install.bat" `
+ -varfileName "setenv-messaging-$global:vsSubdir-$global:buildPathSizeId-$global:build32or64-bit.bat" `
+ -vsEnvironment $global:vsEnvironment `
+ -vsBuildTarget $global:vsBuildTarget
#############################
# Pause on exit. If user ran this script through a graphical launch and there's
@@ -709,4 +668,4 @@ if ($defined64) { # gives him a chance to figure it out.
#
Write-Host "Press any key to continue ..."
-$x = $host.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown")
+[void] $host.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown")
diff --git a/qpid/cpp/docs/man/qpidd.1 b/qpid/cpp/docs/man/qpidd.1 index f48712fad8..ac272552e7 100644 --- a/qpid/cpp/docs/man/qpidd.1 +++ b/qpid/cpp/docs/man/qpidd.1 @@ -17,7 +17,7 @@ .\" under the License. .\" -.TH QPIDD "1" "December 2013" "qpidd (qpid-cpp) version 0.27" "User Commands" +.TH QPIDD "1" "December 2013" "qpidd (qpid-cpp) version 0.29" "User Commands" .SH NAME qpidd \- the Qpid AMQP Message Broker Daemon diff --git a/qpid/cpp/include/qpid/messaging/exceptions.h b/qpid/cpp/include/qpid/messaging/exceptions.h index a9fa8e2506..6c65f4a889 100644 --- a/qpid/cpp/include/qpid/messaging/exceptions.h +++ b/qpid/cpp/include/qpid/messaging/exceptions.h @@ -151,6 +151,16 @@ struct QPID_MESSAGING_CLASS_EXTERN SessionError : public MessagingException QPID_MESSAGING_EXTERN SessionError(const std::string&); }; +/** + * Thrown to indicate that the sesion was closed by this client (probably in + * a different thread) whilst we were waiting on it. This is not really an + * error condition but there is no other way to return this. + */ +struct QPID_MESSAGING_CLASS_EXTERN SessionClosed : public SessionError +{ + QPID_MESSAGING_EXTERN SessionClosed(); +}; + struct QPID_MESSAGING_CLASS_EXTERN TransactionError : public SessionError { QPID_MESSAGING_EXTERN TransactionError(const std::string&); diff --git a/qpid/cpp/src/qpid/client/amqp0_10/SessionImpl.cpp b/qpid/cpp/src/qpid/client/amqp0_10/SessionImpl.cpp index e43abb1a56..057e752c90 100644 --- a/qpid/cpp/src/qpid/client/amqp0_10/SessionImpl.cpp +++ b/qpid/cpp/src/qpid/client/amqp0_10/SessionImpl.cpp @@ -368,6 +368,8 @@ bool SessionImpl::nextReceiver(qpid::messaging::Receiver& receiver, qpid::messag throw qpid::messaging::UnauthorizedAccess(e.what()); } catch (const qpid::SessionException& e) { throw qpid::messaging::SessionError(e.what()); + } catch (const qpid::ClosedException&) { + throw qpid::messaging::SessionClosed(); } catch (const qpid::ConnectionException& e) { throw qpid::messaging::ConnectionError(e.what()); } catch (const qpid::ChannelException& e) { diff --git a/qpid/cpp/src/qpid/linearstore/ISSUES b/qpid/cpp/src/qpid/linearstore/ISSUES index a9908e882e..ccadefc20c 100644 --- a/qpid/cpp/src/qpid/linearstore/ISSUES +++ b/qpid/cpp/src/qpid/linearstore/ISSUES @@ -47,8 +47,6 @@ Current/pending: svn r.1558592 2014-01-15 fixes an issue with using /dev/random as a source of random numbers for Journal serial numbers. svn r.1558913 2014-01-16 replaces use of /dev/urandom with several calls to rand() to construct a 64-bit random number. * Recommend rebuilding and testing for performance again with these two fixes. Marked POST. -# - 1036026 [LinearStore] Qpid linear store unable to create durable queue - framing-error: Queue <q-name>: create() failed: jexception 0x0000 - UNABLE TO REPRODUCE - but Frantizek has additional info - 1039522 Qpid crashes while recovering from linear store around apid::linearstore::journal::JournalFile::getFqFileName() including enq_rec::decode() threw JERR_JREC_BAD_RECTAIL * Possible dup of 1039525 * May be fixed by QPID-5483 - waiting for needinfo, recommend rebuilding with QPID-5483 fix and re-testing. Marked POST. @@ -56,18 +54,6 @@ Current/pending: * Possible dup of 1039522 * May be fixed by QPID-5483 - waiting for needinfo, recommend rebuilding with QPID-5483 fix and re-testing. Marked POST. # - 1049870 [LinearStore] auto-delete property does not survive restart -# 5480 1053749 [linearstore] Recovery of store failure with "JERR_MAP_NOTFOUND: Key not found in map." error message - svn r.1564877 2014-02-05: Proposed fix - * Probability: 6 of 600 (1.0%) using tx-test-soak.sh - * If broker is started a second time after failure, it starts correctly and test completes ok. - * Problem: File is being recycled to EFP with still-locked enqueues in it (ie dequeued transactionally). - * Problem: Record alignment check writes filler records to wrong file when decoding bad record moves across a file boundary - * Test of fix failed on RHEL-7 -# - 1064181 [linearstore] Qpidd closes transactional client session&connection with async_dequeue() failed - * jexception 0x010b LinearFileController::getCurrentSerial() threw JERR_NULL -# - 1064230 [linearstore] Qpidd linearstore recovery sometimes fail to recover messages with recoverMessages() failed - * jexception 0x0701 RecoveryManager::readNextRemainingRecord() threw JERR_JREC_BADRECTAIL - * possible dup of 1063700 Fixed/closed (in commit order): =============================== @@ -104,9 +90,24 @@ NO-JIRA - Added missing Apache copyright/license text 5479 1053701 [linearstore] Using recovered store results in "JERR_JNLF_FILEOFFSOVFL: Attempted to increase submitted offset past file size. (JournalFile::submittedDblkCount)" error message * Probability: 2 of 600 (0.3%) using tx-test-soak.sh * Fixed by checkin for QPID-5480, no longer able to reproduce. VERIFIED + 5480 1053749 [linearstore] Recovery of store failure with "JERR_MAP_NOTFOUND: Key not found in map." error message + svn r.1564877 2014-02-05: Proposed fix + * Probability: 6 of 600 (1.0%) using tx-test-soak.sh + * If broker is started a second time after failure, it starts correctly and test completes ok. + * Problem: File is being recycled to EFP with still-locked enqueues in it (ie dequeued transactionally). + * Problem: Record alignment check writes filler records to wrong file when decoding bad record moves across a file boundary 5603 1063700 [linearstore] broker restart fails under stress test svn r.1574513 2014-03-05: Proposed fix. POST * jexception 0x0701 RecoveryManager::readNextRemainingRecord() threw JERR_JREC_BADRECTAIL + 5607 1064181 [linearstore] Qpidd closes transactional client session&connection with async_dequeue() failed + svn r.1575009 2014-03-06 Proposed fix. POST + * jexception 0x010b LinearFileController::getCurrentSerial() threw JERR_NULL + - 1064230 [linearstore] Qpidd linearstore recovery sometimes fail to recover messages with recoverMessages() failed + * jexception 0x0701 RecoveryManager::readNextRemainingRecord() threw JERR_JREC_BADRECTAIL + * possible dup of 1063700 + - 1036026 [LinearStore] Qpid linear store unable to create durable queue - framing-error: Queue <q-name>: create() failed: jexception 0x0000 + * UNABLE TO REPRODUCE - but Frantizek has additional info + * Retested after checkin 1575009, problem solved. VERIFIED Ordered checkin list: ===================== @@ -135,6 +136,7 @@ no. svn r Q-JIRA RHBZ Date 19. 1564893 5361 - 2014-02-05 20. 1564935 5361 - 2014-02-05 21. 1574513 5603 1063700 2014-03-05 +22. 1575009 5607 1064181 2014-03-06 See above sections for details on these checkins. diff --git a/qpid/cpp/src/qpid/linearstore/journal/jdir.cpp b/qpid/cpp/src/qpid/linearstore/journal/jdir.cpp index 896f44ceff..36f180c21f 100644 --- a/qpid/cpp/src/qpid/linearstore/journal/jdir.cpp +++ b/qpid/cpp/src/qpid/linearstore/journal/jdir.cpp @@ -101,17 +101,9 @@ jdir::clear_dir(const std::string& dirname/*, const std::string& */ , const bool create_flag) { - DIR* dir = ::opendir(dirname.c_str()); - if (!dir) - { - if (errno == 2 && create_flag) // ENOENT (No such file or dir) - { - create_dir(dirname); - return; - } - std::ostringstream oss; - oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); - throw jexception(jerrno::JERR_JDIR_OPENDIR, oss.str(), "jdir", "clear_dir"); + DIR* dir = open_dir(dirname, "clear_dir", true); + if (!dir && create_flag) { + create_dir(dirname); } //#ifndef RHM_JOWRITE struct dirent* entry; @@ -161,13 +153,7 @@ jdir::push_down(const std::string& dirname, const std::string& target_dir/*, con { std::string bak_dir_name = create_bak_dir(dirname/*, bak_dir_base*/); - DIR* dir = ::opendir(dirname.c_str()); - if (!dir) - { - std::ostringstream oss; - oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); - throw jexception(jerrno::JERR_JDIR_OPENDIR, oss.str(), "jdir", "push_down"); - } + DIR* dir = open_dir(dirname, "push_down", false); // Copy contents of targetDirName into bak dir struct dirent* entry; while ((entry = ::readdir(dir)) != 0) @@ -251,60 +237,49 @@ jdir::delete_dir(const std::string& dirname, bool children_only) { struct dirent* entry; struct stat s; - DIR* dir = ::opendir(dirname.c_str()); - if (!dir) - { - if (errno == ENOENT) // dir does not exist. - return; - - std::ostringstream oss; - oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); - throw jexception(jerrno::JERR_JDIR_OPENDIR, oss.str(), "jdir", "delete_dir"); - } - else + DIR* dir = open_dir(dirname, "delete_dir", true); // true = allow dir does not exist, return 0 + if (!dir) return; + while ((entry = ::readdir(dir)) != 0) { - while ((entry = ::readdir(dir)) != 0) + // Ignore . and .. + if (std::strcmp(entry->d_name, ".") != 0 && std::strcmp(entry->d_name, "..") != 0) { - // Ignore . and .. - if (std::strcmp(entry->d_name, ".") != 0 && std::strcmp(entry->d_name, "..") != 0) + std::string full_name(dirname + "/" + entry->d_name); + if (::lstat(full_name.c_str(), &s)) { - std::string full_name(dirname + "/" + entry->d_name); - if (::lstat(full_name.c_str(), &s)) - { - ::closedir(dir); - std::ostringstream oss; - oss << "stat: file=\"" << full_name << "\"" << FORMAT_SYSERR(errno); - throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "jdir", "delete_dir"); - } - if (S_ISREG(s.st_mode) || S_ISLNK(s.st_mode)) // This is a file or slink - { - if(::unlink(full_name.c_str())) - { - ::closedir(dir); - std::ostringstream oss; - oss << "unlink: file=\"" << entry->d_name << "\"" << FORMAT_SYSERR(errno); - throw jexception(jerrno::JERR_JDIR_UNLINK, oss.str(), "jdir", "delete_dir"); - } - } - else if (S_ISDIR(s.st_mode)) // This is a dir - { - delete_dir(full_name); - } - else // all other types, throw up! + ::closedir(dir); + std::ostringstream oss; + oss << "stat: file=\"" << full_name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "jdir", "delete_dir"); + } + if (S_ISREG(s.st_mode) || S_ISLNK(s.st_mode)) // This is a file or slink + { + if(::unlink(full_name.c_str())) { ::closedir(dir); std::ostringstream oss; - oss << "file=\"" << entry->d_name << "\" is not a dir, file or slink."; - oss << " (mode=0x" << std::hex << s.st_mode << std::dec << ")"; - throw jexception(jerrno::JERR_JDIR_BADFTYPE, oss.str(), "jdir", "delete_dir"); + oss << "unlink: file=\"" << entry->d_name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_UNLINK, oss.str(), "jdir", "delete_dir"); } } + else if (S_ISDIR(s.st_mode)) // This is a dir + { + delete_dir(full_name); + } + else // all other types, throw up! + { + ::closedir(dir); + std::ostringstream oss; + oss << "file=\"" << entry->d_name << "\" is not a dir, file or slink."; + oss << " (mode=0x" << std::hex << s.st_mode << std::dec << ")"; + throw jexception(jerrno::JERR_JDIR_BADFTYPE, oss.str(), "jdir", "delete_dir"); + } } + } // FIXME: Find out why this fails with false alarms/errors from time to time... // While commented out, there is no error capture from reading dir entries. // check_err(errno, dir, dirname, "delete_dir"); - } // Now dir is empty, close and delete it close_dir(dir, dirname, "delete_dir"); @@ -321,14 +296,8 @@ jdir::delete_dir(const std::string& dirname, bool children_only) std::string jdir::create_bak_dir(const std::string& dirname) { - DIR* dir = ::opendir(dirname.c_str()); + DIR* dir = open_dir(dirname, "create_bak_dir", false); long dir_num = 0L; - if (!dir) - { - std::ostringstream oss; - oss << "dir=\"" << dirname << "\"" << FORMAT_SYSERR(errno); - throw jexception(jerrno::JERR_JDIR_OPENDIR, oss.str(), "jdir", "create_bak_dir"); - } struct dirent* entry; while ((entry = ::readdir(dir)) != 0) { @@ -407,25 +376,23 @@ void jdir::read_dir(const std::string& name, std::vector<std::string>& dir_list, const bool incl_dirs, const bool incl_files, const bool incl_links, const bool return_fqfn) { struct stat s; if (is_dir(name)) { - DIR* dir = ::opendir(name.c_str()); - if (dir != 0) { - struct dirent* entry; - while ((entry = ::readdir(dir)) != 0) { - if (std::strcmp(entry->d_name, ".") != 0 && std::strcmp(entry->d_name, "..") != 0) { // Ignore . and .. - std::string full_name(name + "/" + entry->d_name); - if (::stat(full_name.c_str(), &s)) - { - ::closedir(dir); - std::ostringstream oss; - oss << "stat: file=\"" << full_name << "\"" << FORMAT_SYSERR(errno); - throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "jdir", "delete_dir"); - } - if ((S_ISREG(s.st_mode) && incl_files) || (S_ISDIR(s.st_mode) && incl_dirs) || (S_ISLNK(s.st_mode) && incl_links)) { - if (return_fqfn) { - dir_list.push_back(name + "/" + entry->d_name); - } else { - dir_list.push_back(entry->d_name); - } + DIR* dir = open_dir(name, "read_dir", false); + struct dirent* entry; + while ((entry = ::readdir(dir)) != 0) { + if (std::strcmp(entry->d_name, ".") != 0 && std::strcmp(entry->d_name, "..") != 0) { // Ignore . and .. + std::string full_name(name + "/" + entry->d_name); + if (::stat(full_name.c_str(), &s)) + { + ::closedir(dir); + std::ostringstream oss; + oss << "stat: file=\"" << full_name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_STAT, oss.str(), "jdir", "delete_dir"); + } + if ((S_ISREG(s.st_mode) && incl_files) || (S_ISDIR(s.st_mode) && incl_dirs) || (S_ISLNK(s.st_mode) && incl_links)) { + if (return_fqfn) { + dir_list.push_back(name + "/" + entry->d_name); + } else { + dir_list.push_back(entry->d_name); } } } @@ -457,6 +424,21 @@ jdir::close_dir(DIR* dir, const std::string& dir_name, const std::string& fn_nam } } +DIR* +jdir::open_dir(const std::string& dir_name, const std::string& fn_name, const bool test_enoent) +{ + DIR* dir = ::opendir(dir_name.c_str()); + if (!dir) { + if (test_enoent && errno == ENOENT) { + return 0; + } + std::ostringstream oss; + oss << "dir=\"" << dir_name << "\"" << FORMAT_SYSERR(errno); + throw jexception(jerrno::JERR_JDIR_OPENDIR, oss.str(), "jdir", fn_name); + } + return dir; +} + std::ostream& operator<<(std::ostream& os, const jdir& jdir) { diff --git a/qpid/cpp/src/qpid/linearstore/journal/jdir.h b/qpid/cpp/src/qpid/linearstore/journal/jdir.h index 86b16f8545..59f21ce499 100644 --- a/qpid/cpp/src/qpid/linearstore/journal/jdir.h +++ b/qpid/cpp/src/qpid/linearstore/journal/jdir.h @@ -353,6 +353,8 @@ namespace journal { * \exception jerrno::JERR_JDIR_CLOSEDIR The directory handle could not be closed. */ static void close_dir(DIR* dir, const std::string& dir_name, const std::string& fn_name); + + static DIR* open_dir(const std::string& dir_name, const std::string& fn_name, const bool test_enoent); }; }}} diff --git a/qpid/cpp/src/qpid/messaging/amqp/ConnectionContext.cpp b/qpid/cpp/src/qpid/messaging/amqp/ConnectionContext.cpp index 6d182f40f8..abf7e58bfe 100644 --- a/qpid/cpp/src/qpid/messaging/amqp/ConnectionContext.cpp +++ b/qpid/cpp/src/qpid/messaging/amqp/ConnectionContext.cpp @@ -572,7 +572,7 @@ void ConnectionContext::checkClosed(boost::shared_ptr<SessionContext> ssn) pn_session_close(ssn->session); throw qpid::messaging::SessionError(text.str()); } else if ((pn_session_state(ssn->session) & IS_CLOSED) == IS_CLOSED) { - throw qpid::messaging::SessionError("Session has ended"); + throw qpid::messaging::SessionClosed(); } } diff --git a/qpid/cpp/src/qpid/messaging/exceptions.cpp b/qpid/cpp/src/qpid/messaging/exceptions.cpp index 5d2683fffe..5054fdc682 100644 --- a/qpid/cpp/src/qpid/messaging/exceptions.cpp +++ b/qpid/cpp/src/qpid/messaging/exceptions.cpp @@ -47,6 +47,8 @@ SendError::SendError(const std::string& msg) : SenderError(msg) {} TargetCapacityExceeded::TargetCapacityExceeded(const std::string& msg) : SendError(msg) {} SessionError::SessionError(const std::string& msg) : MessagingException(msg) {} +SessionClosed::SessionClosed() : SessionError("Session Closed") {} + TransactionError::TransactionError(const std::string& msg) : SessionError(msg) {} TransactionAborted::TransactionAborted(const std::string& msg) : TransactionError(msg) {} UnauthorizedAccess::UnauthorizedAccess(const std::string& msg) : SessionError(msg) {} diff --git a/qpid/cpp/src/tests/legacystore/CMakeLists.txt b/qpid/cpp/src/tests/legacystore/CMakeLists.txt index 85cc4be5c7..7986464eb5 100644 --- a/qpid/cpp/src/tests/legacystore/CMakeLists.txt +++ b/qpid/cpp/src/tests/legacystore/CMakeLists.txt @@ -36,30 +36,6 @@ if (BUILD_TESTING_UNITTESTS) set (qpid_test_boost_libs ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} ${Boost_SYSTEM_LIBRARY}) -# -# define_legacystore_test -# macro to accept the name of a single source file and to create a -# unit test executable that runs the source. -# -MACRO (define_legacystore_test theSourceFile) -add_executable (legacystore_${theSourceFile} - ${theSourceFile} - unit_test - ${platform_test_additions}) -target_link_libraries (legacystore_${theSourceFile} - ${qpid_test_boost_libs} - qpidmessaging qpidtypes qpidbroker qpidcommon legacystore_shared) -set_target_properties (legacystore_${theSourceFile} PROPERTIES COMPILE_DEFINITIONS _IN_QPID_BROKER) -remember_location(legacystore_${theSourceFile}) - -add_test (legacystore_${theSourceFile} ${test_wrap} --boost-test -- ${legacystore_${theSourceFile}_LOCATION}) -ENDMACRO (define_legacystore_test) - -define_legacystore_test (SimpleTest) -define_legacystore_test (OrderingTest) -define_legacystore_test (TransactionalTest) -define_legacystore_test (TwoPhaseCommitTest) - # Journal tests MACRO (define_journal_test mainSourceFile) if ("${ARGV1}" STREQUAL "LONG") diff --git a/qpid/cpp/src/tests/legacystore/OrderingTest.cpp b/qpid/cpp/src/tests/legacystore/OrderingTest.cpp deleted file mode 100644 index 74a9db1c73..0000000000 --- a/qpid/cpp/src/tests/legacystore/OrderingTest.cpp +++ /dev/null @@ -1,171 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "unit_test.h" -#include "MessageUtils.h" - -#include "qpid/broker/Queue.h" -#include "qpid/broker/RecoveryManagerImpl.h" -#include "qpid/broker/PersistableObject.h" -#include "qpid/framing/AMQHeaderBody.h" -#include "qpid/legacystore/MessageStoreImpl.h" -#include "qpid/log/Logger.h" -#include "qpid/sys/Timer.h" - -#include <iostream> - -using namespace qpid; -using namespace qpid::broker; -using namespace qpid::framing; -using namespace mrg::msgstore; - -qpid::broker::Broker::Options opts; -qpid::broker::Broker br(opts); - -QPID_AUTO_TEST_SUITE(OrderingTest) - -#define SET_LOG_LEVEL(level) \ - qpid::log::Options opts(""); \ - opts.selectors.clear(); \ - opts.selectors.push_back(level); \ - qpid::log::Logger::instance().configure(opts); - -const std::string test_filename("OrderingTest"); -const char* tdp = getenv("TMP_DATA_DIR"); -const std::string test_dir(tdp && strlen(tdp) > 0 ? tdp : "/var/tmp/OrderingTest"); - -// === Helper fns === - -const std::string name("OrderingQueue"); -std::auto_ptr<MessageStoreImpl> store; -QueueRegistry queues; -Queue::shared_ptr queue; -std::queue<Uuid> ids; - -class TestConsumer : public Consumer -{ - public: - - TestConsumer(Queue::shared_ptr q, std::queue<Uuid>& i) : Consumer("test", CONSUMER), queue(q), ids(i) {}; - - bool deliver(const QueueCursor& cursor, const Message& message) - { - queue->dequeue(0, cursor); - BOOST_CHECK_EQUAL(ids.front(), MessageUtils::getMessageId(message)); - ids.pop(); - return true; - }; - void notify() {} - void cancel() {} - void acknowledged(const DeliveryRecord&) {} - OwnershipToken* getSession() { return 0; } - private: - Queue::shared_ptr queue; - std::queue<Uuid>& ids; -}; -boost::shared_ptr<TestConsumer> consumer; - -void setup() -{ - store = std::auto_ptr<MessageStoreImpl>(new MessageStoreImpl(&br)); - store->init(test_dir, 4, 1, true); // truncate store - - queue = Queue::shared_ptr(new Queue(name, 0, store.get(), 0)); - queue->create(); - consumer = boost::shared_ptr<TestConsumer>(new TestConsumer(queue, ids)); -} - -void push() -{ - Uuid messageId(true); - ids.push(messageId); - - Message msg = MessageUtils::createMessage("exchange", "routing_key", messageId, true, 0); - - queue->deliver(msg); -} - -bool pop() -{ - return queue->dispatch(consumer); -} - -void restart() -{ - queue.reset(); - store.reset(); - - store = std::auto_ptr<MessageStoreImpl>(new MessageStoreImpl(&br)); - store->init(test_dir, 4, 1); - ExchangeRegistry exchanges; - LinkRegistry links; - sys::Timer t; - DtxManager mgr(t); - mgr.setStore (store.get()); - RecoveredObjects ro; - RecoveryManagerImpl recoveryMgr(queues, exchanges, links, mgr, br.getProtocolRegistry(), ro); - store->recover(recoveryMgr); - - queue = queues.find(name); - consumer = boost::shared_ptr<TestConsumer>(new TestConsumer(queue, ids)); -} - -void check() -{ - BOOST_REQUIRE(queue); - BOOST_CHECK_EQUAL((u_int32_t) ids.size(), queue->getMessageCount()); - while (pop()) ;//keeping popping 'till all messages are dequeued - BOOST_CHECK_EQUAL((u_int32_t) 0, queue->getMessageCount()); - BOOST_CHECK_EQUAL((size_t) 0, ids.size()); -} - - -// === Test suite === - -QPID_AUTO_TEST_CASE(Basic) -{ - SET_LOG_LEVEL("error+"); // This only needs to be set once. - - std::cout << test_filename << ".Basic: " << std::flush; - setup(); - //push on 10 messages - for (int i = 0; i < 10; i++) push(); - restart(); - check(); - std::cout << "ok" << std::endl; -} - -QPID_AUTO_TEST_CASE(Cycle) -{ - std::cout << test_filename << ".Cycle: " << std::flush; - setup(); - //push on 10 messages: - for (int i = 0; i < 10; i++) push(); - //pop 5: - for (int i = 0; i < 5; i++) pop(); - //push on another 5: - for (int i = 0; i < 5; i++) push(); - restart(); - check(); - std::cout << "ok" << std::endl; -} - -QPID_AUTO_TEST_SUITE_END() diff --git a/qpid/cpp/src/tests/legacystore/SimpleTest.cpp b/qpid/cpp/src/tests/legacystore/SimpleTest.cpp deleted file mode 100644 index d3f040817f..0000000000 --- a/qpid/cpp/src/tests/legacystore/SimpleTest.cpp +++ /dev/null @@ -1,500 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "unit_test.h" -#include "MessageUtils.h" - -#include "qpid/broker/DirectExchange.h" -#include "qpid/broker/Queue.h" -#include "qpid/broker/QueueSettings.h" -#include "qpid/broker/RecoveryManagerImpl.h" -#include "qpid/broker/PersistableObject.h" -#include "qpid/framing/AMQHeaderBody.h" -#include "qpid/framing/FieldTable.h" -#include "qpid/framing/FieldValue.h" -#include "qpid/legacystore/MessageStoreImpl.h" -#include "qpid/legacystore/StoreException.h" -#include "qpid/log/Logger.h" -#include "qpid/sys/Timer.h" - -#include <iostream> - -qpid::broker::Broker::Options opts; -qpid::broker::Broker br(opts); - -#define SET_LOG_LEVEL(level) \ - qpid::log::Options opts(""); \ - opts.selectors.clear(); \ - opts.selectors.push_back(level); \ - qpid::log::Logger::instance().configure(opts); - - -using boost::intrusive_ptr; -using boost::static_pointer_cast; -using namespace qpid; -using namespace qpid::broker; -using namespace qpid::framing; -using namespace mrg::msgstore; -using namespace std; - -QPID_AUTO_TEST_SUITE(SimpleTest) - -const string test_filename("SimpleTest"); -const char* tdp = getenv("TMP_DATA_DIR"); -const string test_dir(tdp && strlen(tdp) > 0 ? tdp : "/var/tmp/SimpleTest"); - -// === Helper fns === - -struct DummyHandler : FrameHandler -{ - std::vector<AMQFrame> frames; - - virtual void handle(AMQFrame& frame){ - frames.push_back(frame); - } -}; - -void recover(MessageStoreImpl& store, QueueRegistry& queues, ExchangeRegistry& exchanges, LinkRegistry& links) -{ - sys::Timer t; - DtxManager mgr(t); - mgr.setStore (&store); - RecoveredObjects ro; - RecoveryManagerImpl recovery(queues, exchanges, links, mgr, br.getProtocolRegistry(), ro); - store.recover(recovery); -} - -void recover(MessageStoreImpl& store, ExchangeRegistry& exchanges) -{ - QueueRegistry queues; - LinkRegistry links; - recover(store, queues, exchanges, links); -} - -void recover(MessageStoreImpl& store, QueueRegistry& queues) -{ - ExchangeRegistry exchanges; - LinkRegistry links; - recover(store, queues, exchanges, links); -} - -void bindAndUnbind(const string& exchangeName, const string& queueName, - const string& key, const FieldTable& args) -{ - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - Exchange::shared_ptr exchange(new DirectExchange(exchangeName, true, false, args)); - Queue::shared_ptr queue(new Queue(queueName, 0, &store, 0)); - store.create(*exchange, qpid::framing::FieldTable()); - store.create(*queue, qpid::framing::FieldTable()); - BOOST_REQUIRE(exchange->bind(queue, key, &args)); - store.bind(*exchange, *queue, key, args); - }//db will be closed - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - ExchangeRegistry exchanges; - QueueRegistry queues; - LinkRegistry links; - - recover(store, queues, exchanges, links); - - Exchange::shared_ptr exchange = exchanges.get(exchangeName); - Queue::shared_ptr queue = queues.find(queueName); - // check exchange args are still set - for (FieldTable::ValueMap::const_iterator i = args.begin(); i!=args.end(); i++) { - BOOST_CHECK(exchange->getArgs().get((*i).first)->getData() == (*i).second->getData()); - } - //check it is bound by unbinding - BOOST_REQUIRE(exchange->unbind(queue, key, &args)); - store.unbind(*exchange, *queue, key, args); - } - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - ExchangeRegistry exchanges; - QueueRegistry queues; - LinkRegistry links; - - recover(store, queues, exchanges, links); - - Exchange::shared_ptr exchange = exchanges.get(exchangeName); - Queue::shared_ptr queue = queues.find(queueName); - // check exchange args are still set - for (FieldTable::ValueMap::const_iterator i = args.begin(); i!=args.end(); i++) { - BOOST_CHECK(exchange->getArgs().get((*i).first)->getData() == (*i).second->getData()); - } - //make sure it is no longer bound - BOOST_REQUIRE(!exchange->unbind(queue, key, &args)); - } -} - - -// === Test suite === - -QPID_AUTO_TEST_CASE(CreateDelete) -{ - SET_LOG_LEVEL("error+"); // This only needs to be set once. - - cout << test_filename << ".CreateDelete: " << flush; - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - string name("CreateDeleteQueue"); - Queue queue(name, 0, &store, 0); - store.create(queue, qpid::framing::FieldTable()); -// TODO - check dir exists - BOOST_REQUIRE(queue.getPersistenceId()); - store.destroy(queue); -// TODO - check dir is deleted - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(EmptyRecover) -{ - cout << test_filename << ".EmptyRecover: " << flush; - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - QueueRegistry registry; - registry.setStore (&store); - recover(store, registry); - //nothing to assert, just testing it doesn't blow up - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(QueueCreate) -{ - cout << test_filename << ".QueueCreate: " << flush; - - uint64_t id(0); - string name("MyDurableQueue"); - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - Queue queue(name, 0, &store, 0); - store.create(queue, qpid::framing::FieldTable()); - BOOST_REQUIRE(queue.getPersistenceId()); - id = queue.getPersistenceId(); - }//db will be closed - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - QueueRegistry registry; - registry.setStore (&store); - recover(store, registry); - Queue::shared_ptr queue = registry.find(name); - BOOST_REQUIRE(queue.get()); - BOOST_CHECK_EQUAL(id, queue->getPersistenceId()); - } - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(QueueCreateWithSettings) -{ - cout << test_filename << ".QueueCreateWithSettings: " << flush; - - FieldTable arguments; - arguments.setInt("qpid.max_count", 202); - arguments.setInt("qpid.max_size", 1003); - QueueSettings settings; - settings.populate(arguments, settings.storeSettings); - string name("MyDurableQueue"); - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - Queue queue(name, settings, &store, 0); - queue.create(); - BOOST_REQUIRE(queue.getPersistenceId()); - }//db will be closed - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - QueueRegistry registry; - registry.setStore (&store); - recover(store, registry); - Queue::shared_ptr queue = registry.find(name); - BOOST_REQUIRE(queue); - BOOST_CHECK_EQUAL(settings.maxDepth.getCount(), 202u); - BOOST_CHECK_EQUAL(settings.maxDepth.getSize(), 1003u); - BOOST_CHECK_EQUAL(settings.maxDepth.getCount(), queue->getSettings().maxDepth.getCount()); - BOOST_CHECK_EQUAL(settings.maxDepth.getSize(), queue->getSettings().maxDepth.getSize()); - } - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(QueueDestroy) -{ - cout << test_filename << ".QueueDestroy: " << flush; - - string name("MyDurableQueue"); - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - Queue queue(name, 0, &store, 0); - store.create(queue, qpid::framing::FieldTable()); - store.destroy(queue); - }//db will be closed - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - QueueRegistry registry; - registry.setStore (&store); - recover(store, registry); - BOOST_REQUIRE(!registry.find(name)); - } - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(Enqueue) -{ - cout << test_filename << ".Enqueue: " << flush; - - //TODO: this is largely copy & paste'd from MessageTest in - //qpid tree. ideally need some helper routines for reducing - //this to a simpler less duplicated form - - string name("MyDurableQueue"); - string exchange("MyExchange"); - string routingKey("MyRoutingKey"); - Uuid messageId(true); - string data1("abcdefg"); - string data2("hijklmn"); - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - Queue::shared_ptr queue(new Queue(name, 0, &store, 0)); - queue->create(); - - Message msg = MessageUtils::createMessage(exchange, routingKey, messageId, true, 14); - MessageUtils::addContent(msg, data1); - MessageUtils::addContent(msg, data2); - - msg.addAnnotation("abc", "xyz"); - - queue->deliver(msg); - }//db will be closed - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - QueueRegistry registry; - registry.setStore (&store); - recover(store, registry); - Queue::shared_ptr queue = registry.find(name); - BOOST_REQUIRE(queue); - BOOST_CHECK_EQUAL((u_int32_t) 1, queue->getMessageCount()); - Message msg = MessageUtils::get(*queue); - - BOOST_CHECK_EQUAL(routingKey, msg.getRoutingKey()); - BOOST_CHECK_EQUAL(messageId, MessageUtils::getMessageId(msg)); - BOOST_CHECK_EQUAL(std::string("xyz"), msg.getAnnotation("abc")); - BOOST_CHECK_EQUAL((u_int64_t) 14, msg.getContent().size()); - - DummyHandler handler; - MessageUtils::deliver(msg, handler, 100); - BOOST_CHECK_EQUAL((size_t) 2, handler.frames.size()); - AMQContentBody* contentBody(dynamic_cast<AMQContentBody*>(handler.frames[1].getBody())); - BOOST_REQUIRE(contentBody); - BOOST_CHECK_EQUAL(data1.size() + data2.size(), contentBody->getData().size()); - BOOST_CHECK_EQUAL(data1 + data2, contentBody->getData()); - } - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(Dequeue) -{ - cout << test_filename << ".Dequeue: " << flush; - - //TODO: reduce the duplication in these tests - string name("MyDurableQueue"); - { - string exchange("MyExchange"); - string routingKey("MyRoutingKey"); - Uuid messageId(true); - string data("abcdefg"); - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - Queue::shared_ptr queue(new Queue(name, 0, &store, 0)); - queue->create(); - - Message msg = MessageUtils::createMessage(exchange, routingKey, messageId, true, 7); - MessageUtils::addContent(msg, data); - - queue->deliver(msg); - - QueueCursor cursor; - MessageUtils::get(*queue, &cursor); - queue->dequeue(0, cursor); - }//db will be closed - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - QueueRegistry registry; - registry.setStore (&store); - recover(store, registry); - Queue::shared_ptr queue = registry.find(name); - BOOST_REQUIRE(queue); - BOOST_CHECK_EQUAL((u_int32_t) 0, queue->getMessageCount()); - } - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(ExchangeCreateAndDestroy) -{ - cout << test_filename << ".ExchangeCreateAndDestroy: " << flush; - - uint64_t id(0); - string name("MyDurableExchange"); - string type("direct"); - FieldTable args; - args.setString("a", "A"); - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - ExchangeRegistry registry; - Exchange::shared_ptr exchange = registry.declare(name, type, true, false, args).first; - store.create(*exchange, qpid::framing::FieldTable()); - id = exchange->getPersistenceId(); - BOOST_REQUIRE(id); - }//db will be closed - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - ExchangeRegistry registry; - - recover(store, registry); - - Exchange::shared_ptr exchange = registry.get(name); - BOOST_CHECK_EQUAL(id, exchange->getPersistenceId()); - BOOST_CHECK_EQUAL(type, exchange->getType()); - BOOST_REQUIRE(exchange->isDurable()); - BOOST_CHECK_EQUAL(*args.get("a"), *exchange->getArgs().get("a")); - store.destroy(*exchange); - } - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - ExchangeRegistry registry; - - recover(store, registry); - - try { - Exchange::shared_ptr exchange = registry.get(name); - BOOST_FAIL("Expected exchange not to be found"); - } catch (const SessionException& e) { - BOOST_CHECK_EQUAL((framing::ReplyCode) 404, e.code); - } - } - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(ExchangeBindAndUnbind) -{ - cout << test_filename << ".ExchangeBindAndUnbind: " << flush; - - bindAndUnbind("MyDurableExchange", "MyDurableQueue", "my-routing-key", FieldTable()); - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(ExchangeBindAndUnbindWithArgs) -{ - cout << test_filename << ".ExchangeBindAndUnbindWithArgs: " << flush; - - FieldTable args; - args.setString("a", "A"); - args.setString("b", "B"); - bindAndUnbind("MyDurableExchange", "MyDurableQueue", "my-routing-key", args); - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(ExchangeImplicitUnbind) -{ - cout << test_filename << ".ExchangeImplicitUnbind: " << flush; - - string exchangeName("MyDurableExchange"); - string queueName1("MyDurableQueue1"); - string queueName2("MyDurableQueue2"); - string key("my-routing-key"); - FieldTable args; - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1, true); // truncate store - Exchange::shared_ptr exchange(new DirectExchange(exchangeName, true, false, args)); - Queue::shared_ptr queue1(new Queue(queueName1, 0, &store, 0)); - Queue::shared_ptr queue2(new Queue(queueName2, 0, &store, 0)); - store.create(*exchange, qpid::framing::FieldTable()); - store.create(*queue1, qpid::framing::FieldTable()); - store.create(*queue2, qpid::framing::FieldTable()); - store.bind(*exchange, *queue1, key, args); - store.bind(*exchange, *queue2, key, args); - //delete queue1: - store.destroy(*queue1); - }//db will be closed - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - ExchangeRegistry exchanges; - QueueRegistry queues; - LinkRegistry links; - - //ensure recovery works ok: - recover(store, queues, exchanges, links); - - Exchange::shared_ptr exchange = exchanges.get(exchangeName); - BOOST_REQUIRE(!queues.find(queueName1).get()); - BOOST_REQUIRE(queues.find(queueName2).get()); - - //delete exchange: - store.destroy(*exchange); - } - { - MessageStoreImpl store(&br); - store.init(test_dir, 4, 1); - ExchangeRegistry exchanges; - QueueRegistry queues; - LinkRegistry links; - - //ensure recovery works ok: - recover(store, queues, exchanges, links); - - try { - Exchange::shared_ptr exchange = exchanges.get(exchangeName); - BOOST_FAIL("Expected exchange not to be found"); - } catch (const SessionException& e) { - BOOST_CHECK_EQUAL((framing::ReplyCode) 404, e.code); - } - Queue::shared_ptr queue = queues.find(queueName2); - store.destroy(*queue); - } - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_SUITE_END() diff --git a/qpid/cpp/src/tests/legacystore/TransactionalTest.cpp b/qpid/cpp/src/tests/legacystore/TransactionalTest.cpp deleted file mode 100644 index d1bc34d5a7..0000000000 --- a/qpid/cpp/src/tests/legacystore/TransactionalTest.cpp +++ /dev/null @@ -1,354 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "unit_test.h" -#include "MessageUtils.h" - -#include "qpid/broker/Queue.h" -#include "qpid/broker/RecoveryManagerImpl.h" -#include "qpid/broker/PersistableObject.h" -#include "qpid/framing/AMQHeaderBody.h" -#include "qpid/legacystore/MessageStoreImpl.h" -#include "qpid/legacystore/StoreException.h" -#include "qpid/log/Statement.h" -#include "qpid/log/Logger.h" -#include "qpid/sys/Timer.h" - -#include <iostream> - -using namespace mrg::msgstore; -using namespace qpid; -using namespace qpid::broker; -using namespace qpid::framing; -using namespace std; - -namespace { -qpid::broker::Broker::Options opts; -qpid::broker::Broker br(opts); -} - -QPID_AUTO_TEST_SUITE(TransactionalTest) - -#define SET_LOG_LEVEL(level) \ - qpid::log::Options opts(""); \ - opts.selectors.clear(); \ - opts.selectors.push_back(level); \ - qpid::log::Logger::instance().configure(opts); - -const string test_filename("TransactionalTest"); -const char* tdp = getenv("TMP_DATA_DIR"); -const string test_dir(tdp && strlen(tdp) > 0 ? tdp : "/var/tmp/TransactionalTest"); - -// Test txn context which has special setCompleteFailure() method which prevents entire "txn complete" process from hapenning -class TestTxnCtxt : public TxnCtxt -{ - public: - TestTxnCtxt(IdSequence* _loggedtx) : TxnCtxt(_loggedtx) {} - void setCompleteFailure(const unsigned num_queues_rem) { - // Remove queue members from back of impactedQueues until queues_rem reamin. - // to end to simulate multi-queue txn complete failure. - while (impactedQueues.size() > num_queues_rem) impactedQueues.erase(impactedQueues.begin()); - } - void resetPreparedXidStorePtr() { preparedXidStorePtr = 0; } -}; - -// Test store which has special begin() which returns a TestTPCTxnCtxt, and a method to check for -// remaining open transactions. -// begin(), commit(), and abort() all hide functions in MessageStoreImpl. To avoid the compiler -// warnings/errors these are renamed with a 'TMS' prefix. -class TestMessageStore: public MessageStoreImpl -{ - public: - TestMessageStore(qpid::broker::Broker* br, const char* envpath = 0) : MessageStoreImpl(br, envpath) {} - std::auto_ptr<qpid::broker::TransactionContext> TMSbegin() { - checkInit(); - // pass sequence number for c/a - return auto_ptr<TransactionContext>(new TestTxnCtxt(&messageIdSequence)); - } - void TMScommit(TransactionContext& ctxt, const bool complete_prepared_list) { - checkInit(); - TxnCtxt* txn(check(&ctxt)); - if (!txn->isTPC()) { - localPrepare(dynamic_cast<TxnCtxt*>(txn)); - if (!complete_prepared_list) dynamic_cast<TestTxnCtxt*>(txn)->resetPreparedXidStorePtr(); - } - completed(*dynamic_cast<TxnCtxt*>(txn), true); - } - void TMSabort(TransactionContext& ctxt, const bool complete_prepared_list) - { - checkInit(); - TxnCtxt* txn(check(&ctxt)); - if (!txn->isTPC()) { - localPrepare(dynamic_cast<TxnCtxt*>(txn)); - if (!complete_prepared_list) dynamic_cast<TestTxnCtxt*>(txn)->resetPreparedXidStorePtr(); - } - completed(*dynamic_cast<TxnCtxt*>(txn), false); - } -}; - -// === Helper fns === - -const string nameA("queueA"); -const string nameB("queueB"); -//const Uuid messageId(true); -std::auto_ptr<MessageStoreImpl> store; -std::auto_ptr<QueueRegistry> queues; -Queue::shared_ptr queueA; -Queue::shared_ptr queueB; - -template <class T> -void setup() -{ - store = std::auto_ptr<T>(new T(&br)); - store->init(test_dir, 4, 1, true); // truncate store - - //create two queues: - queueA = Queue::shared_ptr(new Queue(nameA, 0, store.get(), 0)); - queueA->create(); - queueB = Queue::shared_ptr(new Queue(nameB, 0, store.get(), 0)); - queueB->create(); -} - -template <class T> -void restart() -{ - queueA.reset(); - queueB.reset(); - queues.reset(); - store.reset(); - - store = std::auto_ptr<T>(new T(&br)); - store->init(test_dir, 4, 1); - queues = std::auto_ptr<QueueRegistry>(new QueueRegistry); - ExchangeRegistry exchanges; - LinkRegistry links; - sys::Timer t; - DtxManager mgr(t); - mgr.setStore (store.get()); - RecoveredObjects ro; - RecoveryManagerImpl recovery(*queues, exchanges, links, mgr, br.getProtocolRegistry(), ro); - store->recover(recovery); - - queueA = queues->find(nameA); - queueB = queues->find(nameB); -} - -Message createMessage(const string& id, const string& exchange="exchange", const string& key="routing_key") -{ - return MessageUtils::createMessage(exchange, key, Uuid(), true, 0, id); -} - -void checkMsg(Queue::shared_ptr& queue, u_int32_t size, const string& msgid = "<none>") -{ - BOOST_REQUIRE(queue); - BOOST_CHECK_EQUAL(size, queue->getMessageCount()); - if (size > 0) { - Message msg = MessageUtils::get(*queue); - BOOST_REQUIRE(msg); - BOOST_CHECK_EQUAL(msgid, MessageUtils::getCorrelationId(msg)); - } -} - -void swap(bool commit) -{ - setup<MessageStoreImpl>(); - - //create message and enqueue it onto first queue: - Message msgA = createMessage("Message", "exchange", "routing_key"); - queueA->deliver(msgA); - - QueueCursor cursorB; - Message msgB = MessageUtils::get(*queueA, &cursorB); - BOOST_REQUIRE(msgB); - //move the message from one queue to the other as a transaction - std::auto_ptr<TransactionContext> txn = store->begin(); - TxBuffer tx; - queueB->deliver(msgB, &tx);//note: need to enqueue it first to avoid message being deleted - - queueA->dequeue(txn.get(), cursorB); - tx.prepare(txn.get()); - if (commit) { - store->commit(*txn); - } else { - store->abort(*txn); - } - - restart<MessageStoreImpl>(); - - // Check outcome - BOOST_REQUIRE(queueA); - BOOST_REQUIRE(queueB); - - Queue::shared_ptr x;//the queue from which the message was swapped - Queue::shared_ptr y;//the queue on which the message is expected to be - - if (commit) { - x = queueA; - y = queueB; - } else { - x = queueB; - y = queueA; - } - - checkMsg(x, 0); - checkMsg(y, 1, "Message"); - checkMsg(y, 0); -} - -void testMultiQueueTxn(const unsigned num_queues_rem, const bool complete_prepared_list, const bool commit) -{ - setup<TestMessageStore>(); - TestMessageStore* tmsp = static_cast<TestMessageStore*>(store.get()); - std::auto_ptr<TransactionContext> txn(tmsp->TMSbegin()); - TxBuffer tx; - - //create two messages and enqueue them onto both queues: - Message msgA = createMessage("MessageA", "exchange", "routing_key"); - queueA->deliver(msgA, &tx); - queueB->deliver(msgA, &tx); - Message msgB = createMessage("MessageB", "exchange", "routing_key"); - queueA->deliver(msgB, &tx); - queueB->deliver(msgB, &tx); - - tx.prepare(txn.get()); - static_cast<TestTxnCtxt*>(txn.get())->setCompleteFailure(num_queues_rem); - if (commit) - tmsp->TMScommit(*txn, complete_prepared_list); - else - tmsp->TMSabort(*txn, complete_prepared_list); - restart<TestMessageStore>(); - - // Check outcome - if (commit) - { - checkMsg(queueA, 2, "MessageA"); - checkMsg(queueB, 2, "MessageA"); - checkMsg(queueA, 1, "MessageB"); - checkMsg(queueB, 1, "MessageB"); - } - checkMsg(queueA, 0); - checkMsg(queueB, 0); -} - -// === Test suite === - -QPID_AUTO_TEST_CASE(Commit) -{ - SET_LOG_LEVEL("error+"); // This only needs to be set once. - - cout << test_filename << ".Commit: " << flush; - swap(true); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(Abort) -{ - cout << test_filename << ".Abort: " << flush; - swap(false); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueCommit) -{ - cout << test_filename << ".MultiQueueCommit: " << flush; - testMultiQueueTxn(2, true, true); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueAbort) -{ - cout << test_filename << ".MultiQueueAbort: " << flush; - testMultiQueueTxn(2, true, false); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueNoQueueCommitRecover) -{ - cout << test_filename << ".MultiQueueNoQueueCommitRecover: " << flush; - testMultiQueueTxn(0, false, true); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueNoQueueAbortRecover) -{ - cout << test_filename << ".MultiQueueNoQueueAbortRecover: " << flush; - testMultiQueueTxn(0, false, false); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueSomeQueueCommitRecover) -{ - cout << test_filename << ".MultiQueueSomeQueueCommitRecover: " << flush; - testMultiQueueTxn(1, false, true); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueSomeQueueAbortRecover) -{ - cout << test_filename << ".MultiQueueSomeQueueAbortRecover: " << flush; - testMultiQueueTxn(1, false, false); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueAllQueueCommitRecover) -{ - cout << test_filename << ".MultiQueueAllQueueCommitRecover: " << flush; - testMultiQueueTxn(2, false, true); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueAllQueueAbortRecover) -{ - cout << test_filename << ".MultiQueueAllQueueAbortRecover: " << flush; - testMultiQueueTxn(2, false, false); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(LockedRecordTest) -{ - cout << test_filename << ".LockedRecordTest: " << flush; - - setup<MessageStoreImpl>(); - queueA->deliver(createMessage("Message", "exchange", "routingKey")); - std::auto_ptr<TransactionContext> txn = store->begin(); - - QueueCursor cursor; - Message msg = MessageUtils::get(*queueA, &cursor); - queueA->dequeue(txn.get(), cursor); - - try { - store->dequeue(0, msg.getPersistentContext(), *queueA); - BOOST_ERROR("Did not throw JERR_MAP_LOCKED exception as expected."); - } - catch (const mrg::msgstore::StoreException& e) { - if (std::strstr(e.what(), "JERR_MAP_LOCKED") == 0) - BOOST_ERROR("Unexpected StoreException: " << e.what()); - } - catch (const std::exception& e) { - BOOST_ERROR("Unexpected exception: " << e.what()); - } - store->commit(*txn); - checkMsg(queueA, 0); - - cout << "ok" << endl; -} - -QPID_AUTO_TEST_SUITE_END() diff --git a/qpid/cpp/src/tests/legacystore/TwoPhaseCommitTest.cpp b/qpid/cpp/src/tests/legacystore/TwoPhaseCommitTest.cpp deleted file mode 100644 index 25bb9dc607..0000000000 --- a/qpid/cpp/src/tests/legacystore/TwoPhaseCommitTest.cpp +++ /dev/null @@ -1,678 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include "unit_test.h" -#include "MessageUtils.h" - -#include "qpid/broker/Queue.h" -#include "qpid/broker/RecoveryManagerImpl.h" -#include "qpid/broker/PersistableObject.h" -#include "qpid/framing/AMQHeaderBody.h" -#include "qpid/legacystore/MessageStoreImpl.h" -#include "qpid/legacystore/TxnCtxt.h" -#include "qpid/log/Logger.h" -#include "qpid/log/Statement.h" -#include "qpid/sys/Timer.h" - -#include <iostream> - -using namespace mrg::msgstore; -using namespace qpid; -using namespace qpid::broker; -using namespace qpid::framing; -using namespace std; - - -qpid::broker::Broker::Options opts; -qpid::broker::Broker br(opts); - - -QPID_AUTO_TEST_SUITE(TwoPhaseCommitTest) - -#define SET_LOG_LEVEL(level) \ - qpid::log::Options opts(""); \ - opts.selectors.clear(); \ - opts.selectors.push_back(level); \ - qpid::log::Logger::instance().configure(opts); - - -const string test_filename("TwoPhaseCommitTest"); -const char* tdp = getenv("TMP_DATA_DIR"); -string test_dir(tdp && strlen(tdp) > 0 ? tdp : "/var/tmp/TwoPhaseCommitTest"); - -// === Helper fns === - -class TwoPhaseCommitTest -{ - - class Strategy - { - public: - virtual void init() = 0; - virtual void run(TPCTransactionContext* txn) = 0; - virtual void check(bool committed) = 0; - virtual ~Strategy(){} - }; - - class Swap : public Strategy - { - TwoPhaseCommitTest* const test; - const string messageId; - Message msg; - public: - Swap(TwoPhaseCommitTest* const test_, const string& messageId_): test(test_), messageId(messageId_) {} - void init(){ msg = test->deliver(messageId, test->queueA); } - void run(TPCTransactionContext* txn) { test->swap(txn, test->queueA, test->queueB); } - void check(bool committed) { test->swapCheck(committed, messageId, test->queueA, test->queueB); } - }; - - class Enqueue : public Strategy - { - TwoPhaseCommitTest* const test; - Message msg1; - Message msg2; - Message msg3; - public: - Enqueue(TwoPhaseCommitTest* const test_): test(test_) {} - void init() {} - void run(TPCTransactionContext* txn) { - msg1 = test->enqueue(txn, "Enqueue1", test->queueA); - msg2 = test->enqueue(txn, "Enqueue2", test->queueA); - msg3 = test->enqueue(txn, "Enqueue3", test->queueA); - } - void check(bool committed) { - if (committed) { - test->checkMsg(test->queueA, 3, "Enqueue1"); - test->checkMsg(test->queueA, 2, "Enqueue2"); - test->checkMsg(test->queueA, 1, "Enqueue3"); - } - test->checkMsg(test->queueA, 0); - } - }; - - class Dequeue : public Strategy - { - TwoPhaseCommitTest* const test; - Message msg1; - Message msg2; - Message msg3; - public: - Dequeue(TwoPhaseCommitTest* const test_): test(test_) {} - void init() { - msg1 = test->deliver("Dequeue1", test->queueA); - msg2 = test->deliver("Dequeue2", test->queueA); - msg3 = test->deliver("Dequeue3", test->queueA); - } - void run(TPCTransactionContext* txn) { - test->dequeue(txn, test->queueA); - test->dequeue(txn, test->queueA); - test->dequeue(txn, test->queueA); - } - void check(bool committed) { - if (!committed) { - test->checkMsg(test->queueA, 3, "Dequeue1"); - test->checkMsg(test->queueA, 2, "Dequeue2"); - test->checkMsg(test->queueA, 1, "Dequeue3"); - } - test->checkMsg(test->queueA, 0); - } - }; - - class MultiQueueTxn : public Strategy - { - TwoPhaseCommitTest* const test; - Message msg1; - Message msg2; - std::set<Queue::shared_ptr> queueset; - public: - MultiQueueTxn(TwoPhaseCommitTest* const test_): test(test_) {} - virtual void init() {} - virtual void run(TPCTransactionContext* txn) { - queueset.insert(test->queueA); - queueset.insert(test->queueB); - msg1 = test->enqueue(txn, "Message1", queueset); - msg2 = test->enqueue(txn, "Message2", queueset); - queueset.clear(); - } - virtual void check(bool committed) { - TestMessageStore* sptr = static_cast<TestMessageStore*>(test->store.get()); - if (committed) - { - test->checkMsg(test->queueA, 2, "Message1"); - test->checkMsg(test->queueB, 2, "Message1"); - test->checkMsg(test->queueA, 1, "Message2"); - test->checkMsg(test->queueB, 1, "Message2"); - } - test->checkMsg(test->queueA, 0); - test->checkMsg(test->queueB, 0); - // Check there are no remaining open txns in store - BOOST_CHECK_EQUAL(u_int32_t(0), sptr->getRemainingTxns(*(test->queueA))); - BOOST_CHECK_EQUAL(u_int32_t(0), sptr->getRemainingTxns(*(test->queueB))); - BOOST_CHECK_EQUAL(u_int32_t(0), sptr->getRemainingPreparedListTxns()); - } - }; - - // Test txn context which has special setCompleteFailure() method which prevents entire "txn complete" process from hapenning - class TestTPCTxnCtxt : public TPCTxnCtxt - { - public: - TestTPCTxnCtxt(const std::string& _xid, IdSequence* _loggedtx) : TPCTxnCtxt(_xid, _loggedtx) {} - void setCompleteFailure(const unsigned num_queues_rem, const bool complete_prepared_list) { - // Remove queue members from back of impactedQueues until queues_rem reamin. - // to end to simulate multi-queue txn complete failure. - while (impactedQueues.size() > num_queues_rem) impactedQueues.erase(impactedQueues.begin()); - // If prepared list is not to be committed, set pointer to 0 - if (!complete_prepared_list) preparedXidStorePtr = 0; - } - }; - - // Test store which has sepcial begin() which returns a TestTPCTxnCtxt, and a method to check for - // reamining open transactions - class TestMessageStore: public MessageStoreImpl - { - public: - TestMessageStore(qpid::broker::Broker* br, const char* envpath = 0) : MessageStoreImpl(br, envpath) {} - std::auto_ptr<qpid::broker::TPCTransactionContext> TMSbegin(const std::string& xid) { - checkInit(); - IdSequence* jtx = &messageIdSequence; - // pass sequence number for c/a - return auto_ptr<TPCTransactionContext>(new TestTPCTxnCtxt(xid, jtx)); - } - u_int32_t getRemainingTxns(const PersistableQueue& queue) { - return static_cast<JournalImpl*>(queue.getExternalQueueStore())->get_open_txn_cnt(); - } - u_int32_t getRemainingPreparedListTxns() { - return tplStorePtr->get_open_txn_cnt(); - } - }; - - const string nameA; - const string nameB; - std::auto_ptr<MessageStoreImpl> store; - std::auto_ptr<DtxManager> dtxmgr; - std::auto_ptr<QueueRegistry> queues; - std::auto_ptr<LinkRegistry> links; - Queue::shared_ptr queueA; - Queue::shared_ptr queueB; - Message msg1; - Message msg2; - Message msg4; - std::auto_ptr<TxBuffer> tx; - - void recoverPrepared(bool commit) - { - setup<MessageStoreImpl>(); - - Swap swap(this, "RecoverPrepared"); - swap.init(); - std::auto_ptr<TPCTransactionContext> txn(store->begin("my-xid")); - swap.run(txn.get()); - if (tx.get()) { - tx->prepare(txn.get()); - tx.reset(); - } - - store->prepare(*txn); - restart<MessageStoreImpl>(); - - //check that the message is not available from either queue - BOOST_CHECK_EQUAL((u_int32_t) 0, queueA->getMessageCount()); - BOOST_CHECK_EQUAL((u_int32_t) 0, queueB->getMessageCount()); - - //commit/abort the txn - through the dtx manager, not directly on the store - if (commit) { - dtxmgr->commit("my-xid", false); - } else { - dtxmgr->rollback("my-xid"); - } - - swap.check(commit); - restart<MessageStoreImpl>(); - swap.check(commit); - } - - void testMultiQueueTxn(const unsigned num_queues_rem, const bool complete_prepared_list, const bool commit) - { - setup<TestMessageStore>(); - MultiQueueTxn mqtTest(this); - mqtTest.init(); - std::auto_ptr<TPCTransactionContext> txn(static_cast<TestMessageStore*>(store.get())->begin("my-xid")); - mqtTest.run(txn.get()); - if (tx.get()) { - tx->prepare(txn.get()); - tx.reset(); - } - store->prepare(*txn); - - // As the commits and aborts should happen through DtxManager, and it is too complex to - // pass all these test params through, we bypass DtxManager and use the store directly. - // This will prevent the queues from seeing committed txns, however. To test the success - // or failure of - static_cast<TestTPCTxnCtxt*>(txn.get())->setCompleteFailure(num_queues_rem, complete_prepared_list); - if (commit) - store->commit(*txn); - else - store->abort(*txn); - restart<TestMessageStore>(); - mqtTest.check(commit); - } - - void commit(Strategy& strategy) - { - setup<MessageStoreImpl>(); - strategy.init(); - - std::auto_ptr<TPCTransactionContext> txn(store->begin("my-xid")); - strategy.run(txn.get()); - if (tx.get()) { - tx->prepare(txn.get()); - tx.reset(); - } - store->prepare(*txn); - store->commit(*txn); - restart<MessageStoreImpl>(); - strategy.check(true); - } - - void abort(Strategy& strategy, bool prepare) - { - setup<MessageStoreImpl>(); - strategy.init(); - - std::auto_ptr<TPCTransactionContext> txn(store->begin("my-xid")); - strategy.run(txn.get()); - if (tx.get()) { - tx->prepare(txn.get()); - tx.reset(); - } - if (prepare) store->prepare(*txn); - store->abort(*txn); - restart<MessageStoreImpl>(); - strategy.check(false); - } - - void swap(TPCTransactionContext* txn, Queue::shared_ptr& from, Queue::shared_ptr& to) - { - QueueCursor c; - Message msg1 = MessageUtils::get(*from, &c);//just dequeues in memory - //move the message from one queue to the other as part of a - //distributed transaction - if (!tx.get()) tx = std::auto_ptr<TxBuffer>(new TxBuffer); - to->deliver(msg1, tx.get());//note: need to enqueue it first to avoid message being deleted - from->dequeue(txn, c); - } - - void dequeue(TPCTransactionContext* txn, Queue::shared_ptr& queue) - { - QueueCursor c; - Message msg2 = MessageUtils::get(*queue, &c);//just dequeues in memory - queue->dequeue(txn, c); - } - - Message enqueue(TPCTransactionContext* /*txn*/, const string& msgid, Queue::shared_ptr& queue) - { - Message msg = createMessage(msgid); - if (!tx.get()) tx = std::auto_ptr<TxBuffer>(new TxBuffer); - queue->deliver(msg, tx.get()); - return msg; - } - - Message enqueue(TPCTransactionContext* /*txn*/, const string& msgid, std::set<Queue::shared_ptr>& queueset) - { - if (!tx.get()) tx = std::auto_ptr<TxBuffer>(new TxBuffer); - Message msg = createMessage(msgid); - for (std::set<Queue::shared_ptr>::iterator i = queueset.begin(); i != queueset.end(); i++) { - (*i)->deliver(msg, tx.get()); - } - return msg; - } - - Message deliver(const string& msgid, Queue::shared_ptr& queue) - { - Message m = createMessage(msgid); - queue->deliver(m); - return m; - } - - template <class T> - void setup() - { - store = std::auto_ptr<T>(new T(&br)); - store->init(test_dir, 4, 1, true); // truncate store - - //create two queues: - queueA = Queue::shared_ptr(new Queue(nameA, 0, store.get(), 0)); - queueA->create(); - queueB = Queue::shared_ptr(new Queue(nameB, 0, store.get(), 0)); - queueB->create(); - } - - Message createMessage(const string& id, const string& exchange="exchange", const string& key="routing_key") - { - Message msg = MessageUtils::createMessage(exchange, key, Uuid(), true, 0, id); - return msg; - } - - template <class T> - void restart() - { - queueA.reset(); - queueB.reset(); - store.reset(); - queues.reset(); - links.reset(); - - store = std::auto_ptr<T>(new T(&br)); - store->init(test_dir, 4, 1); - sys::Timer t; - ExchangeRegistry exchanges; - queues = std::auto_ptr<QueueRegistry>(new QueueRegistry); - links = std::auto_ptr<LinkRegistry>(new LinkRegistry); - dtxmgr = std::auto_ptr<DtxManager>(new DtxManager(t)); - dtxmgr->setStore (store.get()); - RecoveredObjects ro; - RecoveryManagerImpl recovery(*queues, exchanges, *links, *dtxmgr, br.getProtocolRegistry(), ro); - store->recover(recovery); - - queueA = queues->find(nameA); - queueB = queues->find(nameB); - } - - void checkMsg(Queue::shared_ptr& queue, u_int32_t size, const string& msgid = "<none>") - { - BOOST_REQUIRE(queue); - BOOST_CHECK_EQUAL(size, queue->getMessageCount()); - if (size > 0) { - Message msg = MessageUtils::get(*queue); - BOOST_REQUIRE(msg); - BOOST_CHECK_EQUAL(msgid, MessageUtils::getCorrelationId(msg)); - } - } - - void swapCheck(bool swapped, const string& msgid, Queue::shared_ptr& from, Queue::shared_ptr& to) - { - BOOST_REQUIRE(from); - BOOST_REQUIRE(to); - - Queue::shared_ptr x; //the queue from which the message was swapped - Queue::shared_ptr y; //the queue on which the message is expected to be - - if (swapped) { - x = from; - y = to; - } else { - x = to; - y = from; - } - - checkMsg(x, 0); - checkMsg(y, 1, msgid); - checkMsg(y, 0); - } - -public: - TwoPhaseCommitTest() : nameA("queueA"), nameB("queueB") {} - - void testCommitEnqueue() - { - Enqueue enqueue(this); - commit(enqueue); - } - - void testCommitDequeue() - { - Dequeue dequeue(this); - commit(dequeue); - } - - void testCommitSwap() - { - Swap swap(this, "SwapMessageId"); - commit(swap); - } - - void testPrepareAndAbortEnqueue() - { - Enqueue enqueue(this); - abort(enqueue, true); - } - - void testPrepareAndAbortDequeue() - { - Dequeue dequeue(this); - abort(dequeue, true); - } - - void testPrepareAndAbortSwap() - { - Swap swap(this, "SwapMessageId"); - abort(swap, true); - } - - void testAbortNoPrepareEnqueue() - { - Enqueue enqueue(this); - abort(enqueue, false); - } - - void testAbortNoPrepareDequeue() - { - Dequeue dequeue(this); - abort(dequeue, false); - } - - void testAbortNoPrepareSwap() - { - Swap swap(this, "SwapMessageId"); - abort(swap, false); - } - - void testRecoverPreparedThenCommitted() - { - recoverPrepared(true); - } - - void testRecoverPreparedThenAborted() - { - recoverPrepared(false); - } - - void testMultiQueueCommit() - { - testMultiQueueTxn(2, true, true); - } - - void testMultiQueueAbort() - { - testMultiQueueTxn(2, true, false); - } - - void testMultiQueueNoQueueCommitRecover() - { - testMultiQueueTxn(0, false, true); - } - - void testMultiQueueNoQueueAbortRecover() - { - testMultiQueueTxn(0, false, false); - } - - void testMultiQueueSomeQueueCommitRecover() - { - testMultiQueueTxn(1, false, true); - } - - void testMultiQueueSomeQueueAbortRecover() - { - testMultiQueueTxn(1, false, false); - } - - void testMultiQueueAllQueueCommitRecover() - { - testMultiQueueTxn(2, false, true); - } - - void testMultiQueueAllQueueAbortRecover() - { - testMultiQueueTxn(2, false, false); - } -}; - -TwoPhaseCommitTest tpct; - -// === Test suite === - -QPID_AUTO_TEST_CASE(CommitEnqueue) -{ - SET_LOG_LEVEL("error+"); // This only needs to be set once. - - cout << test_filename << ".CommitEnqueue: " << flush; - tpct.testCommitEnqueue(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(CommitDequeue) -{ - cout << test_filename << ".CommitDequeue: " << flush; - tpct.testCommitDequeue(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(CommitSwap) -{ - cout << test_filename << ".CommitSwap: " << flush; - tpct.testCommitSwap(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(PrepareAndAbortEnqueue) -{ - cout << test_filename << ".PrepareAndAbortEnqueue: " << flush; - tpct.testPrepareAndAbortEnqueue(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(PrepareAndAbortDequeue) -{ - cout << test_filename << ".PrepareAndAbortDequeue: " << flush; - tpct.testPrepareAndAbortDequeue(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(PrepareAndAbortSwap) -{ - cout << test_filename << ".PrepareAndAbortSwap: " << flush; - tpct.testPrepareAndAbortSwap(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(AbortNoPrepareEnqueue) -{ - cout << test_filename << ".AbortNoPrepareEnqueue: " << flush; - tpct.testAbortNoPrepareEnqueue(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(AbortNoPrepareDequeue) -{ - cout << test_filename << ".AbortNoPrepareDequeue: " << flush; - tpct.testAbortNoPrepareDequeue(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(AbortNoPrepareSwap) -{ - cout << test_filename << ".AbortNoPrepareSwap: " << flush; - tpct.testAbortNoPrepareSwap(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(RecoverPreparedThenCommitted) -{ - cout << test_filename << ".RecoverPreparedThenCommitted: " << flush; - tpct.testRecoverPreparedThenCommitted(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(RecoverPreparedThenAborted) -{ - cout << test_filename << ".RecoverPreparedThenAborted: " << flush; - tpct.testRecoverPreparedThenAborted(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueCommit) -{ - cout << test_filename << ".MultiQueueCommit: " << flush; - tpct.testMultiQueueCommit(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueAbort) -{ - cout << test_filename << ".MultiQueueAbort: " << flush; - tpct.testMultiQueueAbort(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueNoQueueCommitRecover) -{ - cout << test_filename << ".MultiQueueNoQueueCommitRecover: " << flush; - tpct.testMultiQueueNoQueueCommitRecover(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueNoQueueAbortRecover) -{ - cout << test_filename << ".MultiQueueNoQueueAbortRecover: " << flush; - tpct.testMultiQueueNoQueueAbortRecover(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueSomeQueueCommitRecover) -{ - cout << test_filename << ".MultiQueueSomeQueueCommitRecover: " << flush; - tpct.testMultiQueueSomeQueueCommitRecover(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueSomeQueueAbortRecover) -{ - cout << test_filename << ".MultiQueueSomeQueueAbortRecover: " << flush; - tpct.testMultiQueueSomeQueueAbortRecover(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueAllQueueCommitRecover) -{ - cout << test_filename << ".MultiQueueAllQueueCommitRecover: " << flush; - tpct.testMultiQueueAllQueueCommitRecover(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_CASE(MultiQueueAllQueueAbortRecover) -{ - cout << test_filename << ".MultiQueueAllQueueAbortRecover: " << flush; - tpct.testMultiQueueAllQueueAbortRecover(); - cout << "ok" << endl; -} - -QPID_AUTO_TEST_SUITE_END() diff --git a/qpid/cpp/src/tests/linearstore/linearstoredirsetup.sh b/qpid/cpp/src/tests/linearstore/linearstoredirsetup.sh index 3cad50b1c5..ef39767e9b 100755 --- a/qpid/cpp/src/tests/linearstore/linearstoredirsetup.sh +++ b/qpid/cpp/src/tests/linearstore/linearstoredirsetup.sh @@ -19,26 +19,37 @@ # under the License. # - -STORE_DIR=/tmp -LINEARSTOREDIR=~/RedHat/linearstore - -rm -rf $STORE_DIR/qls -rm -rf $STORE_DIR/p002 -rm $STORE_DIR/p004 - -mkdir $STORE_DIR/qls -mkdir $STORE_DIR/p002 -touch $STORE_DIR/p004 -mkdir $STORE_DIR/qls/p001 -touch $STORE_DIR/qls/p003 -ln -s $STORE_DIR/p002 $STORE_DIR/qls/p002 -ln -s $STORE_DIR/p004 $STORE_DIR/qls/p004 - -${LINEARSTOREDIR}/tools/src/py/linearstore/efptool.py $STORE_DIR/qls/ -a -p 1 -s 2048 -n 25 -${LINEARSTOREDIR}/tools/src/py/linearstore/efptool.py $STORE_DIR/qls/ -a -p 1 -s 512 -n 25 -${LINEARSTOREDIR}/tools/src/py/linearstore/efptool.py $STORE_DIR/qls/ -a -p 2 -s 2048 -n 25 - +# This script sets up a test directory which contains both +# recoverable and non-recoverable files and directories for +# the empty file pool (EFP). + +# NOTE: The following is based on typical development tree paths, not installed paths + +BASE_DIR=${HOME}/RedHat +STORE_DIR=${BASE_DIR} +PYTHON_TOOLS_DIR=${BASE_DIR}/qpid/tools/src/linearstore +export PYTHONPATH=${BASE_DIR}/qpid/python:${BASE_DIR}/qpid/extras/qmf/src/py:${BASE_DIR}/qpid/tools/src/py + +# Remove old dirs (if present) +rm -rf ${STORE_DIR}/qls +rm -rf ${STORE_DIR}/p002 +rm ${STORE_DIR}/p004 + +# Create new dir tree and links +mkdir ${STORE_DIR}/p002_ext +touch ${STORE_DIR}/p004_ext +mkdir ${STORE_DIR}/qls +mkdir ${STORE_DIR}/qls/p001 +touch ${STORE_DIR}/qls/p003 +ln -s ${STORE_DIR}/p002_ext ${STORE_DIR}/qls/p002 +ln -s ${STORE_DIR}/p004_ext ${STORE_DIR}/qls/p004 + +# Populate efp dirs with empty files +${PYTHON_TOOLS_DIR}/efptool.py $STORE_DIR/qls/ -a -p 1 -s 2048 -n 25 +${PYTHON_TOOLS_DIR}/efptool.py $STORE_DIR/qls/ -a -p 1 -s 512 -n 25 +${PYTHON_TOOLS_DIR}/efptool.py $STORE_DIR/qls/ -a -p 2 -s 2048 -n 25 + +# Show the result for information ${LINEARSTOREDIR}/tools/src/py/linearstore/efptool.py $STORE_DIR/qls/ -l tree -la $STORE_DIR/qls diff --git a/qpid/cpp/src/tests/linearstore/tx-test-soak.sh b/qpid/cpp/src/tests/linearstore/tx-test-soak.sh index fa05e0a4a8..7d5581961f 100755 --- a/qpid/cpp/src/tests/linearstore/tx-test-soak.sh +++ b/qpid/cpp/src/tests/linearstore/tx-test-soak.sh @@ -19,7 +19,6 @@ # under the License. # - # tx-test-soak # # Basic test methodology: @@ -30,6 +29,8 @@ # 5. Run qpid-txtest against broker in check mode, which checks that all expected messages are present. # 6. Wash, rinse, repeat... The number of runs is determined by ${NUM_RUNS} +# NOTE: The following is based on typical development tree paths, not installed paths + NUM_RUNS=1000 BASE_DIR=${HOME}/RedHat CMAKE_BUILD_DIR=${BASE_DIR}/q.cm @@ -43,13 +44,18 @@ BROKER_MANAGEMENT="no" # "no" or "yes" TRUNCATE_INTERVAL=10 MAX_DISK_PERC_USED=90 -# Consts (don't adjust these...) +# Constants (don't adjust these) export BASE_DIR RELATIVE_BASE_DIR=`python -c "import os,os.path; print os.path.relpath(os.environ['BASE_DIR'], os.environ['PWD'])"` +export PYTHONPATH=${BASE_DIR}/qpid/python:${BASE_DIR}/qpid/extras/qmf/src/py:${BASE_DIR}/qpid/tools/src/py LOG_FILE_NAME=log.txt QPIDD_FN=qpidd QPIDD=${CMAKE_BUILD_DIR}/src/${QPIDD_FN} -TXTEST=${CMAKE_BUILD_DIR}/src/tests/qpid-txtest +TXTEST_FN=qpid-txtest +TXTEST=${CMAKE_BUILD_DIR}/src/tests/${TXTEST_FN} +ANALYZE_FN=qpid_qls_analyze.py +ANALYZE=${BASE_DIR}/qpid/tools/src/py/${ANALYZE_FN} +ANALYZE_ARGS="--efp --show-recs --stats" QPIDD_BASE_ARGS="--load-module ${STORE_MODULE} -m ${BROKER_MANAGEMENT} --auth no --default-flow-stop-threshold 0 --default-flow-resume-threshold 0 --default-queue-limit 0 --store-dir ${BASE_DIR} --log-enable ${BROKER_LOG_LEVEL} --log-to-stderr no --log-to-stdout no" TXTEST_INIT_STR="--init yes --transfer no --check no" TXTEST_RUN_STR="--init no --transfer yes --check no" @@ -181,6 +187,17 @@ check_ready_to_run() { fi } +# Analyze store files +# $1: Log suffix flag: either "A" or "B". If "A", client is started in test mode, otherwise client evaluates recovery. +analyze_store() { + ${ANALYZE} ${ANALYZE_ARGS} ${BASE_DIR}/qls &> ${RESULT_DIR}/qls_analysis.$1.log + echo >> ${RESULT_DIR}/qls_analysis.$1.log + echo "----------------------------------------------------------" >> ${RESULT_DIR}/qls_analysis.$1.log + echo "With transactional reconsiliation:" >> ${RESULT_DIR}/qls_analysis.$1.log + echo >> ${RESULT_DIR}/qls_analysis.$1.log + ${ANALYZE} ${ANALYZE_ARGS} --txn ${BASE_DIR}/qls &>> ${RESULT_DIR}/qls_analysis.$1.log +} + ulimit -c unlimited # Allow core files to be created RESULT_BASE_DIR_SUFFIX=`date "${TIMESTAMP_FORMAT}"` @@ -219,7 +236,8 @@ for rn in `seq ${NUM_RUNS}`; do sleep ${RUN_TIME} kill_process ${SIG_KILL} ${QPIDD_PID} sleep 2 - tar -czf ${RESULT_DIR}/qls_B.tar.gz ${RELATIVE_BASE_DIR}/qls + analyze_store "A" + tar -czf ${RESULT_DIR}/qls_A.tar.gz ${RELATIVE_BASE_DIR}/qls # === PART B: Recovery and check === start_broker "B" @@ -234,11 +252,14 @@ for rn in `seq ${NUM_RUNS}`; do kill_process ${SIG_KILL} ${PID} sleep 2 fi - tar -czf ${RESULT_DIR}/qls_C.tar.gz ${RELATIVE_BASE_DIR}/qls + analyze_store "B" + tar -czf ${RESULT_DIR}/qls_B.tar.gz ${RELATIVE_BASE_DIR}/qls # === Check for errors, cores and exceptions in logs === grep -Hn "jexception" ${RESULT_DIR}/qpidd.A.log | tee -a ${LOG_FILE} grep -Hn "jexception" ${RESULT_DIR}/qpidd.B.log | tee -a ${LOG_FILE} + grep -Hn "Traceback (most recent call last):" ${RESULT_DIR}/qls_analysis.A.log | tee -a ${LOG_FILE} + grep -Hn "Traceback (most recent call last):" ${RESULT_DIR}/qls_analysis.B.log | tee -a ${LOG_FILE} grep "${SUCCESS_MSG}" ${RESULT_DIR}/txtest.B.log &> /dev/null if [[ "$?" != "0" ]]; then echo "ERROR in run ${rn}" >> ${LOG_FILE} diff --git a/qpid/doc/book/src/cpp-broker/Active-Passive-Cluster.xml b/qpid/doc/book/src/cpp-broker/Active-Passive-Cluster.xml index 116fd8daef..6cb1b768ab 100644 --- a/qpid/doc/book/src/cpp-broker/Active-Passive-Cluster.xml +++ b/qpid/doc/book/src/cpp-broker/Active-Passive-Cluster.xml @@ -47,7 +47,7 @@ under the License. <para> This approach relies on an external <firstterm>cluster resource manager</firstterm> to detect failures, choose the new primary and handle network partitions. <ulink - url="https://fedorahosted.org/cluster/wiki/RGManager">Rgmanager</ulink> is supported + url="https://fedorahosted.org/cluster/wiki/RGManager">rgmanager</ulink> is supported initially, but others may be supported in the future. </para> <section id="ha-at-least-once"> @@ -81,7 +81,7 @@ under the License. <footnote> <para> Clients must use "at-least-once" reliability to enable re-send of unacknowledged - messages. This is the default behavior, no options need be set to enable it. For + messages. This is the default behaviour, no options need be set to enable it. For details of client addressing options see "Using the Qpid Messaging API" in <citetitle>Programming in Apache Qpid</citetitle>. </para> @@ -102,7 +102,7 @@ under the License. </para> <para> When a new primary is promoted after a fail-over it is initially in - "recovering" mode. In this mode, it delays acknowledgment of messages + "recovering" mode. In this mode, it delays acknowledgement of messages on behalf of all the backups that were connected to the previous primary. This protects those messages against a failure of the new primary until the backups have a chance to connect and catch up. @@ -277,18 +277,15 @@ ssl_addr = "ssl:" host [":" port]' <entry><literal>ha-public-url <replaceable>URL</replaceable></literal> </entry> <entry> <para> - The URL <footnoteref linkend="ha-url-grammar"/> is advertised to - clients as the "known-hosts" for fail-over. It can be a list or - a single virtual IP address. A virtual IP address is recommended. + This option is only needed for backwards compatibility if you + have been using the <literal>amq.failover</literal> exchange. + This exchange is now obsolete, it is recommended to use a + virtual IP address instead. </para> <para> - Using this option you can put client and broker traffic on - separate networks, which is recommended. - </para> - <para> - Note: When HA clustering is enabled the broker option - <literal>known-hosts-url</literal> is ignored and over-ridden by - the <literal>ha-public-url</literal> setting. + If set, this URL is advertized by the + <literal>amq.failover</literal> exchange and overrides the + broker option <literal>known-hosts-url</literal> </para> </entry> </row> @@ -426,7 +423,7 @@ NOTE: fencing is not shown, you must configure fencing appropriately for your cl --> <cluster name="qpid-test" config_version="18"> - <!-- The cluster has 3 nodes. Each has a unique nodid and one vote + <!-- The cluster has 3 nodes. Each has a unique nodeid and one vote for quorum. --> <clusternodes> <clusternode name="node1.example.com" nodeid="1"/> @@ -498,20 +495,17 @@ NOTE: fencing is not shown, you must configure fencing appropriately for your cl </para> <para> The <literal>resources</literal> section also defines a virtual IP - address for clients. + address for clients: <literal>20.0.20.200</literal>. </para> <para> - To take advantage of the virtual IP addresses, <filename>qpidd.conf</filename> - should contain these lines: + <filename>qpidd.conf</filename> should contain these lines: </para> <programlisting> ha-cluster=yes - ha-public-url=20.0.10.200 ha-brokers-url=20.0.20.1,20.0.20.2,20.0.20.3 </programlisting> <para> - This configuration allows clients to connect to a single address: - 20.0.10.200. The brokers connect to each other directly via the addresses + The brokers connect to each other directly via the addresses listed in <command>ha-brokers-url</command>. Note the client and broker addresses are on separate sub-nets, this is recommended but not required. </para> @@ -622,10 +616,7 @@ NOTE: fencing is not shown, you must configure fencing appropriately for your cl <para> Clients can only connect to the primary broker. Backup brokers reject any connection attempt by a client. Clients rejected by a backup broker will - automatically fail-over until they connect to the primary. if - <literal>ha-public-url</literal> contains multiple addresses, the client - will them all in rotation. If it is a virtual IP address the clients will - retry on the same address until it is reconnected. + automatically fail-over until they connect to the primary. </para> <para> Clients are configured with the URL for the cluster (details below for @@ -704,10 +695,10 @@ NOTE: fencing is not shown, you must configure fencing appropriately for your cl Heartbeats are disabled by default. You can enable them by specifying a heartbeat interval (in seconds) for the connection via the <literal>heartbeat</literal> option. For example: - <programlisting> - qpid::messaging::Connection c("node1,node2,node3","{reconnect:true,heartbeat:10}"); - </programlisting> </para> + <programlisting> + qpid::messaging::Connection c("node1,node2,node3","{reconnect:true,heartbeat:10}"); + </programlisting> </section> <section id="ha-python-client"> <title>Python clients</title> @@ -886,6 +877,3 @@ NOTE: fencing is not shown, you must configure fencing appropriately for your cl </section> </section> - -<!-- LocalWords: scalability rgmanager multicast RGManager mailto LVQ qpidd IP dequeued Transactional username API ---> diff --git a/qpid/doc/book/src/java-broker/Java-Broker-Security-ACLs.xml b/qpid/doc/book/src/java-broker/Java-Broker-Security-ACLs.xml index 03537115a4..bd0d543c05 100644 --- a/qpid/doc/book/src/java-broker/Java-Broker-Security-ACLs.xml +++ b/qpid/doc/book/src/java-broker/Java-Broker-Security-ACLs.xml @@ -25,9 +25,7 @@ <title>Access Control Lists</title> <para> In Qpid, Access Control Lists (ACLs) specify which actions can be performed by each authenticated user. - To enable, an <emphasis>Access Control Provider</emphasis> needs to be configured on the <emphasis>Broker</emphasis> - level or/and ACL configuration should be provided on a <emphasis>Virtual Host</emphasis> level. - The first imposes the ACL broker wide, and the second is applied to individual virtual hosts. + To enable, an <emphasis>Access Control Provider</emphasis> needs to be configured on the <emphasis>Broker</emphasis>. The <emphasis>Access Control Provider</emphasis> of type "AclFile" uses local file to specify the ACL rules. By convention, this file should have a .acl extension. </para> @@ -41,12 +39,7 @@ <para>Only one <emphasis>Access Control Provider</emphasis> can be used by the Broker. If several <emphasis>Access Control Providers</emphasis> are configured on Broker level - only one of them will be used (the latest one). <xref linkend="Java-Broker-Virtual-Hosts-Configuration-File-ACL"/> - shows how to configure ACL on <emphasis>Virtual Host</emphasis> using virtual host configuration xml. - If both Broker <emphasis>Access Control Provider</emphasis> and <emphasis>Virtual Host</emphasis> ACL are configured, - the <emphasis>Virtual Host</emphasis> ACL is used for authorization of operations on <emphasis>Virtual Host</emphasis> and - Virtual Host objects and Broker level ACL is used to authorization of operations on Broker and Broker children - (excluding Virtual Hosts having ACL configured). + only one of them will be used (the latest one). </para> <para> @@ -114,6 +107,20 @@ properties. Most projects probably won't need this degree of flexibility. A reasonable approach is to choose to apply permissions at a certain level of abstraction (e.g. QUEUE) and apply them consistently across the whole system. </para> + <note> + <para> + Some rules can be restricted to the virtual host if property virtualhost_name is specified. + <example> + <title>Restrict rules to specific virtual hosts</title> + <programlisting> + ACL ALLOW bob CREATE QUEUE virtualhost_name="test" + ACL ALLOW bob ALL EXCHANGE virtualhost_name="prod" + </programlisting> + </example> + In the example above the first rule allows user "bob" to create queues on virtual host "test" only. + The second rule allows user "bob" any action with exchanges on virtual host "prod". + </para> + </note> </section> <section role="h4" id="Java-Broker-Security-ACLs-Syntax"> @@ -125,7 +132,7 @@ ACL rules follow this syntax: </para> <programlisting> - ACL {permission} {<group-name>|<user-name>>|ALL} {action|ALL} [object|ALL] [property="<property-value>"] + ACL {permission} {<group-name>|<user-name>|ALL} {action|ALL} [object|ALL] [property="<property-value>"] </programlisting> <para> @@ -163,90 +170,144 @@ </table> <table id="table-Java-Broker-Security-ACLs-Syntax_actions"> <title>List of ACL actions</title> - <tgroup cols="2"> + <tgroup cols="4"> + <thead> + <row> + <entry><para>Action</para></entry> + <entry><para>Description</para></entry> + <entry><para>Supported object types</para></entry> + <entry><para>Supported properties</para></entry> + </row> + </thead> <tbody> <row> <entry> <command>CONSUME</command> </entry> <entry> <para> Applied when subscriptions are created </para> </entry> + <entry><para>QUEUE</para></entry> + <entry><para>name, autodelete, temporary, durable, exclusive, alternate, owner, virtualhost_name</para></entry> </row> <row> <entry> <command>PUBLISH</command> </entry> <entry> <para> Applied on a per message basis on publish message transfers</para> </entry> + <entry><para>EXCHANGE</para></entry> + <entry><para>name, routingkey, immediate, virtualhost_name</para></entry> </row> <row> <entry> <command>CREATE</command> </entry> <entry> <para> Applied when an object is created, such as bindings, queues, exchanges</para> </entry> + <entry><para>EXCHANGE, QUEUE, USER, GROUP</para></entry> + <entry><para>see properties on the corresponding object type</para></entry> </row> <row> <entry> <command>ACCESS</command> </entry> <entry> <para> Applied when an object is read or accessed</para> </entry> + <entry><para>VIRTUALHOST, MANAGEMENT</para></entry> + <entry><para>name (for VIRTUALHOST only)</para></entry> </row> <row> <entry> <command>BIND</command> </entry> <entry> <para> Applied when queues are bound to exchanges</para> </entry> + <entry><para>EXCHANGE</para></entry> + <entry><para>name, routingKey, queuename, virtualhost_name, temporary, durable</para></entry> </row> <row> <entry> <command>UNBIND</command> </entry> <entry> <para> Applied when queues are unbound from exchanges</para> </entry> + <entry><para>EXCHANGE</para></entry> + <entry><para>name, routingKey, queuename, virtualhost_name, temporary, durable</para></entry> </row> <row> <entry> <command>DELETE</command> </entry> <entry> <para> Applied when objects are deleted </para> </entry> + <entry><para>EXCHANGE, QUEUE, USER, GROUP</para></entry> + <entry><para>see properties on the corresponding object type</para></entry> </row> <row> <entry> <command>PURGE</command> </entry> <entry> <para>Applied when purge the contents of a queue</para> </entry> + <entry><para>QUEUE</para></entry> + <entry><para> </para></entry> </row> <row> <entry> <command>UPDATE</command> </entry> <entry> <para> Applied when an object is updated </para> </entry> + <entry><para>EXCHANGE, QUEUE, USER, GROUP</para></entry> + <entry><para>see EXCHANGE and QUEUE properties</para></entry> </row> <row> <entry> <command>CONFIGURE</command> </entry> - <entry> <para> Applied when an object is configured via REST management interfaces(Java Broker only).</para> </entry> + <entry> <para> Applied when an object is configured via REST management interfaces.</para> </entry> + <entry><para>BROKER</para></entry> + <entry><para> </para></entry> + </row> + <row> + <entry><command>ACCESS_LOGS</command> </entry> + <entry><para>Allows/denies to the specific user an operation to download broker log file(s) over REST interfaces</para> </entry> + <entry><para>BROKER</para></entry> + <entry><para> </para></entry> </row> </tbody> </tgroup> </table> <table id="table-Java-Broker-Security-ACLs-Syntax_objects"> <title>List of ACL objects</title> - <tgroup cols="2"> + <tgroup cols="4"> + <thead> + <row> + <entry><para>Object type</para></entry> + <entry><para>Description</para></entry> + <entry><para>Supported actions</para></entry> + <entry><para>Supported properties</para></entry> + </row> + </thead> <tbody> <row> <entry> <command>VIRTUALHOST</command> </entry> - <entry> <para>A virtualhost (Java Broker only)</para> </entry> + <entry> <para>A virtualhost</para> </entry> + <entry><para>ALL, ACCESS</para> </entry> + <entry><para>name</para> </entry> </row> <row> <entry> <command>MANAGEMENT </command> </entry> - <entry> <para>Management - for web and JMX (Java Broker only)</para> </entry> + <entry> <para>Management - for web and JMX</para> </entry> + <entry><para>ALL, ACCESS</para> </entry> + <entry><para> </para></entry> </row> <row> <entry> <command>QUEUE</command> </entry> <entry> <para>A queue </para> </entry> + <entry><para>ALL, CREATE, DELETE, PURGE, CONSUME, UPDATE</para></entry> + <entry><para>name, autodelete, temporary, durable, exclusive, alternate, owner, virtualhost_name</para></entry> </row> <row> <entry> <command>EXCHANGE</command> </entry> - <entry> <para>An exchange </para> </entry> + <entry><para>An exchange</para></entry> + <entry><para>ALL, ACCESS, CREATE, DELETE, BIND, UNBIND, PUBLISH, UPDATE</para></entry> + <entry><para>name, autodelete, temporary, durable, type, virtualhost_name, queuename(only for BIND and UNBIND), routingkey(only for BIND and UNBIND, PUBLISH)</para></entry> </row> <row> <entry> <command>USER</command> </entry> - <entry> <para>A user (Java Broker only)</para> </entry> + <entry> <para>A user</para> </entry> + <entry><para>ALL, CREATE, DELETE, UPDATE</para></entry> + <entry><para>name</para></entry> </row> <row> <entry> <command>GROUP</command> </entry> - <entry> <para>A group (Java Broker only)</para> </entry> + <entry> <para>A group</para> </entry> + <entry><para>ALL, CREATE, DELETE, UPDATE</para></entry> + <entry><para>name</para></entry> </row> <row> <entry> <command>METHOD</command> </entry> - <entry> <para>Management or agent or broker method (Java Broker only)</para> </entry> - </row> - <row> - <entry> <command>LINK</command> </entry> - <entry> <para>A federation or inter-broker link (not currently used in Java Broker)</para> </entry> + <entry> <para>Management or agent or broker method</para> </entry> + <entry><para>ALL, ACCESS, UPDATE</para></entry> + <entry><para>name, component, virtualhost_name</para></entry> </row> <row> <entry> <command>BROKER</command> </entry> <entry> <para>The broker</para> </entry> + <entry><para>ALL, CONFIGURE, ACCESS_LOGS</para></entry> + <entry><para> </para></entry> </row> </tbody> </tgroup> @@ -268,10 +329,6 @@ <entry> <para> String. Specifies routing key </para> </entry> </row> <row> - <entry> <command>passive</command> </entry> - <entry> <para> Boolean. Indicates the presence of a <parameter>passive</parameter> flag </para> </entry> - </row> - <row> <entry> <command>autodelete</command> </entry> <entry> <para> Boolean. Indicates whether or not the object gets deleted when the connection is closed </para> </entry> </row> @@ -297,15 +354,7 @@ </row> <row> <entry> <command>component</command> </entry> - <entry> <para> String. JMX component name (Java Broker only)</para> </entry> - </row> - <row> - <entry> <command>schemapackage</command> </entry> - <entry> <para> String. QMF schema package name (Not used in Java Broker)</para> </entry> - </row> - <row> - <entry> <command>schemaclass</command> </entry> - <entry> <para> String. QMF schema class name (Not used in Java Broker)</para> </entry> + <entry> <para> String. JMX component name</para> </entry> </row> <row> <entry> <command>from_network</command> </entry> @@ -322,9 +371,6 @@ (e.g. 192.168.1.0/24; see <ulink url="http://tools.ietf.org/html/rfc4632">RFC 4632</ulink>) or wildcards (e.g. 192.169.1.*). </para> - <para> - Java Broker only. - </para> </entry> </row> <row> @@ -354,8 +400,21 @@ $JAVA_HOME/lib/security/java.security. The latter is preferred because it is JVM vendor-independent. </para> + </entry> + </row> + <row> + <entry><command>virtualhost_name</command></entry> + <entry> <para> - Java Broker only. + String. A name of virtual host to which the rule is applied. + </para> + </entry> + </row> + <row> + <entry><command>immediate</command></entry> + <entry> + <para> + Boolean. A property can be used to restrict PUBLISH action to publishing only messages with given immediate flag. </para> </entry> </row> @@ -363,28 +422,24 @@ </tgroup> </table> <table id="table-Java-Broker-Security-ACLs-Syntax_javacomponents"> - <title>List of ACL rules</title> - <tgroup cols="3"> + <title>List of ACL JMX Components</title> + <tgroup cols="2"> <tbody> <row> <entry> <command>UserManagement</command> </entry> <entry> <para>User maintainance; create/delete/view users, change passwords etc</para> </entry> - <entry> <para>permissionable at broker level only</para> </entry> </row> <row> <entry> <command>ConfigurationManagement</command> </entry> <entry> <para>Dynammically reload configuration from disk.</para> </entry> - <entry> <para>permissionable at broker level only</para> </entry> </row> <row> <entry> <command>LoggingManagement</command> </entry> <entry> <para>Dynammically control Qpid logging level</para> </entry> - <entry> <para>permissionable at broker level only</para> </entry> </row> <row> <entry> <command>ServerInformation</command> </entry> <entry> <para>Read-only information regarding the Qpid: version number etc</para> </entry> - <entry> <para>permissionable at broker level only</para> </entry> </row> <row> <entry> <command>VirtualHost.Queue</command> </entry> diff --git a/qpid/doc/book/src/java-broker/Java-Broker-Virtual-Hosts-Configuration.xml b/qpid/doc/book/src/java-broker/Java-Broker-Virtual-Hosts-Configuration.xml index 6a4c8485e3..43007a3242 100644 --- a/qpid/doc/book/src/java-broker/Java-Broker-Virtual-Hosts-Configuration.xml +++ b/qpid/doc/book/src/java-broker/Java-Broker-Virtual-Hosts-Configuration.xml @@ -43,36 +43,7 @@ </para> </section> - <section id="Java-Broker-Virtual-Hosts-Configuration-File-ACL"> - <title>Configuring ACL</title> - <para><xref linkend="Java-Broker-Security-ACLs"/> provides the details of ACL, rules, formats, etc.</para> - <para> - To apply an ACL on a single virtualhost named <replaceable>test</replaceable>, add the following to the virtualhosts.xml: - </para> - - <programlisting> -<virtualhost> -... - <name>test</name> - <test> - ... - <security> <co id="Java-Broker-Virtual-Hosts-Configuration-Security-ACL-1"/> - ... - <acl><replaceable>${conf}/vhost_test.acl</replaceable></acl> <co id="Java-Broker-Virtual-Hosts-Configuration-Security-ACL-2"/> - ... - </security> - ... - </test> -</virtualhost> - </programlisting> - <calloutlist> - <callout arearefs="Java-Broker-Virtual-Hosts-Configuration-Security-ACL-1"><para>A security section of configuration is used to declare the ACL</para></callout> - <callout arearefs="Java-Broker-Virtual-Hosts-Configuration-Security-ACL-2"><para>A path to an ACL file is configured (assuming that <replaceable>conf</replaceable> has been set to a suitable - location such as ${QPID_HOME}/etc)</para></callout> - </calloutlist> - </section> - - <section role="h3" id="Java-Broker-Stores-Memory-Store-Configuration"> + <section role="h3" id="Java-Broker-Stores-Memory-Store-Configuration"> <title>Configuring MemoryMessageStore</title> <para> An example of MemoryMessageStore configuration for a virtual host is shown below: diff --git a/qpid/doc/book/src/java-broker/commonEntities.xml b/qpid/doc/book/src/java-broker/commonEntities.xml index 2e7a181d65..8dd3d61f55 100644 --- a/qpid/doc/book/src/java-broker/commonEntities.xml +++ b/qpid/doc/book/src/java-broker/commonEntities.xml @@ -22,7 +22,7 @@ <!ENTITY qpidDownloadUrl "http://qpid.apache.org/download.html"> <!ENTITY qpidCppBook "../../AMQP-Messaging-Broker-CPP-Book/html/"> -<!ENTITY qpidCurrentRelease "0.27"> +<!ENTITY qpidCurrentRelease "0.29"> <!ENTITY windowsBrokerDownloadFileName "qpid-java-broker-&qpidCurrentRelease;.zip"> <!ENTITY windowsExtractedBrokerDirName "qpid-broker-&qpidCurrentRelease;"> diff --git a/qpid/doc/book/src/jms-client-0-8/commonEntities.xml b/qpid/doc/book/src/jms-client-0-8/commonEntities.xml index b3d1658fa8..d3247f0483 100644 --- a/qpid/doc/book/src/jms-client-0-8/commonEntities.xml +++ b/qpid/doc/book/src/jms-client-0-8/commonEntities.xml @@ -19,7 +19,7 @@ --> -<!ENTITY qpidVersion "0.27"> +<!ENTITY qpidVersion "0.29"> <!ENTITY qpidDownloadUrlDesc "the Apache Qpid project web site"> <!ENTITY qpidDownloadUrl "http://qpid.apache.org/download.html"> diff --git a/qpid/extras/qmf/setup.py b/qpid/extras/qmf/setup.py index a181866cd1..3022d4b2cd 100755 --- a/qpid/extras/qmf/setup.py +++ b/qpid/extras/qmf/setup.py @@ -20,7 +20,7 @@ from distutils.core import setup setup(name="qpid-qmf", - version="0.27", + version="0.29", author="Apache Qpid", author_email="dev@qpid.apache.org", packages=["qmf"], diff --git a/qpid/java/bdbstore/pom.xml b/qpid/java/bdbstore/pom.xml index d7d12f0b33..5d34559a46 100644 --- a/qpid/java/bdbstore/pom.xml +++ b/qpid/java/bdbstore/pom.xml @@ -102,6 +102,20 @@ </includes> </resource> </resources> + + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> </build> </project> diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBConfiguredObjectRecord.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBConfiguredObjectRecord.java new file mode 100644 index 0000000000..f13e4dd08b --- /dev/null +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBConfiguredObjectRecord.java @@ -0,0 +1,107 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.store.berkeleydb; + +import org.apache.qpid.server.store.ConfiguredObjectRecord; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class BDBConfiguredObjectRecord implements ConfiguredObjectRecord +{ + private final UUID _id; + private final String _type; + private final Map<String,Object> _attributes; + private Map<String, ConfiguredObjectRecord> _parents = new HashMap<String, ConfiguredObjectRecord>(); + + public BDBConfiguredObjectRecord(final UUID id, final String type, final Map<String, Object> attributes) + { + _id = id; + _type = type; + _attributes = Collections.unmodifiableMap(attributes); + } + + public UUID getId() + { + return _id; + } + + public String getType() + { + return _type; + } + + public Map<String, Object> getAttributes() + { + return _attributes; + } + + void addParent(String parentType, ConfiguredObjectRecord parent) + { + _parents.put(parentType, parent); + } + + @Override + public Map<String, ConfiguredObjectRecord> getParents() + { + return Collections.unmodifiableMap(_parents); + } + + @Override + public boolean equals(final Object o) + { + if (this == o) + { + return true; + } + if (o == null || getClass() != o.getClass()) + { + return false; + } + + final BDBConfiguredObjectRecord that = (BDBConfiguredObjectRecord) o; + + if (_attributes != null ? !_attributes.equals(that._attributes) : that._attributes != null) + { + return false; + } + if (_id != null ? !_id.equals(that._id) : that._id != null) + { + return false; + } + if (_type != null ? !_type.equals(that._type) : that._type != null) + { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = _id != null ? _id.hashCode() : 0; + result = 31 * result + (_type != null ? _type.hashCode() : 0); + return result; + } +} diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHost.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHost.java index 7e42d09ba6..492ec9d7bf 100644 --- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHost.java +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBHAVirtualHost.java @@ -242,10 +242,10 @@ public class BDBHAVirtualHost extends AbstractVirtualHost DurableConfigurationRecoverer configRecoverer = new DurableConfigurationRecoverer(getName(), getDurableConfigurationRecoverers(), new DefaultUpgraderProvider(BDBHAVirtualHost.this, getExchangeRegistry()), getEventLogger()); - _messageStore.recoverConfigurationStore(configRecoverer); + _messageStore.recoverConfigurationStore(getModel(), configRecoverer); VirtualHostConfigRecoveryHandler recoveryHandler = new VirtualHostConfigRecoveryHandler(BDBHAVirtualHost.this); - _messageStore.recoverMessageStore(recoveryHandler, recoveryHandler); + _messageStore.recoverMessageStore(getModel(), recoveryHandler, recoveryHandler); } catch (Exception e) { diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBMessageStore.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBMessageStore.java index 2022f36bd9..ec6ae23367 100644 --- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBMessageStore.java +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBMessageStore.java @@ -25,6 +25,8 @@ import java.lang.ref.SoftReference; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -34,6 +36,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.log4j.Logger; import org.apache.qpid.server.message.EnqueueableMessage; +import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.queue.AMQQueue; import org.apache.qpid.server.store.ConfigurationRecoveryHandler; import org.apache.qpid.server.store.ConfiguredObjectRecord; @@ -54,11 +57,13 @@ import org.apache.qpid.server.store.StoredMessage; import org.apache.qpid.server.store.TransactionLogRecoveryHandler; import org.apache.qpid.server.store.TransactionLogRecoveryHandler.QueueEntryRecoveryHandler; import org.apache.qpid.server.store.TransactionLogResource; +import org.apache.qpid.server.store.berkeleydb.entry.HierarchyKey; import org.apache.qpid.server.store.berkeleydb.entry.PreparedTransaction; import org.apache.qpid.server.store.berkeleydb.entry.QueueEntryKey; import org.apache.qpid.server.store.berkeleydb.entry.Xid; import org.apache.qpid.server.store.berkeleydb.tuple.ConfiguredObjectBinding; import org.apache.qpid.server.store.berkeleydb.tuple.ContentBinding; +import org.apache.qpid.server.store.berkeleydb.tuple.HierarchyKeyBinding; import org.apache.qpid.server.store.berkeleydb.tuple.MessageMetaDataBinding; import org.apache.qpid.server.store.berkeleydb.tuple.PreparedTransactionBinding; import org.apache.qpid.server.store.berkeleydb.tuple.QueueEntryBinding; @@ -94,9 +99,11 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore { private static final Logger LOGGER = Logger.getLogger(BDBMessageStore.class); - public static final int VERSION = 7; + public static final int VERSION = 8; private static final int LOCK_RETRY_ATTEMPTS = 5; private static String CONFIGURED_OBJECTS_DB_NAME = "CONFIGURED_OBJECTS"; + private static String CONFIGURED_OBJECT_HIERARCHY_DB_NAME = "CONFIGURED_OBJECT_HIERARCHY"; + private static String MESSAGE_META_DATA_DB_NAME = "MESSAGE_METADATA"; private static String MESSAGE_CONTENT_DB_NAME = "MESSAGE_CONTENT"; private static String DELIVERY_DB_NAME = "QUEUE_ENTRIES"; @@ -106,7 +113,7 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore private static String LINKDB_NAME = "LINKS"; private static String XID_DB_NAME = "XIDS"; private static String CONFIG_VERSION_DB_NAME = "CONFIG_VERSION"; - private static final String[] CONFIGURATION_STORE_DATABASE_NAMES = new String[] { CONFIGURED_OBJECTS_DB_NAME, CONFIG_VERSION_DB_NAME }; + private static final String[] CONFIGURATION_STORE_DATABASE_NAMES = new String[] { CONFIGURED_OBJECTS_DB_NAME, CONFIG_VERSION_DB_NAME , CONFIGURED_OBJECT_HIERARCHY_DB_NAME}; private static final String[] MESSAGE_STORE_DATABASE_NAMES = new String[] { MESSAGE_META_DATA_DB_NAME, MESSAGE_CONTENT_DB_NAME, DELIVERY_DB_NAME, BRIDGEDB_NAME, LINKDB_NAME, XID_DB_NAME }; private EnvironmentFacade _environmentFacade; @@ -163,7 +170,7 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore } @Override - public void recoverConfigurationStore(ConfigurationRecoveryHandler recoveryHandler) + public void recoverConfigurationStore(ConfiguredObject<?> parent, ConfigurationRecoveryHandler recoveryHandler) { _configurationStoreStateManager.attainState(State.ACTIVATING); @@ -172,7 +179,7 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore dbConfig.setAllowCreate(true); try { - new Upgrader(_environmentFacade.getEnvironment(), _virtualHostName).upgradeIfNecessary(); + new Upgrader(_environmentFacade.getEnvironment(), parent).upgradeIfNecessary(); _environmentFacade.openDatabases(dbConfig, CONFIGURATION_STORE_DATABASE_NAMES); } catch(DatabaseException e) @@ -216,7 +223,7 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore } @Override - public synchronized void recoverMessageStore(MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) throws StoreException + public synchronized void recoverMessageStore(ConfiguredObject<?> parent, MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) throws StoreException { _messageStoreStateManager.attainState(State.ACTIVATING); DatabaseConfig dbConfig = new DatabaseConfig(); @@ -224,13 +231,13 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore dbConfig.setAllowCreate(true); try { - new Upgrader(_environmentFacade.getEnvironment(), _virtualHostName).upgradeIfNecessary(); + new Upgrader(_environmentFacade.getEnvironment(), parent).upgradeIfNecessary(); _environmentFacade.openDatabases(dbConfig, MESSAGE_STORE_DATABASE_NAMES); _totalStoreSize = getSizeOnDisk(); } catch(DatabaseException e) { - throw _environmentFacade.handleDatabaseException("Cannot activate message store", e); + throw _environmentFacade.handleDatabaseException("Cannot upgrade message store or open datatbases", e); } if(messageRecoveryHandler != null) @@ -352,10 +359,11 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore @SuppressWarnings("resource") private void updateConfigVersion(int newConfigVersion) throws StoreException { + Transaction txn = null; Cursor cursor = null; try { - Transaction txn = _environmentFacade.getEnvironment().beginTransaction(null, null); + txn = _environmentFacade.getEnvironment().beginTransaction(null, null); cursor = getConfigVersionDb().openCursor(txn, null); DatabaseEntry key = new DatabaseEntry(); ByteBinding.byteToEntry((byte) 0,key); @@ -373,10 +381,12 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore cursor.close(); cursor = null; txn.commit(); + txn = null; } finally { closeCursorSafely(cursor); + abortTransactionIgnoringException("Error setting config version", txn);; } } @@ -412,24 +422,57 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore private void loadConfiguredObjects(ConfigurationRecoveryHandler crh) throws DatabaseException, StoreException { - Cursor cursor = null; + Cursor objectsCursor = null; + Cursor hierarchyCursor = null; try { - cursor = getConfiguredObjectsDb().openCursor(null, null); + objectsCursor = getConfiguredObjectsDb().openCursor(null, null); DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); - while (cursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS) + + Map<UUID, BDBConfiguredObjectRecord> configuredObjects = + new HashMap<UUID, BDBConfiguredObjectRecord>(); + + while (objectsCursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS) { UUID id = UUIDTupleBinding.getInstance().entryToObject(key); - ConfiguredObjectRecord configuredObject = new ConfiguredObjectBinding(id).entryToObject(value); - crh.configuredObject(configuredObject.getId(),configuredObject.getType(),configuredObject.getAttributes()); + BDBConfiguredObjectRecord configuredObject = + (BDBConfiguredObjectRecord) new ConfiguredObjectBinding(id).entryToObject(value); + configuredObjects.put(configuredObject.getId(), configuredObject); + } + + // set parents + hierarchyCursor = getConfiguredObjectHierarchyDb().openCursor(null, null); + while (hierarchyCursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS) + { + HierarchyKey hk = HierarchyKeyBinding.getInstance().entryToObject(key); + UUID parentId = UUIDTupleBinding.getInstance().entryToObject(value); + BDBConfiguredObjectRecord child = configuredObjects.get(hk.getChildId()); + if(child != null) + { + ConfiguredObjectRecord parent = configuredObjects.get(parentId); + if(parent != null) + { + child.addParent(hk.getParentType(), parent); + } + else if(hk.getParentType().equals("Exchange")) + { + // TODO - remove this hack for the pre-defined exchanges + child.addParent(hk.getParentType(), new BDBConfiguredObjectRecord(parentId, "Exchange", Collections.<String,Object>emptyMap())); + } + } } + for (ConfiguredObjectRecord record : configuredObjects.values()) + { + crh.configuredObject(record); + } } finally { - closeCursorSafely(cursor); + closeCursorSafely(objectsCursor); + closeCursorSafely(hierarchyCursor); } } @@ -448,7 +491,6 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore } } - private void recoverMessages(MessageStoreRecoveryHandler msrh) throws StoreException { StoredMessageRecoveryHandler mrh = msrh.begin(); @@ -568,9 +610,8 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore dtxrh.completeDtxRecordRecovery(); } - public void removeMessage(long messageId, boolean sync) throws StoreException + void removeMessage(long messageId, boolean sync) throws StoreException { - boolean complete = false; com.sleepycat.je.Transaction tx = null; @@ -715,116 +756,130 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore } @Override - public void create(UUID id, String type, Map<String, Object> attributes) throws StoreException + public void create(ConfiguredObjectRecord configuredObject) throws StoreException { if (_configurationStoreStateManager.isInState(State.ACTIVE)) { - ConfiguredObjectRecord configuredObject = new ConfiguredObjectRecord(id, type, attributes); - storeConfiguredObjectEntry(configuredObject); + com.sleepycat.je.Transaction txn = null; + try + { + txn = _environmentFacade.getEnvironment().beginTransaction(null, null); + storeConfiguredObjectEntry(txn, configuredObject); + txn.commit(); + txn = null; + } + catch (DatabaseException e) + { + throw _environmentFacade.handleDatabaseException("Error creating configured object " + configuredObject + + " in database: " + e.getMessage(), e); + } + finally + { + if (txn != null) + { + abortTransactionIgnoringException("Error creating configured object", txn); + } + } } } @Override - public void remove(UUID id, String type) throws StoreException + public UUID[] remove(final ConfiguredObjectRecord... objects) throws StoreException { - if (LOGGER.isDebugEnabled()) + com.sleepycat.je.Transaction txn = null; + try { - LOGGER.debug("public void remove(id = " + id + ", type="+type+"): called"); + txn = _environmentFacade.getEnvironment().beginTransaction(null, null); + + Collection<UUID> removed = new ArrayList<UUID>(objects.length); + for(ConfiguredObjectRecord record : objects) + { + if(removeConfiguredObject(txn, record) == OperationStatus.SUCCESS) + { + removed.add(record.getId()); + } + } + + txn.commit(); + txn = null; + return removed.toArray(new UUID[removed.size()]); } - OperationStatus status = removeConfiguredObject(null, id); - if (status == OperationStatus.NOTFOUND) + catch (DatabaseException e) { - throw new StoreException("Configured object of type " + type + " with id " + id + " not found"); + throw _environmentFacade.handleDatabaseException("Error deleting configured objects from database", e); } - } - - @Override - public UUID[] removeConfiguredObjects(final UUID... objects) throws StoreException - { - com.sleepycat.je.Transaction txn = _environmentFacade.getEnvironment().beginTransaction(null, null); - Collection<UUID> removed = new ArrayList<UUID>(objects.length); - for(UUID id : objects) + finally { - if(removeConfiguredObject(txn, id) == OperationStatus.SUCCESS) + if (txn != null) { - removed.add(id); + abortTransactionIgnoringException("Error deleting configured objects", txn); } } - commitTransaction(txn); - return removed.toArray(new UUID[removed.size()]); + } - private void commitTransaction(com.sleepycat.je.Transaction txn) throws StoreException + @Override + public void update(boolean createIfNecessary, ConfiguredObjectRecord... records) throws StoreException { + com.sleepycat.je.Transaction txn = null; try { + txn = _environmentFacade.getEnvironment().beginTransaction(null, null); + for(ConfiguredObjectRecord record : records) + { + update(createIfNecessary, record, txn); + } txn.commit(); + txn = null; } - catch(DatabaseException e) + catch (DatabaseException e) { - throw _environmentFacade.handleDatabaseException("Cannot commit transaction on configured objects removal", e); + throw _environmentFacade.handleDatabaseException("Error updating configuration details within the store: " + e,e); } - } - - @Override - public void update(UUID id, String type, Map<String, Object> attributes) throws StoreException - { - update(false, id, type, attributes, null); - } - - @Override - public void update(boolean createIfNecessary, ConfiguredObjectRecord... records) throws StoreException - { - com.sleepycat.je.Transaction txn = _environmentFacade.getEnvironment().beginTransaction(null, null); - for(ConfiguredObjectRecord record : records) + finally { - update(createIfNecessary, record.getId(), record.getType(), record.getAttributes(), txn); + if (txn != null) + { + abortTransactionIgnoringException("Error updating configuration details within the store", txn); + } } - commitTransaction(txn); + } - private void update(boolean createIfNecessary, UUID id, String type, Map<String, Object> attributes, com.sleepycat.je.Transaction txn) throws StoreException + private void update(boolean createIfNecessary, ConfiguredObjectRecord record, com.sleepycat.je.Transaction txn) throws StoreException { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Updating " + type + ", id: " + id); + LOGGER.debug("Updating " + record.getType() + ", id: " + record.getId()); } - try - { - DatabaseEntry key = new DatabaseEntry(); - UUIDTupleBinding keyBinding = UUIDTupleBinding.getInstance(); - keyBinding.objectToEntry(id, key); + DatabaseEntry key = new DatabaseEntry(); + UUIDTupleBinding keyBinding = UUIDTupleBinding.getInstance(); + keyBinding.objectToEntry(record.getId(), key); - DatabaseEntry value = new DatabaseEntry(); - DatabaseEntry newValue = new DatabaseEntry(); - ConfiguredObjectBinding configuredObjectBinding = ConfiguredObjectBinding.getInstance(); + DatabaseEntry value = new DatabaseEntry(); + DatabaseEntry newValue = new DatabaseEntry(); + ConfiguredObjectBinding configuredObjectBinding = ConfiguredObjectBinding.getInstance(); - OperationStatus status = getConfiguredObjectsDb().get(txn, key, value, LockMode.DEFAULT); - if (status == OperationStatus.SUCCESS || (createIfNecessary && status == OperationStatus.NOTFOUND)) + OperationStatus status = getConfiguredObjectsDb().get(txn, key, value, LockMode.DEFAULT); + final boolean isNewRecord = status == OperationStatus.NOTFOUND; + if (status == OperationStatus.SUCCESS || (createIfNecessary && isNewRecord)) + { + // write the updated entry to the store + configuredObjectBinding.objectToEntry(record, newValue); + status = getConfiguredObjectsDb().put(txn, key, newValue); + if (status != OperationStatus.SUCCESS) { - ConfiguredObjectRecord newQueueRecord = new ConfiguredObjectRecord(id, type, attributes); - - // write the updated entry to the store - configuredObjectBinding.objectToEntry(newQueueRecord, newValue); - status = getConfiguredObjectsDb().put(txn, key, newValue); - if (status != OperationStatus.SUCCESS) - { - throw new StoreException("Error updating configuration details within the store: " + status); - } + throw new StoreException("Error updating configuration details within the store: " + status); } - else if (status != OperationStatus.NOTFOUND) + if(isNewRecord) { - throw new StoreException("Error finding configuration details within the store: " + status); + writeHierarchyRecords(txn, record); } } - catch (DatabaseException e) + else if (status != OperationStatus.NOTFOUND) { - if (txn != null) - { - abortTransactionIgnoringException("Error updating configuration details within the store: " + e.getMessage(), txn); - } - throw _environmentFacade.handleDatabaseException("Error updating configuration details within the store: " + e,e); + throw new StoreException("Error finding configuration details within the store: " + status); } } @@ -1018,7 +1073,7 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore * * @throws StoreException If the operation fails for any reason. */ - public void abortTran(final com.sleepycat.je.Transaction tx) throws StoreException + private void abortTran(final com.sleepycat.je.Transaction tx) throws StoreException { if (LOGGER.isDebugEnabled()) { @@ -1091,7 +1146,7 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore * * @return A fresh message id. */ - public long getNewMessageId() + private long getNewMessageId() { return _messageId.incrementAndGet(); } @@ -1106,7 +1161,7 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore * * @throws StoreException If the operation fails for any reason, or if the specified message does not exist. */ - protected void addContent(final com.sleepycat.je.Transaction tx, long messageId, int offset, + private void addContent(final com.sleepycat.je.Transaction tx, long messageId, int offset, ByteBuffer contentBody) throws StoreException { DatabaseEntry key = new DatabaseEntry(); @@ -1183,7 +1238,7 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore * * @throws StoreException If the operation fails for any reason, or if the specified message does not exist. */ - public StorableMessageMetaData getMessageMetaData(long messageId) throws StoreException + StorableMessageMetaData getMessageMetaData(long messageId) throws StoreException { if (LOGGER.isDebugEnabled()) { @@ -1226,7 +1281,7 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore * * @throws StoreException If the operation fails for any reason, or if the specified message does not exist. */ - public int getContent(long messageId, int offset, ByteBuffer dst) throws StoreException + int getContent(long messageId, int offset, ByteBuffer dst) throws StoreException { DatabaseEntry contentKeyEntry = new DatabaseEntry(); LongBinding.longToEntry(messageId, contentKeyEntry); @@ -1291,20 +1346,17 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore } } - /** - * Makes the specified configured object persistent. - * - * @param configuredObject Details of the configured object to store. - * @throws StoreException If the operation fails for any reason. - */ - private void storeConfiguredObjectEntry(ConfiguredObjectRecord configuredObject) throws StoreException + private void storeConfiguredObjectEntry(final Transaction txn, ConfiguredObjectRecord configuredObject) throws StoreException { if (_configurationStoreStateManager.isInState(State.ACTIVE)) { - LOGGER.debug("Storing configured object: " + configuredObject); + if (LOGGER.isDebugEnabled()) + { + LOGGER.debug("Storing configured object: " + configuredObject); + } DatabaseEntry key = new DatabaseEntry(); - UUIDTupleBinding keyBinding = UUIDTupleBinding.getInstance(); - keyBinding.objectToEntry(configuredObject.getId(), key); + UUIDTupleBinding uuidBinding = UUIDTupleBinding.getInstance(); + uuidBinding.objectToEntry(configuredObject.getId(), key); DatabaseEntry value = new DatabaseEntry(); ConfiguredObjectBinding queueBinding = ConfiguredObjectBinding.getInstance(); @@ -1312,12 +1364,13 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore queueBinding.objectToEntry(configuredObject, value); try { - OperationStatus status = getConfiguredObjectsDb().put(null, key, value); + OperationStatus status = getConfiguredObjectsDb().put(txn, key, value); if (status != OperationStatus.SUCCESS) { throw new StoreException("Error writing configured object " + configuredObject + " to database: " + status); } + writeHierarchyRecords(txn, configuredObject); } catch (DatabaseException e) { @@ -1326,26 +1379,54 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore } } } - - private OperationStatus removeConfiguredObject(Transaction tx, UUID id) throws StoreException + + private void writeHierarchyRecords(final Transaction txn, final ConfiguredObjectRecord configuredObject) { + OperationStatus status; + HierarchyKeyBinding hierarchyBinding = HierarchyKeyBinding.getInstance(); + DatabaseEntry hierarchyKey = new DatabaseEntry(); + DatabaseEntry hierarchyValue = new DatabaseEntry(); - LOGGER.debug("Removing configured object: " + id); + for(Map.Entry<String, ConfiguredObjectRecord> parent : configuredObject.getParents().entrySet()) + { + + hierarchyBinding.objectToEntry(new HierarchyKey(configuredObject.getId(), parent.getKey()), hierarchyKey); + UUIDTupleBinding.getInstance().objectToEntry(parent.getValue().getId(), hierarchyValue); + status = getConfiguredObjectHierarchyDb().put(txn, hierarchyKey, hierarchyValue); + if (status != OperationStatus.SUCCESS) + { + throw new StoreException("Error writing configured object " + configuredObject + " parent record to database: " + + status); + } + } + } + + private OperationStatus removeConfiguredObject(Transaction tx, ConfiguredObjectRecord record) throws StoreException + { + UUID id = record.getId(); + Map<String, ConfiguredObjectRecord> parents = record.getParents(); + + if (LOGGER.isDebugEnabled()) + { + LOGGER.debug("Removing configured object: " + id); + } DatabaseEntry key = new DatabaseEntry(); UUIDTupleBinding uuidBinding = UUIDTupleBinding.getInstance(); uuidBinding.objectToEntry(id, key); - try - { - return getConfiguredObjectsDb().delete(tx, key); - } - catch (DatabaseException e) + OperationStatus status = getConfiguredObjectsDb().delete(tx, key); + if(status == OperationStatus.SUCCESS) { - throw _environmentFacade.handleDatabaseException("Error deleting of configured object with id " + id + " from database", e); + for(String parentType : parents.keySet()) + { + DatabaseEntry hierarchyKey = new DatabaseEntry(); + HierarchyKeyBinding keyBinding = HierarchyKeyBinding.getInstance(); + keyBinding.objectToEntry(new HierarchyKey(record.getId(), parentType), hierarchyKey); + getConfiguredObjectHierarchyDb().delete(tx, hierarchyKey); + } } + return status; } - - private class StoredBDBMessage implements StoredMessage<StorableMessageMetaData> { @@ -1687,14 +1768,19 @@ public class BDBMessageStore implements MessageStore, DurableConfigurationStore return _type; } - private Database getMessageContentDb() + private Database getConfiguredObjectsDb() { - return _environmentFacade.getOpenDatabase(MESSAGE_CONTENT_DB_NAME); + return _environmentFacade.getOpenDatabase(CONFIGURED_OBJECTS_DB_NAME); } - private Database getConfiguredObjectsDb() + private Database getConfiguredObjectHierarchyDb() { - return _environmentFacade.getOpenDatabase(CONFIGURED_OBJECTS_DB_NAME); + return _environmentFacade.getOpenDatabase(CONFIGURED_OBJECT_HIERARCHY_DB_NAME); + } + + private Database getMessageContentDb() + { + return _environmentFacade.getOpenDatabase(MESSAGE_CONTENT_DB_NAME); } private Database getConfigVersionDb() diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/entry/HierarchyKey.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/entry/HierarchyKey.java new file mode 100644 index 0000000000..d1c341447e --- /dev/null +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/entry/HierarchyKey.java @@ -0,0 +1,79 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.store.berkeleydb.entry; + +import java.util.UUID; + +public class HierarchyKey +{ + private final UUID _childId; + private final String _parentType; + + public HierarchyKey(final UUID childId, final String parentType) + { + _childId = childId; + _parentType = parentType; + } + + public UUID getChildId() + { + return _childId; + } + + public String getParentType() + { + return _parentType; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) + { + return true; + } + if (o == null || getClass() != o.getClass()) + { + return false; + } + + final HierarchyKey that = (HierarchyKey) o; + + if (!_childId.equals(that._childId)) + { + return false; + } + if (!_parentType.equals(that._parentType)) + { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = _childId.hashCode(); + result = 31 * result + _parentType.hashCode(); + return result; + } +} diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/tuple/ConfiguredObjectBinding.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/tuple/ConfiguredObjectBinding.java index bc3beeb78b..38a2215fe7 100644 --- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/tuple/ConfiguredObjectBinding.java +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/tuple/ConfiguredObjectBinding.java @@ -32,6 +32,7 @@ import com.sleepycat.bind.tuple.TupleBinding; import com.sleepycat.bind.tuple.TupleInput; import com.sleepycat.bind.tuple.TupleOutput; import org.apache.qpid.server.store.StoreException; +import org.apache.qpid.server.store.berkeleydb.BDBConfiguredObjectRecord; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.JsonProcessingException; @@ -80,7 +81,7 @@ public class ConfiguredObjectBinding extends TupleBinding<ConfiguredObjectRecord _uuid = uuid; } - public ConfiguredObjectRecord entryToObject(TupleInput tupleInput) + public BDBConfiguredObjectRecord entryToObject(TupleInput tupleInput) { String type = tupleInput.readString(); String json = tupleInput.readString(); @@ -88,7 +89,7 @@ public class ConfiguredObjectBinding extends TupleBinding<ConfiguredObjectRecord try { Map<String,Object> value = mapper.readValue(json, Map.class); - ConfiguredObjectRecord configuredObject = new ConfiguredObjectRecord(_uuid, type, value); + BDBConfiguredObjectRecord configuredObject = new BDBConfiguredObjectRecord(_uuid, type, value); return configuredObject; } catch (IOException e) diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/tuple/HierarchyKeyBinding.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/tuple/HierarchyKeyBinding.java new file mode 100644 index 0000000000..13adaabfc8 --- /dev/null +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/tuple/HierarchyKeyBinding.java @@ -0,0 +1,59 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.store.berkeleydb.tuple; + +import com.sleepycat.bind.tuple.TupleBinding; +import com.sleepycat.bind.tuple.TupleInput; +import com.sleepycat.bind.tuple.TupleOutput; +import org.apache.qpid.server.store.berkeleydb.entry.HierarchyKey; +import org.apache.qpid.server.store.berkeleydb.entry.QueueEntryKey; + +import java.util.UUID; + +public class HierarchyKeyBinding extends TupleBinding<HierarchyKey> +{ + + private static final HierarchyKeyBinding INSTANCE = new HierarchyKeyBinding(); + + public static HierarchyKeyBinding getInstance() + { + return INSTANCE; + } + + /** private constructor forces getInstance instead */ + private HierarchyKeyBinding() { } + + public HierarchyKey entryToObject(TupleInput tupleInput) + { + UUID childId = new UUID(tupleInput.readLong(), tupleInput.readLong()); + String parentType = tupleInput.readString(); + + return new HierarchyKey(childId, parentType); + } + + public void objectToEntry(HierarchyKey hk, TupleOutput tupleOutput) + { + UUID uuid = hk.getChildId(); + tupleOutput.writeLong(uuid.getMostSignificantBits()); + tupleOutput.writeLong(uuid.getLeastSignificantBits()); + tupleOutput.writeString(hk.getParentType()); + } +}
\ No newline at end of file diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/StoreUpgrade.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/StoreUpgrade.java index adcaef35ef..0ff90a6d77 100644 --- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/StoreUpgrade.java +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/StoreUpgrade.java @@ -22,7 +22,9 @@ package org.apache.qpid.server.store.berkeleydb.upgrade; import com.sleepycat.je.Environment; +import org.apache.qpid.server.model.ConfiguredObject; + public interface StoreUpgrade { - void performUpgrade(Environment environment, UpgradeInteractionHandler handler, String virtualHostName); + void performUpgrade(Environment environment, UpgradeInteractionHandler handler, ConfiguredObject<?> parent); } diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom4To5.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom4To5.java index 87f8afde4a..3588b96e88 100644 --- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom4To5.java +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom4To5.java @@ -39,6 +39,7 @@ import org.apache.qpid.framing.AMQShortString; import org.apache.qpid.framing.ContentHeaderBody; import org.apache.qpid.framing.FieldTable; import org.apache.qpid.framing.abstraction.MessagePublishInfo; +import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.protocol.v0_8.MessageMetaData; import org.apache.qpid.server.store.StoreException; import org.apache.qpid.server.store.StorableMessageMetaData; @@ -74,7 +75,7 @@ public class UpgradeFrom4To5 extends AbstractStoreUpgrade private static final Logger _logger = Logger.getLogger(UpgradeFrom4To5.class); - public void performUpgrade(final Environment environment, final UpgradeInteractionHandler handler, String virtualHostName) + public void performUpgrade(final Environment environment, final UpgradeInteractionHandler handler, ConfiguredObject<?> parent) { Transaction transaction = null; reportStarting(environment, 4); diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom5To6.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom5To6.java index 46f2afd741..366b6a1c97 100644 --- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom5To6.java +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom5To6.java @@ -40,6 +40,7 @@ import org.apache.qpid.exchange.ExchangeDefaults; import org.apache.qpid.framing.AMQShortString; import org.apache.qpid.framing.FieldTable; import org.apache.qpid.server.model.Binding; +import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.model.Exchange; import org.apache.qpid.server.model.LifetimePolicy; import org.apache.qpid.server.model.Queue; @@ -118,11 +119,11 @@ public class UpgradeFrom5To6 extends AbstractStoreUpgrade * Queue, Exchange, Bindings entries are stored now as configurable objects * in "CONFIGURED_OBJECTS" table. */ - public void performUpgrade(final Environment environment, final UpgradeInteractionHandler handler, String virtualHostName) + public void performUpgrade(final Environment environment, final UpgradeInteractionHandler handler, ConfiguredObject<?> parent) { reportStarting(environment, 5); upgradeMessages(environment, handler); - upgradeConfiguredObjectsAndDependencies(environment, handler, virtualHostName); + upgradeConfiguredObjectsAndDependencies(environment, handler, parent.getName()); renameDatabases(environment, null); reportFinished(environment, 6); } diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom6To7.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom6To7.java index ce00fd1a48..9dcd291b9d 100644 --- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom6To7.java +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom6To7.java @@ -27,6 +27,8 @@ import com.sleepycat.je.DatabaseConfig; import com.sleepycat.je.DatabaseEntry; import com.sleepycat.je.Environment; import com.sleepycat.je.OperationStatus; + +import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.store.StoreException; public class UpgradeFrom6To7 extends AbstractStoreUpgrade @@ -35,7 +37,7 @@ public class UpgradeFrom6To7 extends AbstractStoreUpgrade private static final int DEFAULT_CONFIG_VERSION = 0; @Override - public void performUpgrade(Environment environment, UpgradeInteractionHandler handler, String virtualHostName) + public void performUpgrade(Environment environment, UpgradeInteractionHandler handler, ConfiguredObject<?> parent) { reportStarting(environment, 6); DatabaseConfig dbConfig = new DatabaseConfig(); diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom7To8.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom7To8.java new file mode 100644 index 0000000000..413acc90c4 --- /dev/null +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom7To8.java @@ -0,0 +1,160 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.store.berkeleydb.upgrade; + +import com.sleepycat.bind.tuple.ByteBinding; +import com.sleepycat.bind.tuple.IntegerBinding; +import com.sleepycat.bind.tuple.TupleBinding; +import com.sleepycat.bind.tuple.TupleInput; +import com.sleepycat.bind.tuple.TupleOutput; +import com.sleepycat.je.*; + +import org.apache.qpid.server.model.ConfiguredObject; +import org.apache.qpid.server.store.ConfiguredObjectRecord; +import org.apache.qpid.server.store.StoreException; +import org.apache.qpid.server.store.berkeleydb.BDBConfiguredObjectRecord; +import org.apache.qpid.server.store.berkeleydb.entry.HierarchyKey; +import org.apache.qpid.server.store.berkeleydb.tuple.ConfiguredObjectBinding; +import org.apache.qpid.server.store.berkeleydb.tuple.HierarchyKeyBinding; +import org.apache.qpid.server.store.berkeleydb.tuple.UUIDTupleBinding; +import org.codehaus.jackson.map.ObjectMapper; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class UpgradeFrom7To8 extends AbstractStoreUpgrade +{ + + @Override + public void performUpgrade(Environment environment, UpgradeInteractionHandler handler, ConfiguredObject<?> parent) + { + reportStarting(environment, 7); + + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setTransactional(true); + dbConfig.setAllowCreate(true); + + Database hierarchyDb = environment.openDatabase(null, "CONFIGURED_OBJECT_HIERARCHY", dbConfig); + Database configuredObjectsDb = environment.openDatabase(null, "CONFIGURED_OBJECTS", dbConfig); + + Cursor objectsCursor = null; + + Transaction txn = environment.beginTransaction(null, null); + + try + { + objectsCursor = configuredObjectsDb.openCursor(txn, null); + DatabaseEntry key = new DatabaseEntry(); + DatabaseEntry value = new DatabaseEntry(); + + Map<UUID, BDBConfiguredObjectRecord> configuredObjects = + new HashMap<UUID, BDBConfiguredObjectRecord>(); + + while (objectsCursor.getNext(key, value, LockMode.RMW) == OperationStatus.SUCCESS) + { + UUID id = UUIDTupleBinding.getInstance().entryToObject(key); + TupleInput input = TupleBinding.entryToInput(value); + String type = input.readString(); + + if(!type.endsWith("Binding")) + { + UUIDTupleBinding.getInstance().objectToEntry(parent.getId(),value); + TupleOutput tupleOutput = new TupleOutput(); + tupleOutput.writeLong(id.getMostSignificantBits()); + tupleOutput.writeLong(id.getLeastSignificantBits()); + tupleOutput.writeString("VirtualHost"); + TupleBinding.outputToEntry(tupleOutput, key); + hierarchyDb.put(txn, key, value); + } + else + { + String json = input.readString(); + ObjectMapper mapper = new ObjectMapper(); + try + { + DatabaseEntry hierarchyKey = new DatabaseEntry(); + DatabaseEntry hierarchyValue = new DatabaseEntry(); + + Map<String,Object> attributes = mapper.readValue(json, Map.class); + Object queueIdString = attributes.remove("queue"); + if(queueIdString instanceof String) + { + UUID queueId = UUID.fromString(queueIdString.toString()); + UUIDTupleBinding.getInstance().objectToEntry(queueId,hierarchyValue); + TupleOutput tupleOutput = new TupleOutput(); + tupleOutput.writeLong(id.getMostSignificantBits()); + tupleOutput.writeLong(id.getLeastSignificantBits()); + tupleOutput.writeString("Queue"); + TupleBinding.outputToEntry(tupleOutput, hierarchyKey); + hierarchyDb.put(txn, hierarchyKey, hierarchyValue); + } + Object exchangeIdString = attributes.remove("exchange"); + if(exchangeIdString instanceof String) + { + UUID exchangeId = UUID.fromString(exchangeIdString.toString()); + UUIDTupleBinding.getInstance().objectToEntry(exchangeId,hierarchyValue); + TupleOutput tupleOutput = new TupleOutput(); + tupleOutput.writeLong(id.getMostSignificantBits()); + tupleOutput.writeLong(id.getLeastSignificantBits()); + tupleOutput.writeString("Exchange"); + TupleBinding.outputToEntry(tupleOutput, hierarchyKey); + hierarchyDb.put(txn, hierarchyKey, hierarchyValue); + } + TupleOutput tupleOutput = new TupleOutput(); + tupleOutput.writeString(type); + StringWriter writer = new StringWriter(); + mapper.writeValue(writer,attributes); + tupleOutput.writeString(writer.getBuffer().toString()); + TupleBinding.outputToEntry(tupleOutput, value); + objectsCursor.putCurrent(value); + } + catch (IOException e) + { + throw new StoreException(e); + } + + } + + + } + + + } + finally + { + if(objectsCursor != null) + { + objectsCursor.close(); + } + } + txn.commit(); + + hierarchyDb.close(); + configuredObjectsDb.close(); + + + + reportFinished(environment, 8); + } +} diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/Upgrader.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/Upgrader.java index 7852e2d703..e80d60609f 100644 --- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/Upgrader.java +++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/store/berkeleydb/upgrade/Upgrader.java @@ -26,6 +26,8 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import org.apache.log4j.Logger; +import org.apache.qpid.server.model.ConfiguredObject; +import org.apache.qpid.server.model.VirtualHost; import org.apache.qpid.server.store.StoreException; import org.apache.qpid.server.store.berkeleydb.BDBMessageStore; @@ -45,12 +47,12 @@ public class Upgrader static final String VERSION_DB_NAME = "DB_VERSION"; private Environment _environment; - private String _virtualHostName; + private ConfiguredObject<?> _parent; - public Upgrader(Environment environment, String virtualHostName) + public Upgrader(Environment environment, ConfiguredObject<?> parent) { _environment = environment; - _virtualHostName = virtualHostName; + _parent = parent; } public void upgradeIfNecessary() @@ -158,7 +160,7 @@ public class Upgrader + "UpgradeFrom"+fromVersion+"To"+toVersion); Constructor<StoreUpgrade> ctr = upgradeClass.getConstructor(); StoreUpgrade upgrade = ctr.newInstance(); - upgrade.performUpgrade(_environment, UpgradeInteractionHandler.DEFAULT_HANDLER, _virtualHostName); + upgrade.performUpgrade(_environment, UpgradeInteractionHandler.DEFAULT_HANDLER, _parent); } catch (ClassNotFoundException e) { diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/tuple/ConfiguredObjectBindingTest.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/tuple/ConfiguredObjectBindingTest.java index 5a5d39081c..965cad1cb5 100644 --- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/tuple/ConfiguredObjectBindingTest.java +++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/tuple/ConfiguredObjectBindingTest.java @@ -29,6 +29,7 @@ import org.apache.qpid.server.store.ConfiguredObjectRecord; import com.sleepycat.bind.tuple.TupleInput; import com.sleepycat.bind.tuple.TupleOutput; +import org.apache.qpid.server.store.ConfiguredObjectRecordImpl; public class ConfiguredObjectBindingTest extends TestCase { @@ -46,7 +47,7 @@ public class ConfiguredObjectBindingTest extends TestCase { super.setUp(); _configuredObjectBinding = ConfiguredObjectBinding.getInstance(); - _object = new ConfiguredObjectRecord(UUIDGenerator.generateRandomUUID(), DUMMY_TYPE_STRING, + _object = new ConfiguredObjectRecordImpl(UUIDGenerator.generateRandomUUID(), DUMMY_TYPE_STRING, DUMMY_ATTRIBUTES_MAP); } diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/AbstractUpgradeTestCase.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/AbstractUpgradeTestCase.java index b2b28b3c2d..ce143aba1b 100644 --- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/AbstractUpgradeTestCase.java +++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/AbstractUpgradeTestCase.java @@ -25,12 +25,16 @@ import static org.apache.qpid.server.store.berkeleydb.BDBStoreUpgradeTestPrepare import static org.apache.qpid.server.store.berkeleydb.BDBStoreUpgradeTestPreparer.PRIORITY_QUEUE_NAME; import static org.apache.qpid.server.store.berkeleydb.BDBStoreUpgradeTestPreparer.QUEUE_NAME; import static org.apache.qpid.server.store.berkeleydb.BDBStoreUpgradeTestPreparer.QUEUE_WITH_DLQ_NAME; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.File; import java.io.InputStream; +import java.util.UUID; import org.apache.qpid.server.logging.LogSubject; import org.apache.qpid.server.logging.subjects.TestBlankSubject; +import org.apache.qpid.server.model.VirtualHost; import org.apache.qpid.test.utils.QpidTestCase; import org.apache.qpid.util.FileUtils; @@ -167,8 +171,11 @@ public abstract class AbstractUpgradeTestCase extends QpidTestCase return count.longValue(); } - public String getVirtualHostName() + public VirtualHost getVirtualHost() { - return getName(); + VirtualHost virtualHost = mock(VirtualHost.class); + when(virtualHost.getName()).thenReturn(getName()); + when(virtualHost.getId()).thenReturn(UUID.randomUUID()); + return virtualHost; } } diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom4to5Test.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom4to5Test.java index 500fb0a919..d0f9455d9a 100644 --- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom4to5Test.java +++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom4to5Test.java @@ -73,7 +73,7 @@ public class UpgradeFrom4to5Test extends AbstractUpgradeTestCase public void testPerformUpgradeWithHandlerAnsweringYes() throws Exception { UpgradeFrom4To5 upgrade = new UpgradeFrom4To5(); - upgrade.performUpgrade(_environment, new StaticAnswerHandler(UpgradeInteractionResponse.YES), getVirtualHostName()); + upgrade.performUpgrade(_environment, new StaticAnswerHandler(UpgradeInteractionResponse.YES), getVirtualHost()); assertQueues(new HashSet<String>(Arrays.asList(QUEUE_NAMES))); @@ -103,7 +103,7 @@ public class UpgradeFrom4to5Test extends AbstractUpgradeTestCase public void testPerformUpgradeWithHandlerAnsweringNo() throws Exception { UpgradeFrom4To5 upgrade = new UpgradeFrom4To5(); - upgrade.performUpgrade(_environment, new StaticAnswerHandler(UpgradeInteractionResponse.NO), getVirtualHostName()); + upgrade.performUpgrade(_environment, new StaticAnswerHandler(UpgradeInteractionResponse.NO), getVirtualHost()); HashSet<String> queues = new HashSet<String>(Arrays.asList(QUEUE_NAMES)); assertTrue(NON_DURABLE_QUEUE_NAME + " should be in the list of queues" , queues.remove(NON_DURABLE_QUEUE_NAME)); diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom5To6Test.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom5To6Test.java index 701fd94115..0460b1ce4c 100644 --- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom5To6Test.java +++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgradeFrom5To6Test.java @@ -87,7 +87,7 @@ public class UpgradeFrom5To6Test extends AbstractUpgradeTestCase public void testPerformUpgrade() throws Exception { UpgradeFrom5To6 upgrade = new UpgradeFrom5To6(); - upgrade.performUpgrade(_environment, UpgradeInteractionHandler.DEFAULT_HANDLER, getVirtualHostName()); + upgrade.performUpgrade(_environment, UpgradeInteractionHandler.DEFAULT_HANDLER, getVirtualHost()); assertDatabaseRecordCounts(); assertContent(); @@ -101,7 +101,7 @@ public class UpgradeFrom5To6Test extends AbstractUpgradeTestCase corruptDatabase(); UpgradeFrom5To6 upgrade = new UpgradeFrom5To6(); - upgrade.performUpgrade(_environment, new StaticAnswerHandler(UpgradeInteractionResponse.YES), getVirtualHostName()); + upgrade.performUpgrade(_environment, new StaticAnswerHandler(UpgradeInteractionResponse.YES), getVirtualHost()); assertDatabaseRecordCounts(); @@ -117,7 +117,7 @@ public class UpgradeFrom5To6Test extends AbstractUpgradeTestCase UpgradeInteractionHandler discardMessageInteractionHandler = new StaticAnswerHandler(UpgradeInteractionResponse.NO); - upgrade.performUpgrade(_environment, discardMessageInteractionHandler, getVirtualHostName()); + upgrade.performUpgrade(_environment, discardMessageInteractionHandler, getVirtualHost()); assertDatabaseRecordCount(NEW_METADATA_DB_NAME, 12); assertDatabaseRecordCount(NEW_CONTENT_DB_NAME, 12); @@ -135,7 +135,7 @@ public class UpgradeFrom5To6Test extends AbstractUpgradeTestCase { populateOldXidEntries(environment); UpgradeFrom5To6 upgrade = new UpgradeFrom5To6(); - upgrade.performUpgrade(environment, UpgradeInteractionHandler.DEFAULT_HANDLER, getVirtualHostName()); + upgrade.performUpgrade(environment, UpgradeInteractionHandler.DEFAULT_HANDLER, getVirtualHost()); assertXidEntries(environment); } finally @@ -171,11 +171,11 @@ public class UpgradeFrom5To6Test extends AbstractUpgradeTestCase NewRecordImpl[] newDequeues = newTransaction.getDequeues(); assertEquals("Unxpected new enqueus number", 1, newEnqueues.length); NewRecordImpl enqueue = newEnqueues[0]; - assertEquals("Unxpected queue id", UUIDGenerator.generateQueueUUID("TEST1", getVirtualHostName()), enqueue.getId()); + assertEquals("Unxpected queue id", UUIDGenerator.generateQueueUUID("TEST1", getVirtualHost().getName()), enqueue.getId()); assertEquals("Unxpected message id", 1, enqueue.getMessageNumber()); assertEquals("Unxpected new dequeues number", 1, newDequeues.length); NewRecordImpl dequeue = newDequeues[0]; - assertEquals("Unxpected queue id", UUIDGenerator.generateQueueUUID("TEST2", getVirtualHostName()), dequeue.getId()); + assertEquals("Unxpected queue id", UUIDGenerator.generateQueueUUID("TEST2", getVirtualHost().getName()), dequeue.getId()); assertEquals("Unxpected message id", 2, dequeue.getMessageNumber()); } @@ -347,13 +347,13 @@ public class UpgradeFrom5To6Test extends AbstractUpgradeTestCase { String exchangeName = (String) deserialized.get(Exchange.NAME); assertNotNull(exchangeName); - assertEquals("Unexpected key", key, UUIDGenerator.generateExchangeUUID(exchangeName, getVirtualHostName())); + assertEquals("Unexpected key", key, UUIDGenerator.generateExchangeUUID(exchangeName, getVirtualHost().getName())); } else if (type.equals(Queue.class.getName())) { String queueName = (String) deserialized.get(Queue.NAME); assertNotNull(queueName); - assertEquals("Unexpected key", key, UUIDGenerator.generateQueueUUID(queueName, getVirtualHostName())); + assertEquals("Unexpected key", key, UUIDGenerator.generateQueueUUID(queueName, getVirtualHost().getName())); } else if (type.equals(Binding.class.getName())) { @@ -368,15 +368,15 @@ public class UpgradeFrom5To6Test extends AbstractUpgradeTestCase private Map<String, Object> createExpectedQueueBindingMapAndID(String queue, String bindingName, String exchangeName, Map<String, String> argumentMap, List<UUID> expectedBindingIDs) { Map<String, Object> expectedQueueBinding = new HashMap<String, Object>(); - expectedQueueBinding.put(Binding.QUEUE, UUIDGenerator.generateQueueUUID(queue, getVirtualHostName()).toString()); + expectedQueueBinding.put(Binding.QUEUE, UUIDGenerator.generateQueueUUID(queue, getVirtualHost().getName()).toString()); expectedQueueBinding.put(Binding.NAME, bindingName); - expectedQueueBinding.put(Binding.EXCHANGE, UUIDGenerator.generateExchangeUUID(exchangeName, getVirtualHostName()).toString()); + expectedQueueBinding.put(Binding.EXCHANGE, UUIDGenerator.generateExchangeUUID(exchangeName, getVirtualHost().getName()).toString()); if (argumentMap != null) { expectedQueueBinding.put(Binding.ARGUMENTS, argumentMap); } - expectedBindingIDs.add(UUIDGenerator.generateBindingUUID(exchangeName, queue, bindingName, getVirtualHostName())); + expectedBindingIDs.add(UUIDGenerator.generateBindingUUID(exchangeName, queue, bindingName, getVirtualHost().getName())); return expectedQueueBinding; } diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgraderFailOnNewerVersionTest.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgraderFailOnNewerVersionTest.java index 810f4a1fca..c407be50c6 100644 --- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgraderFailOnNewerVersionTest.java +++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgraderFailOnNewerVersionTest.java @@ -43,7 +43,7 @@ public class UpgraderFailOnNewerVersionTest extends AbstractUpgradeTestCase public void setUp() throws Exception { super.setUp(); - _upgrader = new Upgrader(_environment, getVirtualHostName()); + _upgrader = new Upgrader(_environment, getVirtualHost()); } private int getStoreVersion() diff --git a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgraderTest.java b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgraderTest.java index 3465f3582f..4b9a8d19a8 100644 --- a/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgraderTest.java +++ b/qpid/java/bdbstore/src/test/java/org/apache/qpid/server/store/berkeleydb/upgrade/UpgraderTest.java @@ -51,7 +51,7 @@ public class UpgraderTest extends AbstractUpgradeTestCase public void setUp() throws Exception { super.setUp(); - _upgrader = new Upgrader(_environment, getVirtualHostName()); + _upgrader = new Upgrader(_environment, getVirtualHost()); } private int getStoreVersion(Environment environment) @@ -108,7 +108,7 @@ public class UpgraderTest extends AbstractUpgradeTestCase Environment emptyEnvironment = createEnvironment(nonExistentStoreLocation); try { - _upgrader = new Upgrader(emptyEnvironment, getVirtualHostName()); + _upgrader = new Upgrader(emptyEnvironment, getVirtualHost()); _upgrader.upgradeIfNecessary(); List<String> databaseNames = emptyEnvironment.getDatabaseNames(); diff --git a/qpid/java/bdbstore/systests/pom.xml b/qpid/java/bdbstore/systests/pom.xml index e8620d3426..fe718f9dac 100644 --- a/qpid/java/bdbstore/systests/pom.xml +++ b/qpid/java/bdbstore/systests/pom.xml @@ -33,6 +33,7 @@ <test.log4j.configuration.file>${project.basedir}${file.separator}..${file.separator}..${file.separator}test-profiles${file.separator}log4j-test.xml</test.log4j.configuration.file> <test.working.directory>${basedir}/../..</test.working.directory> <test.resource.directory>${basedir}/../..</test.resource.directory> + <test.systest.resource.directory>${basedir}/../../systests</test.systest.resource.directory> </properties> <dependencies> @@ -65,6 +66,66 @@ <groupId>com.sleepycat</groupId> <artifactId>je</artifactId> </dependency> + + <dependency> + <groupId>org.apache.qpid</groupId> + <artifactId>qpid-bdbstore</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <!--version specified in parent pluginManagement --> + <executions> + <!-- copy the bdbstore bin contents to where the tests expect them --> + <execution> + <id>copy-bdbstore-bin-resources</id> + <phase>generate-resources</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${qpid.home}</outputDirectory> + <resources> + <resource> + <directory>${basedir}/..</directory> + <includes> + <include>bin/</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <!-- fix the fact that the maven-resources-plugin copy-resources doesn't maintain file permissions in unix --> + <execution> + <id>fix-bdb-script-permissions</id> + <phase>package</phase> + <configuration> + <target> + <chmod perm="755"> + <fileset dir="${qpid.home}"> + <include name="bin/**"/> + </fileset> + </chmod> + </target> + </configuration> + <goals><goal>run</goal></goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </project> diff --git a/qpid/java/bdbstore/systests/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBMessageStoreTest.java b/qpid/java/bdbstore/systests/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBMessageStoreTest.java index c8fcfe0826..ba84d0682a 100644 --- a/qpid/java/bdbstore/systests/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBMessageStoreTest.java +++ b/qpid/java/bdbstore/systests/src/main/java/org/apache/qpid/server/store/berkeleydb/BDBMessageStoreTest.java @@ -236,6 +236,7 @@ public class BDBMessageStoreTest extends MessageStoreTest { messageStore.closeMessageStore(); + BDBMessageStore newStore = new BDBMessageStore(); MessageStoreRecoveryHandler recoveryHandler = mock(MessageStoreRecoveryHandler.class); @@ -243,7 +244,7 @@ public class BDBMessageStoreTest extends MessageStoreTest VirtualHost<?> virtualHost = getVirtualHostModel(); newStore.openMessageStore(virtualHost.getName(), virtualHost.getMessageStoreSettings()); - newStore.recoverMessageStore(recoveryHandler, null); + newStore.recoverMessageStore(getVirtualHostModel(), recoveryHandler, null); return newStore; } diff --git a/qpid/java/broker-core/src/main/java/broker.bnd b/qpid/java/broker-core/src/main/java/broker.bnd index 1a371a5efd..8b3f663ec6 100755 --- a/qpid/java/broker-core/src/main/java/broker.bnd +++ b/qpid/java/broker-core/src/main/java/broker.bnd @@ -17,7 +17,7 @@ # under the License. # -ver: 0.27.0 +ver: 0.29.0 Bundle-SymbolicName: qpid-broker Bundle-Version: ${ver} diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/Broker.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/Broker.java index ffa1d65ec2..0ec6a31253 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/Broker.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/Broker.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Properties; import java.util.Set; +import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.log4j.PropertyConfigurator; import org.apache.qpid.server.configuration.ConfigurationEntryStore; @@ -41,7 +42,6 @@ import org.apache.qpid.server.logging.messages.BrokerMessages; import org.apache.qpid.server.registry.ApplicationRegistry; import org.apache.qpid.server.registry.IApplicationRegistry; import org.apache.qpid.server.security.SecurityManager; -import org.apache.qpid.server.security.auth.TaskPrincipal; import javax.security.auth.Subject; @@ -52,6 +52,7 @@ public class Broker private volatile Thread _shutdownHookThread; private volatile IApplicationRegistry _applicationRegistry; private EventLogger _eventLogger; + private boolean _configuringOwnLogging = false; protected static class InitException extends RuntimeException { @@ -71,11 +72,20 @@ public class Broker } finally { - if (_applicationRegistry != null) + try { - _applicationRegistry.close(); + if (_applicationRegistry != null) + { + _applicationRegistry.close(); + } + } + finally + { + if (_configuringOwnLogging) + { + LogManager.shutdown(); + } } - } } @@ -166,6 +176,7 @@ public class Broker private void configureLogging(File logConfigFile, int logWatchTime) throws InitException, IOException { + _configuringOwnLogging = true; if (logConfigFile.exists() && logConfigFile.canRead()) { _eventLogger.message(BrokerMessages.LOG_CONFIG(logConfigFile.getAbsolutePath())); diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/configuration/startup/AuthenticationProviderRecoverer.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/configuration/startup/AuthenticationProviderRecoverer.java index 8eec88d556..46f3cd458b 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/configuration/startup/AuthenticationProviderRecoverer.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/configuration/startup/AuthenticationProviderRecoverer.java @@ -32,6 +32,7 @@ import org.apache.qpid.server.model.AuthenticationProvider; import org.apache.qpid.server.model.Broker; import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.model.PreferencesProvider; +import org.apache.qpid.server.model.User; import org.apache.qpid.server.model.adapter.AuthenticationProviderFactory; public class AuthenticationProviderRecoverer implements ConfiguredObjectRecoverer<AuthenticationProvider> @@ -68,9 +69,24 @@ public class AuthenticationProviderRecoverer implements ConfiguredObjectRecovere Map<String, Collection<ConfigurationEntry>> childEntries, String type) { - ConfiguredObjectRecoverer<?> recoverer = recovererProvider.getRecoverer(type); + ConfiguredObjectRecoverer<?> recoverer = null; + + if(authenticationProvider instanceof RecovererProvider) + { + recoverer = ((RecovererProvider)authenticationProvider).getRecoverer(type); + } + + if(recoverer == null) + { + recoverer = recovererProvider.getRecoverer(type); + } + if (recoverer == null) { + if(authenticationProvider instanceof RecovererProvider) + { + ((RecovererProvider)authenticationProvider).getRecoverer(type); + } throw new IllegalConfigurationException("Cannot recover entry for the type '" + type + "' from broker"); } Collection<ConfigurationEntry> entries = childEntries.get(type); @@ -85,6 +101,10 @@ public class AuthenticationProviderRecoverer implements ConfiguredObjectRecovere { authenticationProvider.setPreferencesProvider((PreferencesProvider)object); } + else if(object instanceof User) + { + authenticationProvider.recoverUser((User)object); + } else { throw new IllegalConfigurationException("Cannot associate " + object + " with authentication provider " + authenticationProvider); diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/configuration/store/StoreConfigurationChangeListener.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/configuration/store/StoreConfigurationChangeListener.java index e030d50e56..addc42e6f9 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/configuration/store/StoreConfigurationChangeListener.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/configuration/store/StoreConfigurationChangeListener.java @@ -87,7 +87,7 @@ public class StoreConfigurationChangeListener implements ConfigurationChangeList private ConfigurationEntry toConfigurationEntry(ConfiguredObject object) { - Class<? extends ConfiguredObject> objectType = getConfiguredObjectType(object); + Class<? extends ConfiguredObject> objectType = object.getCategoryClass(); Set<UUID> childrenIds = getChildrenIds(object, objectType); ConfigurationEntry entry = new ConfigurationEntry(object.getId(), objectType.getSimpleName(), object.getActualAttributes(), childrenIds, _store); @@ -120,98 +120,6 @@ public class StoreConfigurationChangeListener implements ConfigurationChangeList return childrenIds; } - private Class<? extends ConfiguredObject> getConfiguredObjectType(ConfiguredObject object) - { - if (object instanceof Broker) - { - return Broker.class; - } - else if (object instanceof VirtualHost) - { - return VirtualHost.class; - } - else if (object instanceof Port) - { - return Port.class; - } - else if (object instanceof AuthenticationProvider) - { - return AuthenticationProvider.class; - } - return getConfiguredObjectTypeFromImplementedInterfaces(object.getClass()); - } - - @SuppressWarnings("unchecked") - private Class<? extends ConfiguredObject> getConfiguredObjectTypeFromImplementedInterfaces(Class<?> objectClass) - { - // get all implemented interfaces extending ConfiguredObject - Set<Class<?>> interfaces = getImplementedInterfacesExtendingSuper(objectClass, ConfiguredObject.class); - - if (interfaces.size() == 0) - { - throw new StoreException("Can not identify the configured object type"); - } - - if (interfaces.size() == 1) - { - return (Class<? extends ConfiguredObject>)interfaces.iterator().next(); - } - - Set<Class<?>> superInterfaces = new HashSet<Class<?>>(); - - // find all super interfaces - for (Class<?> interfaceClass : interfaces) - { - for (Class<?> interfaceClass2 : interfaces) - { - if (interfaceClass != interfaceClass2) - { - if (interfaceClass.isAssignableFrom(interfaceClass2)) - { - superInterfaces.add(interfaceClass); - } - } - } - } - - // remove super interfaces - for (Class<?> superInterface : superInterfaces) - { - interfaces.remove(superInterface); - } - - if (interfaces.size() == 1) - { - return (Class<? extends ConfiguredObject>)interfaces.iterator().next(); - } - else - { - throw new StoreException("Can not identify the configured object type as an it implements" - + " more than one configured object interfaces: " + interfaces); - } - - } - - private Set<Class<?>> getImplementedInterfacesExtendingSuper(Class<?> classInstance, Class<?> superInterface) - { - Set<Class<?>> interfaces = new HashSet<Class<?>>(); - Class<?>[] classInterfaces = classInstance.getInterfaces(); - for (Class<?> interfaceClass : classInterfaces) - { - if (interfaceClass!= superInterface && superInterface.isAssignableFrom(interfaceClass)) - { - interfaces.add(interfaceClass); - } - } - Class<?> superClass = classInstance.getSuperclass(); - if (superClass != null) - { - Set<Class<?>> superClassInterfaces = getImplementedInterfacesExtendingSuper(superClass, superInterface); - interfaces.addAll(superClassInterfaces); - } - return interfaces; - } - @Override public String toString() { diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/exchange/AbstractExchange.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/exchange/AbstractExchange.java index a30806d810..25f20ba1ee 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/exchange/AbstractExchange.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/exchange/AbstractExchange.java @@ -612,7 +612,7 @@ public abstract class AbstractExchange<T extends AbstractExchange<T>> } // Check access - _virtualHost.getSecurityManager().authoriseUnbind(this, bindingKey, queue); + _virtualHost.getSecurityManager().authoriseUnbind(binding); BindingImpl b = _bindingsMap.remove(new BindingIdentifier(bindingKey,queue)); diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/AuthenticationProvider.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/AuthenticationProvider.java index 8e1ea39cec..fc0a8ab7e5 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/AuthenticationProvider.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/AuthenticationProvider.java @@ -55,4 +55,6 @@ public interface AuthenticationProvider<X extends AuthenticationProvider<X>> ext * @param preferencesProvider */ void setPreferencesProvider(PreferencesProvider preferencesProvider); + + void recoverUser(User user); } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/ConfiguredObject.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/ConfiguredObject.java index b674f1e7db..7e3e5c9bbe 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/ConfiguredObject.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/ConfiguredObject.java @@ -21,6 +21,7 @@ package org.apache.qpid.server.model; import org.apache.qpid.server.model.adapter.AbstractConfiguredObject; +import org.apache.qpid.server.store.ConfiguredObjectRecord; import java.security.AccessControlException; import java.util.Collection; @@ -276,4 +277,10 @@ public interface ConfiguredObject<X extends ConfiguredObject<X>> ConfiguredObject... otherParents); void setAttributes(Map<String, Object> attributes) throws IllegalStateException, AccessControlException, IllegalArgumentException; + + Class<? extends ConfiguredObject> getCategoryClass(); + + // TODO - remove this when objects become responsible for their own storage + ConfiguredObjectRecord asObjectRecord(); + } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/VirtualHost.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/VirtualHost.java index 6e8932d3ba..103602edf5 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/VirtualHost.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/VirtualHost.java @@ -26,9 +26,10 @@ import java.util.Map; import org.apache.qpid.server.configuration.updater.TaskExecutor; import org.apache.qpid.server.message.MessageInstance; -import org.apache.qpid.server.security.SecurityManager; import org.apache.qpid.server.store.MessageStore; +import java.util.UUID; + @ManagedObject( managesChildren = true ) public interface VirtualHost<X extends VirtualHost<X>> extends ConfiguredObject<X> { @@ -173,15 +174,11 @@ public interface VirtualHost<X extends VirtualHost<X>> extends ConfiguredObject< void executeTransaction(TransactionalOperation op); - /** - * A temporary hack to expose host security manager. - * TODO We need to add and implement an authorization provider configured object instead - */ - SecurityManager getSecurityManager(); - // TODO - remove this TaskExecutor getTaskExecutor(); + Exchange getExchange(UUID id); + MessageStore getMessageStore(); String getType(); diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/adapter/AbstractConfiguredObject.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/adapter/AbstractConfiguredObject.java index d263bb45b2..52208f7d7f 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/adapter/AbstractConfiguredObject.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/adapter/AbstractConfiguredObject.java @@ -26,6 +26,7 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.security.AccessControlException; import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.*; import org.apache.qpid.server.model.*; @@ -37,6 +38,7 @@ import org.apache.qpid.server.configuration.updater.SetAttributeTask; import org.apache.qpid.server.configuration.updater.TaskExecutor; import org.apache.qpid.server.security.SecurityManager; import org.apache.qpid.server.security.auth.AuthenticatedPrincipal; +import org.apache.qpid.server.store.ConfiguredObjectRecord; import org.apache.qpid.server.util.MapValueConverter; import org.apache.qpid.server.util.ServerScopedRuntimeException; @@ -212,7 +214,7 @@ public abstract class AbstractConfiguredObject<X extends ConfiguredObject<X>> im final AuthenticatedPrincipal currentUser = SecurityManager.getCurrentUser(); if(currentUser != null) { - _attributes.put(CREATED_BY, currentUser); + _attributes.put(CREATED_BY, currentUser.getName()); } } if(!_attributes.containsKey(CREATED_TIME)) @@ -256,6 +258,11 @@ public abstract class AbstractConfiguredObject<X extends ConfiguredObject<X>> im return _name; } + public Class<? extends ConfiguredObject> getCategoryClass() + { + return getCategory(getClass()); + } + public State getDesiredState() { return null; //TODO @@ -558,6 +565,61 @@ public abstract class AbstractConfiguredObject<X extends ConfiguredObject<X>> im return getClass().getSimpleName() + " [id=" + _id + ", name=" + getName() + "]"; } + public ConfiguredObjectRecord asObjectRecord() + { + return new ConfiguredObjectRecord() + { + @Override + public UUID getId() + { + return AbstractConfiguredObject.this.getId(); + } + + @Override + public String getType() + { + return getCategoryClass().getSimpleName(); + } + + @Override + public Map<String, Object> getAttributes() + { + return Subject.doAs(SecurityManager.getSubjectWithAddedSystemRights(), new PrivilegedAction<Map<String, Object>>() + { + @Override + public Map<String, Object> run() + { + Map<String,Object> actualAttributes = new HashMap<String, Object>(getActualAttributes()); + for(Map.Entry<String,Object> entry : actualAttributes.entrySet()) + { + if(entry.getValue() instanceof ConfiguredObject) + { + entry.setValue(((ConfiguredObject)entry.getValue()).getId()); + } + } + actualAttributes.remove(ID); + return actualAttributes; + } + }); + } + + @Override + public Map<String, ConfiguredObjectRecord> getParents() + { + Map<String, ConfiguredObjectRecord> parents = new LinkedHashMap<String, ConfiguredObjectRecord>(); + for(Class<? extends ConfiguredObject> parentClass : Model.getInstance().getParentTypes(getCategoryClass())) + { + ConfiguredObject parent = getParent(parentClass); + if(parent != null) + { + parents.put(parentClass.getSimpleName(), parent.asObjectRecord()); + } + } + return parents; + } + }; + } + @SuppressWarnings("unchecked") @Override public <C extends ConfiguredObject> C createChild(Class<C> childClass, Map<String, Object> attributes, ConfiguredObject... otherParents) diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/adapter/VirtualHostAdapter.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/adapter/VirtualHostAdapter.java index 2e86d834bb..fafe081226 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/adapter/VirtualHostAdapter.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/model/adapter/VirtualHostAdapter.java @@ -42,7 +42,6 @@ import org.apache.qpid.server.plugin.ExchangeType; import org.apache.qpid.server.protocol.AMQConnectionModel; import org.apache.qpid.server.queue.AMQQueue; import org.apache.qpid.server.queue.ConflationQueue; -import org.apache.qpid.server.security.SecurityManager; import org.apache.qpid.server.security.access.Operation; import org.apache.qpid.server.stats.StatisticsGatherer; import org.apache.qpid.server.store.MessageStore; @@ -1042,12 +1041,6 @@ public final class VirtualHostAdapter extends AbstractConfiguredObject<VirtualHo } @Override - public SecurityManager getSecurityManager() - { - return _virtualHost.getSecurityManager(); - } - - @Override public MessageStore getMessageStore() { return _virtualHost.getMessageStore(); @@ -1089,9 +1082,15 @@ public final class VirtualHostAdapter extends AbstractConfiguredObject<VirtualHo } } + @Override public TaskExecutor getTaskExecutor() { return super.getTaskExecutor(); } + @Override + public Exchange getExchange(UUID id) + { + return _virtualHost.getExchange(id); + } } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/SecurityManager.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/SecurityManager.java index 77886e9030..478499fe6c 100755 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/SecurityManager.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/SecurityManager.java @@ -18,27 +18,6 @@ */ package org.apache.qpid.server.security; -import org.apache.log4j.Logger; - -import org.apache.qpid.server.binding.BindingImpl; -import org.apache.qpid.server.consumer.ConsumerImpl; -import org.apache.qpid.server.exchange.ExchangeImpl; - -import org.apache.qpid.server.model.*; -import org.apache.qpid.server.plugin.AccessControlFactory; -import org.apache.qpid.server.plugin.QpidServiceLoader; -import org.apache.qpid.server.protocol.AMQConnectionModel; -import org.apache.qpid.server.queue.AMQQueue; -import org.apache.qpid.server.security.access.FileAccessControlProviderConstants; -import org.apache.qpid.server.security.access.ObjectProperties; -import org.apache.qpid.server.security.access.ObjectType; -import org.apache.qpid.server.security.access.Operation; -import org.apache.qpid.server.security.access.OperationLoggingDetails; -import org.apache.qpid.server.security.auth.AuthenticatedPrincipal; -import org.apache.qpid.server.security.auth.TaskPrincipal; - -import javax.security.auth.Subject; - import static org.apache.qpid.server.security.access.ObjectType.BROKER; import static org.apache.qpid.server.security.access.ObjectType.EXCHANGE; import static org.apache.qpid.server.security.access.ObjectType.GROUP; @@ -62,54 +41,48 @@ import java.security.AccessController; import java.security.Principal; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import javax.security.auth.Subject; + +import org.apache.qpid.server.binding.BindingImpl; +import org.apache.qpid.server.consumer.ConsumerImpl; +import org.apache.qpid.server.exchange.ExchangeImpl; +import org.apache.qpid.server.model.AccessControlProvider; +import org.apache.qpid.server.model.Broker; +import org.apache.qpid.server.model.ConfigurationChangeListener; +import org.apache.qpid.server.model.ConfiguredObject; +import org.apache.qpid.server.model.State; +import org.apache.qpid.server.protocol.AMQConnectionModel; +import org.apache.qpid.server.queue.AMQQueue; +import org.apache.qpid.server.security.access.ObjectProperties; +import org.apache.qpid.server.security.access.ObjectProperties.Property; +import org.apache.qpid.server.security.access.ObjectType; +import org.apache.qpid.server.security.access.Operation; +import org.apache.qpid.server.security.access.OperationLoggingDetails; +import org.apache.qpid.server.security.auth.AuthenticatedPrincipal; +import org.apache.qpid.server.security.auth.TaskPrincipal; + public class SecurityManager implements ConfigurationChangeListener { - private static final Logger _logger = Logger.getLogger(SecurityManager.class); - private static final Subject SYSTEM = new Subject(true, Collections.singleton(new SystemPrincipal()), Collections.emptySet(), Collections.emptySet()); + private final ConcurrentHashMap<String, AccessControl> _plugins = new ConcurrentHashMap<String, AccessControl>(); + private final boolean _managementMode; + private final Broker<?> _broker; - private ConcurrentHashMap<String, AccessControl> _globalPlugins = new ConcurrentHashMap<String, AccessControl>(); - private ConcurrentHashMap<String, AccessControl> _hostPlugins = new ConcurrentHashMap<String, AccessControl>(); - - private boolean _managementMode; - - private Broker _broker; + private final ConcurrentHashMap<PublishAccessCheckCacheEntry, PublishAccessCheck> _publishAccessCheckCache = new ConcurrentHashMap<SecurityManager.PublishAccessCheckCacheEntry, SecurityManager.PublishAccessCheck>(); - /* - * Used by the Broker. - */ - public SecurityManager(Broker broker, boolean managementMode) + public SecurityManager(Broker<?> broker, boolean managementMode) { _managementMode = managementMode; _broker = broker; } - /* - * Used by the VirtualHost to allow deferring to the broker level security plugins if required. - */ - public SecurityManager(SecurityManager parent, String aclFile, String vhostName) - { - _managementMode = parent._managementMode; - _broker = parent._broker; - if(!_managementMode) - { - configureVirtualHostAclPlugin(aclFile, vhostName); - - // our global plugins are the parent's host plugins - _globalPlugins = parent._hostPlugins; - } - } - public static Subject getSubjectWithAddedSystemRights() { Subject subject = Subject.getSubject(AccessController.getContext()); @@ -135,50 +108,11 @@ public class SecurityManager implements ConfigurationChangeListener return subject; } - private void configureVirtualHostAclPlugin(String aclFile, String vhostName) - { - if(aclFile != null) - { - Map<String, Object> attributes = new HashMap<String, Object>(); - - attributes.put(AccessControlProvider.TYPE, FileAccessControlProviderConstants.ACL_FILE_PROVIDER_TYPE); - attributes.put(FileAccessControlProviderConstants.PATH, aclFile); - - for (AccessControlFactory provider : (new QpidServiceLoader<AccessControlFactory>()).instancesOf(AccessControlFactory.class)) - { - AccessControl accessControl = provider.createInstance(attributes, _broker); - accessControl.open(); - if(accessControl != null) - { - String pluginTypeName = getPluginTypeName(accessControl); - _hostPlugins.put(pluginTypeName, accessControl); - - if(_logger.isDebugEnabled()) - { - _logger.debug("Added access control to host plugins with name: " + vhostName); - } - - break; - } - } - } - - if(_logger.isDebugEnabled()) - { - _logger.debug("Configured " + _hostPlugins.size() + " access control plugins"); - } - } - private String getPluginTypeName(AccessControl accessControl) { return accessControl.getClass().getName(); } - public static Logger getLogger() - { - return _logger; - } - public static boolean isSystemProcess() { Subject subject = Subject.getSubject(AccessController.getContext()); @@ -234,71 +168,14 @@ public class SecurityManager implements ConfigurationChangeListener return true; } - Map<String, AccessControl> remainingPlugins = _globalPlugins.isEmpty() - ? Collections.<String, AccessControl>emptyMap() - : _hostPlugins.isEmpty() ? _globalPlugins : new HashMap<String, AccessControl>(_globalPlugins); - - if(!_hostPlugins.isEmpty()) - { - for (Entry<String, AccessControl> hostEntry : _hostPlugins.entrySet()) - { - // Create set of global only plugins - AccessControl globalPlugin = remainingPlugins.get(hostEntry.getKey()); - if (globalPlugin != null) - { - remainingPlugins.remove(hostEntry.getKey()); - } - - Result host = checker.allowed(hostEntry.getValue()); - - if (host == Result.DENIED) - { - // Something vetoed the access, we're done - return false; - } - - // host allow overrides global allow, so only check global on abstain or defer - if (host != Result.ALLOWED) - { - if (globalPlugin == null) - { - if (host == Result.DEFER) - { - host = hostEntry.getValue().getDefault(); - } - if (host == Result.DENIED) - { - return false; - } - } - else - { - Result global = checker.allowed(globalPlugin); - if (global == Result.DEFER) - { - global = globalPlugin.getDefault(); - } - if (global == Result.ABSTAIN && host == Result.DEFER) - { - global = hostEntry.getValue().getDefault(); - } - if (global == Result.DENIED) - { - return false; - } - } - } - } - } - - for (AccessControl plugin : remainingPlugins.values()) + for (AccessControl plugin : _plugins.values()) { Result remaining = checker.allowed(plugin); - if (remaining == Result.DEFER) + if (remaining == Result.DEFER) { remaining = plugin.getDefault(); } - if (remaining == Result.DENIED) + if (remaining == Result.DENIED) { return false; } @@ -308,28 +185,23 @@ public class SecurityManager implements ConfigurationChangeListener return true; } - public void authoriseCreateBinding(BindingImpl binding) + public void authoriseCreateBinding(final BindingImpl binding) { - final ExchangeImpl exch = binding.getExchange(); - final AMQQueue queue = binding.getAMQQueue(); - final String bindingKey = binding.getBindingKey(); - - boolean allowed = - checkAllPlugins(new AccessCheck() + boolean allowed = checkAllPlugins(new AccessCheck() { Result allowed(AccessControl plugin) { - return plugin.authorise(BIND, EXCHANGE, new ObjectProperties(exch, queue, bindingKey)); + return plugin.authorise(BIND, EXCHANGE, new ObjectProperties(binding)); } }); if(!allowed) { - throw new AccessControlException("Permission denied: binding " + bindingKey); + throw new AccessControlException("Permission denied: binding " + binding.getBindingKey()); } } - public void authoriseMethod(final Operation operation, final String componentName, final String methodName) + public void authoriseMethod(final Operation operation, final String componentName, final String methodName, final String virtualHostName) { boolean allowed = checkAllPlugins(new AccessCheck() { @@ -339,8 +211,11 @@ public class SecurityManager implements ConfigurationChangeListener properties.setName(methodName); if (componentName != null) { - // Only set the property if there is a component name - properties.put(ObjectProperties.Property.COMPONENT, componentName); + properties.put(ObjectProperties.Property.COMPONENT, componentName); + } + if (virtualHostName != null) + { + properties.put(ObjectProperties.Property.VIRTUALHOST_NAME, virtualHostName); } return plugin.authorise(operation, METHOD, properties); } @@ -367,15 +242,19 @@ public class SecurityManager implements ConfigurationChangeListener public void authoriseCreateConnection(final AMQConnectionModel connection) { + final String virtualHostName = connection.getVirtualHostName(); if(!checkAllPlugins(new AccessCheck() { Result allowed(AccessControl plugin) { - return plugin.authorise(Operation.ACCESS, VIRTUALHOST, ObjectProperties.EMPTY); + // We put the name into the properties under both name and virtualhost_name so the user may express predicates using either. + ObjectProperties properties = new ObjectProperties(virtualHostName); + properties.put(Property.VIRTUALHOST_NAME, virtualHostName); + return plugin.authorise(Operation.ACCESS, VIRTUALHOST, properties); } })) { - throw new AccessControlException("Permission denied: " + connection.getVirtualHostName()); + throw new AccessControlException("Permission denied: " + virtualHostName); } } @@ -403,10 +282,7 @@ public class SecurityManager implements ConfigurationChangeListener { Result allowed(AccessControl plugin) { - return plugin.authorise(CREATE, EXCHANGE, new ObjectProperties(exchange.isAutoDelete(), - exchange.isDurable(), - exchangeName, - exchange.getTypeName())); + return plugin.authorise(CREATE, EXCHANGE, new ObjectProperties(exchange)); } })) { @@ -421,11 +297,7 @@ public class SecurityManager implements ConfigurationChangeListener { Result allowed(AccessControl plugin) { - return plugin.authorise(CREATE, QUEUE, new ObjectProperties(queue.getAttribute(Queue.LIFETIME_POLICY) != LifetimePolicy.PERMANENT, - Boolean.TRUE.equals(queue.getAttribute(Queue.DURABLE)), - queue.getAttribute(Queue.EXCLUSIVE) != ExclusivityPolicy.NONE, - queueName, - queue.getOwner())); + return plugin.authorise(CREATE, QUEUE, new ObjectProperties(queue)); } })) { @@ -470,7 +342,7 @@ public class SecurityManager implements ConfigurationChangeListener { Result allowed(AccessControl plugin) { - return plugin.authorise(UPDATE, EXCHANGE, new ObjectProperties(exchange.getName())); + return plugin.authorise(UPDATE, EXCHANGE, new ObjectProperties(exchange)); } })) { @@ -484,7 +356,7 @@ public class SecurityManager implements ConfigurationChangeListener { Result allowed(AccessControl plugin) { - return plugin.authorise(DELETE, EXCHANGE, new ObjectProperties(exchange.getName())); + return plugin.authorise(DELETE, EXCHANGE, new ObjectProperties(exchange)); } })) { @@ -522,39 +394,15 @@ public class SecurityManager implements ConfigurationChangeListener } } - private ConcurrentHashMap<String, ConcurrentHashMap<String, PublishAccessCheck>> _immediatePublishPropsCache - = new ConcurrentHashMap<String, ConcurrentHashMap<String, PublishAccessCheck>>(); - private ConcurrentHashMap<String, ConcurrentHashMap<String, PublishAccessCheck>> _publishPropsCache - = new ConcurrentHashMap<String, ConcurrentHashMap<String, PublishAccessCheck>>(); - - public void authorisePublish(final boolean immediate, String routingKey, String exchangeName) + public void authorisePublish(final boolean immediate, String routingKey, String exchangeName, String virtualHostName) { - if(routingKey == null) + PublishAccessCheckCacheEntry key = new PublishAccessCheckCacheEntry(immediate, routingKey, exchangeName, virtualHostName); + PublishAccessCheck check = _publishAccessCheckCache.get(key); + if (check == null) { - routingKey = ""; + check = new PublishAccessCheck(new ObjectProperties(virtualHostName, exchangeName, routingKey, immediate)); + _publishAccessCheckCache.putIfAbsent(key, check); } - if(exchangeName == null) - { - exchangeName = ""; - } - PublishAccessCheck check; - ConcurrentHashMap<String, ConcurrentHashMap<String, PublishAccessCheck>> cache = - immediate ? _immediatePublishPropsCache : _publishPropsCache; - - ConcurrentHashMap<String, PublishAccessCheck> exchangeMap = cache.get(exchangeName); - if(exchangeMap == null) - { - cache.putIfAbsent(exchangeName, new ConcurrentHashMap<String, PublishAccessCheck>()); - exchangeMap = cache.get(exchangeName); - } - - check = exchangeMap.get(routingKey); - if(check == null) - { - check = new PublishAccessCheck(new ObjectProperties(exchangeName, routingKey, immediate)); - exchangeMap.put(routingKey, check); - } - if(!checkAllPlugins(check)) { throw new AccessControlException("Permission denied, publish to: exchange-name '" + exchangeName + "'"); @@ -575,17 +423,17 @@ public class SecurityManager implements ConfigurationChangeListener } } - public void authoriseUnbind(final ExchangeImpl exch, final String routingKey, final AMQQueue queue) + public void authoriseUnbind(final BindingImpl binding) { if(! checkAllPlugins(new AccessCheck() { Result allowed(AccessControl plugin) { - return plugin.authorise(UNBIND, EXCHANGE, new ObjectProperties(exch, queue, routingKey)); + return plugin.authorise(UNBIND, EXCHANGE, new ObjectProperties(binding)); } })) { - throw new AccessControlException("Permission denied: unbinding " + routingKey); + throw new AccessControlException("Permission denied: unbinding " + binding.getBindingKey()); } } @@ -618,29 +466,29 @@ public class SecurityManager implements ConfigurationChangeListener { if(newState == State.ACTIVE) { - synchronized (_hostPlugins) + synchronized (_plugins) { AccessControl accessControl = ((AccessControlProvider)object).getAccessControl(); String pluginTypeName = getPluginTypeName(accessControl); - _hostPlugins.put(pluginTypeName, accessControl); + _plugins.put(pluginTypeName, accessControl); } } else if(newState == State.DELETED) { - synchronized (_hostPlugins) + synchronized (_plugins) { AccessControl control = ((AccessControlProvider)object).getAccessControl(); String pluginTypeName = getPluginTypeName(control); // Remove the type->control mapping for this type key only if the // given control is actually referred to. - if(_hostPlugins.containsValue(control)) + if(_plugins.containsValue(control)) { // If we are removing this control, check if another of the same // type already exists on the broker and use it in instead. AccessControl other = null; - Collection<AccessControlProvider> providers = _broker.getAccessControlProviders(); + Collection<AccessControlProvider<?>> providers = _broker.getAccessControlProviders(); for(AccessControlProvider p : providers) { if(p == object || p.getState() != State.ACTIVE) @@ -660,12 +508,12 @@ public class SecurityManager implements ConfigurationChangeListener if(other != null) { //Another control of this type was found, use it instead - _hostPlugins.replace(pluginTypeName, control, other); + _plugins.replace(pluginTypeName, control, other); } else { //No other was found, remove the type entirely - _hostPlugins.remove(pluginTypeName); + _plugins.remove(pluginTypeName); } } } @@ -718,4 +566,90 @@ public class SecurityManager implements ConfigurationChangeListener }); } + public static class PublishAccessCheckCacheEntry + { + private final boolean _immediate; + private final String _routingKey; + private final String _exchangeName; + private final String _virtualHostName; + + public PublishAccessCheckCacheEntry(boolean immediate, String routingKey, String exchangeName, String virtualHostName) + { + super(); + _immediate = immediate; + _routingKey = routingKey; + _exchangeName = exchangeName; + _virtualHostName = virtualHostName; + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = 1; + result = prime * result + ((_exchangeName == null) ? 0 : _exchangeName.hashCode()); + result = prime * result + (_immediate ? 1231 : 1237); + result = prime * result + ((_routingKey == null) ? 0 : _routingKey.hashCode()); + result = prime * result + ((_virtualHostName == null) ? 0 : _virtualHostName.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) + { + return true; + } + if (obj == null) + { + return false; + } + if (getClass() != obj.getClass()) + { + return false; + } + PublishAccessCheckCacheEntry other = (PublishAccessCheckCacheEntry) obj; + if (_exchangeName == null) + { + if (other._exchangeName != null) + { + return false; + } + } + else if (!_exchangeName.equals(other._exchangeName)) + { + return false; + } + if (_immediate != other._immediate) + { + return false; + } + if (_routingKey == null) + { + if (other._routingKey != null) + { + return false; + } + } + else if (!_routingKey.equals(other._routingKey)) + { + return false; + } + if (_virtualHostName == null) + { + if (other._virtualHostName != null) + { + return false; + } + } + else if (!_virtualHostName.equals(other._virtualHostName)) + { + return false; + } + return true; + } + + + } } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/access/ObjectProperties.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/access/ObjectProperties.java index ae0241314f..6122cef5c1 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/access/ObjectProperties.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/access/ObjectProperties.java @@ -26,8 +26,10 @@ import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.qpid.server.binding.BindingImpl; import org.apache.qpid.server.exchange.ExchangeImpl; import org.apache.qpid.server.model.LifetimePolicy; +import org.apache.qpid.server.model.VirtualHost; import org.apache.qpid.server.queue.AMQQueue; /** @@ -39,7 +41,7 @@ import org.apache.qpid.server.queue.AMQQueue; */ public class ObjectProperties { - public static final String STAR= "*"; + public static final String WILD_CARD = "*"; public static final ObjectProperties EMPTY = new ObjectProperties(); @@ -65,7 +67,8 @@ public class ObjectProperties PACKAGE, CLASS, FROM_NETWORK, - FROM_HOSTNAME; + FROM_HOSTNAME, + VIRTUALHOST_NAME; private static final Map<String, Property> _canonicalNameToPropertyMap = new HashMap<String, ObjectProperties.Property>(); @@ -152,60 +155,49 @@ public class ObjectProperties { put(Property.OWNER, queue.getOwner()); } - + put(Property.VIRTUALHOST_NAME, queue.getParent(VirtualHost.class).getName()); } - public ObjectProperties(ExchangeImpl exch, AMQQueue queue, String routingKey) + public ObjectProperties(BindingImpl binding) { - this(queue); + ExchangeImpl<?> exch = binding.getExchange(); + AMQQueue<?> queue = binding.getAMQQueue(); + String routingKey = binding.getBindingKey(); setName(exch.getName()); - put(Property.QUEUE_NAME, queue.getName()); + put(Property.QUEUE_NAME, queue.getName()); put(Property.ROUTING_KEY, routingKey); - } - - public ObjectProperties(String exchangeName, String routingKey, Boolean immediate) - { - this(exchangeName, routingKey); + put(Property.VIRTUALHOST_NAME, queue.getParent(VirtualHost.class).getName()); - put(Property.IMMEDIATE, immediate); + // The temporary attribute (inherited from the binding's queue) seems to exist to allow the user to + // express rules about the binding of temporary queues (whose names cannot be predicted). + put(Property.TEMPORARY, queue.getLifetimePolicy() != LifetimePolicy.PERMANENT); + put(Property.DURABLE, queue.isDurable()); } - public ObjectProperties(String exchangeName, String routingKey) + public ObjectProperties(String virtualHostName, String exchangeName, String routingKey, Boolean immediate) { super(); setName(exchangeName); put(Property.ROUTING_KEY, routingKey); + put(Property.IMMEDIATE, immediate); + put(Property.VIRTUALHOST_NAME, virtualHostName); } - public ObjectProperties(Boolean autoDelete, Boolean durable, String exchangeName, - String exchangeType) - { - super(); - - setName(exchangeName); - - put(Property.AUTO_DELETE, autoDelete); - put(Property.TEMPORARY, autoDelete); - put(Property.DURABLE, durable); - put(Property.TYPE, exchangeType); - } - - public ObjectProperties(Boolean autoDelete, Boolean durable, Boolean exclusive, - String queueName, String owner) + public ObjectProperties(ExchangeImpl<?> exchange) { super(); - setName(queueName); + setName(exchange.getName()); - put(Property.AUTO_DELETE, autoDelete); - put(Property.TEMPORARY, autoDelete); - put(Property.DURABLE, durable); - put(Property.EXCLUSIVE, exclusive); - put(Property.OWNER, owner); + put(Property.AUTO_DELETE, exchange.isAutoDelete()); + put(Property.TEMPORARY, exchange.getLifetimePolicy() != LifetimePolicy.PERMANENT); + put(Property.DURABLE, exchange.isDurable()); + put(Property.TYPE, exchange.getTypeName()); + put(Property.VIRTUALHOST_NAME, exchange.getParent(VirtualHost.class).getName()); } public ObjectProperties(Boolean exclusive, Boolean noAck, Boolean noLocal, Boolean nowait, AMQQueue queue) @@ -283,8 +275,8 @@ public class ObjectProperties { return (StringUtils.isEmpty(ruleValue) || StringUtils.equals(thisValue, ruleValue)) - || ruleValue.equals(STAR) - || (ruleValue.endsWith(STAR) + || ruleValue.equals(WILD_CARD) + || (ruleValue.endsWith(WILD_CARD) && thisValue != null && thisValue.length() >= ruleValue.length() - 1 && thisValue.startsWith(ruleValue.substring(0, ruleValue.length() - 1))); diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/access/OperationLoggingDetails.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/access/OperationLoggingDetails.java index a683199abc..f36695cb86 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/access/OperationLoggingDetails.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/access/OperationLoggingDetails.java @@ -32,6 +32,42 @@ public class OperationLoggingDetails extends ObjectProperties } @Override + public int hashCode() + { + return super.hashCode() + ((_description == null) ? 0 : _description.hashCode()); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) + { + return true; + } + if (!super.equals(obj)) + { + return false; + } + if (getClass() != obj.getClass()) + { + return false; + } + OperationLoggingDetails other = (OperationLoggingDetails) obj; + if (_description == null) + { + if (other._description != null) + { + return false; + } + } + else if (!_description.equals(other._description)) + { + return false; + } + return true; + } + + @Override public String toString() { StringBuilder sb = new StringBuilder("("); diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/AbstractAuthenticationManager.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/AbstractAuthenticationManager.java index 17c9a19e50..2274ddb189 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/AbstractAuthenticationManager.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/AbstractAuthenticationManager.java @@ -21,6 +21,7 @@ package org.apache.qpid.server.security.auth.manager; import org.apache.log4j.Logger; +import org.apache.qpid.server.configuration.IllegalConfigurationException; import org.apache.qpid.server.configuration.updater.TaskExecutor; import org.apache.qpid.server.model.*; import org.apache.qpid.server.model.adapter.AbstractConfiguredObject; @@ -85,8 +86,11 @@ public abstract class AbstractAuthenticationManager<T extends AbstractAuthentica _preferencesProvider = preferencesProvider; } - - + @Override + public void recoverUser(final User user) + { + throw new IllegalConfigurationException("Cannot associate " + user + " with authentication provider " + this); + } @Override public String setName(final String currentName, final String desiredName) diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/ScramSHA1AuthenticationManager.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/ScramSHA1AuthenticationManager.java new file mode 100644 index 0000000000..097d0bfb9d --- /dev/null +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/ScramSHA1AuthenticationManager.java @@ -0,0 +1,696 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.security.auth.manager; + +import org.apache.qpid.server.configuration.ConfigurationEntry; +import org.apache.qpid.server.configuration.ConfiguredObjectRecoverer; +import org.apache.qpid.server.configuration.RecovererProvider; +import org.apache.qpid.server.configuration.updater.ChangeAttributesTask; +import org.apache.qpid.server.configuration.updater.TaskExecutor; +import org.apache.qpid.server.model.*; +import org.apache.qpid.server.model.adapter.AbstractConfiguredObject; +import org.apache.qpid.server.security.SecurityManager; +import org.apache.qpid.server.security.access.Operation; +import org.apache.qpid.server.security.auth.AuthenticationResult; +import org.apache.qpid.server.security.auth.UsernamePrincipal; +import org.apache.qpid.server.security.auth.sasl.scram.ScramSHA1SaslServer; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import javax.security.auth.login.AccountNotFoundException; +import javax.security.sasl.SaslException; +import javax.security.sasl.SaslServer; +import javax.xml.bind.DatatypeConverter; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.security.AccessControlException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.Principal; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +public class ScramSHA1AuthenticationManager + extends AbstractAuthenticationManager<ScramSHA1AuthenticationManager> + implements PasswordCredentialManagingAuthenticationProvider<ScramSHA1AuthenticationManager>, + RecovererProvider +{ + public static final String SCRAM_USER_TYPE = "scram"; + private static final Charset ASCII = Charset.forName("ASCII"); + public static final String HMAC_SHA_1 = "HmacSHA1"; + private final SecureRandom _random = new SecureRandom(); + private int _iterationCount = 4096; + private Map<String, ScramAuthUser> _users = new ConcurrentHashMap<String, ScramAuthUser>(); + + + protected ScramSHA1AuthenticationManager(final Broker broker, + final Map<String, Object> defaults, + final Map<String, Object> attributes, + final boolean recovering) + { + super(broker, defaults, attributes); + } + + @Override + public void initialise() + { + + } + + @Override + public String getMechanisms() + { + return ScramSHA1SaslServer.MECHANISM; + } + + @Override + public SaslServer createSaslServer(final String mechanism, + final String localFQDN, + final Principal externalPrincipal) + throws SaslException + { + return new ScramSHA1SaslServer(this); + } + + @Override + public AuthenticationResult authenticate(final SaslServer server, final byte[] response) + { + try + { + // Process response from the client + byte[] challenge = server.evaluateResponse(response != null ? response : new byte[0]); + + if (server.isComplete()) + { + final String userId = server.getAuthorizationID(); + return new AuthenticationResult(new UsernamePrincipal(userId)); + } + else + { + return new AuthenticationResult(challenge, AuthenticationResult.AuthenticationStatus.CONTINUE); + } + } + catch (SaslException e) + { + return new AuthenticationResult(AuthenticationResult.AuthenticationStatus.ERROR, e); + } + } + + @Override + public AuthenticationResult authenticate(final String username, final String password) + { + ScramAuthUser user = getUser(username); + if(user != null) + { + final String[] usernamePassword = user.getPassword().split(","); + byte[] salt = DatatypeConverter.parseBase64Binary(usernamePassword[0]); + try + { + if(Arrays.equals(DatatypeConverter.parseBase64Binary(usernamePassword[1]),createSaltedPassword(salt, password))) + { + return new AuthenticationResult(new UsernamePrincipal(username)); + } + } + catch (SaslException e) + { + return new AuthenticationResult(AuthenticationResult.AuthenticationStatus.ERROR,e); + } + + } + + return new AuthenticationResult(AuthenticationResult.AuthenticationStatus.ERROR); + + + } + + @Override + public void delete() + { + + } + + @Override + public void close() + { + + } + + public int getIterationCount() + { + return _iterationCount; + } + + public byte[] getSalt(final String username) + { + ScramAuthUser user = getUser(username); + + if(user == null) + { + // don't disclose that the user doesn't exist, just generate random data so the failure is indistinguishable + // from the "wrong password" case + + byte[] salt = new byte[32]; + _random.nextBytes(salt); + return salt; + } + else + { + return DatatypeConverter.parseBase64Binary(user.getPassword().split(",")[0]); + } + } + + private static final byte[] INT_1 = new byte[]{0, 0, 0, 1}; + + public byte[] getSaltedPassword(final String username) throws SaslException + { + ScramAuthUser user = getUser(username); + if(user == null) + { + throw new SaslException("Authentication Failed"); + } + else + { + return DatatypeConverter.parseBase64Binary(user.getPassword().split(",")[1]); + } + } + + private ScramAuthUser getUser(final String username) + { + return _users.get(username); + } + + private byte[] createSaltedPassword(byte[] salt, String password) throws SaslException + { + Mac mac = createSha1Hmac(password.getBytes(ASCII)); + + mac.update(salt); + mac.update(INT_1); + byte[] result = mac.doFinal(); + + byte[] previous = null; + for(int i = 1; i < getIterationCount(); i++) + { + mac.update(previous != null? previous: result); + previous = mac.doFinal(); + for(int x = 0; x < result.length; x++) + { + result[x] ^= previous[x]; + } + } + + return result; + + } + + private Mac createSha1Hmac(final byte[] keyBytes) + throws SaslException + { + try + { + SecretKeySpec key = new SecretKeySpec(keyBytes, HMAC_SHA_1); + Mac mac = Mac.getInstance(HMAC_SHA_1); + mac.init(key); + return mac; + } + catch (NoSuchAlgorithmException e) + { + throw new SaslException(e.getMessage(), e); + } + catch (InvalidKeyException e) + { + throw new SaslException(e.getMessage(), e); + } + } + + @Override + public boolean createUser(final String username, final String password, final Map<String, String> attributes) + { + if (getTaskExecutor().isTaskExecutorThread()) + { + getSecurityManager().authoriseUserOperation(Operation.CREATE, username); + if(_users.containsKey(username)) + { + throw new IllegalArgumentException("User '"+username+"' already exists"); + } + try + { + Map<String,Object> userAttrs = new HashMap<String, Object>(); + userAttrs.put(User.ID, UUID.randomUUID()); + userAttrs.put(User.NAME, username); + userAttrs.put(User.PASSWORD, createStoredPassword(password)); + userAttrs.put(User.TYPE, SCRAM_USER_TYPE); + ScramAuthUser user = new ScramAuthUser(userAttrs); + _users.put(username, user); + + return true; + } + catch (SaslException e) + { + throw new IllegalArgumentException(e); + } + } + else + { + return getTaskExecutor().submitAndWait(new TaskExecutor.Task<Boolean>() + { + @Override + public Boolean call() + { + return createUser(username, password, attributes); + } + }); + } + + } + + private SecurityManager getSecurityManager() + { + return getBroker().getSecurityManager(); + } + + @Override + public void deleteUser(final String user) throws AccountNotFoundException + { + if (getTaskExecutor().isTaskExecutorThread()) + { + + final ScramAuthUser authUser = getUser(user); + if(authUser != null) + { + authUser.setState(State.ACTIVE, State.DELETED); + } + else + { + throw new AccountNotFoundException("No such user: '" + user + "'"); + } + } + else + { + AccountNotFoundException e = + getTaskExecutor().submitAndWait(new TaskExecutor.Task<AccountNotFoundException>() { + + @Override + public AccountNotFoundException call() + { + try + { + deleteUser(user); + return null; + } + catch (AccountNotFoundException e) + { + return e; + } + + } + }); + + if(e != null) + { + throw e; + } + } + } + + @Override + public void setPassword(final String username, final String password) throws AccountNotFoundException + { + if (getTaskExecutor().isTaskExecutorThread()) + { + final ScramAuthUser authUser = getUser(username); + if(authUser != null) + { + authUser.setPassword(password); + } + else + { + throw new AccountNotFoundException("No such user: '" + username + "'"); + } + } + else + { + AccountNotFoundException e = + getTaskExecutor().submitAndWait(new TaskExecutor.Task<AccountNotFoundException>() + { + + @Override + public AccountNotFoundException call() + { + try + { + setPassword(username, password); + return null; + } + catch (AccountNotFoundException e) + { + return e; + } + + } + }); + + if (e != null) + { + throw e; + } + } + + } + + @Override + public Map<String, Map<String, String>> getUsers() + { + if (getTaskExecutor().isTaskExecutorThread()) + { + Map<String, Map<String,String>> users = new HashMap<String, Map<String, String>>(); + for(String user : _users.keySet()) + { + users.put(user, Collections.<String,String>emptyMap()); + } + return users; + } + else + { + return getTaskExecutor().submitAndWait(new TaskExecutor.Task<Map<String, Map<String, String>>>() + { + @Override + public Map<String, Map<String, String>> call() + { + return getUsers(); + } + }); + } + } + + @Override + public void reload() throws IOException + { + + } + + private static Map<Class<? extends ConfiguredObject>, ConfiguredObject<?>> parentsMap(final ScramSHA1AuthenticationManager scramSHA1AuthenticationManager) + { + + final Map<Class<? extends ConfiguredObject>, ConfiguredObject<?>> map = new HashMap<Class<? extends ConfiguredObject>, ConfiguredObject<?>>(); + map.put(AuthenticationProvider.class, scramSHA1AuthenticationManager); + return map; + } + + @Override + public ConfiguredObjectRecoverer<? extends ConfiguredObject> getRecoverer(final String type) + { + if("User".equals(type)) + { + return new UserRecoverer(); + } + else + { + return null; + } + } + + private class ScramAuthUser extends AbstractConfiguredObject<ScramAuthUser> implements User<ScramAuthUser> + { + + + protected ScramAuthUser(final Map<String, Object> attributes) + { + super(parentsMap(ScramSHA1AuthenticationManager.this), + Collections.<String,Object>emptyMap(), + attributes, ScramSHA1AuthenticationManager.this.getTaskExecutor()); + + if(!ASCII.newEncoder().canEncode(getName())) + { + throw new IllegalArgumentException("Scram SHA1 user names are restricted to characters in the ASCII charset"); + } + } + + @Override + protected boolean setState(final State currentState, final State desiredState) + { + if(desiredState == State.DELETED) + { + getSecurityManager().authoriseUserOperation(Operation.DELETE, getName()); + _users.remove(getName()); + ScramSHA1AuthenticationManager.this.childRemoved(this); + return true; + } + else + { + return false; + } + } + + @Override + public void setAttributes(final Map<String, Object> attributes) + throws IllegalStateException, AccessControlException, IllegalArgumentException + { + if (getTaskExecutor().isTaskExecutorThread()) + { + Map<String,Object> modifiedAttributes = new HashMap<String, Object>(attributes); + final String newPassword = (String) attributes.get(User.PASSWORD); + if(attributes.containsKey(User.PASSWORD) && !newPassword.equals(getActualAttributes().get(User.PASSWORD))) + { + try + { + modifiedAttributes.put(User.PASSWORD, createStoredPassword(newPassword)); + } + catch (SaslException e) + { + throw new IllegalArgumentException(e); + } + } + super.setAttributes(modifiedAttributes); + } + else + { + getTaskExecutor().submitAndWait(new ChangeAttributesTask(this, attributes)); + } + + } + + @Override + public Object getAttribute(final String name) + { + if(PASSWORD.equals(name)) + { + return null; // for security reasons we don't expose the password + } + return super.getAttribute(name); + } + + @Override + public String getPassword() + { + return (String) getActualAttributes().get(PASSWORD); + } + + @Override + public void setPassword(final String password) + { + getSecurityManager().authoriseUserOperation(Operation.UPDATE, getName()); + + try + { + changeAttribute(User.PASSWORD, getAttribute(User.PASSWORD), createStoredPassword(password)); + } + catch (SaslException e) + { + throw new IllegalArgumentException(e); + } + } + + @Override + public String setName(final String currentName, final String desiredName) + throws IllegalStateException, AccessControlException + { + throw new IllegalStateException("Names cannot be updated"); + } + + @Override + public State getState() + { + return State.ACTIVE; + } + + @Override + public boolean isDurable() + { + return true; + } + + @Override + public void setDurable(final boolean durable) + throws IllegalStateException, AccessControlException, IllegalArgumentException + { + + } + + @Override + public LifetimePolicy getLifetimePolicy() + { + return LifetimePolicy.PERMANENT; + } + + @Override + public LifetimePolicy setLifetimePolicy(final LifetimePolicy expected, final LifetimePolicy desired) + throws IllegalStateException, AccessControlException, IllegalArgumentException + { + if(expected == desired && expected == LifetimePolicy.PERMANENT) + { + return LifetimePolicy.PERMANENT; + } + throw new IllegalArgumentException("Cannot change lifetime policy of a user"); + + } + + @Override + public <C extends ConfiguredObject> Collection<C> getChildren(final Class<C> clazz) + { + return Collections.emptySet(); + } + + @Override + public Map<String, Object> getPreferences() + { + PreferencesProvider preferencesProvider = getPreferencesProvider(); + if (preferencesProvider == null) + { + return null; + } + return preferencesProvider.getPreferences(this.getName()); + } + + @Override + public Object getPreference(String name) + { + Map<String, Object> preferences = getPreferences(); + if (preferences == null) + { + return null; + } + return preferences.get(name); + } + + @Override + public Map<String, Object> setPreferences(Map<String, Object> preferences) + { + PreferencesProvider preferencesProvider = getPreferencesProvider(); + if (preferencesProvider == null) + { + return null; + } + return preferencesProvider.setPreferences(this.getName(), preferences); + } + + @Override + public boolean deletePreferences() + { + PreferencesProvider preferencesProvider = getPreferencesProvider(); + if (preferencesProvider == null) + { + return false; + } + String[] deleted = preferencesProvider.deletePreferences(this.getName()); + return deleted.length == 1; + } + + @Override + public Collection<String> getAttributeNames() + { + return getAttributeNames(getClass()); + } + } + + @Override + public void recoverUser(final User user) + { + _users.put(user.getName(), (ScramAuthUser) user); + } + + protected String createStoredPassword(final String password) throws SaslException + { + byte[] salt = new byte[32]; + _random.nextBytes(salt); + byte[] passwordBytes = createSaltedPassword(salt, password); + return DatatypeConverter.printBase64Binary(salt) + "," + DatatypeConverter.printBase64Binary(passwordBytes); + } + + @Override + public <C extends ConfiguredObject> C addChild(final Class<C> childClass, + final Map<String, Object> attributes, + final ConfiguredObject... otherParents) + { + if(childClass == User.class) + { + String username = (String) attributes.get("name"); + String password = (String) attributes.get("password"); + Principal p = new UsernamePrincipal(username); + + if(createUser(username, password,null)) + { + @SuppressWarnings("unchecked") + C principalAdapter = (C) _users.get(username); + return principalAdapter; + } + else + { + return null; + + } + } + return super.addChild(childClass, attributes, otherParents); + } + + public <C extends ConfiguredObject> Collection<C> getChildren(Class<C> clazz) + { + if(clazz == User.class) + { + return new ArrayList(_users.values()); + } + else + { + return super.getChildren(clazz); + } + } + + private class UserRecoverer implements ConfiguredObjectRecoverer<ScramAuthUser> + { + @Override + public ScramAuthUser create(final RecovererProvider recovererProvider, + final ConfigurationEntry entry, + final ConfiguredObject... parents) + { + + Map<String,Object> attributes = new HashMap<String, Object>(entry.getAttributes()); + attributes.put(User.ID,entry.getId()); + return new ScramAuthUser(attributes); + } + } +} diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/ScramSHA1AuthenticationManagerFactory.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/ScramSHA1AuthenticationManagerFactory.java new file mode 100644 index 0000000000..dd6f77e474 --- /dev/null +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/manager/ScramSHA1AuthenticationManagerFactory.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.security.auth.manager; + +import org.apache.qpid.server.model.AuthenticationProvider; +import org.apache.qpid.server.model.Broker; +import org.apache.qpid.server.plugin.AuthenticationManagerFactory; +import org.apache.qpid.server.util.ResourceBundleLoader; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +public class ScramSHA1AuthenticationManagerFactory implements AuthenticationManagerFactory +{ + + public static final String PROVIDER_TYPE = "SCRAM-SHA1"; + + public static final String ATTRIBUTE_NAME = "name"; + + public static final Collection<String> ATTRIBUTES = Collections.<String> unmodifiableList(Arrays.asList( + AuthenticationProvider.TYPE + )); + + @Override + public ScramSHA1AuthenticationManager createInstance(Broker broker, + Map<String, Object> attributes, + final boolean recovering) + { + if (attributes == null || !PROVIDER_TYPE.equals(attributes.get(AuthenticationProvider.TYPE))) + { + return null; + } + + + return new ScramSHA1AuthenticationManager(broker, Collections.<String,Object>emptyMap(),attributes, false); + } + + @Override + public Collection<String> getAttributeNames() + { + return ATTRIBUTES; + } + + @Override + public String getType() + { + return PROVIDER_TYPE; + } + + @Override + public Map<String, String> getAttributeDescriptions() + { + return Collections.emptyMap(); + } +} diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/sasl/scram/ScramSHA1SaslServer.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/sasl/scram/ScramSHA1SaslServer.java new file mode 100644 index 0000000000..71ef386e3e --- /dev/null +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/security/auth/sasl/scram/ScramSHA1SaslServer.java @@ -0,0 +1,273 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.security.auth.sasl.scram; + +import org.apache.qpid.server.security.auth.manager.ScramSHA1AuthenticationManager; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import javax.security.sasl.SaslException; +import javax.security.sasl.SaslServer; +import javax.xml.bind.DatatypeConverter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.security.InvalidKeyException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.UUID; + +public class ScramSHA1SaslServer implements SaslServer +{ + public static final String MECHANISM = "SCRAM-SHA-1"; + + private static final Charset ASCII = Charset.forName("ASCII"); + + private final ScramSHA1AuthenticationManager _authManager; + private State _state = State.INITIAL; + private String _nonce; + private String _username; + private byte[] _gs2Header; + private String _serverFirstMessage; + private String _clientFirstMessageBare; + private byte[] _serverSignature; + + public ScramSHA1SaslServer(final ScramSHA1AuthenticationManager authenticationManager) + { + _authManager = authenticationManager; + } + + enum State + { + INITIAL, + SERVER_FIRST_MESSAGE_SENT, + COMPLETE + } + + @Override + public String getMechanismName() + { + return MECHANISM; + } + + @Override + public byte[] evaluateResponse(final byte[] response) throws SaslException + { + byte[] challenge; + switch (_state) + { + case INITIAL: + challenge = generateServerFirstMessage(response); + _state = State.SERVER_FIRST_MESSAGE_SENT; + break; + case SERVER_FIRST_MESSAGE_SENT: + challenge = generateServerFinalMessage(response); + _state = State.COMPLETE; + break; + default: + throw new SaslException("No response expected in state " + _state); + + } + return challenge; + } + + private byte[] generateServerFirstMessage(final byte[] response) throws SaslException + { + String clientFirstMessage = new String(response, ASCII); + if(!clientFirstMessage.startsWith("n")) + { + throw new SaslException("Cannot parse gs2-header"); + } + String[] parts = clientFirstMessage.split(","); + if(parts.length < 4) + { + throw new SaslException("Cannot parse client first message"); + } + _gs2Header = ("n,"+parts[1]+",").getBytes(ASCII); + _clientFirstMessageBare = clientFirstMessage.substring(_gs2Header.length); + if(!parts[2].startsWith("n=")) + { + throw new SaslException("Cannot parse client first message"); + } + _username = decodeUsername(parts[2].substring(2)); + if(!parts[3].startsWith("r=")) + { + throw new SaslException("Cannot parse client first message"); + } + _nonce = parts[3].substring(2) + UUID.randomUUID().toString(); + + int count = _authManager.getIterationCount(); + byte[] saltBytes = _authManager.getSalt(_username); + _serverFirstMessage = "r="+_nonce+",s="+ DatatypeConverter.printBase64Binary(saltBytes)+",i=" + count; + return _serverFirstMessage.getBytes(ASCII); + } + + private String decodeUsername(String username) throws SaslException + { + if(username.contains("=")) + { + String check = username; + while (check.contains("=")) + { + check = check.substring(check.indexOf('=') + 1); + if (!(check.startsWith("2C") || check.startsWith("3D"))) + { + throw new SaslException("Invalid username"); + } + } + username = username.replace("=2C", ","); + username = username.replace("=3D","="); + } + return username; + } + + + private byte[] generateServerFinalMessage(final byte[] response) throws SaslException + { + try + { + String clientFinalMessage = new String(response, ASCII); + String[] parts = clientFinalMessage.split(","); + if(!parts[0].startsWith("c=")) + { + throw new SaslException("Cannot parse client final message"); + } + if(!Arrays.equals(_gs2Header,DatatypeConverter.parseBase64Binary(parts[0].substring(2)))) + { + throw new SaslException("Client final message channel bind data invalid"); + } + if(!parts[1].startsWith("r=")) + { + throw new SaslException("Cannot parse client final message"); + } + if(!parts[1].substring(2).equals(_nonce)) + { + throw new SaslException("Client final message has incorrect nonce value"); + } + if(!parts[parts.length-1].startsWith("p=")) + { + throw new SaslException("Client final message does not have proof"); + } + + String clientFinalMessageWithoutProof = clientFinalMessage.substring(0,clientFinalMessage.length()-(1+parts[parts.length-1].length())); + byte[] proofBytes = DatatypeConverter.parseBase64Binary(parts[parts.length-1].substring(2)); + + String authMessage = _clientFirstMessageBare + "," + _serverFirstMessage + "," + clientFinalMessageWithoutProof; + + byte[] saltedPassword = _authManager.getSaltedPassword(_username); + + byte[] clientKey = computeHmacSHA1(saltedPassword, "Client Key"); + + byte[] storedKey = MessageDigest.getInstance("SHA1").digest(clientKey); + + byte[] clientSignature = computeHmacSHA1(storedKey, authMessage); + + byte[] clientProof = clientKey.clone(); + for(int i = 0 ; i < clientProof.length; i++) + { + clientProof[i] ^= clientSignature[i]; + } + + if(!Arrays.equals(clientProof, proofBytes)) + { + throw new SaslException("Authentication failed"); + } + byte[] serverKey = computeHmacSHA1(saltedPassword, "Server Key"); + String finalResponse = "v=" + DatatypeConverter.printBase64Binary(computeHmacSHA1(serverKey, authMessage)); + + return finalResponse.getBytes(ASCII); + } + catch (NoSuchAlgorithmException e) + { + throw new SaslException(e.getMessage(), e); + } + catch (UnsupportedEncodingException e) + { + throw new SaslException(e.getMessage(), e); + } + } + + @Override + public boolean isComplete() + { + return _state == State.COMPLETE; + } + + @Override + public String getAuthorizationID() + { + return _username; + } + + @Override + public byte[] unwrap(final byte[] incoming, final int offset, final int len) throws SaslException + { + throw new IllegalStateException("No security layer supported"); + } + + @Override + public byte[] wrap(final byte[] outgoing, final int offset, final int len) throws SaslException + { + throw new IllegalStateException("No security layer supported"); + } + + @Override + public Object getNegotiatedProperty(final String propName) + { + return null; + } + + @Override + public void dispose() throws SaslException + { + + } + + private byte[] computeHmacSHA1(final byte[] key, final String string) + throws SaslException, UnsupportedEncodingException + { + Mac mac = createSha1Hmac(key); + mac.update(string.getBytes(ASCII)); + return mac.doFinal(); + } + + + private Mac createSha1Hmac(final byte[] keyBytes) + throws SaslException + { + try + { + SecretKeySpec key = new SecretKeySpec(keyBytes, "HmacSHA1"); + Mac mac = Mac.getInstance("HmacSHA1"); + mac.init(key); + return mac; + } + catch (NoSuchAlgorithmException e) + { + throw new SaslException(e.getMessage(), e); + } + catch (InvalidKeyException e) + { + throw new SaslException(e.getMessage(), e); + } + } + +} diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractDurableConfiguredObjectRecoverer.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractDurableConfiguredObjectRecoverer.java index dbe8bf22a0..da84580a94 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractDurableConfiguredObjectRecoverer.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractDurableConfiguredObjectRecoverer.java @@ -27,10 +27,9 @@ public abstract class AbstractDurableConfiguredObjectRecoverer<T> implements Dur { @Override public void load(final DurableConfigurationRecoverer durableConfigurationRecoverer, - final UUID id, - final Map<String, Object> attributes) + final ConfiguredObjectRecord record) { - final UnresolvedObject obj = createUnresolvedObject(id, getType(), attributes); + final UnresolvedObject obj = createUnresolvedObject(record); UnresolvedDependency[] dependencies = obj.getUnresolvedDependencies(); for(final UnresolvedDependency dependency : dependencies) { @@ -53,7 +52,7 @@ public abstract class AbstractDurableConfiguredObjectRecoverer<T> implements Dur dependency.resolve(o); if(obj.getUnresolvedDependencies().length == 0) { - durableConfigurationRecoverer.resolve(getType(), id, obj.resolve()); + durableConfigurationRecoverer.resolve(getType(), record.getId(), obj.resolve()); } } }); @@ -61,17 +60,15 @@ public abstract class AbstractDurableConfiguredObjectRecoverer<T> implements Dur } if(obj.getUnresolvedDependencies().length == 0) { - durableConfigurationRecoverer.resolve(getType(), id, obj.resolve()); + durableConfigurationRecoverer.resolve(getType(), record.getId(), obj.resolve()); } else { - durableConfigurationRecoverer.addUnresolvedObject(getType(), id, obj); + durableConfigurationRecoverer.addUnresolvedObject(getType(), record.getId(), obj); } } - public abstract UnresolvedObject<T> createUnresolvedObject(final UUID id, - final String type, - final Map<String, Object> attributes); + public abstract UnresolvedObject<T> createUnresolvedObject(final ConfiguredObjectRecord record); } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractJDBCMessageStore.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractJDBCMessageStore.java index a6424a3d28..ad3e685004 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractJDBCMessageStore.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractJDBCMessageStore.java @@ -24,6 +24,8 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.lang.ref.SoftReference; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -37,6 +39,8 @@ import java.sql.Types; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -49,6 +53,7 @@ import org.apache.qpid.server.message.EnqueueableMessage; import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.plugin.MessageMetaDataType; import org.apache.qpid.server.queue.AMQQueue; +import org.apache.qpid.transport.ConnectionOpen; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.JsonParseException; @@ -71,20 +76,22 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC private static final String META_DATA_TABLE_NAME = "QPID_MESSAGE_METADATA"; private static final String MESSAGE_CONTENT_TABLE_NAME = "QPID_MESSAGE_CONTENT"; - private static final String LINKS_TABLE_NAME = "QPID_LINKS"; - private static final String BRIDGES_TABLE_NAME = "QPID_BRIDGES"; private static final String XID_TABLE_NAME = "QPID_XIDS"; private static final String XID_ACTIONS_TABLE_NAME = "QPID_XID_ACTIONS"; private static final String CONFIGURED_OBJECTS_TABLE_NAME = "QPID_CONFIGURED_OBJECTS"; + private static final String CONFIGURED_OBJECT_HIERARCHY_TABLE_NAME = "QPID_CONFIGURED_OBJECT_HIERARCHY"; + private static final int DEFAULT_CONFIG_VERSION = 0; public static final Set<String> CONFIGURATION_STORE_TABLE_NAMES = new HashSet<String>(Arrays.asList(CONFIGURED_OBJECTS_TABLE_NAME, CONFIGURATION_VERSION_TABLE_NAME)); - public static final Set<String> MESSAGE_STORE_TABLE_NAMES = new HashSet<String>(Arrays.asList(DB_VERSION_TABLE_NAME, META_DATA_TABLE_NAME, MESSAGE_CONTENT_TABLE_NAME, QUEUE_ENTRY_TABLE_NAME, BRIDGES_TABLE_NAME, LINKS_TABLE_NAME, XID_TABLE_NAME, XID_ACTIONS_TABLE_NAME)); - + public static final Set<String> MESSAGE_STORE_TABLE_NAMES = new HashSet<String>(Arrays.asList(DB_VERSION_TABLE_NAME, + META_DATA_TABLE_NAME, MESSAGE_CONTENT_TABLE_NAME, + QUEUE_ENTRY_TABLE_NAME, + XID_TABLE_NAME, XID_ACTIONS_TABLE_NAME)); - private static final int DB_VERSION = 7; + private static final int DB_VERSION = 8; private final AtomicLong _messageId = new AtomicLong(0); @@ -116,34 +123,6 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC private static final String DELETE_FROM_META_DATA = "DELETE FROM " + META_DATA_TABLE_NAME + " WHERE message_id = ?"; private static final String SELECT_ALL_FROM_META_DATA = "SELECT message_id, meta_data FROM " + META_DATA_TABLE_NAME; - private static final String SELECT_FROM_LINKS = - "SELECT create_time, arguments FROM " + LINKS_TABLE_NAME + " WHERE id_lsb = ? and id_msb"; - private static final String DELETE_FROM_LINKS = "DELETE FROM " + LINKS_TABLE_NAME - + " WHERE id_lsb = ? and id_msb = ?"; - private static final String SELECT_ALL_FROM_LINKS = "SELECT id_lsb, id_msb, create_time, " - + "arguments FROM " + LINKS_TABLE_NAME; - private static final String FIND_LINK = "SELECT id_lsb, id_msb FROM " + LINKS_TABLE_NAME + " WHERE id_lsb = ? and" - + " id_msb = ?"; - private static final String INSERT_INTO_LINKS = "INSERT INTO " + LINKS_TABLE_NAME + "( id_lsb, " - + "id_msb, create_time, arguments ) values (?, ?, ?, ?)"; - private static final String SELECT_FROM_BRIDGES = - "SELECT create_time, link_id_lsb, link_id_msb, arguments FROM " - + BRIDGES_TABLE_NAME + " WHERE id_lsb = ? and id_msb = ?"; - private static final String DELETE_FROM_BRIDGES = "DELETE FROM " + BRIDGES_TABLE_NAME - + " WHERE id_lsb = ? and id_msb = ?"; - private static final String SELECT_ALL_FROM_BRIDGES = "SELECT id_lsb, id_msb, " - + " create_time," - + " link_id_lsb, link_id_msb, " - + "arguments FROM " + BRIDGES_TABLE_NAME - + " WHERE link_id_lsb = ? and link_id_msb = ?"; - private static final String FIND_BRIDGE = "SELECT id_lsb, id_msb FROM " + BRIDGES_TABLE_NAME + - " WHERE id_lsb = ? and id_msb = ?"; - private static final String INSERT_INTO_BRIDGES = "INSERT INTO " + BRIDGES_TABLE_NAME + "( id_lsb, id_msb, " - + "create_time, " - + "link_id_lsb, link_id_msb, " - + "arguments )" - + " values (?, ?, ?, ?, ?, ?)"; - private static final String INSERT_INTO_XIDS = "INSERT INTO "+ XID_TABLE_NAME +" ( format, global_id, branch_id ) values (?, ?, ?)"; private static final String DELETE_FROM_XIDS = "DELETE FROM " + XID_TABLE_NAME @@ -167,6 +146,14 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC + " where id = ?"; private static final String SELECT_FROM_CONFIGURED_OBJECTS = "SELECT id, object_type, attributes FROM " + CONFIGURED_OBJECTS_TABLE_NAME; + + private static final String INSERT_INTO_CONFIGURED_OBJECT_HIERARCHY = "INSERT INTO " + CONFIGURED_OBJECT_HIERARCHY_TABLE_NAME + + " ( child_id, parent_type, parent_id) VALUES (?,?,?)"; + + private static final String DELETE_FROM_CONFIGURED_OBJECT_HIERARCHY = "DELETE FROM " + CONFIGURED_OBJECT_HIERARCHY_TABLE_NAME + + " where child_id = ?"; + private static final String SELECT_FROM_CONFIGURED_OBJECT_HIERARCHY = "SELECT child_id, parent_type, parent_id FROM " + CONFIGURED_OBJECT_HIERARCHY_TABLE_NAME; + protected static final Charset UTF8_CHARSET = Charset.forName("UTF-8"); @@ -198,6 +185,7 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC private StateManager _configurationStoreStateManager; private boolean _initialized; + public AbstractJDBCMessageStore() { _messageStoreStateManager = new StateManager(_eventManager); @@ -233,13 +221,13 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC } @Override - public void recoverConfigurationStore(ConfigurationRecoveryHandler recoveryHandler) + public void recoverConfigurationStore(ConfiguredObject<?> parent, ConfigurationRecoveryHandler recoveryHandler) { _configurationStoreStateManager.attainState(State.ACTIVATING); - try { createOrOpenConfigurationStoreDatabase(); + upgradeIfVersionTableExists(parent); recoveryHandler.beginConfigurationRecovery(this, getConfigVersion()); loadConfiguredObjects(recoveryHandler); setConfigVersion(recoveryHandler.completeConfigurationRecovery()); @@ -251,6 +239,25 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC _configurationStoreStateManager.attainState(State.ACTIVE); } + private void upgradeIfVersionTableExists(ConfiguredObject<?> parent) + throws SQLException { + Connection conn = newAutoCommitConnection(); + try + { + if (tableExists(DB_VERSION_TABLE_NAME, conn)) + { + upgradeIfNecessary(parent); + } + } + finally + { + if (conn != null) + { + conn.close(); + } + } + } + @Override public void openMessageStore(String virtualHostName, Map<String, Object> messageStoreSettings) { @@ -260,13 +267,13 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC } @Override - public void recoverMessageStore(MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) + public void recoverMessageStore(ConfiguredObject<?> parent, MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) { _messageStoreStateManager.attainState(State.ACTIVATING); try { createOrOpenMessageStoreDatabase(); - upgradeIfNecessary(); + upgradeIfNecessary(parent); } catch (SQLException e) { @@ -302,7 +309,7 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC _messageStoreStateManager.attainState(State.ACTIVE); } - protected void upgradeIfNecessary() throws SQLException + protected void upgradeIfNecessary(ConfiguredObject<?> parent) throws SQLException { Connection conn = newAutoCommitConnection(); try @@ -323,6 +330,8 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC { case 6: upgradeFromV6(); + case 7: + upgradeFromV7(parent); case DB_VERSION: return; default: @@ -351,6 +360,135 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC updateDbVersion(7); } + + private void upgradeFromV7(ConfiguredObject<?> parent) throws SQLException + { + Connection connection = newConnection(); + try + { + Map<UUID,Map<String,Object>> bindingsToUpdate = new HashMap<UUID, Map<String, Object>>(); + List<UUID> others = new ArrayList<UUID>(); + final ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(_module); + + PreparedStatement stmt = connection.prepareStatement(SELECT_FROM_CONFIGURED_OBJECTS); + try + { + ResultSet rs = stmt.executeQuery(); + try + { + while (rs.next()) + { + UUID id = UUID.fromString(rs.getString(1)); + String objectType = rs.getString(2); + Map<String,Object> attributes = objectMapper.readValue(getBlobAsString(rs, 3),Map.class); + if(objectType.endsWith("Binding")) + { + bindingsToUpdate.put(id,attributes); + } + else + { + others.add(id); + } + } + } + catch (JsonMappingException e) + { + throw new StoreException("Error recovering persistent state: " + e.getMessage(), e); + } + catch (JsonParseException e) + { + throw new StoreException("Error recovering persistent state: " + e.getMessage(), e); + } + catch (IOException e) + { + throw new StoreException("Error recovering persistent state: " + e.getMessage(), e); + } + finally + { + rs.close(); + } + } + finally + { + stmt.close(); + } + + stmt = connection.prepareStatement(INSERT_INTO_CONFIGURED_OBJECT_HIERARCHY); + try + { + for (UUID id : others) + { + stmt.setString(1, id.toString()); + stmt.setString(2, "VirtualHost"); + stmt.setString(3, parent.getId().toString()); + stmt.execute(); + } + for(Map.Entry<UUID, Map<String,Object>> bindingEntry : bindingsToUpdate.entrySet()) + { + stmt.setString(1, bindingEntry.getKey().toString()); + stmt.setString(2,"Queue"); + stmt.setString(3, bindingEntry.getValue().remove("queue").toString()); + stmt.execute(); + + stmt.setString(1, bindingEntry.getKey().toString()); + stmt.setString(2,"Exchange"); + stmt.setString(3, bindingEntry.getValue().remove("exchange").toString()); + stmt.execute(); + } + } + finally + { + stmt.close(); + } + stmt = connection.prepareStatement(UPDATE_CONFIGURED_OBJECTS); + try + { + for(Map.Entry<UUID, Map<String,Object>> bindingEntry : bindingsToUpdate.entrySet()) + { + stmt.setString(1, "Binding"); + byte[] attributesAsBytes = objectMapper.writeValueAsBytes(bindingEntry.getValue()); + + ByteArrayInputStream bis = new ByteArrayInputStream(attributesAsBytes); + stmt.setBinaryStream(2, bis, attributesAsBytes.length); + stmt.setString(3, bindingEntry.getKey().toString()); + stmt.execute(); + } + } + catch (JsonMappingException e) + { + throw new StoreException("Error recovering persistent state: " + e.getMessage(), e); + } + catch (JsonGenerationException e) + { + throw new StoreException("Error recovering persistent state: " + e.getMessage(), e); + } + catch (IOException e) + { + throw new StoreException("Error recovering persistent state: " + e.getMessage(), e); + } + finally + { + stmt.close(); + } + stmt = connection.prepareStatement(UPDATE_DB_VERSION); + try + { + stmt.setInt(1, 8); + stmt.execute(); + } + finally + { + stmt.close(); + } + connection.commit(); + } + finally + { + connection.close(); + } + } + private void updateDbVersion(int newVersion) throws SQLException { Connection conn = newAutoCommitConnection(); @@ -392,8 +530,6 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC createQueueEntryTable(conn); createMetaDataTable(conn); createMessageContentTable(conn); - createLinkTable(conn); - createBridgeTable(conn); createXidTable(conn); createXidActionTable(conn); conn.close(); @@ -405,6 +541,7 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC createConfigVersionTable(conn); createConfiguredObjectsTable(conn); + createConfiguredObjectHierarchyTable(conn); conn.close(); } @@ -480,6 +617,23 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC } } + private void createConfiguredObjectHierarchyTable(final Connection conn) throws SQLException + { + if(!tableExists(CONFIGURED_OBJECT_HIERARCHY_TABLE_NAME, conn)) + { + Statement stmt = conn.createStatement(); + try + { + stmt.execute("CREATE TABLE " + CONFIGURED_OBJECT_HIERARCHY_TABLE_NAME + + " ( child_id VARCHAR(36) not null, parent_type varchar(255), parent_id VARCHAR(36), PRIMARY KEY (child_id, parent_type))"); + } + finally + { + stmt.close(); + } + } + } + private void createQueueEntryTable(final Connection conn) throws SQLException @@ -546,45 +700,7 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC } - private void createLinkTable(final Connection conn) throws SQLException - { - if(!tableExists(LINKS_TABLE_NAME, conn)) - { - Statement stmt = conn.createStatement(); - try - { - stmt.execute("CREATE TABLE "+ LINKS_TABLE_NAME +" ( id_lsb " + getSqlBigIntType() + " not null," - + " id_msb " + getSqlBigIntType() + " not null," - + " create_time " + getSqlBigIntType() + " not null," - + " arguments "+getSqlBlobType()+", PRIMARY KEY ( id_lsb, id_msb ))"); - } - finally - { - stmt.close(); - } - } - } - private void createBridgeTable(final Connection conn) throws SQLException - { - if(!tableExists(BRIDGES_TABLE_NAME, conn)) - { - Statement stmt = conn.createStatement(); - try - { - stmt.execute("CREATE TABLE "+ BRIDGES_TABLE_NAME +" ( id_lsb " + getSqlBigIntType() + " not null," - + " id_msb " + getSqlBigIntType() + " not null," - + " create_time " + getSqlBigIntType() + " not null," - + " link_id_lsb " + getSqlBigIntType() + " not null," - + " link_id_msb " + getSqlBigIntType() + " not null," - + " arguments "+getSqlBlobType()+", PRIMARY KEY ( id_lsb, id_msb ))"); - } - finally - { - stmt.close(); - } - } - } private void createXidTable(final Connection conn) throws SQLException { @@ -826,38 +942,28 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC @Override - public void create(UUID id, String type, Map<String,Object> attributes) throws StoreException - { - if (_configurationStoreStateManager.isInState(State.ACTIVE)) - { - insertConfiguredObject(new ConfiguredObjectRecord(id, type, attributes)); - } - - } - - @Override - public void remove(UUID id, String type) throws StoreException - { - int results = removeConfiguredObject(id); - if (results == 0) - { - throw new StoreException(type + " with id " + id + " not found"); - } - } - - @Override - public void update(UUID id, String type, Map<String, Object> attributes) throws StoreException + public void create(ConfiguredObjectRecord object) throws StoreException { if (_configurationStoreStateManager.isInState(State.ACTIVE)) { - ConfiguredObjectRecord queueConfiguredObject = loadConfiguredObject(id); - if (queueConfiguredObject != null) + try { - ConfiguredObjectRecord newQueueRecord = new ConfiguredObjectRecord(id, type, attributes); - updateConfiguredObject(newQueueRecord); + Connection conn = newConnection(); + try + { + insertConfiguredObject(object, conn); + conn.commit(); + } + finally + { + conn.close(); + } + } + catch (SQLException e) + { + throw new StoreException("Error creating ConfiguredObject " + object); } } - } /** @@ -1997,66 +2103,65 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC _eventManager.addEventListener(eventListener, events); } - private void insertConfiguredObject(ConfiguredObjectRecord configuredObject) throws StoreException + private void insertConfiguredObject(ConfiguredObjectRecord configuredObject, final Connection conn) throws StoreException { if (_configurationStoreStateManager.isInState(State.ACTIVE)) { try { - Connection conn = newAutoCommitConnection(); + PreparedStatement stmt = conn.prepareStatement(FIND_CONFIGURED_OBJECT); try { - PreparedStatement stmt = conn.prepareStatement(FIND_CONFIGURED_OBJECT); + stmt.setString(1, configuredObject.getId().toString()); + ResultSet rs = stmt.executeQuery(); + boolean exists; try { - stmt.setString(1, configuredObject.getId().toString()); - ResultSet rs = stmt.executeQuery(); + exists = rs.next(); + + } + finally + { + rs.close(); + } + // If we don't have any data in the result set then we can add this configured object + if (!exists) + { + PreparedStatement insertStmt = conn.prepareStatement(INSERT_INTO_CONFIGURED_OBJECTS); try { - // If we don't have any data in the result set then we can add this configured object - if (!rs.next()) + insertStmt.setString(1, configuredObject.getId().toString()); + insertStmt.setString(2, configuredObject.getType()); + if(configuredObject.getAttributes() == null) { - PreparedStatement insertStmt = conn.prepareStatement(INSERT_INTO_CONFIGURED_OBJECTS); - try - { - insertStmt.setString(1, configuredObject.getId().toString()); - insertStmt.setString(2, configuredObject.getType()); - if(configuredObject.getAttributes() == null) - { - insertStmt.setNull(3, Types.BLOB); - } - else - { - final Map<String, Object> attributes = configuredObject.getAttributes(); - final ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(_module); - byte[] attributesAsBytes = objectMapper.writeValueAsBytes(attributes); - - ByteArrayInputStream bis = new ByteArrayInputStream(attributesAsBytes); - insertStmt.setBinaryStream(3, bis, attributesAsBytes.length); - } - insertStmt.execute(); - } - finally - { - insertStmt.close(); - } + insertStmt.setNull(3, Types.BLOB); } + else + { + final Map<String, Object> attributes = configuredObject.getAttributes(); + final ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(_module); + byte[] attributesAsBytes = objectMapper.writeValueAsBytes(attributes); + + ByteArrayInputStream bis = new ByteArrayInputStream(attributesAsBytes); + insertStmt.setBinaryStream(3, bis, attributesAsBytes.length); + } + insertStmt.execute(); } finally { - rs.close(); + insertStmt.close(); } + + writeHierarchy(configuredObject, conn); } - finally - { - stmt.close(); - } + } finally { - conn.close(); + stmt.close(); } + } catch (JsonMappingException e) { @@ -2075,31 +2180,11 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC throw new StoreException("Error inserting of configured object " + configuredObject + " into database: " + e.getMessage(), e); } } - } - private int removeConfiguredObject(UUID id) throws StoreException - { - int results = 0; - try - { - Connection conn = newAutoCommitConnection(); - try - { - results = removeConfiguredObject(id, conn); - } - finally - { - conn.close(); - } - } - catch (SQLException e) - { - throw new StoreException("Error deleting of configured object with id " + id + " from database: " + e.getMessage(), e); - } - return results; } - public UUID[] removeConfiguredObjects(UUID... objects) throws StoreException + @Override + public UUID[] remove(ConfiguredObjectRecord... objects) throws StoreException { Collection<UUID> removed = new ArrayList<UUID>(objects.length); try @@ -2108,11 +2193,11 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC Connection conn = newAutoCommitConnection(); try { - for(UUID id : objects) + for(ConfiguredObjectRecord record : objects) { - if(removeConfiguredObject(id, conn) != 0) + if(removeConfiguredObject(record.getId(), conn) != 0) { - removed.add(id); + removed.add(record.getId()); } } } @@ -2130,7 +2215,8 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC private int removeConfiguredObject(final UUID id, final Connection conn) throws SQLException { - final int results;PreparedStatement stmt = conn.prepareStatement(DELETE_FROM_CONFIGURED_OBJECTS); + final int results; + PreparedStatement stmt = conn.prepareStatement(DELETE_FROM_CONFIGURED_OBJECTS); try { stmt.setString(1, id.toString()); @@ -2140,30 +2226,18 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC { stmt.close(); } - return results; - } - - private void updateConfiguredObject(final ConfiguredObjectRecord configuredObject) throws StoreException - { - if (_configurationStoreStateManager.isInState(State.ACTIVE)) + stmt = conn.prepareStatement(DELETE_FROM_CONFIGURED_OBJECT_HIERARCHY); + try { - try - { - Connection conn = newAutoCommitConnection(); - try - { - updateConfiguredObject(configuredObject, false, conn); - } - finally - { - conn.close(); - } - } - catch (SQLException e) - { - throw new StoreException("Error updating configured object " + configuredObject + " in database: " + e.getMessage(), e); - } + stmt.setString(1, id.toString()); + stmt.executeUpdate(); + } + finally + { + stmt.close(); } + + return results; } public void update(boolean createIfNecessary, ConfiguredObjectRecord... records) throws StoreException @@ -2258,6 +2332,7 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC { insertStmt.close(); } + writeHierarchy(configuredObject, conn); } } finally @@ -2284,72 +2359,31 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC } - private ConfiguredObjectRecord loadConfiguredObject(final UUID id) throws StoreException + private void writeHierarchy(final ConfiguredObjectRecord configuredObject, final Connection conn) throws SQLException, StoreException { - ConfiguredObjectRecord result = null; + PreparedStatement insertStmt = conn.prepareStatement(INSERT_INTO_CONFIGURED_OBJECT_HIERARCHY); try { - Connection conn = newAutoCommitConnection(); - try + for(Map.Entry<String,ConfiguredObjectRecord> parentEntry : configuredObject.getParents().entrySet()) { - PreparedStatement stmt = conn.prepareStatement(FIND_CONFIGURED_OBJECT); - try - { - stmt.setString(1, id.toString()); - ResultSet rs = stmt.executeQuery(); - try - { - if (rs.next()) - { - String type = rs.getString(1); - String attributes = getBlobAsString(rs, 2); - result = new ConfiguredObjectRecord(id, type, - (new ObjectMapper()).readValue(attributes,Map.class)); - } - } - finally - { - rs.close(); - } - } - finally - { - stmt.close(); - } - } - finally - { - conn.close(); + insertStmt.setString(1, configuredObject.getId().toString()); + insertStmt.setString(2, parentEntry.getKey()); + insertStmt.setString(3, parentEntry.getValue().getId().toString()); + + insertStmt.execute(); } } - catch (JsonMappingException e) - { - throw new StoreException("Error loading of configured object with id " + id + " from database: " - + e.getMessage(), e); - } - catch (JsonParseException e) - { - throw new StoreException("Error loading of configured object with id " + id + " from database: " - + e.getMessage(), e); - } - catch (IOException e) - { - throw new StoreException("Error loading of configured object with id " + id + " from database: " - + e.getMessage(), e); - } - catch (SQLException e) + finally { - throw new StoreException("Error loading of configured object with id " + id + " from database: " - + e.getMessage(), e); + insertStmt.close(); } - return result; } private void loadConfiguredObjects(ConfigurationRecoveryHandler recoveryHandler) throws SQLException, StoreException { Connection conn = newAutoCommitConnection(); - + Map<UUID, ConfiguredObjectRecordImpl> configuredObjects = new HashMap<UUID, ConfiguredObjectRecordImpl>(); final ObjectMapper objectMapper = new ObjectMapper(); try { @@ -2364,8 +2398,11 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC String id = rs.getString(1); String objectType = rs.getString(2); String attributes = getBlobAsString(rs, 3); - recoveryHandler.configuredObject(UUID.fromString(id), objectType, - objectMapper.readValue(attributes,Map.class)); + final ConfiguredObjectRecordImpl configuredObjectRecord = + new ConfiguredObjectRecordImpl(UUID.fromString(id), objectType, + objectMapper.readValue(attributes, Map.class)); + configuredObjects.put(configuredObjectRecord.getId(),configuredObjectRecord); + } } catch (JsonMappingException e) @@ -2389,11 +2426,53 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC { stmt.close(); } + stmt = conn.prepareStatement(SELECT_FROM_CONFIGURED_OBJECT_HIERARCHY); + try + { + ResultSet rs = stmt.executeQuery(); + try + { + while (rs.next()) + { + UUID childId = UUID.fromString(rs.getString(1)); + String parentType = rs.getString(2); + UUID parentId = UUID.fromString(rs.getString(3)); + + ConfiguredObjectRecordImpl child = configuredObjects.get(childId); + ConfiguredObjectRecordImpl parent = configuredObjects.get(parentId); + + if(child != null && parent != null) + { + child.addParent(parentType, parent); + } + else if(child != null && child.getType().endsWith("Binding") && parentType.equals("Exchange")) + { + // TODO - remove this hack for amq. exchanges + child.addParent(parentType, new ConfiguredObjectRecordImpl(parentId, parentType, Collections.<String,Object>emptyMap())); + } + + } + } + finally + { + rs.close(); + } + } + finally + { + stmt.close(); + } + } finally { conn.close(); } + + for(ConfiguredObjectRecord record : configuredObjects.values()) + { + recoveryHandler.configuredObject(record); + } } protected abstract String getBlobAsString(ResultSet rs, int col) throws SQLException; @@ -2441,4 +2520,51 @@ abstract public class AbstractJDBCMessageStore implements MessageStore, DurableC } } + + private static final class ConfiguredObjectRecordImpl implements ConfiguredObjectRecord + { + + private final UUID _id; + private final String _type; + private final Map<String, Object> _attributes; + private final Map<String, ConfiguredObjectRecord> _parents = new HashMap<String, ConfiguredObjectRecord>(); + + private ConfiguredObjectRecordImpl(final UUID id, + final String type, + final Map<String, Object> attributes) + { + _id = id; + _type = type; + _attributes = Collections.unmodifiableMap(attributes); + } + + @Override + public UUID getId() + { + return _id; + } + + @Override + public String getType() + { + return _type; + } + + private void addParent(String parentType, ConfiguredObjectRecord parent) + { + _parents.put(parentType, parent); + } + + @Override + public Map<String, Object> getAttributes() + { + return _attributes; + } + + @Override + public Map<String, ConfiguredObjectRecord> getParents() + { + return Collections.unmodifiableMap(_parents); + } + } } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractMemoryMessageStore.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractMemoryMessageStore.java index afdeb257e0..bf538e4592 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractMemoryMessageStore.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/AbstractMemoryMessageStore.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import org.apache.qpid.server.message.EnqueueableMessage; +import org.apache.qpid.server.model.ConfiguredObject; /** A simple message store that stores the messages in a thread-safe structure in memory. */ abstract public class AbstractMemoryMessageStore extends NullMessageStore @@ -83,7 +84,7 @@ abstract public class AbstractMemoryMessageStore extends NullMessageStore } @Override - public void recoverConfigurationStore(ConfigurationRecoveryHandler recoveryHandler) + public void recoverConfigurationStore(ConfiguredObject<?> parent, ConfigurationRecoveryHandler recoveryHandler) { } @@ -96,7 +97,7 @@ abstract public class AbstractMemoryMessageStore extends NullMessageStore } @Override - public void recoverMessageStore(MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) + public void recoverMessageStore(ConfiguredObject<?> parent, MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) { _stateManager.attainState(State.ACTIVATING); diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfigurationRecoveryHandler.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfigurationRecoveryHandler.java index c6ebe90802..c8aef92a95 100755 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfigurationRecoveryHandler.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfigurationRecoveryHandler.java @@ -27,7 +27,7 @@ public interface ConfigurationRecoveryHandler { void beginConfigurationRecovery(DurableConfigurationStore store, int configVersion); - void configuredObject(UUID id, String type, Map<String, Object> attributes); + void configuredObject(ConfiguredObjectRecord object); /** * diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfiguredObjectRecord.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfiguredObjectRecord.java index 44490385d9..99f8f0f04a 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfiguredObjectRecord.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfiguredObjectRecord.java @@ -20,69 +20,16 @@ */ package org.apache.qpid.server.store; -import java.util.Collections; -import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; -public class ConfiguredObjectRecord +public interface ConfiguredObjectRecord { - private UUID _id; - private String _type; - private Map<String,Object> _attributes; + UUID getId(); - public ConfiguredObjectRecord(UUID id, String type, Map<String,Object> attributes) - { - super(); - _id = id; - _type = type; - _attributes = Collections.unmodifiableMap(new LinkedHashMap<String,Object>(attributes)); - } + String getType(); - public UUID getId() - { - return _id; - } + Map<String,Object> getAttributes(); - public String getType() - { - return _type; - } - - public Map<String,Object> getAttributes() - { - return _attributes; - } - - @Override - public String toString() - { - return "ConfiguredObjectRecord [id=" + _id + ", type=" + _type + ", attributes=" + _attributes + "]"; - } - - @Override - public boolean equals(Object o) - { - if(this == o) - { - return true; - } - if(o == null || getClass() != o.getClass()) - { - return false; - } - - ConfiguredObjectRecord that = (ConfiguredObjectRecord) o; - - return _type.equals(that._type) && _id.equals(that._id) && _attributes.equals(that._attributes); - } - - @Override - public int hashCode() - { - int result = _id.hashCode(); - result = 31 * result + _type.hashCode(); - result = 31 * result + _attributes.hashCode(); - return result; - } + Map<String, ConfiguredObjectRecord> getParents(); } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfiguredObjectRecordImpl.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfiguredObjectRecordImpl.java new file mode 100644 index 0000000000..ed553aa823 --- /dev/null +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/ConfiguredObjectRecordImpl.java @@ -0,0 +1,104 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.store; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; + +public class ConfiguredObjectRecordImpl implements ConfiguredObjectRecord +{ + private UUID _id; + private String _type; + private final Map<String,Object> _attributes; + private final Map<String,ConfiguredObjectRecord> _parents; + + public ConfiguredObjectRecordImpl(UUID id, String type, Map<String, Object> attributes) + { + this(id,type,attributes,Collections.<String,ConfiguredObjectRecord>emptyMap()); + } + + public ConfiguredObjectRecordImpl(UUID id, String type, Map<String, Object> attributes, Map<String,ConfiguredObjectRecord> parents) + { + super(); + _id = id; + _type = type; + _attributes = Collections.unmodifiableMap(new LinkedHashMap<String,Object>(attributes)); + _parents = Collections.unmodifiableMap(new LinkedHashMap<String, ConfiguredObjectRecord>(parents)); + } + + @Override + public UUID getId() + { + return _id; + } + + @Override + public String getType() + { + return _type; + } + + @Override + public Map<String,Object> getAttributes() + { + return _attributes; + } + + @Override + public Map<String, ConfiguredObjectRecord> getParents() + { + return _parents; + } + + @Override + public String toString() + { + return "ConfiguredObjectRecord [id=" + _id + ", type=" + _type + ", attributes=" + _attributes + "]"; + } + + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + if(o == null || getClass() != o.getClass()) + { + return false; + } + + ConfiguredObjectRecordImpl that = (ConfiguredObjectRecordImpl) o; + + return _type.equals(that._type) && _id.equals(that._id) && _attributes.equals(that._attributes); + } + + @Override + public int hashCode() + { + int result = _id.hashCode(); + result = 31 * result + _type.hashCode(); + result = 31 * result + _attributes.hashCode(); + return result; + } +} diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationRecoverer.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationRecoverer.java index 6696426c0e..f8d8ecdd7c 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationRecoverer.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationRecoverer.java @@ -23,6 +23,7 @@ package org.apache.qpid.server.store; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.UUID; @@ -76,19 +77,19 @@ public class DurableConfigurationRecoverer implements ConfigurationRecoveryHandl } @Override - public void configuredObject(final UUID id, final String type, final Map<String, Object> attributes) + public void configuredObject(ConfiguredObjectRecord record) { - _upgrader.configuredObject(id, type, attributes); + _upgrader.configuredObject(record); } - void onConfiguredObject(final UUID id, final String type, final Map<String, Object> attributes) + void onConfiguredObject(ConfiguredObjectRecord record) { - DurableConfiguredObjectRecoverer recoverer = getRecoverer(type); + DurableConfiguredObjectRecoverer recoverer = getRecoverer(record.getType()); if(recoverer == null) { - throw new IllegalConfigurationException("Unknown type for configured object: " + type); + throw new IllegalConfigurationException("Unknown type for configured object: " + record.getType()); } - recoverer.load(this, id, attributes); + recoverer.load(this, record); } private DurableConfiguredObjectRecoverer getRecoverer(final String type) @@ -111,19 +112,12 @@ public class DurableConfigurationRecoverer implements ConfigurationRecoveryHandl private void applyUpgrade() { - final Collection<ConfiguredObjectRecord> updates = new ArrayList<ConfiguredObjectRecord>(); - final Collection<UUID> deletes = new ArrayList<UUID>(); - for(Map.Entry<UUID,ConfiguredObjectRecord> entry : _upgrader.getUpdatedRecords().entrySet()) - { - if(entry.getValue() != null) - { - updates.add(entry.getValue()); - } - else - { - deletes.add(entry.getKey()); - } - } + final Collection<ConfiguredObjectRecord> updates = new HashSet<ConfiguredObjectRecord>(_upgrader.getUpdatedRecords().values()); + final Collection<ConfiguredObjectRecord> deletes = new HashSet<ConfiguredObjectRecord>(_upgrader.getDeletedRecords().values()); + + // Due to the way the upgraders work it is possible that the updates list may contain nulls + updates.remove(null); + deletes.remove(null); if(!updates.isEmpty()) { @@ -131,7 +125,7 @@ public class DurableConfigurationRecoverer implements ConfigurationRecoveryHandl } if(!deletes.isEmpty()) { - _store.removeConfiguredObjects(deletes.toArray(new UUID[deletes.size()])); + _store.remove(deletes.toArray(new ConfiguredObjectRecord[deletes.size()])); } } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStore.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStore.java index 10e56f1f71..918a9b0134 100755 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStore.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStore.java @@ -23,6 +23,8 @@ package org.apache.qpid.server.store; import java.util.Map; import java.util.UUID; +import org.apache.qpid.server.model.ConfiguredObject; + public interface DurableConfigurationStore { String STORE_TYPE = "storeType"; @@ -43,48 +45,40 @@ public interface DurableConfigurationStore /** * Recovers configuration from the store using given recovery handler - * + * @param parent parent * @param recoveryHandler recovery handler */ - void recoverConfigurationStore(ConfigurationRecoveryHandler recoveryHandler) throws StoreException; + void recoverConfigurationStore(ConfiguredObject<?> parent, ConfigurationRecoveryHandler recoveryHandler) throws StoreException; /** * Makes the specified object persistent. * - * @param id The id of the object to persist. - * @param type The type of the object to persist - * @param attributes the attributes of the object to persist + * @param object The object to persist. * * @throws StoreException If the operation fails for any reason. */ - void create(UUID id, String type, Map<String, Object> attributes) throws StoreException; + void create(ConfiguredObjectRecord object) throws StoreException; /** - * Removes the specified persistent configured object. + * Removes the specified persistent configured objects. * - * @param id The id of the object to remove. - * @param type The type of the object to remove + * @param objects The objects to remove. * * @throws StoreException If the operation fails for any reason. */ - void remove(UUID id, String type) throws StoreException; - - public UUID[] removeConfiguredObjects(UUID... objects) throws StoreException; + public UUID[] remove(ConfiguredObjectRecord... objects) throws StoreException; /** - * Updates the specified object in the persistent store, IF it is already present. If the object - * is not present in the store, it will not be added. + * Updates the specified objects in the persistent store, IF it is already present. If the object + * is not present in the store, it will only be added if createIfNecessary is set to true, otherwise an exception + * will be thrown. * - * @param id The id of the object to update. - * @param type The type of the object to update - * @param attributes the updated attributes + * @param createIfNecessary if false then will fail if the object does not exist. + * @param records the records to update * * @throws StoreException If the operation fails for any reason. */ - void update(UUID id, String type, Map<String, Object> attributes) throws StoreException; - - void update(boolean createIfNecessary, ConfiguredObjectRecord... records) throws StoreException; diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStoreHelper.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStoreHelper.java index f914389b0e..9410006d65 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStoreHelper.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStoreHelper.java @@ -20,12 +20,8 @@ */ package org.apache.qpid.server.store; -import java.security.PrivilegedAction; import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -35,12 +31,8 @@ import org.apache.qpid.server.exchange.ExchangeImpl; import org.apache.qpid.server.model.Binding; import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.model.Exchange; -import org.apache.qpid.server.model.LifetimePolicy; import org.apache.qpid.server.model.Queue; import org.apache.qpid.server.queue.AMQQueue; -import org.apache.qpid.server.security.*; - -import javax.security.auth.Subject; public class DurableConfigurationStoreHelper { @@ -58,7 +50,7 @@ public class DurableConfigurationStoreHelper { attributesMap.put(Queue.ALTERNATE_EXCHANGE, queue.getAlternateExchange().getId()); } - store.update(queue.getId(), QUEUE, attributesMap); + store.update(false, new ConfiguredObjectRecordImpl(queue.getId(), QUEUE, attributesMap)); } public static void createQueue(DurableConfigurationStore store, AMQQueue<?> queue) @@ -71,49 +63,33 @@ public class DurableConfigurationStoreHelper attributesMap.put(Queue.ALTERNATE_EXCHANGE, queue.getAlternateExchange().getId()); } - store.create(queue.getId(), QUEUE, attributesMap); + store.create(new ConfiguredObjectRecordImpl(queue.getId(), QUEUE, attributesMap)); } public static void removeQueue(DurableConfigurationStore store, AMQQueue queue) { - store.remove(queue.getId(), QUEUE); + store.remove(queue.asObjectRecord()); } public static void createExchange(DurableConfigurationStore store, ExchangeImpl exchange) { - Map<String, Object> attributesMap = exchange.getActualAttributes(); - attributesMap.remove(ConfiguredObject.ID); - - store.create(exchange.getId(), EXCHANGE, attributesMap); - + store.create(exchange.asObjectRecord()); } - public static void removeExchange(DurableConfigurationStore store, ExchangeImpl exchange) { - store.remove(exchange.getId(), EXCHANGE); + store.remove(exchange.asObjectRecord()); } public static void createBinding(DurableConfigurationStore store, final BindingImpl binding) { - Map<String, Object> attributesMap = binding.getActualAttributes(); - attributesMap.remove(ConfiguredObject.ID); - if(!attributesMap.containsKey(Binding.EXCHANGE)) - { - attributesMap.put(Binding.EXCHANGE, binding.getExchange()); - } - if(!attributesMap.containsKey(Binding.QUEUE)) - { - attributesMap.put(Binding.QUEUE, binding.getQueue()); - } - - store.create(binding.getId(), BINDING, attributesMap); + store.create(binding.asObjectRecord()); } public static void removeBinding(DurableConfigurationStore store, BindingImpl binding) { - store.remove(binding.getId(), BINDING); + store.remove(binding.asObjectRecord()); } } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStoreUpgrader.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStoreUpgrader.java index 1d3e4cc672..45b2ff56d7 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStoreUpgrader.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfigurationStoreUpgrader.java @@ -25,11 +25,12 @@ import java.util.UUID; public interface DurableConfigurationStoreUpgrader { - void configuredObject(UUID id, String type, Map<String, Object> attributes); + void configuredObject(ConfiguredObjectRecord record); void complete(); void setNextUpgrader(DurableConfigurationStoreUpgrader upgrader); Map<UUID, ConfiguredObjectRecord> getUpdatedRecords(); + Map<UUID, ConfiguredObjectRecord> getDeletedRecords(); } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfiguredObjectRecoverer.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfiguredObjectRecoverer.java index e065728bd3..4c618f5089 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfiguredObjectRecoverer.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/DurableConfiguredObjectRecoverer.java @@ -26,8 +26,7 @@ import java.util.UUID; public interface DurableConfiguredObjectRecoverer { public void load(final DurableConfigurationRecoverer durableConfigurationRecoverer, - final UUID id, - final Map<String, Object> attributes); + final ConfiguredObjectRecord record); public String getType(); } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/JsonFileConfigStore.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/JsonFileConfigStore.java index a1512bbc22..97559c8286 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/JsonFileConfigStore.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/JsonFileConfigStore.java @@ -26,22 +26,22 @@ import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.model.Model; import org.apache.qpid.server.model.VirtualHost; +import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.JsonProcessingException; +import org.codehaus.jackson.Version; import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.JsonSerializer; +import org.codehaus.jackson.map.Module; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.SerializationConfig; +import org.codehaus.jackson.map.SerializerProvider; +import org.codehaus.jackson.map.module.SimpleModule; public class JsonFileConfigStore implements DurableConfigurationStore { @@ -61,8 +61,30 @@ public class JsonFileConfigStore implements DurableConfigurationStore private String _backupFileName; private int _configVersion; + private static final Module _module; + static + { + SimpleModule module= new SimpleModule("ConfiguredObjectSerializer", new Version(1,0,0,null)); + + final JsonSerializer<ConfiguredObject> serializer = new JsonSerializer<ConfiguredObject>() + { + @Override + public void serialize(final ConfiguredObject value, + final JsonGenerator jgen, + final SerializerProvider provider) + throws IOException, JsonProcessingException + { + jgen.writeString(value.getId().toString()); + } + }; + module.addSerializer(ConfiguredObject.class, serializer); + + _module = module; + } + public JsonFileConfigStore() { + _objectMapper.registerModule(_module); _objectMapper.enable(SerializationConfig.Feature.INDENT_OUTPUT); } @@ -76,13 +98,13 @@ public class JsonFileConfigStore implements DurableConfigurationStore } @Override - public void recoverConfigurationStore(ConfigurationRecoveryHandler recoveryHandler) + public void recoverConfigurationStore(ConfiguredObject<?> parent, ConfigurationRecoveryHandler recoveryHandler) { recoveryHandler.beginConfigurationRecovery(this,_configVersion); List<ConfiguredObjectRecord> records = new ArrayList<ConfiguredObjectRecord>(_objectsById.values()); for(ConfiguredObjectRecord record : records) { - recoveryHandler.configuredObject(record.getId(), record.getType(), record.getAttributes()); + recoveryHandler.configuredObject(record); } int oldConfigVersion = _configVersion; _configVersion = recoveryHandler.completeConfigurationRecovery(); @@ -287,20 +309,21 @@ public class JsonFileConfigStore implements DurableConfigurationStore } } + Map<String,UUID> parentMap = new HashMap<String, UUID>(); if(parentId != null) { - data.put(parentClass.getSimpleName().toLowerCase(),parentId); + parentMap.put(parentClass.getSimpleName(),parentId); for(Class<? extends ConfiguredObject> otherParent : MODEL.getParentTypes(clazz)) { if(otherParent != parentClass) { final String otherParentAttr = otherParent.getSimpleName().toLowerCase(); - Object otherParentId = data.get(otherParentAttr); + Object otherParentId = data.remove(otherParentAttr); if(otherParentId instanceof String) { try { - data.put(otherParentAttr, UUID.fromString((String) otherParentId)); + parentMap.put(otherParent.getSimpleName(), UUID.fromString((String) otherParentId)); } catch(IllegalArgumentException e) { @@ -312,7 +335,7 @@ public class JsonFileConfigStore implements DurableConfigurationStore } } - _objectsById.put(id, new ConfiguredObjectRecord(id, type, data)); + _objectsById.put(id, new ConfiguredObjectRecordImpl(id, type, data, parentMap)); List<UUID> idsForType = _idsByType.get(type); if(idsForType == null) { @@ -324,28 +347,27 @@ public class JsonFileConfigStore implements DurableConfigurationStore } @Override - public synchronized void create(final UUID id, final String type, final Map<String, Object> attributes) throws - StoreException + public synchronized void create(ConfiguredObjectRecord record) throws StoreException { - if(_objectsById.containsKey(id)) + if(_objectsById.containsKey(record.getId())) { - throw new StoreException("Object with id " + id + " already exists"); + throw new StoreException("Object with id " + record.getId() + " already exists"); } - else if(!CLASS_NAME_MAPPING.containsKey(type)) + else if(!CLASS_NAME_MAPPING.containsKey(record.getType())) { - throw new StoreException("Cannot create object of unknown type " + type); + throw new StoreException("Cannot create object of unknown type " + record.getType()); } else { - ConfiguredObjectRecord record = new ConfiguredObjectRecord(id, type, attributes); - _objectsById.put(id, record); - List<UUID> idsForType = _idsByType.get(type); + + _objectsById.put(record.getId(), record); + List<UUID> idsForType = _idsByType.get(record.getType()); if(idsForType == null) { idsForType = new ArrayList<UUID>(); - _idsByType.put(type, idsForType); + _idsByType.put(record.getType(), idsForType); } - idsForType.add(id); + idsForType.add(record.getId()); save(); } } @@ -400,7 +422,21 @@ public class JsonFileConfigStore implements DurableConfigurationStore Map<String,Object> map = new LinkedHashMap<String, Object>(); map.put("id", id); map.putAll(record.getAttributes()); - map.remove(MODEL.getParentTypes(type).iterator().next().getSimpleName().toLowerCase()); + + Collection<Class<? extends ConfiguredObject>> parentTypes = MODEL.getParentTypes(type); + if(parentTypes.size() > 1) + { + Iterator<Class<? extends ConfiguredObject>> iter = parentTypes.iterator(); + // skip the first parent, which is given by structure + iter.next(); + // for all other parents add a fake attribute with name being the parent type in lower case, and the value + // being the parents id + while(iter.hasNext()) + { + String parentType = iter.next().getSimpleName(); + map.put(parentType.toLowerCase(), record.getParents().get(parentType).getId()); + } + } Collection<Class<? extends ConfiguredObject>> childClasses = new ArrayList<Class<? extends ConfiguredObject>>(MODEL.getChildTypes(type)); @@ -418,8 +454,10 @@ public class JsonFileConfigStore implements DurableConfigurationStore for(UUID childId : childIds) { ConfiguredObjectRecord childRecord = _objectsById.get(childId); - final String parentArg = type.getSimpleName().toLowerCase(); - if(id.toString().equals(String.valueOf(childRecord.getAttributes().get(parentArg)))) + + final ConfiguredObjectRecord parent = childRecord.getParents().get(type.getSimpleName()); + String parentId = parent.getId().toString(); + if(id.toString().equals(parentId)) { entities.add(build(childClass,childId)); } @@ -436,34 +474,22 @@ public class JsonFileConfigStore implements DurableConfigurationStore } @Override - public void remove(final UUID id, final String type) throws StoreException - { - removeConfiguredObjects(id); - } - - @Override - public synchronized UUID[] removeConfiguredObjects(final UUID... objects) throws StoreException + public synchronized UUID[] remove(final ConfiguredObjectRecord... objects) throws StoreException { List<UUID> removedIds = new ArrayList<UUID>(); - for(UUID id : objects) + for(ConfiguredObjectRecord requestedRecord : objects) { - ConfiguredObjectRecord record = _objectsById.remove(id); + ConfiguredObjectRecord record = _objectsById.remove(requestedRecord.getId()); if(record != null) { - removedIds.add(id); - _idsByType.get(record.getType()).remove(id); + removedIds.add(record.getId()); + _idsByType.get(record.getType()).remove(record.getId()); } } save(); return removedIds.toArray(new UUID[removedIds.size()]); } - @Override - public void update(final UUID id, final String type, final Map<String, Object> attributes) throws - StoreException - { - update(false, new ConfiguredObjectRecord(id, type, attributes)); - } @Override public void update(final boolean createIfNecessary, final ConfiguredObjectRecord... records) @@ -554,5 +580,59 @@ public class JsonFileConfigStore implements DurableConfigurationStore return map; } + private class ConfiguredObjectRecordImpl implements ConfiguredObjectRecord + { + + private final UUID _id; + private final String _type; + private final Map<String, Object> _attributes; + private final Map<String, UUID> _parents; + + private ConfiguredObjectRecordImpl(final UUID id, final String type, final Map<String, Object> attributes, + final Map<String, UUID> parents) + { + _id = id; + _type = type; + _attributes = attributes; + _parents = parents; + } + + @Override + public UUID getId() + { + return _id; + } + + @Override + public String getType() + { + return _type; + } + + @Override + public Map<String, Object> getAttributes() + { + return _attributes; + } + + @Override + public Map<String, ConfiguredObjectRecord> getParents() + { + Map<String,ConfiguredObjectRecord> parents = new HashMap<String, ConfiguredObjectRecord>(); + for(Map.Entry<String,UUID> entry : _parents.entrySet()) + { + ConfiguredObjectRecord value = _objectsById.get(entry.getValue()); + + if(value == null && entry.getKey().equals("Exchange")) + { + // TODO - remove this hack for the defined exchanges + value = new ConfiguredObjectRecordImpl(entry.getValue(),entry.getKey(),Collections.<String,Object>emptyMap(), Collections.<String,UUID>emptyMap()); + } + + parents.put(entry.getKey(), value); + } + return parents; + } + } } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/MessageStore.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/MessageStore.java index fb99191477..7fb6c4df48 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/MessageStore.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/MessageStore.java @@ -22,6 +22,8 @@ package org.apache.qpid.server.store; import java.util.Map; +import org.apache.qpid.server.model.ConfiguredObject; + /** * MessageStore defines the interface to a storage area, which can be used to preserve the state of messages. * @@ -43,11 +45,11 @@ public interface MessageStore /** * Called after opening to recover messages and transactions with given recovery handlers - * + * @param parent TODO * @param messageRecoveryHandler * @param transactionLogRecoveryHandler */ - void recoverMessageStore(MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler); + void recoverMessageStore(ConfiguredObject<?> parent, MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler); public <T extends StorableMessageMetaData> StoredMessage<T> addMessage(T metaData); diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NonNullUpgrader.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NonNullUpgrader.java index a671e93b26..1abd3de34a 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NonNullUpgrader.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NonNullUpgrader.java @@ -28,6 +28,7 @@ public abstract class NonNullUpgrader implements DurableConfigurationStoreUpgrad { private DurableConfigurationStoreUpgrader _nextUpgrader; private final Map<UUID, ConfiguredObjectRecord> _updates = new HashMap<UUID, ConfiguredObjectRecord>(); + private final Map<UUID, ConfiguredObjectRecord> _deletes = new HashMap<UUID, ConfiguredObjectRecord>(); public final void setNextUpgrader(final DurableConfigurationStoreUpgrader upgrader) { @@ -50,6 +51,10 @@ public abstract class NonNullUpgrader implements DurableConfigurationStoreUpgrad { return _updates; } + protected Map<UUID, ConfiguredObjectRecord> getDeleteMap() + { + return _deletes; + } @Override public final Map<UUID, ConfiguredObjectRecord> getUpdatedRecords() @@ -59,4 +64,11 @@ public abstract class NonNullUpgrader implements DurableConfigurationStoreUpgrad return updates; } + @Override + public final Map<UUID, ConfiguredObjectRecord> getDeletedRecords() + { + final Map<UUID, ConfiguredObjectRecord> deletes = new HashMap<UUID, ConfiguredObjectRecord>(_deletes); + deletes.putAll(_nextUpgrader.getDeletedRecords()); + return deletes; + } } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NullMessageStore.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NullMessageStore.java index c095675602..66c3fe6cae 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NullMessageStore.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NullMessageStore.java @@ -22,6 +22,8 @@ package org.apache.qpid.server.store; import java.util.Map; import java.util.UUID; +import org.apache.qpid.server.model.ConfiguredObject; + public abstract class NullMessageStore implements MessageStore, DurableConfigurationStore { @Override @@ -30,12 +32,7 @@ public abstract class NullMessageStore implements MessageStore, DurableConfigura } @Override - public void recoverConfigurationStore(ConfigurationRecoveryHandler recoveryHandler) - { - } - - @Override - public void update(UUID id, String type, Map<String, Object> attributes) + public void recoverConfigurationStore(ConfiguredObject<?> parent, ConfigurationRecoveryHandler recoveryHandler) { } @@ -44,20 +41,19 @@ public abstract class NullMessageStore implements MessageStore, DurableConfigura { } - - @Override - public void remove(UUID id, String type) - { - } - @Override - public UUID[] removeConfiguredObjects(final UUID... objects) + public UUID[] remove(final ConfiguredObjectRecord... objects) { - return objects; + final UUID[] removed = new UUID[objects.length]; + for(int i = 0; i < objects.length; i++) + { + removed[i] = objects[i].getId(); + } + return removed; } @Override - public void create(UUID id, String type, Map<String, Object> attributes) + public void create(ConfiguredObjectRecord record) { } @@ -95,7 +91,7 @@ public abstract class NullMessageStore implements MessageStore, DurableConfigura } @Override - public void recoverMessageStore(MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) + public void recoverMessageStore(ConfiguredObject<?> parent, MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) { } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NullUpgrader.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NullUpgrader.java index c8a812fa89..b17dd4a6f3 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NullUpgrader.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/store/NullUpgrader.java @@ -34,9 +34,9 @@ public final class NullUpgrader implements DurableConfigurationStoreUpgrader } @Override - public void configuredObject(final UUID id, final String type, final Map<String, Object> attributes) + public void configuredObject(final ConfiguredObjectRecord record) { - _durableConfigurationRecoverer.onConfiguredObject(id, type, attributes); + _durableConfigurationRecoverer.onConfiguredObject(record); } @Override @@ -55,4 +55,10 @@ public final class NullUpgrader implements DurableConfigurationStoreUpgrader { return Collections.emptyMap(); } + + @Override + public Map<UUID, ConfiguredObjectRecord> getDeletedRecords() + { + return Collections.emptyMap(); + } } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/AbstractVirtualHost.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/AbstractVirtualHost.java index a14b1ad8c1..e41ee051f3 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/AbstractVirtualHost.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/AbstractVirtualHost.java @@ -137,7 +137,7 @@ public abstract class AbstractVirtualHost implements VirtualHost, IConnectionReg _eventLogger.message(VirtualHostMessages.CREATED(_name)); - _securityManager = new SecurityManager(parentSecurityManager, virtualHost.getSecurityAcl(), _name); + _securityManager = parentSecurityManager; _connectionRegistry = new ConnectionRegistry(); _connectionRegistry.addRegistryChangeListener(this); diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/BindingRecoverer.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/BindingRecoverer.java index e6577e04e4..58721ea8b2 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/BindingRecoverer.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/BindingRecoverer.java @@ -28,9 +28,11 @@ import org.apache.log4j.Logger; import org.apache.qpid.server.binding.BindingImpl; import org.apache.qpid.server.exchange.ExchangeImpl; import org.apache.qpid.server.exchange.ExchangeRegistry; +import org.apache.qpid.server.model.Exchange; import org.apache.qpid.server.model.Queue; import org.apache.qpid.server.queue.AMQQueue; import org.apache.qpid.server.store.AbstractDurableConfiguredObjectRecoverer; +import org.apache.qpid.server.store.ConfiguredObjectRecord; import org.apache.qpid.server.store.UnresolvedDependency; import org.apache.qpid.server.store.UnresolvedObject; @@ -49,11 +51,9 @@ public class BindingRecoverer extends AbstractDurableConfiguredObjectRecoverer<B } @Override - public UnresolvedObject<BindingImpl> createUnresolvedObject(final UUID id, - final String type, - final Map<String, Object> attributes) + public UnresolvedObject<BindingImpl> createUnresolvedObject(ConfiguredObjectRecord record) { - return new UnresolvedBinding(id, attributes); + return new UnresolvedBinding(record); } @Override @@ -76,12 +76,11 @@ public class BindingRecoverer extends AbstractDurableConfiguredObjectRecoverer<B private ExchangeImpl _exchange; private AMQQueue _queue; - public UnresolvedBinding(final UUID id, - final Map<String, Object> attributeMap) + public UnresolvedBinding(final ConfiguredObjectRecord record) { - _bindingId = id; - _exchangeId = UUID.fromString(String.valueOf(attributeMap.get(org.apache.qpid.server.model.Binding.EXCHANGE))); - _queueId = UUID.fromString(String.valueOf(attributeMap.get(org.apache.qpid.server.model.Binding.QUEUE))); + _bindingId = record.getId(); + _exchangeId = record.getParents().get(Exchange.class.getSimpleName()).getId(); + _queueId = record.getParents().get(Queue.class.getSimpleName()).getId(); _exchange = _exchangeRegistry.getExchange(_exchangeId); if(_exchange == null) { @@ -93,8 +92,8 @@ public class BindingRecoverer extends AbstractDurableConfiguredObjectRecoverer<B _unresolvedDependencies.add(new QueueDependency()); } - _bindingName = (String) attributeMap.get(org.apache.qpid.server.model.Binding.NAME); - _bindingArgumentsMap = (Map<String, Object>) attributeMap.get(org.apache.qpid.server.model.Binding.ARGUMENTS); + _bindingName = (String) record.getAttributes().get(org.apache.qpid.server.model.Binding.NAME); + _bindingArgumentsMap = (Map<String, Object>) record.getAttributes().get(org.apache.qpid.server.model.Binding.ARGUMENTS); } @Override diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/DefaultUpgraderProvider.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/DefaultUpgraderProvider.java index 2d824cbd2d..f6fa6344d3 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/DefaultUpgraderProvider.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/DefaultUpgraderProvider.java @@ -39,6 +39,7 @@ import org.apache.qpid.server.model.Queue; import org.apache.qpid.server.model.UUIDGenerator; import org.apache.qpid.server.queue.QueueArgumentsConverter; import org.apache.qpid.server.store.ConfiguredObjectRecord; +import org.apache.qpid.server.store.ConfiguredObjectRecordImpl; import org.apache.qpid.server.store.DurableConfigurationRecoverer; import org.apache.qpid.server.store.DurableConfigurationStoreUpgrader; import org.apache.qpid.server.store.NonNullUpgrader; @@ -133,9 +134,9 @@ public class DefaultUpgraderProvider implements UpgraderProvider } @Override - public void configuredObject(final UUID id, final String type, Map<String, Object> attributes) + public void configuredObject(final ConfiguredObjectRecord record) { - _records.put(id, new ConfiguredObjectRecord(id, type, attributes)); + _records.put(record.getId(), record); } private void removeSelectorArguments(Map<String, Object> binding) @@ -147,9 +148,9 @@ public class DefaultUpgraderProvider implements UpgraderProvider binding.put(Binding.ARGUMENTS, arguments); } - private boolean isTopicExchange(Map<String, Object> binding) + private boolean isTopicExchange(ConfiguredObjectRecord entry) { - UUID exchangeId = UUID.fromString((String)binding.get(Binding.EXCHANGE)); + UUID exchangeId = entry.getParents().get("Exchange").getId(); if(_records.containsKey(exchangeId)) { @@ -188,17 +189,17 @@ public class DefaultUpgraderProvider implements UpgraderProvider String type = record.getType(); Map<String, Object> attributes = record.getAttributes(); UUID id = record.getId(); - if(type.equals(Binding.class.getName()) && hasSelectorArguments(attributes) && !isTopicExchange(attributes)) + if(type.equals(Binding.class.getName()) && hasSelectorArguments(attributes) && !isTopicExchange(record)) { attributes = new LinkedHashMap<String, Object>(attributes); removeSelectorArguments(attributes); - record = new ConfiguredObjectRecord(id, type, attributes); + record = new ConfiguredObjectRecordImpl(id, type, attributes, record.getParents()); getUpdateMap().put(id, record); entry.setValue(record); } - getNextUpgrader().configuredObject(id, type, attributes); + getNextUpgrader().configuredObject(record); } getNextUpgrader().complete(); @@ -212,12 +213,14 @@ public class DefaultUpgraderProvider implements UpgraderProvider */ private class Version1Upgrader extends NonNullUpgrader { + + @Override - public void configuredObject(final UUID id, String type, final Map<String, Object> attributes) + public void configuredObject(final ConfiguredObjectRecord record) { - type = type.substring(1+type.lastIndexOf('.')); - getUpdateMap().put(id, new ConfiguredObjectRecord(id, type, attributes)); - + String type = record.getType().substring(1 + record.getType().lastIndexOf('.')); + getUpdateMap().put(record.getId(), + new ConfiguredObjectRecordImpl(record.getId(), type, record.getAttributes(), record.getParents())); } @Override @@ -226,22 +229,24 @@ public class DefaultUpgraderProvider implements UpgraderProvider for(Map.Entry<UUID, ConfiguredObjectRecord> entry : getUpdateMap().entrySet()) { final ConfiguredObjectRecord record = entry.getValue(); - if(isBinding(record.getType()) && (unknownExchange((String) record.getAttributes().get(Binding.EXCHANGE)) - || unknownQueue((String) record.getAttributes().get(Binding.QUEUE)))) + final ConfiguredObjectRecord exchangeParent = record.getParents().get(Exchange.class.getSimpleName()); + final ConfiguredObjectRecord queueParent = record.getParents().get(Queue.class.getSimpleName()); + if(isBinding(record.getType()) && (exchangeParent == null || unknownExchange(exchangeParent.getId()) + || queueParent == null || unknownQueue(queueParent.getId()))) { + getDeleteMap().put(entry.getKey(), entry.getValue()); entry.setValue(null); } else { - getNextUpgrader().configuredObject(record.getId(), record.getType(), record.getAttributes()); + getNextUpgrader().configuredObject(record); } } getNextUpgrader().complete(); } - private boolean unknownExchange(final String exchangeIdString) + private boolean unknownExchange(final UUID exchangeId) { - UUID exchangeId = UUID.fromString(exchangeIdString); if (_defaultExchangeIds.containsValue(exchangeId)) { return false; @@ -251,9 +256,8 @@ public class DefaultUpgraderProvider implements UpgraderProvider || _exchangeRegistry.getExchange(exchangeId) != null); } - private boolean unknownQueue(final String queueIdString) + private boolean unknownQueue(final UUID queueId) { - UUID queueId = UUID.fromString(queueIdString); ConfiguredObjectRecord localRecord = getUpdateMap().get(queueId); return !((localRecord != null && localRecord.getType().equals(Queue.class.getSimpleName())) || _virtualHost.getQueue(queueId) != null); @@ -277,22 +281,24 @@ public class DefaultUpgraderProvider implements UpgraderProvider private static final String ARGUMENTS = "arguments"; @Override - public void configuredObject(UUID id, String type, Map<String, Object> attributes) + public void configuredObject(ConfiguredObjectRecord record) { - if(Queue.class.getSimpleName().equals(type)) + + if(Queue.class.getSimpleName().equals(record.getType())) { Map<String, Object> newAttributes = new LinkedHashMap<String, Object>(); - if(attributes.get(ARGUMENTS) instanceof Map) + if(record.getAttributes().get(ARGUMENTS) instanceof Map) { - newAttributes.putAll(QueueArgumentsConverter.convertWireArgsToModel((Map<String, Object>) attributes + newAttributes.putAll(QueueArgumentsConverter.convertWireArgsToModel((Map<String, Object>) record.getAttributes() .get(ARGUMENTS))); } - newAttributes.putAll(attributes); - attributes = newAttributes; - getUpdateMap().put(id, new ConfiguredObjectRecord(id,type,attributes)); + newAttributes.putAll(record.getAttributes()); + + record = new ConfiguredObjectRecordImpl(record.getId(), record.getType(), newAttributes, record.getParents()); + getUpdateMap().put(record.getId(), record); } - getNextUpgrader().configuredObject(id,type,attributes); + getNextUpgrader().configuredObject(record); } @Override @@ -311,16 +317,17 @@ public class DefaultUpgraderProvider implements UpgraderProvider { @Override - public void configuredObject(UUID id, String type, Map<String, Object> attributes) + public void configuredObject(ConfiguredObjectRecord record) { - if(Queue.class.getSimpleName().equals(type)) + + if(Queue.class.getSimpleName().equals(record.getType())) { - Map<String, Object> newAttributes = new LinkedHashMap<String, Object>(attributes); - if(attributes.get(EXCLUSIVE) instanceof Boolean) + Map<String, Object> newAttributes = new LinkedHashMap<String, Object>(record.getAttributes()); + if(record.getAttributes().get(EXCLUSIVE) instanceof Boolean) { - boolean isExclusive = (Boolean) attributes.get(EXCLUSIVE); + boolean isExclusive = (Boolean) record.getAttributes().get(EXCLUSIVE); newAttributes.put(EXCLUSIVE, isExclusive ? "CONTAINER" : "NONE"); - if(!isExclusive && attributes.containsKey("owner")) + if(!isExclusive && record.getAttributes().containsKey("owner")) { newAttributes.remove("owner"); } @@ -329,15 +336,16 @@ public class DefaultUpgraderProvider implements UpgraderProvider { newAttributes.remove("owner"); } - if(!attributes.containsKey("durable")) + if(!record.getAttributes().containsKey("durable")) { newAttributes.put("durable","true"); } - attributes = newAttributes; - getUpdateMap().put(id, new ConfiguredObjectRecord(id,type,attributes)); + + record = new ConfiguredObjectRecordImpl(record.getId(),record.getType(),newAttributes, record.getParents()); + getUpdateMap().put(record.getId(), record); } - getNextUpgrader().configuredObject(id,type,attributes); + getNextUpgrader().configuredObject(record); } @Override @@ -352,15 +360,16 @@ public class DefaultUpgraderProvider implements UpgraderProvider private Map<String, String> _missingAmqpExchanges = new HashMap<String, String>(DEFAULT_EXCHANGES); @Override - public void configuredObject(UUID id, String type, Map<String, Object> attributes) + public void configuredObject(ConfiguredObjectRecord record) { - if(Exchange.class.getSimpleName().equals(type)) + if(Exchange.class.getSimpleName().equals(record.getType())) { + Map<String, Object> attributes = record.getAttributes(); String name = (String)attributes.get(NAME); _missingAmqpExchanges.remove(name); } - getNextUpgrader().configuredObject(id,type,attributes); + getNextUpgrader().configuredObject(record); } @Override @@ -383,9 +392,11 @@ public class DefaultUpgraderProvider implements UpgraderProvider attributes.put(org.apache.qpid.server.model.Exchange.DURABLE, true); - getUpdateMap().put(id, new ConfiguredObjectRecord(id, Exchange.class.getSimpleName(), attributes)); + ConfiguredObjectRecord virtualHostRecord = new ConfiguredObjectRecordImpl(_virtualHost.getId(), org.apache.qpid.server.model.VirtualHost.class.getSimpleName(), Collections.<String, Object>emptyMap()); + ConfiguredObjectRecord record = new ConfiguredObjectRecordImpl(id, Exchange.class.getSimpleName(), attributes, Collections.singletonMap(virtualHostRecord.getType(), virtualHostRecord)); + getUpdateMap().put(id, record); - getNextUpgrader().configuredObject(id, Exchange.class.getSimpleName(), attributes); + getNextUpgrader().configuredObject(record); } diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/ExchangeRecoverer.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/ExchangeRecoverer.java index 2743b0ef59..4431fc786d 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/ExchangeRecoverer.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/ExchangeRecoverer.java @@ -28,6 +28,7 @@ import org.apache.qpid.server.exchange.ExchangeImpl; import org.apache.qpid.server.exchange.ExchangeFactory; import org.apache.qpid.server.exchange.ExchangeRegistry; import org.apache.qpid.server.store.AbstractDurableConfiguredObjectRecoverer; +import org.apache.qpid.server.store.ConfiguredObjectRecord; import org.apache.qpid.server.store.UnresolvedDependency; import org.apache.qpid.server.store.UnresolvedObject; import org.apache.qpid.server.util.ServerScopedRuntimeException; @@ -50,11 +51,9 @@ public class ExchangeRecoverer extends AbstractDurableConfiguredObjectRecoverer< } @Override - public UnresolvedObject<ExchangeImpl> createUnresolvedObject(final UUID id, - final String type, - final Map<String, Object> attributes) + public UnresolvedObject<ExchangeImpl> createUnresolvedObject(final ConfiguredObjectRecord record) { - return new UnresolvedExchange(id, attributes); + return new UnresolvedExchange(record.getId(), record.getAttributes()); } private class UnresolvedExchange implements UnresolvedObject<ExchangeImpl> diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/QueueRecoverer.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/QueueRecoverer.java index 385f4e219e..eb1bc883ca 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/QueueRecoverer.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/QueueRecoverer.java @@ -32,6 +32,7 @@ import org.apache.qpid.server.model.Queue; import org.apache.qpid.server.queue.AMQQueue; import org.apache.qpid.server.queue.QueueFactory; import org.apache.qpid.server.store.AbstractDurableConfiguredObjectRecoverer; +import org.apache.qpid.server.store.ConfiguredObjectRecord; import org.apache.qpid.server.store.UnresolvedDependency; import org.apache.qpid.server.store.UnresolvedObject; @@ -58,11 +59,9 @@ public class QueueRecoverer extends AbstractDurableConfiguredObjectRecoverer<AMQ } @Override - public UnresolvedObject<AMQQueue> createUnresolvedObject(final UUID id, - final String type, - final Map<String, Object> attributes) + public UnresolvedObject<AMQQueue> createUnresolvedObject(final ConfiguredObjectRecord record) { - return new UnresolvedQueue(id, type, attributes); + return new UnresolvedQueue(record.getId(), record.getAttributes()); } private class UnresolvedQueue implements UnresolvedObject<AMQQueue> @@ -75,7 +74,6 @@ public class QueueRecoverer extends AbstractDurableConfiguredObjectRecoverer<AMQ private ExchangeImpl _alternateExchange; public UnresolvedQueue(final UUID id, - final String type, final Map<String, Object> attributes) { _attributes = attributes; diff --git a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/StandardVirtualHost.java b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/StandardVirtualHost.java index 58a0e689cb..bb3f8fc012 100644 --- a/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/StandardVirtualHost.java +++ b/qpid/java/broker-core/src/main/java/org/apache/qpid/server/virtualhost/StandardVirtualHost.java @@ -96,13 +96,13 @@ public class StandardVirtualHost extends AbstractVirtualHost _messageStore.openMessageStore(virtualHost.getName(), virtualHost.getMessageStoreSettings()); - _durableConfigurationStore.recoverConfigurationStore(configRecoverer); + _durableConfigurationStore.recoverConfigurationStore(getModel(), configRecoverer); // If store does not have entries for standard exchanges (amq.*), the following will create them. initialiseModel(); VirtualHostConfigRecoveryHandler recoveryHandler = new VirtualHostConfigRecoveryHandler(this); - _messageStore.recoverMessageStore(recoveryHandler, recoveryHandler); + _messageStore.recoverMessageStore(getModel(), recoveryHandler, recoveryHandler); attainActivation(); } diff --git a/qpid/java/broker-core/src/main/resources/META-INF/services/org.apache.qpid.server.plugin.AuthenticationManagerFactory b/qpid/java/broker-core/src/main/resources/META-INF/services/org.apache.qpid.server.plugin.AuthenticationManagerFactory index 8ff67030ef..a1139b386c 100644 --- a/qpid/java/broker-core/src/main/resources/META-INF/services/org.apache.qpid.server.plugin.AuthenticationManagerFactory +++ b/qpid/java/broker-core/src/main/resources/META-INF/services/org.apache.qpid.server.plugin.AuthenticationManagerFactory @@ -22,3 +22,5 @@ org.apache.qpid.server.security.auth.manager.ExternalAuthenticationManagerFactor org.apache.qpid.server.security.auth.manager.KerberosAuthenticationManagerFactory org.apache.qpid.server.security.auth.manager.PlainPasswordFileAuthenticationManagerFactory org.apache.qpid.server.security.auth.manager.SimpleLDAPAuthenticationManagerFactory +org.apache.qpid.server.security.auth.manager.ScramSHA1AuthenticationManagerFactory + diff --git a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/configuration/store/StoreConfigurationChangeListenerTest.java b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/configuration/store/StoreConfigurationChangeListenerTest.java index c23c4715e8..c5786fb981 100644 --- a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/configuration/store/StoreConfigurationChangeListenerTest.java +++ b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/configuration/store/StoreConfigurationChangeListenerTest.java @@ -63,7 +63,9 @@ public class StoreConfigurationChangeListenerTest extends QpidTestCase { notifyBrokerStarted(); Broker broker = mock(Broker.class); + when(broker.getCategoryClass()).thenReturn(Broker.class); VirtualHost child = mock(VirtualHost.class); + when(child.getCategoryClass()).thenReturn(VirtualHost.class); _listener.childAdded(broker, child); verify(_store).save(any(ConfigurationEntry.class), any(ConfigurationEntry.class)); } @@ -72,7 +74,9 @@ public class StoreConfigurationChangeListenerTest extends QpidTestCase { notifyBrokerStarted(); Broker broker = mock(Broker.class); + when(broker.getCategoryClass()).thenReturn(Broker.class); VirtualHost child = mock(VirtualHost.class); + when(child.getCategoryClass()).thenReturn(VirtualHost.class); _listener.childRemoved(broker, child); verify(_store).save(any(ConfigurationEntry.class)); } @@ -81,6 +85,7 @@ public class StoreConfigurationChangeListenerTest extends QpidTestCase { notifyBrokerStarted(); Broker broker = mock(Broker.class); + when(broker.getCategoryClass()).thenReturn(Broker.class); _listener.attributeSet(broker, Broker.QUEUE_FLOW_CONTROL_SIZE_BYTES, null, 1); verify(_store).save(any(ConfigurationEntry.class)); } diff --git a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/security/SecurityManagerTest.java b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/security/SecurityManagerTest.java new file mode 100644 index 0000000000..e27981d22c --- /dev/null +++ b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/security/SecurityManagerTest.java @@ -0,0 +1,571 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.security; + +import static org.apache.qpid.server.security.access.ObjectType.BROKER; +import static org.apache.qpid.server.security.access.ObjectType.EXCHANGE; +import static org.apache.qpid.server.security.access.Operation.ACCESS_LOGS; +import static org.apache.qpid.server.security.access.Operation.PUBLISH; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.security.AccessControlException; + +import org.apache.qpid.server.binding.BindingImpl; +import org.apache.qpid.server.consumer.ConsumerImpl; +import org.apache.qpid.server.exchange.ExchangeImpl; +import org.apache.qpid.server.model.AccessControlProvider; +import org.apache.qpid.server.model.Broker; +import org.apache.qpid.server.model.LifetimePolicy; +import org.apache.qpid.server.model.State; +import org.apache.qpid.server.model.VirtualHost; +import org.apache.qpid.server.protocol.AMQConnectionModel; +import org.apache.qpid.server.queue.AMQQueue; +import org.apache.qpid.server.security.access.ObjectProperties; +import org.apache.qpid.server.security.access.ObjectProperties.Property; +import org.apache.qpid.server.security.access.ObjectType; +import org.apache.qpid.server.security.access.Operation; +import org.apache.qpid.server.security.access.OperationLoggingDetails; +import org.apache.qpid.test.utils.QpidTestCase; + +public class SecurityManagerTest extends QpidTestCase +{ + private static final String TEST_EXCHANGE_TYPE = "testExchangeType"; + private static final String TEST_VIRTUAL_HOST = "testVirtualHost"; + private static final String TEST_EXCHANGE = "testExchange"; + private static final String TEST_QUEUE = "testQueue"; + + private AccessControl _accessControl; + private SecurityManager _securityManager; + private VirtualHost<?> _virtualHost; + + @Override + public void setUp() throws Exception + { + super.setUp(); + _accessControl = mock(AccessControl.class); + _virtualHost = mock(VirtualHost.class); + + AccessControlProvider<?> aclProvider = mock(AccessControlProvider.class); + when(aclProvider.getAccessControl()).thenReturn(_accessControl); + + when(_virtualHost.getName()).thenReturn(TEST_VIRTUAL_HOST); + + _securityManager = new SecurityManager(mock(Broker.class), false); + _securityManager.stateChanged(aclProvider, State.INITIALISING, State.ACTIVE); + } + + public void testAuthoriseCreateBinding() + { + ExchangeImpl<?> exchange = mock(ExchangeImpl.class); + when(exchange.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(exchange.getName()).thenReturn(TEST_EXCHANGE); + + AMQQueue<?> queue = mock(AMQQueue.class); + when(queue.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(queue.getName()).thenReturn(TEST_QUEUE); + when(queue.isDurable()).thenReturn(true); + when(queue.getLifetimePolicy()).thenReturn(LifetimePolicy.PERMANENT); + + BindingImpl binding = mock(BindingImpl.class); + when(binding.getExchange()).thenReturn(exchange); + when(binding.getAMQQueue()).thenReturn(queue); + when(binding.getBindingKey()).thenReturn("bindingKey"); + + ObjectProperties properties = new ObjectProperties(); + properties.put(Property.NAME, TEST_EXCHANGE); + properties.put(Property.VIRTUALHOST_NAME, TEST_VIRTUAL_HOST); + properties.put(Property.QUEUE_NAME, TEST_QUEUE); + properties.put(Property.ROUTING_KEY, "bindingKey"); + properties.put(Property.TEMPORARY, false); + properties.put(Property.DURABLE, true); + + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseCreateBinding(binding); + verify(_accessControl).authorise(eq(Operation.BIND), eq(ObjectType.EXCHANGE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseCreateBinding(binding); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.BIND), eq(ObjectType.EXCHANGE), eq(properties)); + } + + + public void testAuthoriseMethod() + { + ObjectProperties properties = new ObjectProperties("testMethod"); + properties.put(ObjectProperties.Property.COMPONENT, "testComponent"); + properties.put(ObjectProperties.Property.VIRTUALHOST_NAME, TEST_VIRTUAL_HOST); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseMethod(Operation.UPDATE, "testComponent", "testMethod", TEST_VIRTUAL_HOST); + verify(_accessControl).authorise(eq(Operation.UPDATE), eq(ObjectType.METHOD), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseMethod(Operation.UPDATE, "testComponent", "testMethod", TEST_VIRTUAL_HOST); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.UPDATE), eq(ObjectType.METHOD), eq(properties)); + } + + public void testAccessManagement() + { + configureAccessPlugin(Result.ALLOWED); + _securityManager.accessManagement(); + verify(_accessControl).authorise(Operation.ACCESS, ObjectType.MANAGEMENT, ObjectProperties.EMPTY); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.accessManagement(); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(Operation.ACCESS, ObjectType.MANAGEMENT, ObjectProperties.EMPTY); + } + + public void testAuthoriseCreateConnection() + { + AMQConnectionModel<?,?> connection = mock(AMQConnectionModel.class); + when(connection.getVirtualHostName()).thenReturn(TEST_VIRTUAL_HOST); + + ObjectProperties properties = new ObjectProperties(); + properties.put(Property.NAME, TEST_VIRTUAL_HOST); + properties.put(Property.VIRTUALHOST_NAME, TEST_VIRTUAL_HOST); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseCreateConnection(connection); + verify(_accessControl).authorise(eq(Operation.ACCESS), eq(ObjectType.VIRTUALHOST), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseCreateConnection(connection); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.ACCESS), eq(ObjectType.VIRTUALHOST), eq(properties)); + } + + public void testAuthoriseCreateConsumer() + { + AMQQueue<?> queue = mock(AMQQueue.class); + when(queue.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(queue.getName()).thenReturn(TEST_QUEUE); + when(queue.isDurable()).thenReturn(true); + when(queue.getLifetimePolicy()).thenReturn(LifetimePolicy.PERMANENT); + + ConsumerImpl consumer = mock(ConsumerImpl.class); + when(consumer.getMessageSource()).thenReturn(queue); + + ObjectProperties properties = new ObjectProperties(); + properties.put(Property.NAME, TEST_QUEUE); + properties.put(Property.VIRTUALHOST_NAME, TEST_VIRTUAL_HOST); + properties.put(Property.AUTO_DELETE, false); + properties.put(Property.TEMPORARY, false); + properties.put(Property.DURABLE, true); + properties.put(Property.EXCLUSIVE, false); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseCreateConsumer(consumer); + verify(_accessControl).authorise(eq(Operation.CONSUME), eq(ObjectType.QUEUE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseCreateConsumer(consumer); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.CONSUME), eq(ObjectType.QUEUE), eq(properties)); + } + + public void testAuthoriseCreateExchange() + { + ExchangeImpl<?> exchange = mock(ExchangeImpl.class); + when(exchange.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(exchange.getName()).thenReturn(TEST_EXCHANGE); + when(exchange.getTypeName()).thenReturn(TEST_EXCHANGE_TYPE); + + ObjectProperties properties = createExpectedExchangeObjectProperties(); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseCreateExchange(exchange); + verify(_accessControl).authorise(eq(Operation.CREATE), eq(ObjectType.EXCHANGE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseCreateExchange(exchange); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.CREATE), eq(ObjectType.EXCHANGE), eq(properties)); + } + + public void testAuthoriseCreateQueue() + { + AMQQueue<?> queue = mock(AMQQueue.class); + when(queue.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(queue.getName()).thenReturn(TEST_QUEUE); + + ObjectProperties properties = createExpectedQueueObjectProperties(); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseCreateQueue(queue); + verify(_accessControl).authorise(eq(Operation.CREATE), eq(ObjectType.QUEUE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseCreateQueue(queue); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.CREATE), eq(ObjectType.QUEUE), eq(properties)); + } + + public void testAuthoriseDeleteQueue() + { + AMQQueue<?> queue = mock(AMQQueue.class); + when(queue.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(queue.getName()).thenReturn(TEST_QUEUE); + + ObjectProperties properties = createExpectedQueueObjectProperties(); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseDelete(queue); + verify(_accessControl).authorise(eq(Operation.DELETE), eq(ObjectType.QUEUE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseDelete(queue); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.DELETE), eq(ObjectType.QUEUE), eq(properties)); + } + + public void testAuthoriseUpdateQueue() + { + AMQQueue<?> queue = mock(AMQQueue.class); + when(queue.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(queue.getName()).thenReturn(TEST_QUEUE); + + ObjectProperties properties = createExpectedQueueObjectProperties(); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseUpdate(queue); + verify(_accessControl).authorise(eq(Operation.UPDATE), eq(ObjectType.QUEUE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseUpdate(queue); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.UPDATE), eq(ObjectType.QUEUE), eq(properties)); + } + + public void testAuthoriseUpdateExchange() + { + ExchangeImpl<?> exchange = mock(ExchangeImpl.class); + when(exchange.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(exchange.getName()).thenReturn(TEST_EXCHANGE); + when(exchange.getTypeName()).thenReturn(TEST_EXCHANGE_TYPE); + + ObjectProperties properties = createExpectedExchangeObjectProperties(); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseUpdate(exchange); + verify(_accessControl).authorise(eq(Operation.UPDATE), eq(ObjectType.EXCHANGE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseUpdate(exchange); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.UPDATE), eq(ObjectType.EXCHANGE), eq(properties)); + } + + public void testAuthoriseDeleteExchange() + { + ExchangeImpl<?> exchange = mock(ExchangeImpl.class); + when(exchange.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(exchange.getName()).thenReturn(TEST_EXCHANGE); + when(exchange.getTypeName()).thenReturn(TEST_EXCHANGE_TYPE); + + ObjectProperties properties = createExpectedExchangeObjectProperties(); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseDelete(exchange); + verify(_accessControl).authorise(eq(Operation.DELETE), eq(ObjectType.EXCHANGE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseDelete(exchange); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.DELETE), eq(ObjectType.EXCHANGE), eq(properties)); + } + + public void testAuthoriseGroupOperation() + { + ObjectProperties properties = new ObjectProperties("testGroup"); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseGroupOperation(Operation.CREATE, "testGroup"); + verify(_accessControl).authorise(eq(Operation.CREATE), eq(ObjectType.GROUP), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseGroupOperation(Operation.CREATE, "testGroup"); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.CREATE), eq(ObjectType.GROUP), eq(properties)); + } + + public void testAuthoriseUserOperation() + { + ObjectProperties properties = new ObjectProperties("testUser"); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseUserOperation(Operation.CREATE, "testUser"); + verify(_accessControl).authorise(eq(Operation.CREATE), eq(ObjectType.USER), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseUserOperation(Operation.CREATE, "testUser"); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.CREATE), eq(ObjectType.USER), eq(properties)); + } + + public void testAuthorisePublish() + { + String routingKey = "routingKey"; + String exchangeName = "exchangeName"; + boolean immediate = true; + ObjectProperties properties = new ObjectProperties(TEST_VIRTUAL_HOST, exchangeName, routingKey, immediate); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authorisePublish(immediate, routingKey, exchangeName, TEST_VIRTUAL_HOST); + verify(_accessControl).authorise(eq(Operation.PUBLISH), eq(ObjectType.EXCHANGE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authorisePublish(immediate, routingKey, exchangeName, TEST_VIRTUAL_HOST); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.PUBLISH), eq(ObjectType.EXCHANGE), eq(properties)); + } + + public void testAuthorisePurge() + { + AMQQueue<?> queue = mock(AMQQueue.class); + when(queue.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(queue.getName()).thenReturn(TEST_QUEUE); + + ObjectProperties properties = createExpectedQueueObjectProperties(); + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authorisePurge(queue); + verify(_accessControl).authorise(eq(Operation.PURGE), eq(ObjectType.QUEUE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authorisePurge(queue); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.PURGE), eq(ObjectType.QUEUE), eq(properties)); + } + + + public void testAuthoriseUnbind() + { + ExchangeImpl<?> exchange = mock(ExchangeImpl.class); + when(exchange.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(exchange.getName()).thenReturn(TEST_EXCHANGE); + + AMQQueue<?> queue = mock(AMQQueue.class); + when(queue.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(queue.getName()).thenReturn(TEST_QUEUE); + when(queue.isDurable()).thenReturn(true); + when(queue.getLifetimePolicy()).thenReturn(LifetimePolicy.PERMANENT); + + BindingImpl binding = mock(BindingImpl.class); + when(binding.getExchange()).thenReturn(exchange); + when(binding.getAMQQueue()).thenReturn(queue); + when(binding.getBindingKey()).thenReturn("bindingKey"); + + ObjectProperties properties = new ObjectProperties(); + properties.put(Property.NAME, TEST_EXCHANGE); + properties.put(Property.VIRTUALHOST_NAME, TEST_VIRTUAL_HOST); + properties.put(Property.QUEUE_NAME, TEST_QUEUE); + properties.put(Property.ROUTING_KEY, "bindingKey"); + properties.put(Property.TEMPORARY, false); + properties.put(Property.DURABLE, true); + + + configureAccessPlugin(Result.ALLOWED); + _securityManager.authoriseUnbind(binding); + verify(_accessControl).authorise(eq(Operation.UNBIND), eq(ObjectType.EXCHANGE), eq(properties)); + + configureAccessPlugin(Result.DENIED); + try + { + _securityManager.authoriseUnbind(binding); + fail("AccessControlException is expected"); + } + catch(AccessControlException e) + { + // pass + } + verify(_accessControl, times(2)).authorise(eq(Operation.UNBIND), eq(ObjectType.EXCHANGE), eq(properties)); + } + + public void testAuthoriseConfiguringBroker() + { + OperationLoggingDetails properties = new OperationLoggingDetails("create virtualhost 'test'"); + + configureAccessPlugin(Result.ALLOWED); + assertTrue(_securityManager.authoriseConfiguringBroker("test", VirtualHost.class, Operation.CREATE)); + verify(_accessControl).authorise(eq(Operation.CONFIGURE), eq(ObjectType.BROKER), eq(properties)); + + configureAccessPlugin(Result.DENIED); + assertFalse(_securityManager.authoriseConfiguringBroker("test", VirtualHost.class, Operation.CREATE)); + verify(_accessControl, times(2)).authorise(eq(Operation.CONFIGURE), eq(ObjectType.BROKER), eq(properties)); + } + + public void testAuthoriseLogsAccess() + { + configureAccessPlugin(Result.ALLOWED); + assertTrue(_securityManager.authoriseLogsAccess()); + verify(_accessControl).authorise(ACCESS_LOGS, BROKER, ObjectProperties.EMPTY); + + configureAccessPlugin(Result.DENIED); + assertFalse(_securityManager.authoriseLogsAccess()); + verify(_accessControl, times(2)).authorise(ACCESS_LOGS, BROKER, ObjectProperties.EMPTY); + } + + private void configureAccessPlugin(Result result) + { + when(_accessControl.authorise(any(Operation.class), any(ObjectType.class), any(ObjectProperties.class))).thenReturn(result); + } + + private ObjectProperties createExpectedExchangeObjectProperties() + { + ObjectProperties properties = new ObjectProperties(); + properties.put(Property.NAME, TEST_EXCHANGE); + properties.put(Property.VIRTUALHOST_NAME, TEST_VIRTUAL_HOST); + properties.put(Property.AUTO_DELETE, false); + properties.put(Property.TEMPORARY, true); + properties.put(Property.DURABLE, false); + properties.put(Property.TYPE, TEST_EXCHANGE_TYPE); + return properties; + } + + private ObjectProperties createExpectedQueueObjectProperties() + { + ObjectProperties properties = new ObjectProperties(); + properties.put(Property.NAME, TEST_QUEUE); + properties.put(Property.VIRTUALHOST_NAME, TEST_VIRTUAL_HOST); + properties.put(Property.AUTO_DELETE, true); + properties.put(Property.TEMPORARY, true); + properties.put(Property.DURABLE, false); + properties.put(Property.EXCLUSIVE, false); + return properties; + } + + + +} diff --git a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/security/auth/manager/ScramSHA1AuthenticationManagerTest.java b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/security/auth/manager/ScramSHA1AuthenticationManagerTest.java new file mode 100644 index 0000000000..033e0afde8 --- /dev/null +++ b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/security/auth/manager/ScramSHA1AuthenticationManagerTest.java @@ -0,0 +1,203 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.qpid.server.security.auth.manager; + +import org.apache.qpid.server.configuration.updater.TaskExecutor; +import org.apache.qpid.server.model.AuthenticationProvider; +import org.apache.qpid.server.model.Broker; +import org.apache.qpid.server.model.State; +import org.apache.qpid.server.model.User; +import org.apache.qpid.server.security.auth.AuthenticationResult; +import org.apache.qpid.test.utils.QpidTestCase; + import org.apache.qpid.server.security.SecurityManager; + +import javax.security.auth.login.AccountNotFoundException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ScramSHA1AuthenticationManagerTest extends QpidTestCase +{ + private ScramSHA1AuthenticationManager _authManager; + private Broker _broker; + private SecurityManager _securityManager; + private TaskExecutor _executor; + + @Override + public void setUp() throws Exception + { + super.setUp(); + _executor = new TaskExecutor(); + _executor.start(); + _broker = mock(Broker.class); + _securityManager = mock(SecurityManager.class); + when(_broker.getTaskExecutor()).thenReturn(_executor); + when(_broker.getSecurityManager()).thenReturn(_securityManager); + final Map<String, Object> attributesMap = new HashMap<String, Object>(); + attributesMap.put(AuthenticationProvider.NAME, getTestName()); + attributesMap.put(AuthenticationProvider.ID, UUID.randomUUID()); + _authManager = new ScramSHA1AuthenticationManager(_broker, Collections.<String,Object>emptyMap(),attributesMap,false); + } + + @Override + public void tearDown() throws Exception + { + _executor.stop(); + super.tearDown(); + } + + public void testAddChildAndThenDelete() + { + // No children should be present before the test starts + assertEquals("No users should be present before the test starts", 0, _authManager.getChildren(User.class).size()); + assertEquals("No users should be present before the test starts", 0, _authManager.getUsers().size()); + + final Map<String, Object> childAttrs = new HashMap<String, Object>(); + + childAttrs.put(User.NAME, getTestName()); + childAttrs.put(User.PASSWORD, "password"); + User user = _authManager.addChild(User.class, childAttrs); + assertNotNull("User should be created but addChild returned null", user); + assertEquals(getTestName(), user.getName()); + // password shouldn't actually be the given string, but instead salt and the hashed value + assertFalse("Password shouldn't actually be the given string, but instead salt and the hashed value", "password".equals(user.getPassword())); + + AuthenticationResult authResult = + _authManager.authenticate(getTestName(), "password"); + + assertEquals("User should authenticate with given password", AuthenticationResult.AuthenticationStatus.SUCCESS, authResult.getStatus()); + + assertEquals("Manager should have exactly one user child",1, _authManager.getChildren(User.class).size()); + assertEquals("Manager should have exactly one user child",1, _authManager.getUsers().size()); + + + user.setDesiredState(State.ACTIVE, State.DELETED); + + assertEquals("No users should be present after child deletion", 0, _authManager.getChildren(User.class).size()); + + + authResult = _authManager.authenticate(getTestName(), "password"); + assertEquals("User should no longer authenticate with given password", AuthenticationResult.AuthenticationStatus.ERROR, authResult.getStatus()); + + } + + public void testCreateUser() + { + assertEquals("No users should be present before the test starts", 0, _authManager.getChildren(User.class).size()); + assertTrue(_authManager.createUser(getTestName(), "password", Collections.<String, String>emptyMap())); + assertEquals("Manager should have exactly one user child",1, _authManager.getChildren(User.class).size()); + User user = _authManager.getChildren(User.class).iterator().next(); + assertEquals(getTestName(), user.getName()); + // password shouldn't actually be the given string, but instead salt and the hashed value + assertFalse("Password shouldn't actually be the given string, but instead salt and the hashed value", "password".equals(user.getPassword())); + final Map<String, Object> childAttrs = new HashMap<String, Object>(); + + childAttrs.put(User.NAME, getTestName()); + childAttrs.put(User.PASSWORD, "password"); + try + { + user = _authManager.addChild(User.class, childAttrs); + fail("Should not be able to create a second user with the same name"); + } + catch(IllegalArgumentException e) + { + // pass + } + try + { + _authManager.deleteUser(getTestName()); + } + catch (AccountNotFoundException e) + { + fail("AccountNotFoundException thrown when none was expected: " + e.getMessage()); + } + try + { + _authManager.deleteUser(getTestName()); + fail("AccountNotFoundException not thrown when was expected"); + } + catch (AccountNotFoundException e) + { + // pass + } + } + + public void testUpdateUser() + { + assertTrue(_authManager.createUser(getTestName(), "password", Collections.<String, String>emptyMap())); + assertTrue(_authManager.createUser(getTestName()+"_2", "password", Collections.<String, String>emptyMap())); + assertEquals("Manager should have exactly two user children",2, _authManager.getChildren(User.class).size()); + + AuthenticationResult authResult = _authManager.authenticate(getTestName(), "password"); + + assertEquals("User should authenticate with given password", AuthenticationResult.AuthenticationStatus.SUCCESS, authResult.getStatus()); + authResult = _authManager.authenticate(getTestName()+"_2", "password"); + assertEquals("User should authenticate with given password", AuthenticationResult.AuthenticationStatus.SUCCESS, authResult.getStatus()); + + for(User user : _authManager.getChildren(User.class)) + { + if(user.getName().equals(getTestName())) + { + user.setAttributes(Collections.singletonMap(User.PASSWORD, "newpassword")); + } + } + + authResult = _authManager.authenticate(getTestName(), "newpassword"); + assertEquals("User should authenticate with updated password", AuthenticationResult.AuthenticationStatus.SUCCESS, authResult.getStatus()); + authResult = _authManager.authenticate(getTestName()+"_2", "password"); + assertEquals("User should authenticate with original password", AuthenticationResult.AuthenticationStatus.SUCCESS, authResult.getStatus()); + + authResult = _authManager.authenticate(getTestName(), "password"); + assertEquals("User not authenticate with original password", AuthenticationResult.AuthenticationStatus.ERROR, authResult.getStatus()); + + for(User user : _authManager.getChildren(User.class)) + { + if(user.getName().equals(getTestName())) + { + user.setPassword("newerpassword"); + } + } + + authResult = _authManager.authenticate(getTestName(), "newerpassword"); + assertEquals("User should authenticate with updated password", AuthenticationResult.AuthenticationStatus.SUCCESS, authResult.getStatus()); + + + + } + + public void testNonASCIIUser() + { + try + { + _authManager.createUser(getTestName()+Character.toString((char)0xa3), "password", Collections.<String, String>emptyMap()); + fail("Expected exception when attempting to create a user with a non ascii name"); + } + catch(IllegalArgumentException e) + { + // pass + } + } + +} diff --git a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/AbstractDurableConfigurationStoreTestCase.java b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/AbstractDurableConfigurationStoreTestCase.java index b4dfbe837d..fbd26208d8 100644 --- a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/AbstractDurableConfigurationStoreTestCase.java +++ b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/AbstractDurableConfigurationStoreTestCase.java @@ -21,7 +21,6 @@ package org.apache.qpid.server.store; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyMap; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; @@ -40,6 +39,9 @@ import org.apache.qpid.common.AMQPFilterTypes; import org.apache.qpid.server.binding.BindingImpl; import org.apache.qpid.server.exchange.ExchangeImpl; import org.apache.qpid.server.logging.EventLogger; +import org.apache.qpid.server.model.Binding; +import org.apache.qpid.server.model.ConfiguredObject; +import org.apache.qpid.server.model.Exchange; import org.apache.qpid.server.model.ExclusivityPolicy; import org.apache.qpid.server.model.LifetimePolicy; import org.apache.qpid.server.model.Queue; @@ -62,6 +64,10 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest private static final String BINDING = org.apache.qpid.server.model.Binding.class.getSimpleName(); private static final String QUEUE = Queue.class.getSimpleName(); + private static final UUID ANY_UUID = UUID.randomUUID(); + private static final Map ANY_MAP = new HashMap(); + + private String _storePath; private String _storeName; @@ -91,12 +97,16 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest setTestSystemProperty("QPID_WORK", TMP_FOLDER); _recoveryHandler = mock(ConfigurationRecoveryHandler.class); - when(_exchange.getName()).thenReturn(EXCHANGE_NAME); when(_exchange.getId()).thenReturn(_exchangeId); when(_exchange.getExchangeType()).thenReturn(mock(ExchangeType.class)); when(_exchange.getEventLogger()).thenReturn(new EventLogger()); + ConfiguredObjectRecord exchangeRecord = mock(ConfiguredObjectRecord.class); + when(exchangeRecord.getId()).thenReturn(_exchangeId); + when(exchangeRecord.getType()).thenReturn(Exchange.class.getSimpleName()); + when(_exchange.asObjectRecord()).thenReturn(exchangeRecord); + _bindingArgs = new HashMap<String, Object>(); String argKey = AMQPFilterTypes.JMS_SELECTOR.toString(); String argValue = "some selector expression"; @@ -124,8 +134,8 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest DurableConfigurationStoreHelper.createExchange(_configStore, exchange); reopenStore(); - verify(_recoveryHandler).configuredObject(eq(_exchangeId), eq(EXCHANGE), - eq(map( org.apache.qpid.server.model.Exchange.NAME, getName(), + verify(_recoveryHandler).configuredObject(matchesRecord(_exchangeId, EXCHANGE, + map( org.apache.qpid.server.model.Exchange.NAME, getName(), org.apache.qpid.server.model.Exchange.TYPE, getName()+"Type", org.apache.qpid.server.model.Exchange.LIFETIME_POLICY, LifetimePolicy.DELETE_ON_NO_OUTBOUND_LINKS.name()))); } @@ -158,7 +168,7 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest DurableConfigurationStoreHelper.removeExchange(_configStore, exchange); reopenStore(); - verify(_recoveryHandler, never()).configuredObject(any(UUID.class), anyString(), anyMap()); + verify(_recoveryHandler, never()).configuredObject(any(ConfiguredObjectRecord.class)); } public void testBindQueue() throws Exception @@ -166,42 +176,86 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest AMQQueue queue = createTestQueue(QUEUE_NAME, "queueOwner", false, null); BindingImpl binding = new BindingImpl(UUIDGenerator.generateRandomUUID(), ROUTING_KEY, queue, _exchange, _bindingArgs); + DurableConfigurationStoreHelper.createQueue(_configStore, queue); DurableConfigurationStoreHelper.createBinding(_configStore, binding); reopenStore(); Map<String,Object> map = new HashMap<String, Object>(); - map.put(org.apache.qpid.server.model.Binding.EXCHANGE, _exchange.getId().toString()); - map.put(org.apache.qpid.server.model.Binding.QUEUE, queue.getId().toString()); - map.put(org.apache.qpid.server.model.Binding.NAME, ROUTING_KEY); - map.put(org.apache.qpid.server.model.Binding.ARGUMENTS,_bindingArgs); + map.put(Binding.NAME, ROUTING_KEY); + map.put(Binding.ARGUMENTS,_bindingArgs); + + Map<String,UUID> parents = new HashMap<String, UUID>(); + + parents.put(Exchange.class.getSimpleName(), _exchange.getId()); + parents.put(Queue.class.getSimpleName(), queue.getId()); + + verify(_recoveryHandler).configuredObject(matchesRecord(binding.getId(), BINDING, map, parents)); + } + + + private ConfiguredObjectRecord matchesRecord(UUID id, + String type, + Map<String, Object> attributes, + final Map<String, UUID> parents) + { + return argThat(new ConfiguredObjectMatcher(id, type, attributes, parents)); + } - verify(_recoveryHandler).configuredObject(eq(binding.getId()), eq(BINDING), - argThat(new IgnoreCreatedByMatcher(map))); + private ConfiguredObjectRecord matchesRecord(UUID id, String type, Map<String, Object> attributes) + { + return argThat(new ConfiguredObjectMatcher(id, type, attributes, ANY_MAP)); } - private static class IgnoreCreatedByMatcher extends ArgumentMatcher<Map<String,Object>> + private static class ConfiguredObjectMatcher extends ArgumentMatcher<ConfiguredObjectRecord> { private final Map<String,Object> _matchingMap; + private final UUID _id; + private final String _name; + private final Map<String,UUID> _parents; - private IgnoreCreatedByMatcher(final Map<String, Object> matchingMap) + private ConfiguredObjectMatcher(final UUID id, final String type, final Map<String, Object> matchingMap, Map<String,UUID> parents) { + _id = id; + _name = type; _matchingMap = matchingMap; + _parents = parents; } @Override public boolean matches(final Object argument) { - if(argument instanceof Map) + if(argument instanceof ConfiguredObjectRecord) { - Map<String,Object> arg = new HashMap<String, Object>((Map<String,Object>) argument); + ConfiguredObjectRecord binding = (ConfiguredObjectRecord) argument; + + Map<String,Object> arg = new HashMap<String, Object>(binding.getAttributes()); arg.remove("createdBy"); arg.remove("createdTime"); - return arg.equals(_matchingMap); - + return (_id == ANY_UUID || _id.equals(binding.getId())) + && _name.equals(binding.getType()) + && (_matchingMap == ANY_MAP || arg.equals(_matchingMap)) + && (_parents == ANY_MAP || matchesParents(binding)); } return false; } + + private boolean matchesParents(ConfiguredObjectRecord binding) + { + Map<String, ConfiguredObjectRecord> bindingParents = binding.getParents(); + if(bindingParents.size() != _parents.size()) + { + return false; + } + for(Map.Entry<String,UUID> entry : _parents.entrySet()) + { + if(!bindingParents.get(entry.getKey()).getId().equals(entry.getValue())) + { + return false; + } + } + return true; + } } public void testUnbindQueue() throws Exception @@ -214,9 +268,8 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest DurableConfigurationStoreHelper.removeBinding(_configStore, binding); reopenStore(); - verify(_recoveryHandler, never()).configuredObject(any(UUID.class), - eq(BINDING), - anyMap()); + verify(_recoveryHandler, never()).configuredObject(matchesRecord(ANY_UUID, BINDING, + ANY_MAP)); } public void testCreateQueueAMQQueue() throws Exception @@ -229,7 +282,7 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest queueAttributes.put(Queue.NAME, getName()); queueAttributes.put(Queue.OWNER, getName()+"Owner"); queueAttributes.put(Queue.EXCLUSIVE, ExclusivityPolicy.CONTAINER.name()); - verify(_recoveryHandler).configuredObject(eq(_queueId), eq(QUEUE), eq(queueAttributes)); + verify(_recoveryHandler).configuredObject(matchesRecord(_queueId, QUEUE, queueAttributes)); } public void testCreateQueueAMQQueueFieldTable() throws Exception @@ -251,7 +304,7 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest queueAttributes.put(Queue.EXCLUSIVE, ExclusivityPolicy.CONTAINER.name()); queueAttributes.putAll(attributes); - verify(_recoveryHandler).configuredObject(eq(_queueId), eq(QUEUE), eq(queueAttributes)); + verify(_recoveryHandler).configuredObject(matchesRecord(_queueId, QUEUE, queueAttributes)); } public void testCreateQueueAMQQueueWithAlternateExchange() throws Exception @@ -269,7 +322,7 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest queueAttributes.put(Queue.EXCLUSIVE, ExclusivityPolicy.CONTAINER.name()); queueAttributes.put(Queue.ALTERNATE_EXCHANGE, alternateExchange.getId().toString()); - verify(_recoveryHandler).configuredObject(eq(_queueId), eq(QUEUE), eq(queueAttributes)); + verify(_recoveryHandler).configuredObject(matchesRecord(_queueId, QUEUE, queueAttributes)); } private ExchangeImpl createTestAlternateExchange() @@ -302,7 +355,7 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest queueAttributes.put(Queue.NAME, getName()); queueAttributes.putAll(attributes); - verify(_recoveryHandler).configuredObject(eq(_queueId), eq(QUEUE), eq(queueAttributes)); + verify(_recoveryHandler).configuredObject(matchesRecord(_queueId, QUEUE, queueAttributes)); } @@ -329,7 +382,7 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest queueAttributes.putAll(attributes); queueAttributes.put(Queue.ALTERNATE_EXCHANGE, alternateExchange.getId().toString()); - verify(_recoveryHandler).configuredObject(eq(_queueId), eq(QUEUE), eq(queueAttributes)); + verify(_recoveryHandler).configuredObject(matchesRecord(_queueId, QUEUE, queueAttributes)); } public void testRemoveQueue() throws Exception @@ -344,9 +397,7 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest // remove queue DurableConfigurationStoreHelper.removeQueue(_configStore,queue); reopenStore(); - verify(_recoveryHandler, never()).configuredObject(any(UUID.class), - eq(org.apache.qpid.server.model.Queue.class.getName()), - anyMap()); + verify(_recoveryHandler, never()).configuredObject(any(ConfiguredObjectRecord.class)); } private AMQQueue createTestQueue(String queueName, @@ -399,6 +450,12 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest }); when(queue.getActualAttributes()).thenReturn(attributes); + + ConfiguredObjectRecord objectRecord = mock(ConfiguredObjectRecord.class); + when(objectRecord.getId()).thenReturn(_queueId); + when(objectRecord.getType()).thenReturn(Queue.class.getSimpleName()); + when(objectRecord.getAttributes()).thenReturn(attributes); + when(queue.asObjectRecord()).thenReturn(objectRecord); return queue; } @@ -415,6 +472,13 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest when(exchange.getTypeName()).thenReturn(getName() + "Type"); when(exchange.isAutoDelete()).thenReturn(true); when(exchange.getId()).thenReturn(_exchangeId); + ConfiguredObjectRecord exchangeRecord = mock(ConfiguredObjectRecord.class); + when(exchangeRecord.getId()).thenReturn(_exchangeId); + when(exchangeRecord.getType()).thenReturn(Exchange.class.getSimpleName()); + Map<String,Object> actualAttributesExceptId = new HashMap<String, Object>(actualAttributes); + actualAttributesExceptId.remove("id"); + when(exchangeRecord.getAttributes()).thenReturn(actualAttributesExceptId); + when(exchange.asObjectRecord()).thenReturn(exchangeRecord); return exchange; } @@ -425,7 +489,7 @@ public abstract class AbstractDurableConfigurationStoreTestCase extends QpidTest _configStore = createConfigStore(); _configStore.openConfigurationStore("testName", _configurationStoreSettings); - _configStore.recoverConfigurationStore(_recoveryHandler); + _configStore.recoverConfigurationStore(mock(ConfiguredObject.class), _recoveryHandler); } protected abstract DurableConfigurationStore createConfigStore() throws Exception; diff --git a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/JsonFileConfigStoreTest.java b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/JsonFileConfigStoreTest.java index 5d2998de86..bca16b6e70 100644 --- a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/JsonFileConfigStoreTest.java +++ b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/JsonFileConfigStoreTest.java @@ -27,16 +27,19 @@ import java.util.Map; import java.util.UUID; import org.apache.qpid.server.model.Binding; +import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.model.Queue; import org.apache.qpid.server.util.ServerScopedRuntimeException; import org.apache.qpid.test.utils.QpidTestCase; import org.apache.qpid.test.utils.TestFileUtils; import org.apache.qpid.util.FileUtils; +import org.mockito.ArgumentMatcher; import org.mockito.InOrder; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyMap; import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -53,6 +56,10 @@ public class JsonFileConfigStoreTest extends QpidTestCase private String _virtualHostName; private File _storeLocation; + + private static final UUID ANY_UUID = UUID.randomUUID(); + private static final Map<String, Object> ANY_MAP = new HashMap<String, Object>(); + @Override public void setUp() throws Exception { @@ -110,10 +117,10 @@ public class JsonFileConfigStoreTest extends QpidTestCase public void testStartFromNoStore() throws Exception { _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); - _store.recoverConfigurationStore(_recoveryHandler); + _store.recoverConfigurationStore(mock(ConfiguredObject.class), _recoveryHandler); InOrder inorder = inOrder(_recoveryHandler); inorder.verify(_recoveryHandler).beginConfigurationRecovery(eq(_store), eq(0)); - inorder.verify(_recoveryHandler,never()).configuredObject(any(UUID.class),anyString(),anyMap()); + inorder.verify(_recoveryHandler,never()).configuredObject(any(ConfiguredObjectRecord.class)); inorder.verify(_recoveryHandler).completeConfigurationRecovery(); _store.closeConfigurationStore(); } @@ -124,11 +131,11 @@ public class JsonFileConfigStoreTest extends QpidTestCase when(_recoveryHandler.completeConfigurationRecovery()).thenReturn(NEW_CONFIG_VERSION); _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); - _store.recoverConfigurationStore(_recoveryHandler); + _store.recoverConfigurationStore(mock(ConfiguredObject.class), _recoveryHandler); _store.closeConfigurationStore(); _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); - _store.recoverConfigurationStore(_recoveryHandler); + _store.recoverConfigurationStore(mock(ConfiguredObject.class), _recoveryHandler); InOrder inorder = inOrder(_recoveryHandler); // first time the config version should be the initial version - 0 @@ -147,12 +154,12 @@ public class JsonFileConfigStoreTest extends QpidTestCase final String queueType = Queue.class.getSimpleName(); final Map<String,Object> queueAttr = Collections.singletonMap("name", (Object) "q1"); - _store.create(queueId, queueType, queueAttr); + _store.create(new ConfiguredObjectRecordImpl(queueId, queueType, queueAttr)); _store.closeConfigurationStore(); _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); - _store.recoverConfigurationStore(_recoveryHandler); - verify(_recoveryHandler).configuredObject(eq(queueId), eq(queueType), eq(queueAttr)); + _store.recoverConfigurationStore(mock(ConfiguredObject.class), _recoveryHandler); + verify(_recoveryHandler).configuredObject(matchesRecord(queueId, queueType, queueAttr)); _store.closeConfigurationStore(); } @@ -163,18 +170,18 @@ public class JsonFileConfigStoreTest extends QpidTestCase final String queueType = Queue.class.getSimpleName(); Map<String,Object> queueAttr = Collections.singletonMap("name", (Object) "q1"); - _store.create(queueId, queueType, queueAttr); + _store.create(new ConfiguredObjectRecordImpl(queueId, queueType, queueAttr)); queueAttr = new HashMap<String,Object>(queueAttr); queueAttr.put("owner", "theowner"); - _store.update(queueId, queueType, queueAttr); + _store.update(false, new ConfiguredObjectRecordImpl(queueId, queueType, queueAttr)); _store.closeConfigurationStore(); _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); - _store.recoverConfigurationStore(_recoveryHandler); - verify(_recoveryHandler).configuredObject(eq(queueId), eq(queueType), eq(queueAttr)); + _store.recoverConfigurationStore(mock(ConfiguredObject.class), _recoveryHandler); + verify(_recoveryHandler).configuredObject(matchesRecord(queueId, queueType, queueAttr)); _store.closeConfigurationStore(); } @@ -186,16 +193,17 @@ public class JsonFileConfigStoreTest extends QpidTestCase final String queueType = Queue.class.getSimpleName(); Map<String,Object> queueAttr = Collections.singletonMap("name", (Object) "q1"); - _store.create(queueId, queueType, queueAttr); + final ConfiguredObjectRecordImpl record = new ConfiguredObjectRecordImpl(queueId, queueType, queueAttr); + _store.create(record); - _store.remove(queueId, queueType); + _store.remove(record); _store.closeConfigurationStore(); _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); - _store.recoverConfigurationStore(_recoveryHandler); - verify(_recoveryHandler, never()).configuredObject(any(UUID.class), anyString(), anyMap()); + _store.recoverConfigurationStore(mock(ConfiguredObject.class), _recoveryHandler); + verify(_recoveryHandler, never()).configuredObject(any(ConfiguredObjectRecord.class)); _store.closeConfigurationStore(); } @@ -204,7 +212,7 @@ public class JsonFileConfigStoreTest extends QpidTestCase _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); try { - _store.create(UUID.randomUUID(), "wibble", Collections.<String, Object>emptyMap()); + _store.create(new ConfiguredObjectRecordImpl(UUID.randomUUID(), "wibble", Collections.<String, Object>emptyMap())); fail("Should not be able to create instance of type wibble"); } catch (StoreException e) @@ -217,10 +225,10 @@ public class JsonFileConfigStoreTest extends QpidTestCase { _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); final UUID id = UUID.randomUUID(); - _store.create(id, "Queue", Collections.<String, Object>emptyMap()); + _store.create(new ConfiguredObjectRecordImpl(id, "Queue", Collections.<String, Object>emptyMap())); try { - _store.create(id, "Exchange", Collections.<String, Object>emptyMap()); + _store.create(new ConfiguredObjectRecordImpl(id, "Exchange", Collections.<String, Object>emptyMap())); fail("Should not be able to create two objects with same id"); } catch (StoreException e) @@ -234,13 +242,13 @@ public class JsonFileConfigStoreTest extends QpidTestCase { _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); final UUID id = UUID.randomUUID(); - _store.create(id, "Queue", Collections.<String, Object>emptyMap()); + _store.create(new ConfiguredObjectRecordImpl(id, "Queue", Collections.<String, Object>emptyMap())); _store.closeConfigurationStore(); _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); try { - _store.update(id, "Exchange", Collections.<String, Object>emptyMap()); + _store.update(false, new ConfiguredObjectRecordImpl(id, "Exchange", Collections.<String, Object>emptyMap())); fail("Should not be able to update object to different type"); } catch (StoreException e) @@ -279,32 +287,76 @@ public class JsonFileConfigStoreTest extends QpidTestCase final Map<String, Object> EMPTY_ATTR = Collections.emptyMap(); final UUID exchangeId = new UUID(0, 2); - final Map<String, Object> bindingAttributes = new HashMap<String, Object>(); - bindingAttributes.put(Binding.EXCHANGE, exchangeId); - bindingAttributes.put(Binding.QUEUE, queueId); - final Map<String, Object> binding2Attributes = new HashMap<String, Object>(); - binding2Attributes.put(Binding.EXCHANGE, exchangeId); - binding2Attributes.put(Binding.QUEUE, queue2Id); final UUID bindingId = new UUID(0, 3); final UUID binding2Id = new UUID(1, 3); - _store.create(queueId, "Queue", EMPTY_ATTR); - _store.create(queue2Id, "Queue", EMPTY_ATTR); - _store.create(exchangeId, "Exchange", EMPTY_ATTR); - _store.update(true, - new ConfiguredObjectRecord(bindingId, "Binding", bindingAttributes), - new ConfiguredObjectRecord(binding2Id, "Binding", binding2Attributes)); + final ConfiguredObjectRecordImpl queueRecord = new ConfiguredObjectRecordImpl(queueId, "Queue", EMPTY_ATTR); + _store.create(queueRecord); + final ConfiguredObjectRecordImpl queue2Record = new ConfiguredObjectRecordImpl(queue2Id, "Queue", EMPTY_ATTR); + _store.create(queue2Record); + final ConfiguredObjectRecordImpl exchangeRecord = new ConfiguredObjectRecordImpl(exchangeId, "Exchange", EMPTY_ATTR); + _store.create(exchangeRecord); + Map<String,ConfiguredObjectRecord> bindingParents = new HashMap<String, ConfiguredObjectRecord>(); + bindingParents.put("Exchange", exchangeRecord); + bindingParents.put("Queue", queueRecord); + final ConfiguredObjectRecordImpl bindingRecord = + new ConfiguredObjectRecordImpl(bindingId, "Binding", EMPTY_ATTR, bindingParents); + + + Map<String,ConfiguredObjectRecord> binding2Parents = new HashMap<String, ConfiguredObjectRecord>(); + binding2Parents.put("Exchange", exchangeRecord); + binding2Parents.put("Queue", queue2Record); + final ConfiguredObjectRecordImpl binding2Record = + new ConfiguredObjectRecordImpl(binding2Id, "Binding", EMPTY_ATTR, binding2Parents); + _store.update(true, bindingRecord, binding2Record); _store.closeConfigurationStore(); _store.openConfigurationStore(_virtualHostName, _configurationStoreSettings); - _store.recoverConfigurationStore(_recoveryHandler); - verify(_recoveryHandler).configuredObject(eq(queueId), eq("Queue"), eq(EMPTY_ATTR)); - verify(_recoveryHandler).configuredObject(eq(queue2Id), eq("Queue"), eq(EMPTY_ATTR)); - verify(_recoveryHandler).configuredObject(eq(exchangeId), eq("Exchange"), eq(EMPTY_ATTR)); - verify(_recoveryHandler).configuredObject(eq(bindingId),eq("Binding"), eq(bindingAttributes)); - verify(_recoveryHandler).configuredObject(eq(binding2Id),eq("Binding"), eq(binding2Attributes)); + _store.recoverConfigurationStore(mock(ConfiguredObject.class), _recoveryHandler); + verify(_recoveryHandler).configuredObject(matchesRecord(queueId, "Queue", EMPTY_ATTR)); + verify(_recoveryHandler).configuredObject(matchesRecord(queue2Id, "Queue", EMPTY_ATTR)); + verify(_recoveryHandler).configuredObject(matchesRecord(exchangeId, "Exchange", EMPTY_ATTR)); + verify(_recoveryHandler).configuredObject(matchesRecord(bindingId, "Binding", EMPTY_ATTR)); + verify(_recoveryHandler).configuredObject(matchesRecord(binding2Id, "Binding", EMPTY_ATTR)); _store.closeConfigurationStore(); } + private ConfiguredObjectRecord matchesRecord(UUID id, String type, Map<String, Object> attributes) + { + return argThat(new ConfiguredObjectMatcher(id, type, attributes)); + } + + private static class ConfiguredObjectMatcher extends ArgumentMatcher<ConfiguredObjectRecord> + { + private final Map<String,Object> _matchingMap; + private final UUID _id; + private final String _name; + + private ConfiguredObjectMatcher(final UUID id, final String type, final Map<String, Object> matchingMap) + { + _id = id; + _name = type; + _matchingMap = matchingMap; + } + + @Override + public boolean matches(final Object argument) + { + if(argument instanceof ConfiguredObjectRecord) + { + ConfiguredObjectRecord binding = (ConfiguredObjectRecord) argument; + + Map<String,Object> arg = new HashMap<String, Object>(binding.getAttributes()); + arg.remove("createdBy"); + arg.remove("createdTime"); + return (_id == ANY_UUID || _id.equals(binding.getId())) + && _name.equals(binding.getType()) + && (_matchingMap == ANY_MAP || arg.equals(_matchingMap)); + + } + return false; + } + } + } diff --git a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/MessageStoreQuotaEventsTestBase.java b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/MessageStoreQuotaEventsTestBase.java index e46a8939f4..fc69a53c85 100644 --- a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/MessageStoreQuotaEventsTestBase.java +++ b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/MessageStoreQuotaEventsTestBase.java @@ -32,6 +32,8 @@ import java.util.UUID; import org.apache.log4j.Logger; import org.apache.qpid.server.message.EnqueueableMessage; +import org.apache.qpid.server.model.ConfiguredObject; +import org.apache.qpid.server.model.VirtualHost; import org.apache.qpid.server.store.MessageStoreRecoveryHandler.StoredMessageRecoveryHandler; import org.apache.qpid.test.utils.QpidTestCase; import org.apache.qpid.util.FileUtils; @@ -64,11 +66,11 @@ public abstract class MessageStoreQuotaEventsTestBase extends QpidTestCase imple Map<String, Object> storeSettings = createStoreSettings(_storeLocation.getAbsolutePath()); _store = createStore(); - ((DurableConfigurationStore)_store).openConfigurationStore("test", storeSettings); + MessageStoreRecoveryHandler recoveryHandler = mock(MessageStoreRecoveryHandler.class); when(recoveryHandler.begin()).thenReturn(mock(StoredMessageRecoveryHandler.class)); _store.openMessageStore("test", storeSettings); - _store.recoverMessageStore(recoveryHandler, null); + _store.recoverMessageStore(mock(ConfiguredObject.class), recoveryHandler, null); _transactionResource = UUID.randomUUID(); _events = new ArrayList<Event>(); diff --git a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/MessageStoreTestCase.java b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/MessageStoreTestCase.java index 45f7a2a39e..1d95133784 100644 --- a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/MessageStoreTestCase.java +++ b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/store/MessageStoreTestCase.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.UUID; import org.apache.qpid.server.message.EnqueueableMessage; +import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.model.UUIDGenerator; import org.apache.qpid.server.store.MessageStoreRecoveryHandler.StoredMessageRecoveryHandler; import org.apache.qpid.server.store.Transaction.Record; @@ -65,7 +66,7 @@ public abstract class MessageStoreTestCase extends QpidTestCase _store = createMessageStore(); _store.openMessageStore("test", _storeSettings); - _store.recoverMessageStore(_messageStoreRecoveryHandler, _logRecoveryHandler); + _store.recoverMessageStore(mock(ConfiguredObject.class), _messageStoreRecoveryHandler, _logRecoveryHandler); } protected abstract Map<String, Object> getStoreSettings() throws Exception; @@ -106,7 +107,7 @@ public abstract class MessageStoreTestCase extends QpidTestCase _store = createMessageStore(); _store.openMessageStore("test", _storeSettings); - _store.recoverMessageStore(_messageStoreRecoveryHandler, _logRecoveryHandler); + _store.recoverMessageStore(mock(ConfiguredObject.class), _messageStoreRecoveryHandler, _logRecoveryHandler); } private Record getTestRecord(long messageNumber) { diff --git a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/virtualhost/DurableConfigurationRecovererTest.java b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/virtualhost/DurableConfigurationRecovererTest.java index e3d56ab4bb..c90c6af220 100644 --- a/qpid/java/broker-core/src/test/java/org/apache/qpid/server/virtualhost/DurableConfigurationRecovererTest.java +++ b/qpid/java/broker-core/src/test/java/org/apache/qpid/server/virtualhost/DurableConfigurationRecovererTest.java @@ -30,6 +30,7 @@ import java.util.UUID; import org.apache.qpid.server.exchange.ExchangeImpl; import org.apache.qpid.server.logging.EventLogger; +import org.apache.qpid.server.store.ConfiguredObjectRecordImpl; import org.apache.qpid.server.store.StoreException; import org.apache.qpid.server.configuration.IllegalConfigurationException; import org.apache.qpid.server.exchange.AMQUnknownExchangeType; @@ -229,17 +230,16 @@ public class DurableConfigurationRecovererTest extends QpidTestCase _durableConfigurationRecoverer.beginConfigurationRecovery(_store, 0); - _durableConfigurationRecoverer.configuredObject(new UUID(1, 0), + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(new UUID(1, 0), "org.apache.qpid.server.model.Binding", createBinding("key", - DIRECT_EXCHANGE_ID, - QUEUE_ID, "x-filter-jms-selector", - "wibble")); + "wibble"), + createBindingParents(DIRECT_EXCHANGE_ID, QUEUE_ID))); final ConfiguredObjectRecord[] expected = { - new ConfiguredObjectRecord(new UUID(1, 0), "Binding", - createBinding("key", DIRECT_EXCHANGE_ID, QUEUE_ID)) + new ConfiguredObjectRecordImpl(new UUID(1, 0), "Binding", + createBinding("key")) }; verifyCorrectUpdates(expected); @@ -254,32 +254,30 @@ public class DurableConfigurationRecovererTest extends QpidTestCase _durableConfigurationRecoverer.beginConfigurationRecovery(_store, 0); - _durableConfigurationRecoverer.configuredObject(new UUID(1, 0), + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(new UUID(1, 0), "org.apache.qpid.server.model.Binding", createBinding("key", - DIRECT_EXCHANGE_ID, - QUEUE_ID, "x-filter-jms-selector", "wibble", "not-a-selector", - "moo")); + "moo"), + createBindingParents(DIRECT_EXCHANGE_ID, QUEUE_ID))); final UUID customExchangeId = new UUID(3,0); - _durableConfigurationRecoverer.configuredObject(new UUID(2, 0), + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(new UUID(2, 0), "org.apache.qpid.server.model.Binding", createBinding("key", - customExchangeId, - QUEUE_ID, "x-filter-jms-selector", "wibble", "not-a-selector", - "moo")); + "moo"), + createBindingParents(customExchangeId,QUEUE_ID))); - _durableConfigurationRecoverer.configuredObject(customExchangeId, + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(customExchangeId, "org.apache.qpid.server.model.Exchange", - createExchange(CUSTOM_EXCHANGE_NAME, HeadersExchange.TYPE)); + createExchange(CUSTOM_EXCHANGE_NAME, HeadersExchange.TYPE))); final ExchangeImpl customExchange = mock(ExchangeImpl.class); @@ -323,10 +321,10 @@ public class DurableConfigurationRecovererTest extends QpidTestCase final ConfiguredObjectRecord[] expected = { - new ConfiguredObjectRecord(new UUID(1, 0), "org.apache.qpid.server.model.Binding", - createBinding("key", DIRECT_EXCHANGE_ID, QUEUE_ID, "not-a-selector", "moo")), - new ConfiguredObjectRecord(new UUID(2, 0), "org.apache.qpid.server.model.Binding", - createBinding("key", customExchangeId, QUEUE_ID, "not-a-selector", "moo")) + new ConfiguredObjectRecordImpl(new UUID(1, 0), "org.apache.qpid.server.model.Binding", + createBinding("key", "not-a-selector", "moo")), + new ConfiguredObjectRecordImpl(new UUID(2, 0), "org.apache.qpid.server.model.Binding", + createBinding("key", "not-a-selector", "moo")) }; verifyCorrectUpdates(expected); @@ -340,17 +338,16 @@ public class DurableConfigurationRecovererTest extends QpidTestCase _durableConfigurationRecoverer.beginConfigurationRecovery(_store, 0); - _durableConfigurationRecoverer.configuredObject(new UUID(1, 0), + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(new UUID(1, 0), "org.apache.qpid.server.model.Binding", createBinding("key", - TOPIC_EXCHANGE_ID, - QUEUE_ID, "x-filter-jms-selector", - "wibble")); + "wibble"), + createBindingParents(TOPIC_EXCHANGE_ID,QUEUE_ID))); final ConfiguredObjectRecord[] expected = { - new ConfiguredObjectRecord(new UUID(1, 0), "Binding", - createBinding("key", TOPIC_EXCHANGE_ID, QUEUE_ID, "x-filter-jms-selector", "wibble")) + new ConfiguredObjectRecordImpl(new UUID(1, 0), "Binding", + createBinding("key", "x-filter-jms-selector", "wibble")) }; verifyCorrectUpdates(expected); @@ -363,16 +360,15 @@ public class DurableConfigurationRecovererTest extends QpidTestCase _durableConfigurationRecoverer.beginConfigurationRecovery(_store, 2); - _durableConfigurationRecoverer.configuredObject(new UUID(1, 0), + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(new UUID(1, 0), "Binding", createBinding("key", - DIRECT_EXCHANGE_ID, - QUEUE_ID, "x-filter-jms-selector", - "wibble")); + "wibble"), + createBindingParents(DIRECT_EXCHANGE_ID,QUEUE_ID))); doThrow(new RuntimeException("Update Should not be called")) - .when(_store).update(anyBoolean(), any(ConfiguredObjectRecord[].class)); + .when(_store).update(anyBoolean(), any(ConfiguredObjectRecordImpl[].class)); _durableConfigurationRecoverer.completeConfigurationRecovery(); } @@ -382,13 +378,13 @@ public class DurableConfigurationRecovererTest extends QpidTestCase _durableConfigurationRecoverer.beginConfigurationRecovery(_store, 2); - _durableConfigurationRecoverer.configuredObject(new UUID(1, 0), + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(new UUID(1, 0), "Binding", createBinding("key", - new UUID(3,0), - QUEUE_ID, "x-filter-jms-selector", - "wibble")); + "wibble"), + createBindingParents(new UUID(3,0), + QUEUE_ID))); try { @@ -410,8 +406,8 @@ public class DurableConfigurationRecovererTest extends QpidTestCase try { final Map<String, Object> emptyArguments = Collections.emptyMap(); - _durableConfigurationRecoverer.configuredObject(new UUID(1, 0), - "Wibble", emptyArguments); + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(new UUID(1, 0), + "Wibble", emptyArguments)); _durableConfigurationRecoverer.completeConfigurationRecovery(); fail("Expected resolution to fail due to unknown object type"); } @@ -474,11 +470,11 @@ public class DurableConfigurationRecovererTest extends QpidTestCase _durableConfigurationRecoverer.beginConfigurationRecovery(_store, 2); - _durableConfigurationRecoverer.configuredObject(queueId, Queue.class.getSimpleName(), - createQueue("testQueue", exchangeId)); - _durableConfigurationRecoverer.configuredObject(exchangeId, + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(queueId, Queue.class.getSimpleName(), + createQueue("testQueue", exchangeId))); + _durableConfigurationRecoverer.configuredObject(new ConfiguredObjectRecordImpl(exchangeId, org.apache.qpid.server.model.Exchange.class.getSimpleName(), - createExchange(CUSTOM_EXCHANGE_NAME, HeadersExchange.TYPE)); + createExchange(CUSTOM_EXCHANGE_NAME, HeadersExchange.TYPE))); _durableConfigurationRecoverer.completeConfigurationRecovery(); @@ -499,16 +495,14 @@ public class DurableConfigurationRecovererTest extends QpidTestCase return null; } - }).when(_store).update(anyBoolean(), any(ConfiguredObjectRecord[].class)); + }).when(_store).update(anyBoolean(), any(ConfiguredObjectRecordImpl[].class)); } - private Map<String,Object> createBinding(String bindingKey, UUID exchangeId, UUID queueId, String... args) + private Map<String,Object> createBinding(String bindingKey, String... args) { Map<String, Object> binding = new LinkedHashMap<String, Object>(); binding.put("name", bindingKey); - binding.put(Binding.EXCHANGE, exchangeId.toString()); - binding.put(Binding.QUEUE, queueId.toString()); Map<String,String> argumentMap = new LinkedHashMap<String, String>(); if(args != null && args.length != 0) { @@ -530,6 +524,15 @@ public class DurableConfigurationRecovererTest extends QpidTestCase return binding; } + private Map<String,ConfiguredObjectRecord> createBindingParents(UUID exchangeId, UUID queueId) + { + Map<String,ConfiguredObjectRecord> parents = new HashMap<String, ConfiguredObjectRecord>(); + parents.put("Exchange", new ConfiguredObjectRecordImpl(exchangeId,"Exchange",Collections.<String,Object>emptyMap())); + parents.put("Queue", new ConfiguredObjectRecordImpl(queueId,"Queue",Collections.<String,Object>emptyMap())); + + return parents; + } + private Map<String, Object> createExchange(String name, ExchangeType<HeadersExchange> type) { diff --git a/qpid/java/broker-plugins/access-control/src/test/java/org/apache/qpid/server/security/access/plugins/DefaultAccessControlTest.java b/qpid/java/broker-plugins/access-control/src/test/java/org/apache/qpid/server/security/access/plugins/DefaultAccessControlTest.java index 3a36ddef2c..072bd6a87f 100644 --- a/qpid/java/broker-plugins/access-control/src/test/java/org/apache/qpid/server/security/access/plugins/DefaultAccessControlTest.java +++ b/qpid/java/broker-plugins/access-control/src/test/java/org/apache/qpid/server/security/access/plugins/DefaultAccessControlTest.java @@ -20,7 +20,9 @@ */ package org.apache.qpid.server.security.access.plugins; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -181,7 +183,7 @@ public class DefaultAccessControlTest extends TestCase final RuleSet rs = new RuleSet(mock(EventLoggerProvider.class)); // grant user4 access right on any method in any component - rs.grant(1, "user4", Permission.ALLOW, Operation.ACCESS, ObjectType.METHOD, new ObjectProperties(ObjectProperties.STAR)); + rs.grant(1, "user4", Permission.ALLOW, Operation.ACCESS, ObjectType.METHOD, new ObjectProperties(ObjectProperties.WILD_CARD)); configureAccessControl(rs); Subject.doAs(TestPrincipalUtils.createTestSubject("user4"), new PrivilegedAction<Object>() { @@ -207,7 +209,7 @@ public class DefaultAccessControlTest extends TestCase final RuleSet rs = new RuleSet(mock(EventLoggerProvider.class)); // grant user5 access right on any methods in "Test" component - ObjectProperties ruleProperties = new ObjectProperties(ObjectProperties.STAR); + ObjectProperties ruleProperties = new ObjectProperties(ObjectProperties.WILD_CARD); ruleProperties.put(ObjectProperties.Property.COMPONENT, "Test"); rs.grant(1, "user5", Permission.ALLOW, Operation.ACCESS, ObjectType.METHOD, ruleProperties); configureAccessControl(rs); @@ -234,6 +236,7 @@ public class DefaultAccessControlTest extends TestCase public void testAccess() throws Exception { final Subject subject = TestPrincipalUtils.createTestSubject("user1"); + final String testVirtualHost = getName(); final InetAddress inetAddress = InetAddress.getLocalHost(); final InetSocketAddress inetSocketAddress = new InetSocketAddress(inetAddress, 1); @@ -249,13 +252,12 @@ public class DefaultAccessControlTest extends TestCase { RuleSet mockRuleSet = mock(RuleSet.class); - - DefaultAccessControl accessControl = new DefaultAccessControl(mockRuleSet); - accessControl.authorise(Operation.ACCESS, ObjectType.VIRTUALHOST, ObjectProperties.EMPTY); + ObjectProperties properties = new ObjectProperties(testVirtualHost); + accessControl.authorise(Operation.ACCESS, ObjectType.VIRTUALHOST, properties); - verify(mockRuleSet).check(subject, Operation.ACCESS, ObjectType.VIRTUALHOST, ObjectProperties.EMPTY, inetAddress); + verify(mockRuleSet).check(subject, Operation.ACCESS, ObjectType.VIRTUALHOST, properties, inetAddress); return null; } }); diff --git a/qpid/java/broker-plugins/access-control/src/test/java/org/apache/qpid/server/security/access/plugins/RuleSetTest.java b/qpid/java/broker-plugins/access-control/src/test/java/org/apache/qpid/server/security/access/plugins/RuleSetTest.java index caf9b2fb61..32037807cd 100644 --- a/qpid/java/broker-plugins/access-control/src/test/java/org/apache/qpid/server/security/access/plugins/RuleSetTest.java +++ b/qpid/java/broker-plugins/access-control/src/test/java/org/apache/qpid/server/security/access/plugins/RuleSetTest.java @@ -21,24 +21,26 @@ package org.apache.qpid.server.security.access.plugins; -import java.security.Principal; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import javax.security.auth.Subject; -import org.apache.qpid.server.logging.EventLogger; +import org.apache.qpid.server.exchange.ExchangeImpl; import org.apache.qpid.server.logging.EventLoggerProvider; +import org.apache.qpid.server.model.VirtualHost; +import org.apache.qpid.server.queue.AMQQueue; import org.apache.qpid.server.security.Result; import org.apache.qpid.server.security.access.ObjectProperties; import org.apache.qpid.server.security.access.ObjectType; import org.apache.qpid.server.security.access.Operation; import org.apache.qpid.server.security.access.Permission; +import org.apache.qpid.server.security.access.ObjectProperties.Property; import org.apache.qpid.server.security.access.config.Rule; import org.apache.qpid.server.security.access.config.RuleSet; import org.apache.qpid.server.security.auth.TestPrincipalUtils; import org.apache.qpid.test.utils.QpidTestCase; -import static org.mockito.Mockito.mock; - /** * This test checks that the {@link RuleSet} object which forms the core of the access control plugin performs correctly. * @@ -51,6 +53,9 @@ import static org.mockito.Mockito.mock; */ public class RuleSetTest extends QpidTestCase { + private static final String DENIED_VH = "deniedVH"; + private static final String ALLOWED_VH = "allowedVH"; + private RuleSet _ruleSet; // Object under test private static final String TEST_USER = "user"; @@ -60,6 +65,8 @@ public class RuleSetTest extends QpidTestCase private String _exchangeName = "amq.direct"; private String _exchangeType = "direct"; private Subject _testSubject = TestPrincipalUtils.createTestSubject(TEST_USER); + private AMQQueue<?> _queue; + private VirtualHost<?> _virtualHost; @Override public void setUp() throws Exception @@ -67,6 +74,11 @@ public class RuleSetTest extends QpidTestCase super.setUp(); _ruleSet = new RuleSet(mock(EventLoggerProvider.class)); + + _virtualHost = mock(VirtualHost.class); + _queue = mock(AMQQueue.class); + when(_queue.getName()).thenReturn(_queueName); + when(_queue.getParent(VirtualHost.class)).thenReturn(_virtualHost); } @Override @@ -83,10 +95,8 @@ public class RuleSetTest extends QpidTestCase public void assertDenyGrantAllow(Subject subject, Operation operation, ObjectType objectType, ObjectProperties properties) { - final Principal identity = subject.getPrincipals().iterator().next(); - assertEquals(Result.DENIED, _ruleSet.check(subject, operation, objectType, properties)); - _ruleSet.grant(0, identity.getName(), Permission.ALLOW, operation, objectType, properties); + _ruleSet.grant(0, TEST_USER, Permission.ALLOW, operation, objectType, properties); assertEquals(1, _ruleSet.getRuleCount()); assertEquals(Result.ALLOWED, _ruleSet.check(subject, operation, objectType, properties)); } @@ -98,17 +108,77 @@ public class RuleSetTest extends QpidTestCase assertEquals(_ruleSet.getDefault(), _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, ObjectProperties.EMPTY)); } - public void testVirtualHostAccess() throws Exception + public void testVirtualHostAccessAllowPermissionWithVirtualHostName() throws Exception { - assertDenyGrantAllow(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST); + _ruleSet.grant(0, TEST_USER, Permission.ALLOW, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ALLOWED_VH)); + assertEquals(Result.ALLOWED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ALLOWED_VH))); + assertEquals(Result.DEFER, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(DENIED_VH))); } + public void testVirtualHostAccessAllowPermissionWithNameSetToWildCard() throws Exception + { + _ruleSet.grant(0, TEST_USER, Permission.ALLOW, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ObjectProperties.WILD_CARD)); + assertEquals(Result.ALLOWED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ALLOWED_VH))); + assertEquals(Result.ALLOWED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(DENIED_VH))); + } + + public void testVirtualHostAccessAllowPermissionWithNoName() throws Exception + { + _ruleSet.grant(0, TEST_USER, Permission.ALLOW, Operation.ACCESS, ObjectType.VIRTUALHOST, ObjectProperties.EMPTY); + assertEquals(Result.ALLOWED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ALLOWED_VH))); + assertEquals(Result.ALLOWED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(DENIED_VH))); + } + + public void testVirtualHostAccessDenyPermissionWithNoName() throws Exception + { + _ruleSet.grant(0, TEST_USER, Permission.DENY, Operation.ACCESS, ObjectType.VIRTUALHOST, ObjectProperties.EMPTY); + assertEquals(Result.DENIED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ALLOWED_VH))); + assertEquals(Result.DENIED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(DENIED_VH))); + } + + public void testVirtualHostAccessDenyPermissionWithNameSetToWildCard() throws Exception + { + _ruleSet.grant(0, TEST_USER, Permission.DENY, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ObjectProperties.WILD_CARD)); + assertEquals(Result.DENIED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ALLOWED_VH))); + assertEquals(Result.DENIED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(DENIED_VH))); + } + + public void testVirtualHostAccessAllowDenyPermissions() throws Exception + { + _ruleSet.grant(0, TEST_USER, Permission.DENY, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(DENIED_VH)); + _ruleSet.grant(1, TEST_USER, Permission.ALLOW, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ALLOWED_VH)); + assertEquals(Result.ALLOWED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(ALLOWED_VH))); + assertEquals(Result.DENIED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(DENIED_VH))); + } + + public void testVirtualHostAccessAllowPermissionWithVirtualHostNameOtherPredicate() throws Exception + { + ObjectProperties properties = new ObjectProperties(); + properties.put(Property.VIRTUALHOST_NAME, ALLOWED_VH); + + _ruleSet.grant(0, TEST_USER, Permission.ALLOW, Operation.ACCESS, ObjectType.VIRTUALHOST, properties); + assertEquals(Result.ALLOWED, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, properties)); + assertEquals(Result.DEFER, _ruleSet.check(_testSubject, Operation.ACCESS, ObjectType.VIRTUALHOST, new ObjectProperties(DENIED_VH))); + } + + public void testQueueCreateNamed() throws Exception { assertDenyGrantAllow(_testSubject, Operation.CREATE, ObjectType.QUEUE, new ObjectProperties(_queueName)); } - public void testQueueCreatenamedNullRoutingKey() + public void testQueueCreateNamedVirtualHost() throws Exception + { + _ruleSet.grant(0, TEST_USER, Permission.ALLOW, Operation.CREATE, ObjectType.QUEUE, new ObjectProperties(Property.VIRTUALHOST_NAME, ALLOWED_VH)); + + when(_virtualHost.getName()).thenReturn(ALLOWED_VH); + assertEquals(Result.ALLOWED, _ruleSet.check(_testSubject, Operation.CREATE, ObjectType.QUEUE, new ObjectProperties(_queue))); + + when(_virtualHost.getName()).thenReturn(DENIED_VH); + assertEquals(Result.DEFER, _ruleSet.check(_testSubject, Operation.CREATE, ObjectType.QUEUE, new ObjectProperties(_queue))); + } + + public void testQueueCreateNamedNullRoutingKey() { ObjectProperties properties = new ObjectProperties(_queueName); properties.put(ObjectProperties.Property.ROUTING_KEY, (String) null); @@ -116,6 +186,21 @@ public class RuleSetTest extends QpidTestCase assertDenyGrantAllow(_testSubject, Operation.CREATE, ObjectType.QUEUE, properties); } + public void testExchangeCreateNamedVirtualHost() + { + _ruleSet.grant(0, TEST_USER, Permission.ALLOW, Operation.CREATE, ObjectType.EXCHANGE, new ObjectProperties(Property.VIRTUALHOST_NAME, ALLOWED_VH)); + + ExchangeImpl<?> exchange = mock(ExchangeImpl.class); + when(exchange.getParent(VirtualHost.class)).thenReturn(_virtualHost); + when(exchange.getTypeName()).thenReturn(_exchangeType); + when(_virtualHost.getName()).thenReturn(ALLOWED_VH); + + assertEquals(Result.ALLOWED, _ruleSet.check(_testSubject, Operation.CREATE, ObjectType.EXCHANGE, new ObjectProperties(exchange))); + + when(_virtualHost.getName()).thenReturn(DENIED_VH); + assertEquals(Result.DEFER, _ruleSet.check(_testSubject, Operation.CREATE, ObjectType.EXCHANGE, new ObjectProperties(exchange))); + } + public void testExchangeCreate() { ObjectProperties properties = new ObjectProperties(_exchangeName); diff --git a/qpid/java/broker-plugins/amqp-0-10-protocol/src/main/java/org/apache/qpid/server/protocol/v0_10/ServerSessionDelegate.java b/qpid/java/broker-plugins/amqp-0-10-protocol/src/main/java/org/apache/qpid/server/protocol/v0_10/ServerSessionDelegate.java index 040be92ceb..999da2da6c 100644 --- a/qpid/java/broker-plugins/amqp-0-10-protocol/src/main/java/org/apache/qpid/server/protocol/v0_10/ServerSessionDelegate.java +++ b/qpid/java/broker-plugins/amqp-0-10-protocol/src/main/java/org/apache/qpid/server/protocol/v0_10/ServerSessionDelegate.java @@ -291,7 +291,7 @@ public class ServerSessionDelegate extends SessionDelegate final VirtualHost virtualHost = getVirtualHost(ssn); try { - virtualHost.getSecurityManager().authorisePublish(messageMetaData.isImmediate(), messageMetaData.getRoutingKey(), exchange.getName()); + virtualHost.getSecurityManager().authorisePublish(messageMetaData.isImmediate(), messageMetaData.getRoutingKey(), exchange.getName(), virtualHost.getName()); } catch (AccessControlException e) { diff --git a/qpid/java/broker-plugins/amqp-0-8-protocol/src/main/java/org/apache/qpid/server/protocol/v0_8/AMQChannel.java b/qpid/java/broker-plugins/amqp-0-8-protocol/src/main/java/org/apache/qpid/server/protocol/v0_8/AMQChannel.java index baf5eceef7..7bde83cc99 100644 --- a/qpid/java/broker-plugins/amqp-0-8-protocol/src/main/java/org/apache/qpid/server/protocol/v0_8/AMQChannel.java +++ b/qpid/java/broker-plugins/amqp-0-8-protocol/src/main/java/org/apache/qpid/server/protocol/v0_8/AMQChannel.java @@ -303,9 +303,10 @@ public class AMQChannel<T extends AMQProtocolSession<T>> public void setPublishFrame(MessagePublishInfo info, final MessageDestination e) { String routingKey = info.getRoutingKey() == null ? null : info.getRoutingKey().asString(); - SecurityManager securityManager = getVirtualHost().getSecurityManager(); + VirtualHost virtualHost = getVirtualHost(); + SecurityManager securityManager = virtualHost.getSecurityManager(); - securityManager.authorisePublish(info.isImmediate(), routingKey, e.getName()); + securityManager.authorisePublish(info.isImmediate(), routingKey, e.getName(), virtualHost.getName()); _currentMessage = new IncomingMessage(info); _currentMessage.setMessageDestination(e); diff --git a/qpid/java/broker-plugins/amqp-0-8-protocol/src/main/java/org/apache/qpid/server/protocol/v0_8/handler/ConnectionOpenMethodHandler.java b/qpid/java/broker-plugins/amqp-0-8-protocol/src/main/java/org/apache/qpid/server/protocol/v0_8/handler/ConnectionOpenMethodHandler.java index a29d56605a..1a29806f62 100644 --- a/qpid/java/broker-plugins/amqp-0-8-protocol/src/main/java/org/apache/qpid/server/protocol/v0_8/handler/ConnectionOpenMethodHandler.java +++ b/qpid/java/broker-plugins/amqp-0-8-protocol/src/main/java/org/apache/qpid/server/protocol/v0_8/handler/ConnectionOpenMethodHandler.java @@ -80,6 +80,8 @@ public class ConnectionOpenMethodHandler implements StateAwareMethodListener<Con } else { + session.setVirtualHost(virtualHost); + // Check virtualhost access try { @@ -95,7 +97,6 @@ public class ConnectionOpenMethodHandler implements StateAwareMethodListener<Con throw body.getConnectionException(AMQConstant.CONNECTION_FORCED, "Virtual host '" + virtualHost.getName() + "' is not active"); } - session.setVirtualHost(virtualHost); // See Spec (0.8.2). Section 3.1.2 Virtual Hosts if (session.getContextKey() == null) diff --git a/qpid/java/broker-plugins/jdbc-provider-bone/src/main/java/resources/virtualhost/store/pool/bonecp/add.html b/qpid/java/broker-plugins/jdbc-provider-bone/src/main/java/resources/virtualhost/store/pool/bonecp/add.html index 0a83bd9f6f..5b053849a3 100644 --- a/qpid/java/broker-plugins/jdbc-provider-bone/src/main/java/resources/virtualhost/store/pool/bonecp/add.html +++ b/qpid/java/broker-plugins/jdbc-provider-bone/src/main/java/resources/virtualhost/store/pool/bonecp/add.html @@ -20,7 +20,7 @@ <td class="tableContainer-labelCell" style="width: 300px;"><strong>Partition Count: </strong></td> <td class="tableContainer-valueCell"> <input data-dojo-type="dijit/form/NumberSpinner" id="formAddVirtualHost.specific.store.pool.parititions" - name="minConnectionsPerPartition" value="4" smallDelta="1" constraints="{min:1,max:1000,places:0}"/> + name="partitionCount" value="4" smallDelta="1" constraints="{min:1,max:1000,places:0}"/> </td> </tr> <tr> diff --git a/qpid/java/broker-plugins/management-http/pom.xml b/qpid/java/broker-plugins/management-http/pom.xml index 4bfaf5e5d2..afb295f7cd 100644 --- a/qpid/java/broker-plugins/management-http/pom.xml +++ b/qpid/java/broker-plugins/management-http/pom.xml @@ -69,6 +69,12 @@ <type>zip</type> </dependency> + <dependency> + <groupId>org.webjars</groupId> + <artifactId>cryptojs</artifactId> + <version>3.1.2</version> + </dependency> + <!-- test dependencies --> <dependency> <groupId>org.apache.qpid</groupId> diff --git a/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/FileServlet.java b/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/FileServlet.java index 618aaed319..72e9bac29b 100644 --- a/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/FileServlet.java +++ b/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/FileServlet.java @@ -89,6 +89,10 @@ public class FileServlet extends HttpServlet } URL resourceURL = getClass().getResource(_resourcePathPrefix + filename); + if(resourceURL == null) + { + resourceURL = getClass().getResource("/META-INF" + _resourcePathPrefix + filename); + } if(resourceURL != null) { response.setStatus(HttpServletResponse.SC_OK); diff --git a/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/rest/MessageServlet.java b/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/rest/MessageServlet.java index 0947ae2a89..b23f0cb168 100644 --- a/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/rest/MessageServlet.java +++ b/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/rest/MessageServlet.java @@ -485,10 +485,10 @@ public class MessageServlet extends AbstractServlet } - private void authorizeMethod(String methodName, VirtualHost host) + private void authorizeMethod(String methodName, VirtualHost<?> vhost) { - SecurityManager securityManager = host.getSecurityManager(); - securityManager.authoriseMethod(Operation.UPDATE, "VirtualHost.Queue", methodName); + SecurityManager securityManager = getBroker().getSecurityManager(); + securityManager.authoriseMethod(Operation.UPDATE, "VirtualHost.Queue", methodName, vhost.getName()); } } diff --git a/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/rest/SaslServlet.java b/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/rest/SaslServlet.java index 2ca67fadc9..af3973c7b3 100644 --- a/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/rest/SaslServlet.java +++ b/qpid/java/broker-plugins/management-http/src/main/java/org/apache/qpid/server/management/plugin/servlet/rest/SaslServlet.java @@ -59,7 +59,7 @@ public class SaslServlet extends AbstractServlet private static final String ATTR_ID = "SaslServlet.ID"; private static final String ATTR_SASL_SERVER = "SaslServlet.SaslServer"; private static final String ATTR_EXPIRY = "SaslServlet.Expiry"; - private static final long SASL_EXCHANGE_EXPIRY = 1000L; + private static final long SASL_EXCHANGE_EXPIRY = 3000L; public SaslServlet() { @@ -260,7 +260,17 @@ public class SaslServlet extends AbstractServlet session.removeAttribute(ATTR_ID); session.removeAttribute(ATTR_SASL_SERVER); session.removeAttribute(ATTR_EXPIRY); + if(challenge != null && challenge.length != 0) + { + Map<String, Object> outputObject = new LinkedHashMap<String, Object>(); + outputObject.put("challenge", new String(Base64.encodeBase64(challenge))); + + final PrintWriter writer = response.getWriter(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); + mapper.writeValue(writer, outputObject); + } response.setStatus(HttpServletResponse.SC_OK); } else diff --git a/qpid/java/broker-plugins/management-http/src/main/java/resources/js/qpid/authorization/sasl.js b/qpid/java/broker-plugins/management-http/src/main/java/resources/js/qpid/authorization/sasl.js index 2d99f886ed..a25375a669 100644 --- a/qpid/java/broker-plugins/management-http/src/main/java/resources/js/qpid/authorization/sasl.js +++ b/qpid/java/broker-plugins/management-http/src/main/java/resources/js/qpid/authorization/sasl.js @@ -18,8 +18,8 @@ * under the License. * */ -define(["dojo/_base/xhr", "dojox/encoding/base64", "dojox/encoding/digests/_base", "dojox/encoding/digests/MD5"], - function (xhr, base64, digestsBase, MD5) { +define(["dojo/_base/xhr", "dojox/encoding/base64", "dojox/encoding/digests/_base", "dojox/encoding/digests/MD5", "dojox/uuid/generateRandomUuid", "dojo/request/script"], + function (xhr, base64, digestsBase, MD5, uuid, script) { var encodeUTF8 = function encodeUTF8(str) { var byteArray = []; @@ -83,34 +83,32 @@ var saslPlain = function saslPlain(user, password, callbackFunction) var saslCramMD5 = function saslCramMD5(user, password, saslMechanism, callbackFunction) { - - // Using dojo.xhrGet, as very little information is being sent - dojo.xhrPost({ - // The URL of the request - url: "rest/sasl", - content: { - mechanism: saslMechanism - }, - handleAs: "json", - failOk: true - }).then(function(data) - { - - var challengeBytes = base64.decode(data.challenge); - var wa=[]; - var bitLength = challengeBytes.length*8; - for(var i=0; i<bitLength; i+=8) + dojo.xhrPost({ + // The URL of the request + url: "rest/sasl", + content: { + mechanism: saslMechanism + }, + handleAs: "json", + failOk: true + }).then(function(data) { - wa[i>>5] |= (challengeBytes[i/8] & 0xFF)<<(i%32); - } - var challengeStr = digestsBase.wordToString(wa).substring(0,challengeBytes.length); - var digest = user + " " + MD5._hmac(challengeStr, password, digestsBase.outputTypes.Hex); - var id = data.id; + var challengeBytes = base64.decode(data.challenge); + var wa=[]; + var bitLength = challengeBytes.length*8; + for(var i=0; i<bitLength; i+=8) + { + wa[i>>5] |= (challengeBytes[i/8] & 0xFF)<<(i%32); + } + var challengeStr = digestsBase.wordToString(wa).substring(0,challengeBytes.length); + + var digest = user + " " + MD5._hmac(challengeStr, password, digestsBase.outputTypes.Hex); + var id = data.id; - var response = base64.encode(encodeUTF8( digest )); + var response = base64.encode(encodeUTF8( digest )); - dojo.xhrPost({ + dojo.xhrPost({ // The URL of the request url: "rest/sasl", content: { @@ -121,20 +119,163 @@ var saslCramMD5 = function saslCramMD5(user, password, saslMechanism, callbackFu failOk: true }).then(callbackFunction, errorHandler); - }, - function(error) - { - if(error.status == 403) + }, + function(error) { - alert("Authentication Failed"); - } - else - { - alert(error); - } - }); + if(error.status == 403) + { + alert("Authentication Failed"); + } + else + { + alert(error); + } + }); + + + }; + var saslScramSha1 = function saslScramSha1(user, password, saslMechanism, callbackFunction) + { + + script.get("webjars/cryptojs/3.1.2/rollups/hmac-sha1.js").then( function() + { + script.get("webjars/cryptojs/3.1.2/components/enc-base64-min.js").then ( function() + { + + var toBase64 = function toBase64( input ) + { + var result = []; + for(var i = 0; i < input.length; i++) + { + result[i] = input.charCodeAt(i); + } + return base64.encode( result ) + }; + + var fromBase64 = function fromBase64( input ) + { + var decoded = base64.decode( input ); + var result = ""; + for(var i = 0; i < decoded.length; i++) + { + result+= String.fromCharCode(decoded[i]); + } + return result; + }; + + var xor = function xor(lhs, rhs) { + var words = []; + for(var i = 0; i < lhs.words.length; i++) + { + words.push(lhs.words[i]^rhs.words[i]); + } + return CryptoJS.lib.WordArray.create(words); + }; + + var hasNonAscii = function hasNonAscii(name) { + for(var i = 0; i < name.length; i++) { + if(name.charCodeAt(i) > 127) { + return true; + } + } + return false; + }; + + var generateSaltedPassword = function generateSaltedPassword(salt, password, iterationCount) + { + var hmac = CryptoJS.algo.HMAC.create(CryptoJS.algo.SHA1, password); + + hmac.update(salt); + hmac.update(CryptoJS.enc.Hex.parse("00000001")); + + var result = hmac.finalize(); + var previous = null; + for(var i = 1 ;i < iterationCount; i++) + { + hmac = CryptoJS.algo.HMAC.create(CryptoJS.algo.SHA1, password); + hmac.update( previous != null ? previous : result ); + previous = hmac.finalize(); + result = xor(result, previous); + } + return result; + + }; + + GS2_HEADER = "n,,"; + + if(!hasNonAscii(user)) { + + user = user.replace(/=/g, "=3D"); + user = user.replace(/,/g, "=2C"); + + clientNonce = uuid(); + clientFirstMessageBare = "n=" + user + ",r=" + clientNonce; + dojo.xhrPost({ + // The URL of the request + url: "rest/sasl", + content: { + mechanism: saslMechanism, + response: toBase64(GS2_HEADER + clientFirstMessageBare) + }, + handleAs: "json", + failOk: true + }).then(function (data) { + var serverFirstMessage = fromBase64(data.challenge); + var id = data.id; + + var parts = serverFirstMessage.split(","); + nonce = parts[0].substring(2); + if (!nonce.substr(0, clientNonce.length) == clientNonce) { + alert("Authentication error - server nonce does not start with client nonce") + } + else { + var salt = CryptoJS.enc.Base64.parse(parts[1].substring(2)); + var iterationCount = parts[2].substring(2); + var saltedPassword = generateSaltedPassword(salt, password, iterationCount) + var clientFinalMessageWithoutProof = "c=" + toBase64(GS2_HEADER) + ",r=" + nonce; + var authMessage = clientFirstMessageBare + "," + serverFirstMessage + "," + clientFinalMessageWithoutProof; + var clientKey = CryptoJS.HmacSHA1("Client Key", saltedPassword); + var storedKey = CryptoJS.SHA1(clientKey); + var clientSignature = CryptoJS.HmacSHA1(authMessage, storedKey); + var clientProof = xor(clientKey, clientSignature); + var serverKey = CryptoJS.HmacSHA1("Server Key", saltedPassword); + serverSignature = CryptoJS.HmacSHA1(authMessage, serverKey); + dojo.xhrPost({ + // The URL of the request + url: "rest/sasl", + content: { + id: id, + response: toBase64(clientFinalMessageWithoutProof + + ",p=" + clientProof.toString(CryptoJS.enc.Base64)) + }, + handleAs: "json", + failOk: true + }).then(function (data) { + var serverFinalMessage = fromBase64(data.challenge); + if (serverSignature.toString(CryptoJS.enc.Base64) == serverFinalMessage.substring(2)) { + callbackFunction(); + } + else { + errorHandler("Server signature did not match"); + } + + + }, errorHandler); + } + + }, errorHandler); + } + else + { + alert("Username '"+name+"' is invalid"); + } + + }, errorHandler); + }, errorHandler); + }; + var containsMechanism = function containsMechanism(mechanisms, mech) { for (var i = 0; i < mechanisms.length; i++) { @@ -157,7 +298,11 @@ SaslClient.authenticate = function(username, password, callbackFunction) }).then(function(data) { var mechMap = data.mechanisms; - if (containsMechanism(mechMap, "CRAM-MD5")) + if(containsMechanism(mechMap, "SCRAM-SHA-1")) + { + saslScramSha1(username, password, "SCRAM-SHA-1", callbackFunction) + } + else if (containsMechanism(mechMap, "CRAM-MD5")) { saslCramMD5(username, password, "CRAM-MD5", callbackFunction); } diff --git a/qpid/java/broker-plugins/management-http/src/main/java/resources/js/qpid/common/util.js b/qpid/java/broker-plugins/management-http/src/main/java/resources/js/qpid/common/util.js index 3d349830ac..6d83a68fb4 100644 --- a/qpid/java/broker-plugins/management-http/src/main/java/resources/js/qpid/common/util.js +++ b/qpid/java/broker-plugins/management-http/src/main/java/resources/js/qpid/common/util.js @@ -140,7 +140,7 @@ define(["dojo/_base/xhr", util.isProviderManagingUsers = function(type) { - return (type === "PlainPasswordFile" || type === "Base64MD5PasswordFile"); + return (type === "PlainPasswordFile" || type === "Base64MD5PasswordFile" || type === "SCRAM-SHA1"); }; util.showSetAttributesDialog = function(attributeWidgetFactories, data, putURL, dialogTitle, appendNameToUrl) diff --git a/qpid/java/broker-plugins/management-jmx/src/main/java/org/apache/qpid/server/jmx/MBeanInvocationHandlerImpl.java b/qpid/java/broker-plugins/management-jmx/src/main/java/org/apache/qpid/server/jmx/MBeanInvocationHandlerImpl.java index 7b0a48cac1..5a7674d4fd 100644 --- a/qpid/java/broker-plugins/management-jmx/src/main/java/org/apache/qpid/server/jmx/MBeanInvocationHandlerImpl.java +++ b/qpid/java/broker-plugins/management-jmx/src/main/java/org/apache/qpid/server/jmx/MBeanInvocationHandlerImpl.java @@ -60,15 +60,15 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler private MBeanServer _mbs; private final boolean _managementRightsInferAllAccess; - private final Broker _broker; + private final Broker<?> _broker; - MBeanInvocationHandlerImpl(Broker broker) + MBeanInvocationHandlerImpl(Broker<?> broker) { _managementRightsInferAllAccess = Boolean.valueOf(System.getProperty(BrokerProperties.PROPERTY_MANAGEMENT_RIGHTS_INFER_ALL_ACCESS, "true")); _broker = broker; } - public static MBeanServerForwarder newProxyInstance(Broker broker) + public static MBeanServerForwarder newProxyInstance(Broker<?> broker) { final InvocationHandler handler = new MBeanInvocationHandlerImpl(broker); final Class<?>[] interfaces = new Class[] { MBeanServerForwarder.class }; @@ -195,28 +195,23 @@ public class MBeanInvocationHandlerImpl implements InvocationHandler String methodName; // Get the component, type and impact, which may be null String type = getType(method, args); - String vhost = getVirtualHost(method, args); + String virtualHostName = getVirtualHost(method, args); int impact = getImpact(method, args); - // Get the security manager for the virtual host (if set) - SecurityManager security; - if (vhost == null) + if (virtualHostName != null) { - security = _broker.getSecurityManager(); - } - else - { - VirtualHost virtualHost = _broker.findVirtualHostByName(vhost); + VirtualHost<?> virtualHost = _broker.findVirtualHostByName(virtualHostName); if (virtualHost == null) { - throw new IllegalArgumentException("Virtual host with name '" + vhost + "' is not found."); + throw new IllegalArgumentException("Virtual host with name '" + virtualHostName + "' is not found."); } - security = virtualHost.getSecurityManager(); } methodName = getMethodName(method, args); Operation operation = (isAccessMethod(methodName) || impact == MBeanOperationInfo.INFO) ? Operation.ACCESS : Operation.UPDATE; - security.authoriseMethod(operation, type, methodName); + + SecurityManager security = _broker.getSecurityManager(); + security.authoriseMethod(operation, type, methodName, virtualHostName); if (_managementRightsInferAllAccess) { diff --git a/qpid/java/build.deps b/qpid/java/build.deps index f4496ce8ab..a0efbea658 100644 --- a/qpid/java/build.deps +++ b/qpid/java/build.deps @@ -57,6 +57,8 @@ jetty-servlet=lib/required/jetty-servlet-8.1.14.v20131031.jar jetty-websocket=lib/required/jetty-websocket-8.1.14.v20131031.jar servlet-api=${geronimo-servlet} +cryptojs=lib/required/cryptojs-3.1.2.jar + dojo-version=1.9.1 dojo=lib/required/dojo-${dojo-version}.zip @@ -80,7 +82,7 @@ broker-core.libs=${commons-cli} ${commons-logging} ${log4j} ${slf4j-log4j} \ #Borrow the broker-core libs, hack for release binary generation broker.libs=${broker-core.libs} -broker-plugins-management-http.libs=${dojo} +broker-plugins-management-http.libs=${dojo} ${cryptojs} broker-plugins.libs=${log4j} ${commons.libs} test.libs=${slf4j-log4j} ${log4j} ${junit} ${slf4j-api} ${mockito-all} diff --git a/qpid/java/client/src/main/java/client.bnd b/qpid/java/client/src/main/java/client.bnd index 63b1e3b34c..c2abcd55b5 100755 --- a/qpid/java/client/src/main/java/client.bnd +++ b/qpid/java/client/src/main/java/client.bnd @@ -17,7 +17,7 @@ # under the License. # -ver: 0.27.0 +ver: 0.29.0 Bundle-SymbolicName: qpid-client Bundle-Version: ${ver} diff --git a/qpid/java/client/src/main/java/org/apache/qpid/client/security/CallbackHandlerRegistry.properties b/qpid/java/client/src/main/java/org/apache/qpid/client/security/CallbackHandlerRegistry.properties index 8855a040ea..109ff9942a 100644 --- a/qpid/java/client/src/main/java/org/apache/qpid/client/security/CallbackHandlerRegistry.properties +++ b/qpid/java/client/src/main/java/org/apache/qpid/client/security/CallbackHandlerRegistry.properties @@ -31,3 +31,5 @@ CRAM-MD5.4=org.apache.qpid.client.security.UsernamePasswordCallbackHandler AMQPLAIN.5=org.apache.qpid.client.security.UsernamePasswordCallbackHandler PLAIN.6=org.apache.qpid.client.security.UsernamePasswordCallbackHandler ANONYMOUS.7=org.apache.qpid.client.security.UsernamePasswordCallbackHandler +SCRAM-SHA1.8=org.apache.qpid.client.security.UsernamePasswordCallbackHandler + diff --git a/qpid/java/client/src/main/java/org/apache/qpid/client/security/DynamicSaslRegistrar.properties b/qpid/java/client/src/main/java/org/apache/qpid/client/security/DynamicSaslRegistrar.properties index b903208927..c2cd671bdf 100644 --- a/qpid/java/client/src/main/java/org/apache/qpid/client/security/DynamicSaslRegistrar.properties +++ b/qpid/java/client/src/main/java/org/apache/qpid/client/security/DynamicSaslRegistrar.properties @@ -19,3 +19,4 @@ AMQPLAIN=org.apache.qpid.client.security.amqplain.AmqPlainSaslClientFactory CRAM-MD5-HASHED=org.apache.qpid.client.security.crammd5hashed.CRAMMD5HashedSaslClientFactory ANONYMOUS=org.apache.qpid.client.security.anonymous.AnonymousSaslClientFactory +SCRAM-SHA1=org.apache.qpid.client.security.scram.ScramSHA1SaslClientFactory diff --git a/qpid/java/common.xml b/qpid/java/common.xml index 889cf3c4d8..87b8f2d175 100644 --- a/qpid/java/common.xml +++ b/qpid/java/common.xml @@ -24,9 +24,9 @@ <property name="project.name" value="qpid"/> <!-- Version used for standard build output --> - <property name="project.version" value="0.27"/> + <property name="project.version" value="0.29"/> <!-- The release version used for maven output. SNAPSHOT added via maven.version.suffix --> - <property name="project.version.maven" value="0.28"/> + <property name="project.version.maven" value="0.30"/> <property name="project.url" value="http://qpid.apache.org"/> <property name="project.groupid" value="org.apache.qpid"/> <property name="project.namever" value="${project.name}-${project.version}"/> diff --git a/qpid/java/common/src/main/java/common.bnd b/qpid/java/common/src/main/java/common.bnd index 76ee9d740f..498562f3c1 100755 --- a/qpid/java/common/src/main/java/common.bnd +++ b/qpid/java/common/src/main/java/common.bnd @@ -17,7 +17,7 @@ # under the License.
#
-ver: 0.27.0
+ver: 0.29.0
Bundle-SymbolicName: qpid-common
Bundle-Version: ${ver}
diff --git a/qpid/java/ivy.retrieve.xml b/qpid/java/ivy.retrieve.xml index dcb97fcf9f..e604023940 100644 --- a/qpid/java/ivy.retrieve.xml +++ b/qpid/java/ivy.retrieve.xml @@ -70,6 +70,7 @@ <dependency org="xalan" name="xalan" rev="2.7.0" transitive="false"/> <dependency org="velocity" name="velocity" rev="1.4" transitive="false"/> <dependency org="velocity" name="velocity-dep" rev="1.4" transitive="false"/> + <dependency org="org.webjars" name="cryptojs" rev="3.1.2" transitive="false"/> <dependency org="org.dojotoolkit" name="dojo" rev="1.9.1" transitive="false"> <artifact name="dojo" type="zip"/> </dependency> diff --git a/qpid/java/management/common/src/main/java/management-common.bnd b/qpid/java/management/common/src/main/java/management-common.bnd index 459c3e60a8..cb080c9680 100644 --- a/qpid/java/management/common/src/main/java/management-common.bnd +++ b/qpid/java/management/common/src/main/java/management-common.bnd @@ -17,7 +17,7 @@ # under the License. # -ver: 0.27.0 +ver: 0.29.0 Bundle-SymbolicName: qpid-management-common Bundle-Version: ${ver} diff --git a/qpid/java/maven/qpid-enforcer-plugin-rules/pom.xml b/qpid/java/maven/qpid-enforcer-plugin-rules/pom.xml new file mode 100644 index 0000000000..47a393b888 --- /dev/null +++ b/qpid/java/maven/qpid-enforcer-plugin-rules/pom.xml @@ -0,0 +1,93 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.qpid.enforcer</groupId> + <artifactId>qpid-enforcer-plugin-rules</artifactId> + <version>1.0-SNAPSHOT</version> + <name>Qpid Maven Enforcer Plugin Rules</name> + <description>Custom maven enforcer plugin rules for the Qpid maven build.</description> + + <properties> + <api.version>1.3.1</api.version> + <maven.version>2.2.1</maven.version> + <plexus.container.version>1.5.5</plexus.container.version> + <junit.version>4.11</junit.version> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.maven.enforcer</groupId> + <artifactId>enforcer-rules</artifactId> + <version>${api.version}</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-io</artifactId> + <version>1.3.2</version> + </dependency> + <dependency> + <groupId>org.apache.maven.enforcer</groupId> + <artifactId>enforcer-api</artifactId> + <version>${api.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-project</artifactId> + <version>${maven.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-core</artifactId> + <version>${maven.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-artifact</artifactId> + <version>${maven.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-plugin-api</artifactId> + <version>${maven.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-container-default</artifactId> + <version>${plexus.container.version}</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junit.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.maven.enforcer</groupId> + <artifactId>enforcer-rules</artifactId> + <version>${api.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + </build> + +</project> diff --git a/qpid/java/maven/qpid-enforcer-plugin-rules/src/main/java/org/apache/qpid/maven/enforcer/rule/RequireFileContentsAreEquivalent.java b/qpid/java/maven/qpid-enforcer-plugin-rules/src/main/java/org/apache/qpid/maven/enforcer/rule/RequireFileContentsAreEquivalent.java new file mode 100644 index 0000000000..11186a5c7f --- /dev/null +++ b/qpid/java/maven/qpid-enforcer-plugin-rules/src/main/java/org/apache/qpid/maven/enforcer/rule/RequireFileContentsAreEquivalent.java @@ -0,0 +1,104 @@ +/** + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package org.apache.qpid.maven.enforcer.rule; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.io.FileUtils; +import org.apache.maven.enforcer.rule.api.EnforcerRule; +import org.apache.maven.enforcer.rule.api.EnforcerRuleException; +import org.apache.maven.enforcer.rule.api.EnforcerRuleHelper; +import org.apache.maven.plugins.enforcer.AbstractStandardEnforcerRule; + +public class RequireFileContentsAreEquivalent extends AbstractStandardEnforcerRule +{ + + final static String WHITESPACE_REGEX = "\\s+"; + final static String EMPTY_STRING = ""; + + File[] files; + + @Override + public void execute(final EnforcerRuleHelper enforcerRuleHelper) throws EnforcerRuleException + { + if (files.length < 2) + { + throw new EnforcerRuleException("The file list must contain at least two files for comparison."); + } + + boolean success = true; + boolean firstTime = true; + String referenceContent = null; + + for (final File file : files) + { + try + { + final String fileContent = FileUtils.readFileToString(file); + if (firstTime) + { + referenceContent = fileContent; + firstTime = false; + } + else if (referenceContent != null && fileContent != null) + { + final String strippedReferenceContent = referenceContent.replaceAll(WHITESPACE_REGEX, EMPTY_STRING); + final String strippedFileContent = fileContent.replaceAll(WHITESPACE_REGEX, EMPTY_STRING); + if (!strippedReferenceContent.equalsIgnoreCase(strippedFileContent)) + { + success = false; + break; + } + } + else + { + throw new EnforcerRuleException("Unable to read file contents"); + } + } + catch (final IOException ioe) + { + throw new EnforcerRuleException("Cannot process file : " + file.getName(), ioe); + } + } + + if (!success) + { + throw new EnforcerRuleException("Files specified are not equal in content"); + } + } + + @Override + public String getCacheId() + { + return Integer.toString(Arrays.hashCode(files)); + } + + @Override + public boolean isCacheable() + { + return true; + } + + @Override + public boolean isResultValid(EnforcerRule arg0) + { + return true; + } + +} diff --git a/qpid/java/maven/qpid-enforcer-plugin-rules/src/test/java/org/apache/qpid/maven/enforcer/rule/TestRequireFileContentsAreEquivalent.java b/qpid/java/maven/qpid-enforcer-plugin-rules/src/test/java/org/apache/qpid/maven/enforcer/rule/TestRequireFileContentsAreEquivalent.java new file mode 100644 index 0000000000..7a5d214972 --- /dev/null +++ b/qpid/java/maven/qpid-enforcer-plugin-rules/src/test/java/org/apache/qpid/maven/enforcer/rule/TestRequireFileContentsAreEquivalent.java @@ -0,0 +1,149 @@ +/** + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package org.apache.qpid.maven.enforcer.rule; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.apache.maven.enforcer.rule.api.EnforcerRuleException; +import org.apache.maven.plugins.enforcer.EnforcerTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +public class TestRequireFileContentsAreEquivalent +{ + final RequireFileContentsAreEquivalent rule = new RequireFileContentsAreEquivalent(); + + final static String TEST_TEXT = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + final static String TEST_TEXT_WHITESPACE = "Lorem ipsum dolor sit amet,\n consectetur adipiscing \t elit. "; + final static String ALTERNATE_TEST_TEXT_WHITESPACE = " Lorem \t ip sum dolor \n sit amet,\n consectetur adipiscing \t elit."; + final static String DIFFERENT_TEST_TEXT = "Donec velit felis, semper dapibus mattis vitae"; + + @Test + public void testDifferentContentFiles() throws Exception + { + final File f1 = createTestFile(1, TEST_TEXT); + final File f2 = createTestFile(2, DIFFERENT_TEST_TEXT); + + rule.files = new File[] + { f1, f2 }; + + try + { + rule.execute(EnforcerTestUtils.getHelper()); + Assert.fail("Files with different content should have failed enforcer rule"); + } + catch (final EnforcerRuleException ere) + { + // do nothing + } + } + + @Test + public void testIdenticalContentFiles() throws Exception + { + final File f1 = createTestFile(1, TEST_TEXT); + final File f2 = createTestFile(2, TEST_TEXT); + + rule.files = new File[] + { f1, f2 }; + + rule.execute(EnforcerTestUtils.getHelper()); + } + + @Test + public void testUsingOneFileTwice() throws Exception + { + final File f1 = createTestFile(1, TEST_TEXT); + + rule.files = new File[] + { f1, f1 }; + + rule.execute(EnforcerTestUtils.getHelper()); + } + + @Test + public void testSimilarFiles() throws Exception + { + final File f1 = createTestFile(1, TEST_TEXT); + final File f2 = createTestFile(2, TEST_TEXT_WHITESPACE); + + rule.files = new File[] + { f1, f2 }; + + rule.execute(EnforcerTestUtils.getHelper()); + } + + @Test + public void testMultipleFilesOneDifferent() throws Exception + { + final File f1 = createTestFile(1, TEST_TEXT); + final File f2 = createTestFile(2, TEST_TEXT_WHITESPACE); + final File f3 = createTestFile(3, ALTERNATE_TEST_TEXT_WHITESPACE); + final File f4 = createTestFile(4, DIFFERENT_TEST_TEXT); + + rule.files = new File[] + { f1, f2, f3, f4 }; + + try + { + rule.execute(EnforcerTestUtils.getHelper()); + Assert.fail("Files with different content should have failed enforcer rule"); + } + catch (final EnforcerRuleException ere) + { + // do nothing + } + } + + @Test + public void testMultipleFilesAllSimilar() throws Exception + { + final File f1 = createTestFile(1, TEST_TEXT); + final File f2 = createTestFile(2, TEST_TEXT); + final File f3 = createTestFile(3, TEST_TEXT_WHITESPACE); + final File f4 = createTestFile(4, ALTERNATE_TEST_TEXT_WHITESPACE); + + rule.files = new File[] + { f1, f2, f3, f4 }; + + rule.execute(EnforcerTestUtils.getHelper()); + } + + @After + public void deleteTestFiles() throws Exception + { + for (File file : rule.files) + { + if (file.exists()) + { + file.delete(); + } + } + } + + private File createTestFile(final int id, final String content) throws IOException + { + final File file = File.createTempFile(TestRequireFileContentsAreEquivalent.class.getName() + + "-testfile" + id, "tmp"); + file.deleteOnExit(); + FileUtils.writeStringToFile(file, content); + return file; + } +} diff --git a/qpid/java/pom.xml b/qpid/java/pom.xml index f5fe52e5e1..7c99ea6c85 100644 --- a/qpid/java/pom.xml +++ b/qpid/java/pom.xml @@ -54,7 +54,7 @@ <notice.text>Apache Qpid${line.separator}Copyright ${project.inceptionYear}-2014${line.separator}Apache Software Foundation${line.separator}This product includes software developed at Apache Software Foundation (http://www.apache.org/)</notice.text> <!-- enforcer plugin config properties --> - <supported-test-profiles-regex>(java-mms.0-9|java-mms.0-9-1|java-mms.0-10|java-bdb.0-9|java-bdb.0-9-1|java-bdb.0-10|java-dby-mem.0-9|java-dby-mem.0-9-1|java-dby-mem.0-10)</supported-test-profiles-regex> + <supported-test-profiles-regex>(java-mms.0-9|java-mms.0-9-1|java-mms.0-10|java-bdb.0-9|java-bdb.0-9-1|java-bdb.0-10|java-dby.0-9|java-dby.0-9-1|java-dby.0-10|java-dby-mem.0-9|java-dby-mem.0-9-1|java-dby-mem.0-10)</supported-test-profiles-regex> <!-- plugin properties--> <license-maven-plugin-output-dir>${project.build.directory}/generated-licenses</license-maven-plugin-output-dir> @@ -64,12 +64,21 @@ <qpid.home>${basedir}</qpid.home> <!-- override for broker tests --> <qpid.home.qbtc.output>${qpid.home}${file.separator}target${file.separator}qbtc-output</qpid.home.qbtc.output> <!-- override for broker tests --> <qpid.work>${project.build.directory}${file.separator}QPID_WORK</qpid.work> + <profile>java-mms.0-10</profile> - <profile.excludes>JavaTransientExcludes Java010Excludes</profile.excludes> + <profile.broker.language>java</profile.broker.language> + <profile.broker.type>internal</profile.broker.type> + <profile.broker.stopped>Exception</profile.broker.stopped> + <profile.broker.ready>BRK-1004</profile.broker.ready> + <profile.broker.command>${qpid.home}${file.separator}bin${file.separator}qpid-server -sp @STORE_PATH -st @STORE_TYPE -l @LOG_CONFIG_FILE</profile.broker.command> + <profile.broker.command.windows>${qpid.home}${file.separator}bin${file.separator}qpid-server.bat -sp @STORE_PATH -st @STORE_TYPE -l @LOG_CONFIG_FILE</profile.broker.command.windows> + <profile.test.excludes>Excludes JavaExcludes ${profile}.excludes ${profile.specific.excludes}</profile.test.excludes> + <profile.specific.excludes>JavaTransientExcludes Java010Excludes</profile.specific.excludes> <profile.broker.version>v0_10</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>false</profile.broker.persistent> <profile.messagestore.type>Memory</profile.messagestore.type> + <profile.broker.clean.between.tests>true</profile.broker.clean.between.tests> </properties> <modules> @@ -146,7 +155,11 @@ <tasks> <echo>Qpid Test Profile Properties</echo> <echo>[profile] ${profile}</echo> - <echo>[profile.excludes] ${profile.excludes}</echo> + <echo>[profile.broker.language] ${profile.broker.language}</echo> + <echo>[profile.broker.type] ${profile.broker.type}</echo> + <echo>[profile.broker.command] ${profile.broker.command}</echo> + <echo>[profile.specific.excludes] ${profile.specific.excludes}</echo> + <echo>[profile.test.excludes] ${profile.test.excludes}</echo> <echo>[profile.broker.version] ${profile.broker.version}</echo> <echo>[profile.qpid.broker_default_amqp_protocol_excludes] ${profile.qpid.broker_default_amqp_protocol_excludes}</echo> <echo>[profile.broker.persistent] ${profile.broker.persistent}</echo> @@ -216,20 +229,21 @@ <test.exclude>true</test.exclude> <test.mem>512M</test.mem> <profile.clustered>false</profile.clustered> - <broker.language>java</broker.language> - <broker.type>internal</broker.type> - <broker.stopped>Exception</broker.stopped> - <broker.ready>BRK-1004</broker.ready> - <broker.command>${qpid.home}${file.separator}bin${file.separator}qpid-server -sp @STORE_PATH -st @STORE_TYPE -l @LOG_CONFIG_FILE</broker.command> - <broker.command.windows>${qpid.home}${file.separator}bin${file.separator}qpid-server.bat -sp @STORE_PATH -st @STORE_TYPE -l @LOG_CONFIG_FILE</broker.command.windows> <!-- Profile Based Values --> - <test.excludes>Excludes JavaExcludes ${profile}.excludes ${profile.excludes}</test.excludes> + <profile>${profile}</profile> + <broker.language>${profile.broker.language}</broker.language> + <broker.type>${profile.broker.type}</broker.type> + <broker.stopped>${profile.broker.stopped}</broker.stopped> + <broker.ready>${profile.broker.ready}</broker.ready> + <broker.command>${profile.broker.command}</broker.command> + <broker.command.windows>${profile.broker.command.windows}</broker.command.windows> + <test.excludes>${profile.test.excludes}</test.excludes> <broker.version>${profile.broker.version}</broker.version> <qpid.broker_default_amqp_protocol_excludes>${profile.qpid.broker_default_amqp_protocol_excludes}</qpid.broker_default_amqp_protocol_excludes> <broker.persistent>${profile.broker.persistent}</broker.persistent> <messagestore.type>${profile.messagestore.type}</messagestore.type> - <profile>${profile}</profile> + <broker.clean.between.tests>${profile.broker.clean.between.tests}</broker.clean.between.tests> <!-- This must be a child of qpid home currently due to the horrible mechanics of QBTC --> <test.output>${qpid.home.qbtc.output}</test.output> @@ -417,7 +431,7 @@ </activation> <properties> <profile>java-mms.0-10</profile> - <profile.excludes>JavaTransientExcludes Java010Excludes</profile.excludes> + <profile.specific.excludes>JavaTransientExcludes Java010Excludes</profile.specific.excludes> <profile.broker.version>v0_10</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>false</profile.broker.persistent> @@ -435,7 +449,7 @@ </activation> <properties> <profile>java-mms.0-9-1</profile> - <profile.excludes>JavaTransientExcludes XAExcludes JavaPre010Excludes</profile.excludes> + <profile.specific.excludes>JavaTransientExcludes XAExcludes JavaPre010Excludes</profile.specific.excludes> <profile.broker.version>v0_9_1</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0,AMQP_0_10</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>false</profile.broker.persistent> @@ -453,7 +467,7 @@ </activation> <properties> <profile>java-mms.0-9</profile> - <profile.excludes>JavaTransientExcludes XAExcludes JavaPre010Excludes</profile.excludes> + <profile.specific.excludes>JavaTransientExcludes XAExcludes JavaPre010Excludes</profile.specific.excludes> <profile.broker.version>v0_9</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0,AMQP_0_10,AMQP_0_9_1</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>false</profile.broker.persistent> @@ -471,7 +485,7 @@ </activation> <properties> <profile>java-bdb.0-10</profile> - <profile.excludes>JavaPersistentExcludes Java010Excludes JavaBDBExcludes</profile.excludes> + <profile.specific.excludes>JavaPersistentExcludes Java010Excludes JavaBDBExcludes</profile.specific.excludes> <profile.broker.version>v0_10</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>true</profile.broker.persistent> @@ -489,7 +503,7 @@ </activation> <properties> <profile>java-bdb.0-9-1</profile> - <profile.excludes>JavaPersistentExcludes XAExcludes JavaPre010Excludes JavaBDBExcludes</profile.excludes> + <profile.specific.excludes>JavaPersistentExcludes XAExcludes JavaPre010Excludes JavaBDBExcludes</profile.specific.excludes> <profile.broker.version>v0_9_1</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0,AMQP_0_10</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>true</profile.broker.persistent> @@ -507,7 +521,7 @@ </activation> <properties> <profile>java-bdb.0-9</profile> - <profile.excludes>JavaPersistentExcludes XAExcludes JavaPre010Excludes JavaBDBExcludes</profile.excludes> + <profile.specific.excludes>JavaPersistentExcludes XAExcludes JavaPre010Excludes JavaBDBExcludes</profile.specific.excludes> <profile.broker.version>v0_9</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0,AMQP_0_10,AMQP_0_9_1</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>true</profile.broker.persistent> @@ -525,7 +539,7 @@ </activation> <properties> <profile>java-dby-mem.0-10</profile> - <profile.excludes>JavaPersistentExcludes JavaDerbyExcludes Java010Excludes</profile.excludes> + <profile.specific.excludes>JavaPersistentExcludes JavaDerbyExcludes Java010Excludes</profile.specific.excludes> <profile.broker.version>v0_10</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>true</profile.broker.persistent> @@ -543,7 +557,7 @@ </activation> <properties> <profile>java-dby-mem.0-9-1</profile> - <profile.excludes>JavaPersistentExcludes JavaDerbyExcludes XAExcludes JavaPre010Excludes</profile.excludes> + <profile.specific.excludes>JavaPersistentExcludes JavaDerbyExcludes XAExcludes JavaPre010Excludes</profile.specific.excludes> <profile.broker.version>v0_9_1</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0,AMQP_0_10</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>true</profile.broker.persistent> @@ -561,7 +575,7 @@ </activation> <properties> <profile>java-dby-mem.0-9</profile> - <profile.excludes>JavaPersistentExcludes JavaDerbyExcludes XAExcludes JavaPre010Excludes</profile.excludes> + <profile.specific.excludes>JavaPersistentExcludes JavaDerbyExcludes XAExcludes JavaPre010Excludes</profile.specific.excludes> <profile.broker.version>v0_9</profile.broker.version> <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0,AMQP_0_10,AMQP_0_9_1</profile.qpid.broker_default_amqp_protocol_excludes> <profile.broker.persistent>true</profile.broker.persistent> @@ -569,6 +583,63 @@ </properties> </profile> + <profile> + <id>java-dby.0-10</id> + <activation> + <property> + <name>profile</name> + <value>java-dby.0-10</value> + </property> + </activation> + <properties> + <profile>java-dby.0-10</profile> + <profile.specific.excludes>JavaPersistentExcludes JavaDerbyExcludes Java010Excludes</profile.specific.excludes> + <profile.broker.version>v0_10</profile.broker.version> + <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0</profile.qpid.broker_default_amqp_protocol_excludes> + <profile.broker.virtualhosts-config>${QPID_HOME}${file.separator}etc${file.separator}virtualhosts-systests-derby.xml</profile.broker.virtualhosts-config> + <profile.broker.persistent>true</profile.broker.persistent> + <profile.messagestore.class.name>org.apache.qpid.server.store.derby.DerbyMessageStore</profile.messagestore.class.name> + </properties> + </profile> + + <profile> + <id>java-dby.0-9-1</id> + <activation> + <property> + <name>profile</name> + <value>java-dby.0-9-1</value> + </property> + </activation> + <properties> + <profile>java-dby.0-9-1</profile> + <profile.specific.excludes>JavaPersistentExcludes JavaDerbyExcludes XAExcludes JavaPre010Excludes</profile.specific.excludes> + <profile.broker.version>v0_9_1</profile.broker.version> + <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0,AMQP_0_10</profile.qpid.broker_default_amqp_protocol_excludes> + <profile.broker.virtualhosts-config>${QPID_HOME}${file.separator}etc${file.separator}virtualhosts-systests-derby.xml</profile.broker.virtualhosts-config> + <profile.broker.persistent>true</profile.broker.persistent> + <profile.messagestore.class.name>org.apache.qpid.server.store.derby.DerbyMessageStore</profile.messagestore.class.name> + </properties> + </profile> + + <profile> + <id>java-dby.0-9</id> + <activation> + <property> + <name>profile</name> + <value>java-dby.0-9</value> + </property> + </activation> + <properties> + <profile>java-dby.0-9</profile> + <profile.specific.excludes>JavaPersistentExcludes JavaDerbyExcludes XAExcludes JavaPre010Excludes</profile.specific.excludes> + <profile.broker.version>v0_9</profile.broker.version> + <profile.qpid.broker_default_amqp_protocol_excludes>AMQP_1_0,AMQP_0_10,AMQP_0_9_1</profile.qpid.broker_default_amqp_protocol_excludes> + <profile.broker.virtualhosts-config>${QPID_HOME}${file.separator}etc${file.separator}virtualhosts-systests-derby.xml</profile.broker.virtualhosts-config> + <profile.broker.persistent>true</profile.broker.persistent> + <profile.messagestore.class.name>org.apache.qpid.server.store.derby.DerbyMessageStore</profile.messagestore.class.name> + </properties> + </profile> + </profiles> </project> diff --git a/qpid/java/qpid-perftests-systests/pom.xml b/qpid/java/qpid-perftests-systests/pom.xml index 6856fd9d17..4557e903ff 100644 --- a/qpid/java/qpid-perftests-systests/pom.xml +++ b/qpid/java/qpid-perftests-systests/pom.xml @@ -86,6 +86,20 @@ <scope>runtime</scope> </dependency> + <dependency> + <groupId>org.apache.qpid</groupId> + <artifactId>qpid-bdbstore</artifactId> + <version>${project.version}</version> + <scope>test</scope> + <optional>true</optional> + </dependency> + + <dependency> + <groupId>com.sleepycat</groupId> + <artifactId>je</artifactId> + <scope>test</scope> + <optional>true</optional> + </dependency> </dependencies> <build> diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/server/logging/AccessControlLoggingTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/server/logging/AccessControlLoggingTest.java index 37f960a65a..a0188626ee 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/server/logging/AccessControlLoggingTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/server/logging/AccessControlLoggingTest.java @@ -49,8 +49,7 @@ public class AccessControlLoggingTest extends AbstractTestLogging public void setUp() throws Exception { // Write out ACL for this test - AbstractACLTestCase.writeACLFileUtil(this, "test", - "ACL ALLOW client ACCESS VIRTUALHOST", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW client ACCESS VIRTUALHOST", "ACL ALLOW client CREATE QUEUE name='allow'", "ACL ALLOW-LOG client CREATE QUEUE name='allow-log'", "ACL DENY client CREATE QUEUE name='deny'", diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/AbstractACLTestCase.java b/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/AbstractACLTestCase.java index 72626e5089..789ad420d8 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/AbstractACLTestCase.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/AbstractACLTestCase.java @@ -24,9 +24,7 @@ import org.apache.qpid.client.AMQConnection; import org.apache.qpid.client.AMQConnectionURL; import org.apache.qpid.jms.ConnectionListener; import org.apache.qpid.protocol.AMQConstant; -import org.apache.qpid.server.model.VirtualHost; import org.apache.qpid.test.utils.QpidBrokerTestCase; -import org.apache.qpid.test.utils.TestBrokerConfiguration; import org.apache.qpid.url.URLSyntaxException; import javax.jms.Connection; @@ -45,12 +43,10 @@ import java.util.concurrent.TimeUnit; /** * Abstract test case for ACLs. - * + * * This base class contains convenience methods to manage ACL files and implements a mechanism that allows each * test method to run its own setup code before the broker starts. - * - * TODO move the pre broker-startup setup method invocation code to {@link QpidBrokerTestCase} - * + * * @see ExternalACLTest * @see ExternalACLJMXTest * @see ExhaustiveACLTest @@ -80,7 +76,7 @@ public abstract class AbstractACLTestCase extends QpidBrokerTestCase implements { throw (Exception) e.getTargetException(); } - + super.setUp(); } @@ -97,26 +93,18 @@ public abstract class AbstractACLTestCase extends QpidBrokerTestCase implements //that we provoked with authentication failures, where the test passes - we can ignore on con close } } - - public void writeACLFile(final String vhost, final String...rules) throws IOException + + public void writeACLFile(final String...rules) throws IOException { - writeACLFileUtil(this, vhost, rules); + writeACLFileUtil(this, rules); } - public static void writeACLFileUtil(QpidBrokerTestCase testcase, String vhost, String...rules) throws IOException + public static void writeACLFileUtil(QpidBrokerTestCase testcase, String...rules) throws IOException { File aclFile = File.createTempFile(testcase.getClass().getSimpleName(), testcase.getName()); aclFile.deleteOnExit(); - TestBrokerConfiguration config = testcase.getBrokerConfiguration(); - if (vhost == null) - { - config.addAclFileConfiguration(aclFile.getAbsolutePath()); - } - else - { - config.setObjectAttribute(vhost, VirtualHost.SECURITY_ACL, aclFile.getAbsolutePath()); - } + testcase.getBrokerConfiguration().addAclFileConfiguration(aclFile.getAbsolutePath()); PrintWriter out = new PrintWriter(new FileWriter(aclFile)); out.println(String.format("# %s", testcase.getName())); @@ -128,7 +116,7 @@ public abstract class AbstractACLTestCase extends QpidBrokerTestCase implements } /** - * Creates a connection to the broker, and sets a connection listener to prevent failover and an exception listener + * Creates a connection to the broker, and sets a connection listener to prevent failover and an exception listener * with a {@link CountDownLatch} to synchronise in the {@link #check403Exception(Throwable)} method and allow the * {@link #tearDown()} method to complete properly. */ @@ -138,8 +126,8 @@ public abstract class AbstractACLTestCase extends QpidBrokerTestCase implements //Prevent Failover connection.setConnectionListener(this); - - //QPID-2081: use a latch to sync on exception causing connection close, to work + + //QPID-2081: use a latch to sync on exception causing connection close, to work //around the connection close race during tearDown() causing sporadic failures _exceptionReceived = new CountDownLatch(1); @@ -196,8 +184,8 @@ public abstract class AbstractACLTestCase extends QpidBrokerTestCase implements assertNotNull("There was no linked exception", t); assertTrue("Wrong linked exception type : " + t.getClass(), t instanceof AMQException); assertEquals("Incorrect error code received", 403, ((AMQException) t).getErrorCode().getCode()); - - //use the latch to ensure the control thread waits long enough for the exception thread + + //use the latch to ensure the control thread waits long enough for the exception thread //to have done enough to mark the connection closed before teardown commences assertTrue("Timed out waiting for conneciton to report close", _exceptionReceived.await(2, TimeUnit.SECONDS)); } diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExhaustiveACLTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExhaustiveACLTest.java index 50c80692dd..07fe88b38f 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExhaustiveACLTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExhaustiveACLTest.java @@ -39,20 +39,20 @@ public class ExhaustiveACLTest extends AbstractACLTestCase /** * Creates a queue. - * + * * Connects to the broker as a particular user and create the named queue on a virtual host, with the provided * parameters. Uses a new {@link Connection} and {@link Session} and closes them afterwards. */ private void createQueue(String vhost, String user, String name, boolean autoDelete, boolean durable) throws Exception { - Connection conn = getConnection(vhost, user, "guest"); + Connection conn = getConnection(vhost, user, "guest"); Session sess = conn.createSession(true, Session.SESSION_TRANSACTED); conn.start(); ((AMQSession<?, ?>) sess).createQueue(new AMQShortString(name), autoDelete, durable, false); sess.commit(); conn.close(); } - + /** * Calls {@link #createQueue(String, String, String, boolean, boolean)} with the provided parameters and checks that * no exceptions were thrown. @@ -61,7 +61,7 @@ public class ExhaustiveACLTest extends AbstractACLTestCase { try { - createQueue(vhost, user, name, autoDelete, durable); + createQueue(vhost, user, name, autoDelete, durable); } catch (AMQException e) { @@ -72,7 +72,7 @@ public class ExhaustiveACLTest extends AbstractACLTestCase /** * Calls {@link #createQueue(String, String, String, boolean, boolean)} with the provided parameters and checks that - * the exception thrown was an {@link AMQConstant#ACCESS_REFUSED} or 403 error code. + * the exception thrown was an {@link AMQConstant#ACCESS_REFUSED} or 403 error code. */ private void createQueueFailure(String vhost, String user, String name, boolean autoDelete, boolean durable) throws Exception { @@ -87,67 +87,64 @@ public class ExhaustiveACLTest extends AbstractACLTestCase assertEquals("Should be an ACCESS_REFUSED error", 403, e.getErrorCode().getCode()); } } - + public void setUpAuthoriseCreateQueueAutodelete() throws Exception { - writeACLFile("test", - "acl allow client access virtualhost", + writeACLFile("acl allow client access virtualhost", "acl allow server access virtualhost", "acl allow client create queue name=\"temp.true.*\" autodelete=true", "acl allow client create queue name=\"temp.false.*\" autodelete=false", - "acl deny client create queue", - "acl allow client delete queue", + "acl deny client create queue", + "acl allow client delete queue", "acl deny all create queue" ); } - + /** * Test creation of temporary queues, with the autodelete property set to true. */ public void testAuthoriseCreateQueueAutodelete() throws Exception { - createQueueSuccess("test", "client", "temp.true.00", true, false); + createQueueSuccess("test", "client", "temp.true.00", true, false); createQueueSuccess("test", "client", "temp.true.01", true, false); createQueueSuccess("test", "client", "temp.true.02", true, true); - createQueueSuccess("test", "client", "temp.false.03", false, false); + createQueueSuccess("test", "client", "temp.false.03", false, false); createQueueSuccess("test", "client", "temp.false.04", false, false); createQueueSuccess("test", "client", "temp.false.05", false, true); - createQueueFailure("test", "client", "temp.true.06", false, false); + createQueueFailure("test", "client", "temp.true.06", false, false); createQueueFailure("test", "client", "temp.false.07", true, false); - createQueueFailure("test", "server", "temp.true.08", true, false); + createQueueFailure("test", "server", "temp.true.08", true, false); createQueueFailure("test", "client", "temp.other.09", false, false); } - + public void setUpAuthoriseCreateQueue() throws Exception { - writeACLFile("test", - "acl allow client access virtualhost", + writeACLFile("acl allow client access virtualhost", "acl allow server access virtualhost", "acl allow client create queue name=\"create.*\"" ); } - + /** * Tests creation of named queues. * - * If a named queue is specified + * If a named queue is specified */ public void testAuthoriseCreateQueue() throws Exception { createQueueSuccess("test", "client", "create.00", true, true); createQueueSuccess("test", "client", "create.01", true, false); createQueueSuccess("test", "client", "create.02", false, true); - createQueueSuccess("test", "client", "create.03", true, false); + createQueueSuccess("test", "client", "create.03", true, false); createQueueFailure("test", "server", "create.04", true, true); createQueueFailure("test", "server", "create.05", true, false); createQueueFailure("test", "server", "create.06", false, true); - createQueueFailure("test", "server", "create.07", true, false); + createQueueFailure("test", "server", "create.07", true, false); } - + public void setUpAuthoriseCreateQueueBoth() throws Exception { - writeACLFile("test", - "acl allow all access virtualhost", + writeACLFile("acl allow all access virtualhost", "acl allow client create queue name=\"create.*\"", "acl allow all create queue temporary=true" ); @@ -156,16 +153,16 @@ public class ExhaustiveACLTest extends AbstractACLTestCase /** * Tests creation of named queues. * - * If a named queue is specified + * If a named queue is specified */ public void testAuthoriseCreateQueueBoth() throws Exception { createQueueSuccess("test", "client", "create.00", true, false); createQueueSuccess("test", "client", "create.01", false, false); createQueueFailure("test", "server", "create.02", false, false); - createQueueFailure("test", "guest", "create.03", false, false); + createQueueFailure("test", "guest", "create.03", false, false); createQueueSuccess("test", "client", "tmp.00", true, false); - createQueueSuccess("test", "server", "tmp.01", true, false); + createQueueSuccess("test", "server", "tmp.01", true, false); createQueueSuccess("test", "guest", "tmp.02", true, false); } } diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExternalACLJMXTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExternalACLJMXTest.java index 389ac5ff4d..7a1a6cd639 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExternalACLJMXTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExternalACLJMXTest.java @@ -56,7 +56,7 @@ public class ExternalACLJMXTest extends AbstractACLTestCase super.setUp(); _jmx.open(); } - + @Override public void tearDown() throws Exception { @@ -66,9 +66,8 @@ public class ExternalACLJMXTest extends AbstractACLTestCase public void setUpDenyAllIsCatchAllRule() throws Exception { - writeACLFile(null, - "ACL ALLOW admin ACCESS MANAGEMENT", - "#No more rules, default catch all (deny all) should apply"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "#No more rules, default catch all (deny all) should apply"); } public void testDenyAllIsCatchAllRule() throws Exception @@ -107,7 +106,7 @@ public class ExternalACLJMXTest extends AbstractACLTestCase */ public void setUpAllowAll() throws Exception { - writeACLFile(null, "ACL ALLOW ALL ALL"); + writeACLFile("ACL ALLOW ALL ALL"); } public void testAllowAll() throws Exception @@ -118,24 +117,17 @@ public class ExternalACLJMXTest extends AbstractACLTestCase // PASS } - /** - * admin user is denied at broker level but allowed at vhost level. - */ - public void setUpVhostAllowOverridesGlobalDeny() throws Exception + public void setUpVhostWithName() throws Exception { - writeACLFile(null, - "ACL ALLOW admin ACCESS MANAGEMENT", - "ACL DENY admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue'"); - writeACLFile(TEST_VHOST, - "ACL ALLOW admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue'"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL ALLOW admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue' virtualhost_name='"+ TEST_VHOST + "'", + "ACL DENY admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue' virtualhost_name='"+ TEST2_VHOST + "'"); } - public void testVhostAllowOverridesGlobalDeny() throws Exception + public void testVhostWithName() throws Exception { - //try a vhost-level method on the allowed vhost _jmx.createQueue(TEST_VHOST, getTestQueueName(), TEST_QUEUE_OWNER, true); - //try a vhost-level method on a different vhost try { _jmx.createQueue(TEST2_VHOST, getTestQueueName(), TEST_QUEUE_OWNER, true); @@ -153,9 +145,8 @@ public class ExternalACLJMXTest extends AbstractACLTestCase */ public void setUpUpdateComponentOnlyAllow() throws Exception { - writeACLFile(null, - "ACL ALLOW admin ACCESS MANAGEMENT", - "ACL ALLOW admin UPDATE METHOD component='VirtualHost.VirtualHostManager'"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL ALLOW admin UPDATE METHOD component='VirtualHost.VirtualHostManager'"); } public void testUpdateComponentOnlyAllow() throws Exception @@ -172,9 +163,8 @@ public class ExternalACLJMXTest extends AbstractACLTestCase */ public void setUpUpdateMethodOnlyAllow() throws Exception { - writeACLFile(null, - "ACL ALLOW admin ACCESS MANAGEMENT", - "ACL ALLOW admin UPDATE METHOD"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL ALLOW admin UPDATE METHOD"); } public void testUpdateMethodOnlyAllow() throws Exception @@ -187,12 +177,12 @@ public class ExternalACLJMXTest extends AbstractACLTestCase /** - * admin user has JMX right, AMPQ right is irrelevant. + * admin user has JMX right, AMQP right is irrelevant. */ public void setUpCreateQueueSuccess() throws Exception { - writeACLFile(null, "ACL ALLOW admin ACCESS MANAGEMENT"); - writeACLFile(TEST_VHOST, "ACL ALLOW admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue'"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL ALLOW admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue'"); } public void testCreateQueueSuccess() throws Exception @@ -202,14 +192,13 @@ public class ExternalACLJMXTest extends AbstractACLTestCase /** - * admin user has JMX right, verifies lack of AMPQ rights is irrelevant. + * admin user has JMX right, verifies lack of AMQP rights is irrelevant. */ public void setUpCreateQueueSuccessNoAMQPRights() throws Exception { - writeACLFile(null, "ACL ALLOW admin ACCESS MANAGEMENT"); - writeACLFile(TEST_VHOST, - "ACL ALLOW admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue'", - "ACL DENY admin CREATE QUEUE"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL ALLOW admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue'", + "ACL DENY admin CREATE QUEUE"); } public void testCreateQueueSuccessNoAMQPRights() throws Exception @@ -219,13 +208,12 @@ public class ExternalACLJMXTest extends AbstractACLTestCase /** - * admin user does not have JMX right, AMPQ right is irrelevant. + * admin user does not have JMX right, AMQP right is irrelevant. */ public void setUpCreateQueueDenied() throws Exception { - writeACLFile(null, "ACL ALLOW admin ACCESS MANAGEMENT"); - writeACLFile(TEST_VHOST, - "ACL DENY admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue'"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL DENY admin UPDATE METHOD component='VirtualHost.VirtualHostManager' name='createNewQueue'"); } public void testCreateQueueDenied() throws Exception @@ -247,9 +235,8 @@ public class ExternalACLJMXTest extends AbstractACLTestCase */ public void setUpServerInformationUpdateDenied() throws Exception { - writeACLFile(null, - "ACL ALLOW admin ACCESS MANAGEMENT", - "ACL DENY admin UPDATE METHOD component='ServerInformation' name='resetStatistics'"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL DENY admin UPDATE METHOD component='ServerInformation' name='resetStatistics'"); } public void testServerInformationUpdateDenied() throws Exception @@ -272,9 +259,8 @@ public class ExternalACLJMXTest extends AbstractACLTestCase */ public void setUpServerInformationAccessGranted() throws Exception { - writeACLFile(null, - "ACL ALLOW admin ACCESS MANAGEMENT", - "ACL ALLOW-LOG admin ACCESS METHOD component='ServerInformation' name='getManagementApiMajorVersion'"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL ALLOW-LOG admin ACCESS METHOD component='ServerInformation' name='getManagementApiMajorVersion'"); } public void testServerInformationAccessGranted() throws Exception @@ -299,9 +285,8 @@ public class ExternalACLJMXTest extends AbstractACLTestCase */ public void setUpServerInformationUpdateMethodPermission() throws Exception { - writeACLFile(null, - "ACL ALLOW admin ACCESS MANAGEMENT", - "ACL ALLOW admin UPDATE METHOD component='ServerInformation' name='resetStatistics'"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL ALLOW admin UPDATE METHOD component='ServerInformation' name='resetStatistics'"); } public void testServerInformationUpdateMethodPermission() throws Exception @@ -317,9 +302,8 @@ public class ExternalACLJMXTest extends AbstractACLTestCase */ public void setUpServerInformationAllMethodPermissions() throws Exception { - writeACLFile(null, - "ACL ALLOW admin ACCESS MANAGEMENT", - "ACL ALLOW admin ALL METHOD component='ServerInformation'"); + writeACLFile("ACL ALLOW admin ACCESS MANAGEMENT", + "ACL ALLOW admin ALL METHOD component='ServerInformation'"); } public void testServerInformationAllMethodPermissions() throws Exception diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExternalACLTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExternalACLTest.java index 00c85e80c8..0e8f3cb7d8 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExternalACLTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/server/security/acl/ExternalACLTest.java @@ -18,12 +18,6 @@ */ package org.apache.qpid.server.security.acl; -import org.apache.qpid.AMQException; -import org.apache.qpid.client.AMQDestination; -import org.apache.qpid.client.AMQSession; -import org.apache.qpid.protocol.AMQConstant; -import org.apache.qpid.url.URLSyntaxException; - import javax.jms.Connection; import javax.jms.Destination; import javax.jms.JMSException; @@ -37,8 +31,14 @@ import javax.jms.Topic; import javax.jms.TopicSubscriber; import javax.naming.NamingException; +import org.apache.qpid.AMQException; +import org.apache.qpid.client.AMQDestination; +import org.apache.qpid.client.AMQSession; +import org.apache.qpid.protocol.AMQConstant; +import org.apache.qpid.url.URLSyntaxException; + /** - * Tests the V2 ACLs. The tests perform basic AMQP operations like creating queues or excahnges and publishing and consuming messages, using + * Tests the V2 ACLs. The tests perform basic AMQP operations like creating queues or exchanges and publishing and consuming messages, using * JMS to contact the broker. */ public class ExternalACLTest extends AbstractACLTestCase @@ -46,65 +46,113 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpAccessAuthorizedSuccess() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST"); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST"); } public void testAccessAuthorizedSuccess() throws Exception { + Connection conn = getConnection("test", "client", "guest"); + conn.close(); + } + + public void setUpAccessNoRightsFailure() throws Exception + { + writeACLFile("ACL DENY-LOG client ACCESS VIRTUALHOST"); + } + + public void testAccessNoRightsFailure() throws Exception + { try { - Connection conn = getConnection("test", "client", "guest"); - Session sess = conn.createSession(true, Session.SESSION_TRANSACTED); - conn.start(); + getConnection("test", "client", "guest"); + fail("Connection was created."); + } + catch (JMSException e) + { + assertAccessDeniedException(e); + } + } - //Do something to show connection is active. - sess.rollback(); + private void assertAccessDeniedException(JMSException e) + { + assertEquals("Unexpected exception message", "Error creating connection: Permission denied: test", e.getMessage()); + + // JMSException -> linkedException -> cause = AMQException (403 or 320) + Exception linkedException = e.getLinkedException(); + assertNotNull("There was no linked exception", linkedException); + Throwable cause = linkedException.getCause(); + assertNotNull("Cause was null", cause); + assertTrue("Wrong linked exception type", cause instanceof AMQException); + AMQConstant errorCode = isBroker010() ? AMQConstant.CONNECTION_FORCED : AMQConstant.ACCESS_REFUSED; + assertEquals("Incorrect error code received", errorCode, ((AMQException) cause).getErrorCode()); + } + + public void setUpAccessVirtualHostWithName() throws Exception + { + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST name='test'", "ACL DENY-LOG guest ACCESS VIRTUALHOST name='test'", + "ACL ALLOW-LOG server ACCESS VIRTUALHOST name='*'"); + } - conn.close(); + public void testAccessVirtualHostWithName() throws Exception + { + Connection conn = getConnection("test", "client", "guest"); + conn.close(); + + try + { + getConnection("test", "guest", "guest"); + fail("Access should be denied"); } - catch (Exception e) + catch (JMSException e) { - fail("Connection was not created due to:" + e); + assertAccessDeniedException(e); } + + Connection conn2 = getConnection("test", "server", "guest"); + conn2.close(); } - public void setUpAccessNoRightsFailure() throws Exception + public void setUpClientCreateVirtualHostQueue() throws Exception { - writeACLFile("test", "ACL DENY-LOG client ACCESS VIRTUALHOST"); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL ALLOW-LOG client CREATE QUEUE virtualhost_name='test'", + "ACL ALLOW-LOG client CONSUME QUEUE", + "ACL ALLOW-LOG client BIND EXCHANGE", + "ACL ALLOW-LOG guest ACCESS VIRTUALHOST", + "ACL DENY-LOG guest CREATE QUEUE virtualhost_name='test'"); } - public void testAccessNoRightsFailure() throws Exception + public void testClientCreateVirtualHostQueue() throws NamingException, JMSException, AMQException, Exception { + Connection conn = getConnection("test", "client", "guest"); + Session sess = conn.createSession(false, Session.AUTO_ACKNOWLEDGE); + Destination dest = sess.createQueue(getTestQueueName()); + sess.createConsumer(dest); + conn.close(); + try { - Connection conn = getConnection("test", "guest", "guest"); - Session sess = conn.createSession(true, Session.SESSION_TRANSACTED); - conn.start(); - sess.rollback(); + conn = getConnection("test", "guest", "guest"); + sess = conn.createSession(false, Session.AUTO_ACKNOWLEDGE); + sess.createConsumer(dest); - fail("Connection was created."); + fail("Queue creation for user 'guest' is denied"); } catch (JMSException e) { - // JMSException -> linkedException -> cause = AMQException (403 or 320) - Exception linkedException = e.getLinkedException(); - assertNotNull("There was no linked exception", linkedException); - Throwable cause = linkedException.getCause(); - assertNotNull("Cause was null", cause); - assertTrue("Wrong linked exception type", cause instanceof AMQException); - AMQConstant errorCode = isBroker010() ? AMQConstant.CONNECTION_FORCED : AMQConstant.ACCESS_REFUSED; - assertEquals("Incorrect error code received", errorCode, ((AMQException) cause).getErrorCode()); + check403Exception(e.getLinkedException()); } } + public void setUpClientDeleteQueueSuccess() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST", - "ACL ALLOW-LOG client CREATE QUEUE durable=\"true\"" , - "ACL ALLOW-LOG client CONSUME QUEUE name=\"clientid:kipper\"", - "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.topic\" durable=true routingKey=kipper", - "ACL ALLOW-LOG client DELETE QUEUE durable=\"true\"", - "ACL ALLOW-LOG client UNBIND EXCHANGE name=\"amq.topic\" durable=true routingKey=kipper"); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL ALLOW-LOG client CREATE QUEUE durable=\"true\"", + "ACL ALLOW-LOG client CONSUME QUEUE name=\"clientid:kipper\"" , + "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.topic\" durable=true routingKey=kipper", + "ACL ALLOW-LOG client DELETE QUEUE durable=\"true\"", + "ACL ALLOW-LOG client UNBIND EXCHANGE name=\"amq.topic\" durable=true routingKey=kipper"); } public void testClientDeleteQueueSuccess() throws Exception @@ -128,12 +176,12 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpClientDeleteQueueFailure() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST", - "ACL ALLOW-LOG client CREATE QUEUE durable=\"true\"" , - "ACL ALLOW-LOG client CONSUME QUEUE name=\"clientid:kipper\"", - "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.topic\" durable=true routingKey=kipper", - "ACL DENY-LOG client DELETE QUEUE durable=\"true\"", - "ACL DENY-LOG client UNBIND EXCHANGE name=\"amq.topic\" durable=true routingKey=kipper"); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL ALLOW-LOG client CREATE QUEUE durable=\"true\"", + "ACL ALLOW-LOG client CONSUME QUEUE name=\"clientid:kipper\"" , + "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.topic\" durable=true routingKey=kipper", + "ACL DENY-LOG client DELETE QUEUE durable=\"true\"", + "ACL DENY-LOG client UNBIND EXCHANGE name=\"amq.topic\" durable=true routingKey=kipper"); } public void testClientDeleteQueueFailure() throws Exception @@ -177,10 +225,10 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpClientConsumeFromNamedQueueValid() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST", - "ACL ALLOW-LOG client CREATE QUEUE name=\"example.RequestQueue\"" , - "ACL ALLOW-LOG client CONSUME QUEUE name=\"example.RequestQueue\"", - "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.direct\" routingKey=\"example.RequestQueue\""); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL ALLOW-LOG client CREATE QUEUE name=\"example.RequestQueue\"", + "ACL ALLOW-LOG client CONSUME QUEUE name=\"example.RequestQueue\"" , + "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.direct\" routingKey=\"example.RequestQueue\""); } @@ -197,10 +245,10 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpClientConsumeFromNamedQueueFailure() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST", - "ACL ALLOW-LOG client CREATE QUEUE" , - "ACL ALLOW-LOG client BIND EXCHANGE", - "ACL DENY-LOG client CONSUME QUEUE name=\"IllegalQueue\""); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL ALLOW-LOG client CREATE QUEUE", + "ACL ALLOW-LOG client BIND EXCHANGE" , + "ACL DENY-LOG client CONSUME QUEUE name=\"IllegalQueue\""); } public void testClientConsumeFromNamedQueueFailure() throws NamingException, Exception @@ -224,11 +272,11 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpClientCreateTemporaryQueueSuccess() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST", - "ACL ALLOW-LOG client CREATE QUEUE temporary=\"true\"" , - "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.direct\" temporary=true", - "ACL ALLOW-LOG client DELETE QUEUE temporary=\"true\"", - "ACL ALLOW-LOG client UNBIND EXCHANGE name=\"amq.direct\" temporary=true"); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL ALLOW-LOG client CREATE QUEUE temporary=\"true\"", + "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.direct\" temporary=true" , + "ACL ALLOW-LOG client DELETE QUEUE temporary=\"true\"", + "ACL ALLOW-LOG client UNBIND EXCHANGE name=\"amq.direct\" temporary=true"); } public void testClientCreateTemporaryQueueSuccess() throws JMSException, URLSyntaxException, Exception @@ -243,8 +291,8 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpClientCreateTemporaryQueueFailed() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST", - "ACL DENY-LOG client CREATE QUEUE temporary=\"true\""); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL DENY-LOG client CREATE QUEUE temporary=\"true\""); } public void testClientCreateTemporaryQueueFailed() throws NamingException, Exception @@ -268,9 +316,8 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpClientCreateNamedQueueFailure() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST", - "ACL ALLOW-LOG client CREATE QUEUE name=\"ValidQueue\"", - "ACL ALLOW-LOG client CONSUME QUEUE"); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL ALLOW-LOG client CREATE QUEUE name=\"ValidQueue\""); } public void testClientCreateNamedQueueFailure() throws NamingException, JMSException, AMQException, Exception @@ -294,10 +341,10 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpClientPublishUsingTransactionSuccess() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST", - "ACL ALLOW-LOG client CREATE QUEUE" , - "ACL ALLOW-LOG client BIND EXCHANGE", - "ACL ALLOW-LOG client PUBLISH EXCHANGE name=\"amq.direct\" routingKey=\"example.RequestQueue\""); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL ALLOW-LOG client CREATE QUEUE", + "ACL ALLOW-LOG client BIND EXCHANGE" , + "ACL ALLOW-LOG client PUBLISH EXCHANGE name=\"amq.direct\" routingKey=\"example.RequestQueue\""); } public void testClientPublishUsingTransactionSuccess() throws Exception @@ -329,19 +376,18 @@ public class ExternalACLTest extends AbstractACLTestCase // We tolerate a dependency from this test to that file because its // contents are expected to change rarely. - writeACLFile("test", "ACL ALLOW-LOG messaging-users ACCESS VIRTUALHOST", - "# Server side", - "ACL ALLOW-LOG server CREATE QUEUE name=\"example.RequestQueue\"" , - "ACL ALLOW-LOG server BIND EXCHANGE", - "ACL ALLOW-LOG server PUBLISH EXCHANGE name=\"amq.direct\" routingKey=\"TempQueue*\"", - "ACL ALLOW-LOG server CONSUME QUEUE name=\"example.RequestQueue\"", - "# Client side", - "ACL ALLOW-LOG client PUBLISH EXCHANGE name=\"amq.direct\" routingKey=\"example.RequestQueue\"", - "ACL ALLOW-LOG client CONSUME QUEUE temporary=true", - "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.direct\" temporary=true", - "ACL ALLOW-LOG client UNBIND EXCHANGE name=\"amq.direct\" temporary=true", - "ACL ALLOW-LOG client CREATE QUEUE temporary=true", - "ACL ALLOW-LOG client DELETE QUEUE temporary=true"); + writeACLFile("ACL ALLOW-LOG messaging-users ACCESS VIRTUALHOST", "# Server side", + "ACL ALLOW-LOG server CREATE QUEUE name=\"example.RequestQueue\"", + "ACL ALLOW-LOG server BIND EXCHANGE" , + "ACL ALLOW-LOG server PUBLISH EXCHANGE name=\"amq.direct\" routingKey=\"TempQueue*\"", + "ACL ALLOW-LOG server CONSUME QUEUE name=\"example.RequestQueue\"", + "# Client side", + "ACL ALLOW-LOG client PUBLISH EXCHANGE name=\"amq.direct\" routingKey=\"example.RequestQueue\"", + "ACL ALLOW-LOG client CONSUME QUEUE temporary=true", + "ACL ALLOW-LOG client BIND EXCHANGE name=\"amq.direct\" temporary=true", + "ACL ALLOW-LOG client UNBIND EXCHANGE name=\"amq.direct\" temporary=true", + "ACL ALLOW-LOG client CREATE QUEUE temporary=true", + "ACL ALLOW-LOG client DELETE QUEUE temporary=true"); } @@ -386,9 +432,9 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpClientDeleteQueueSuccessWithOnlyAllPermissions() throws Exception { - writeACLFile("test", "ACL ALLOW-LOG client ACCESS VIRTUALHOST", - "ACL ALLOW-LOG client ALL QUEUE", - "ACL ALLOW-LOG client ALL EXCHANGE"); + writeACLFile("ACL ALLOW-LOG client ACCESS VIRTUALHOST", + "ACL ALLOW-LOG client ALL QUEUE", + "ACL ALLOW-LOG client ALL EXCHANGE"); } public void testClientDeleteQueueSuccessWithOnlyAllPermissions() throws Exception @@ -412,7 +458,7 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpFirewallAllow() throws Exception { - writeACLFile("test", "ACL ALLOW client ACCESS VIRTUALHOST from_network=\"127.0.0.1\""); + writeACLFile("ACL ALLOW client ACCESS VIRTUALHOST from_network=\"127.0.0.1\""); } public void testFirewallAllow() throws Exception @@ -423,7 +469,7 @@ public class ExternalACLTest extends AbstractACLTestCase public void setUpFirewallDeny() throws Exception { - writeACLFile("test", "ACL DENY client ACCESS VIRTUALHOST from_network=\"127.0.0.1\""); + writeACLFile("ACL DENY client ACCESS VIRTUALHOST from_network=\"127.0.0.1\""); } public void testFirewallDeny() throws Exception diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/server/store/MessageStoreTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/server/store/MessageStoreTest.java index 36e86fbe7b..d89f5cc66e 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/server/store/MessageStoreTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/server/store/MessageStoreTest.java @@ -185,6 +185,12 @@ public class MessageStoreTest extends QpidTestCase try { _virtualHost = BrokerTestHelper.createVirtualHost(new VirtualHostRegistry(new EventLogger()), getVirtualHostModel()); + when(_virtualHostModel.getId()).thenReturn(_virtualHost.getId()); + + ConfiguredObjectRecord objectRecord = mock(ConfiguredObjectRecord.class); + when(objectRecord.getId()).thenReturn(_virtualHost.getId()); + when(objectRecord.getType()).thenReturn(org.apache.qpid.server.model.VirtualHost.class.getSimpleName()); + when(_virtualHostModel.asObjectRecord()).thenReturn(objectRecord); } catch (Exception e) { diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/server/store/QuotaMessageStore.java b/qpid/java/systests/src/main/java/org/apache/qpid/server/store/QuotaMessageStore.java index 06b7091a47..cae6e29b96 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/server/store/QuotaMessageStore.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/server/store/QuotaMessageStore.java @@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.qpid.server.message.EnqueueableMessage; import org.apache.qpid.server.message.MessageContentSource; +import org.apache.qpid.server.model.ConfiguredObject; public class QuotaMessageStore extends NullMessageStore @@ -81,7 +82,7 @@ public class } @Override - public void recoverMessageStore(MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) + public void recoverMessageStore(ConfiguredObject<?> parent, MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) { _stateManager.attainState(State.ACTIVATING); _stateManager.attainState(State.ACTIVE); diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/server/store/SlowMessageStore.java b/qpid/java/systests/src/main/java/org/apache/qpid/server/store/SlowMessageStore.java index 44a5252355..8ec576c7ca 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/server/store/SlowMessageStore.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/server/store/SlowMessageStore.java @@ -29,7 +29,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.log4j.Logger; import org.apache.qpid.server.message.EnqueueableMessage; -import org.apache.qpid.server.plugin.DurableConfigurationStoreFactory; +import org.apache.qpid.server.model.ConfiguredObject; import org.apache.qpid.server.plugin.MessageStoreFactory; public class SlowMessageStore implements MessageStore, DurableConfigurationStore @@ -64,9 +64,9 @@ public class SlowMessageStore implements MessageStore, DurableConfigurationStore } @Override - public void recoverConfigurationStore(ConfigurationRecoveryHandler recoveryHandler) + public void recoverConfigurationStore(ConfiguredObject<?> parent, ConfigurationRecoveryHandler recoveryHandler) { - _realDurableConfigurationStore.recoverConfigurationStore(recoveryHandler); + _realDurableConfigurationStore.recoverConfigurationStore(parent, recoveryHandler); } private void configureDelays(Map<String, Object> delays) @@ -183,7 +183,6 @@ public class SlowMessageStore implements MessageStore, DurableConfigurationStore _realDurableConfigurationStore.closeConfigurationStore(); } - @Override public <M extends StorableMessageMetaData> StoredMessage<M> addMessage(M metaData) { @@ -191,39 +190,24 @@ public class SlowMessageStore implements MessageStore, DurableConfigurationStore } @Override - public void create(UUID id, String type, Map<String, Object> attributes) throws StoreException + public void create(ConfiguredObjectRecord record) throws StoreException { doPreDelay("create"); - _realDurableConfigurationStore.create(id, type, attributes); + _realDurableConfigurationStore.create(record); doPostDelay("create"); } - @Override - public void remove(UUID id, String type) throws StoreException - { - doPreDelay("remove"); - _realDurableConfigurationStore.remove(id, type); - doPostDelay("remove"); - } @Override - public UUID[] removeConfiguredObjects(final UUID... objects) throws StoreException + public UUID[] remove(final ConfiguredObjectRecord... objects) throws StoreException { doPreDelay("remove"); - UUID[] removed = _realDurableConfigurationStore.removeConfiguredObjects(objects); + UUID[] removed = _realDurableConfigurationStore.remove(objects); doPostDelay("remove"); return removed; } @Override - public void update(UUID id, String type, Map<String, Object> attributes) throws StoreException - { - doPreDelay("update"); - _realDurableConfigurationStore.update(id, type, attributes); - doPostDelay("update"); - } - - @Override public void update(boolean createIfNecessary, ConfiguredObjectRecord... records) throws StoreException { doPreDelay("update"); @@ -310,9 +294,9 @@ public class SlowMessageStore implements MessageStore, DurableConfigurationStore } @Override - public void recoverMessageStore(MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) + public void recoverMessageStore(ConfiguredObject<?> parent, MessageStoreRecoveryHandler messageRecoveryHandler, TransactionLogRecoveryHandler transactionLogRecoveryHandler) { - _realMessageStore.recoverMessageStore(messageRecoveryHandler, transactionLogRecoveryHandler); + _realMessageStore.recoverMessageStore(parent, messageRecoveryHandler, transactionLogRecoveryHandler); } @Override diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/BrokerACLTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/BrokerACLTest.java index ec389e55f1..da9466e9a9 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/BrokerACLTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/BrokerACLTest.java @@ -63,8 +63,7 @@ public class BrokerACLTest extends QpidRestTestCase super.customizeConfiguration(); getRestTestHelper().configureTemporaryPasswordFile(this, ALLOWED_USER, DENIED_USER); - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_USER + " CONFIGURE BROKER", "ACL DENY-LOG " + DENIED_USER + " CONFIGURE BROKER", "ACL DENY-LOG ALL ALL"); diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/ExchangeRestACLTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/ExchangeRestACLTest.java index b63df34b98..f20d9dfa47 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/ExchangeRestACLTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/ExchangeRestACLTest.java @@ -47,8 +47,7 @@ public class ExchangeRestACLTest extends QpidRestTestCase super.customizeConfiguration(); getRestTestHelper().configureTemporaryPasswordFile(this, ALLOWED_USER, DENIED_USER); - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_USER + " CREATE QUEUE", "ACL ALLOW-LOG " + ALLOWED_USER + " CREATE EXCHANGE", "ACL DENY-LOG " + DENIED_USER + " CREATE EXCHANGE", diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/GroupRestACLTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/GroupRestACLTest.java index aff1eac9cf..e6d5f8b1e0 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/GroupRestACLTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/GroupRestACLTest.java @@ -94,8 +94,7 @@ public class GroupRestACLTest extends QpidRestTestCase public void testCreateGroup() throws Exception { - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_GROUP + " CREATE GROUP", "ACL DENY-LOG " + DENIED_GROUP + " CREATE GROUP"); @@ -121,8 +120,7 @@ public class GroupRestACLTest extends QpidRestTestCase public void testDeleteGroup() throws Exception { - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_GROUP + " DELETE GROUP", "ACL DENY-LOG " + DENIED_GROUP + " DELETE GROUP"); @@ -148,8 +146,7 @@ public class GroupRestACLTest extends QpidRestTestCase public void testUpdateGroupAddMember() throws Exception { - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_GROUP + " UPDATE GROUP", "ACL DENY-LOG " + DENIED_GROUP + " UPDATE GROUP"); @@ -169,8 +166,7 @@ public class GroupRestACLTest extends QpidRestTestCase public void testUpdateGroupDeleteMember() throws Exception { - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_GROUP + " UPDATE GROUP", "ACL DENY-LOG " + DENIED_GROUP + " UPDATE GROUP"); diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/LogViewerACLTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/LogViewerACLTest.java index 7d94ee27ad..dd6c7d6a28 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/LogViewerACLTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/LogViewerACLTest.java @@ -38,8 +38,7 @@ public class LogViewerACLTest extends QpidRestTestCase super.customizeConfiguration(); getRestTestHelper().configureTemporaryPasswordFile(this, ALLOWED_USER, DENIED_USER); - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_USER + " ACCESS_LOGS BROKER", "ACL DENY-LOG " + DENIED_USER + " ACCESS_LOGS BROKER", "ACL DENY-LOG ALL ALL"); diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/QueueRestACLTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/QueueRestACLTest.java index 52f70a5dd2..b712327329 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/QueueRestACLTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/QueueRestACLTest.java @@ -42,8 +42,7 @@ public class QueueRestACLTest extends QpidRestTestCase super.customizeConfiguration(); getRestTestHelper().configureTemporaryPasswordFile(this, ALLOWED_USER, DENIED_USER); - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_USER + " CREATE QUEUE", "ACL DENY-LOG " + DENIED_USER + " CREATE QUEUE", "ACL ALLOW-LOG " + ALLOWED_USER + " UPDATE QUEUE", diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/UserPreferencesRestACLTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/UserPreferencesRestACLTest.java index b23e44a4d3..4b4854e4b3 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/UserPreferencesRestACLTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/UserPreferencesRestACLTest.java @@ -76,7 +76,7 @@ public class UserPreferencesRestACLTest extends QpidRestTestCase super.customizeConfiguration(); getRestTestHelper().configureTemporaryPasswordFile(this, ALLOWED_USER, DENIED_USER); - AbstractACLTestCase.writeACLFileUtil(this, null, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_USER + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_USER + " UPDATE USER", "ACL DENY-LOG " + DENIED_USER + " UPDATE USER", "ACL DENY-LOG ALL ALL"); TestBrokerConfiguration brokerConfiguration = getBrokerConfiguration(); diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/UserRestACLTest.java b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/UserRestACLTest.java index d2f0401db5..7c5042c750 100644 --- a/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/UserRestACLTest.java +++ b/qpid/java/systests/src/main/java/org/apache/qpid/systest/rest/acl/UserRestACLTest.java @@ -96,8 +96,7 @@ public class UserRestACLTest extends QpidRestTestCase public void testAddUser() throws Exception { - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_GROUP + " CREATE USER", "ACL DENY-LOG " + DENIED_GROUP + " CREATE USER"); @@ -121,8 +120,7 @@ public class UserRestACLTest extends QpidRestTestCase public void testDeleteUser() throws Exception { - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_GROUP + " DELETE USER", "ACL DENY-LOG " + DENIED_GROUP + " DELETE USER"); @@ -142,8 +140,7 @@ public class UserRestACLTest extends QpidRestTestCase public void testUpdateUser() throws Exception { - AbstractACLTestCase.writeACLFileUtil(this, null, - "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", + AbstractACLTestCase.writeACLFileUtil(this, "ACL ALLOW-LOG ALL ACCESS MANAGEMENT", "ACL ALLOW-LOG " + ALLOWED_GROUP + " UPDATE USER", "ACL DENY-LOG " + DENIED_GROUP + " UPDATE USER"); diff --git a/qpid/packaging/windows/INSTALL_NOTES.html b/qpid/packaging/windows/INSTALL_NOTES.html index 862e43c714..8c142b3c2a 100644 --- a/qpid/packaging/windows/INSTALL_NOTES.html +++ b/qpid/packaging/windows/INSTALL_NOTES.html @@ -1,11 +1,11 @@ <html>
<head>
-<title>Apache Qpid C++ 0.27 Installation Notes</title>
+<title>Apache Qpid C++ 0.29 Installation Notes</title>
</head>
<body>
-<H1>Apache Qpid C++ 0.27 Installation Notes</H1>
+<H1>Apache Qpid C++ 0.29 Installation Notes</H1>
-<p>Thank you for installing Apache Qpid version 0.27 for Windows.
+<p>Thank you for installing Apache Qpid version 0.29 for Windows.
If the requisite features were installed, you can now run a broker,
use the example programs, and design your own messaging programs while
reading the Qpid C++ API reference documentation.</p>
@@ -83,7 +83,7 @@ default; therefore, to gain support for durable items the persistence plugin must be loaded into the broker. This can be done using the
<code>--load-module</code> option to load the needed plugins. For example:
<pre>
-cd "C:\Program Files\Apache\qpidc-0.27"
+cd "C:\Program Files\Apache\qpidc-0.29"
qpidd.exe --load-module plugins\broker\store.dll --load-module plugins\broker\msclfs_store.dll
</pre>
The <code>--load-module</code> option can also take a full path. The option
diff --git a/qpid/packaging/windows/installer.proj b/qpid/packaging/windows/installer.proj index 4c24f17bbc..84eb652b0b 100644 --- a/qpid/packaging/windows/installer.proj +++ b/qpid/packaging/windows/installer.proj @@ -32,7 +32,7 @@ <source_root>$(MSBuildProjectDirectory)\..\..</source_root>
<staging_dir>$(MSBuildProjectDirectory)\stage</staging_dir>
<bits Condition="'$(bits)' == ''">32</bits>
- <qpid_version>0.27</qpid_version>
+ <qpid_version>0.29</qpid_version>
<OutputName>qpidc</OutputName>
<OutputType>Package</OutputType>
<WixToolPath>C:\Program Files (x86)\Windows Installer XML v3.5\bin</WixToolPath>
diff --git a/qpid/python/setup.py b/qpid/python/setup.py index 67b6d25472..c56ff6afba 100755 --- a/qpid/python/setup.py +++ b/qpid/python/setup.py @@ -298,7 +298,7 @@ class install_lib(_install_lib): return outfiles + extra setup(name="qpid-python", - version="0.27", + version="0.29", author="Apache Qpid", author_email="dev@qpid.apache.org", packages=["mllib", "qpid", "qpid.messaging", "qpid.tests", diff --git a/qpid/tests/setup.py b/qpid/tests/setup.py index 605a529d59..101ec322a2 100755 --- a/qpid/tests/setup.py +++ b/qpid/tests/setup.py @@ -20,7 +20,7 @@ from distutils.core import setup setup(name="qpid-tests", - version="0.27", + version="0.29", author="Apache Qpid", author_email="dev@qpid.apache.org", packages=["qpid_tests", "qpid_tests.broker_0_10", "qpid_tests.broker_0_9", diff --git a/qpid/tools/setup.py b/qpid/tools/setup.py index 068af322d0..cb40192d0b 100755 --- a/qpid/tools/setup.py +++ b/qpid/tools/setup.py @@ -19,8 +19,22 @@ # from distutils.core import setup +pypi_long_description = """ + +=========== +qpidtoollibs +=========== + +qpidtoollibs provides a useful BrokerAgent object for managing a Qpid broker +using Qpid Management Framework (QMF). + +This library depends on the qpid.messaging python client to send AMQP messaging +containing QMF commands to the Qpid broker. + +""" + setup(name="qpid-tools", - version="0.27", + version="0.29", author="Apache Qpid", author_email="dev@qpid.apache.org", package_dir={'' : 'src/py'}, @@ -35,4 +49,8 @@ setup(name="qpid-tools", "src/py/qmf-tool"], url="http://qpid.apache.org/", license="Apache Software License", - description="Diagnostic and management tools for Apache Qpid brokers.") + description="Diagnostic and management tools for Apache Qpid brokers.", + long_description=pypi_long_description, + install_requires=[ + "qpid-python >= 0.26", + ]) diff --git a/qpid/tools/src/py/qls/anal.py b/qpid/tools/src/py/qls/anal.py new file mode 100644 index 0000000000..865dfa16c7 --- /dev/null +++ b/qpid/tools/src/py/qls/anal.py @@ -0,0 +1,593 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +""" +Module: qls.anal + +Classes for recovery and analysis of a Qpid Linear Store (QLS). +""" + +import os.path +import qls.err +import qls.jrnl +import qls.utils + +class HighCounter(object): + def __init__(self): + self.num = 0 + def check(self, num): + if self.num < num: + self.num = num + def get(self): + return self.num + def get_next(self): + self.num += 1 + return self.num + +class JournalRecoveryManager(object): + TPL_DIR_NAME = 'tpl' + JRNL_DIR_NAME = 'jrnl' + def __init__(self, directory, args): + if not os.path.exists(directory): + raise qls.err.InvalidQlsDirectoryNameError(directory) + self.directory = directory + self.args = args + self.tpl = None + self.journals = {} + self.high_rid_counter = HighCounter() + self.prepared_list = None + def report(self, print_stats_flag): + self._reconcile_transactions(self.prepared_list, self.args.txn) + if self.tpl is not None: + self.tpl.report(print_stats_flag) + for queue_name in sorted(self.journals.keys()): + self.journals[queue_name].report(print_stats_flag) + def run(self): + tpl_dir = os.path.join(self.directory, JournalRecoveryManager.TPL_DIR_NAME) + if os.path.exists(tpl_dir): + self.tpl = Journal(tpl_dir, None, self.args) + self.tpl.recover(self.high_rid_counter) + if self.args.show_recs or self.args.show_all_recs: + print + jrnl_dir = os.path.join(self.directory, JournalRecoveryManager.JRNL_DIR_NAME) + self.prepared_list = self.tpl.txn_map.get_prepared_list() if self.tpl is not None else {} + if os.path.exists(jrnl_dir): + for dir_entry in sorted(os.listdir(jrnl_dir)): + jrnl = Journal(os.path.join(jrnl_dir, dir_entry), self.prepared_list, self.args) + jrnl.recover(self.high_rid_counter) + self.journals[jrnl.get_queue_name()] = jrnl + if self.args.show_recs or self.args.show_all_recs: + print + def _reconcile_transactions(self, prepared_list, txn_flag): + print 'Transaction reconciliation report:' + print '==================================' + print len(prepared_list), 'open transaction(s) found in prepared transaction list:' + for xid in prepared_list.keys(): + commit_flag = prepared_list[xid] + if commit_flag is None: + status = '[Prepared, neither committed nor aborted - assuming commit]' + elif commit_flag: + status = '[Prepared, but interrupted during commit phase]' + else: + status = '[Prepared, but interrupted during abort phase]' + print ' ', qls.utils.format_xid(xid), status + if prepared_list[xid] is None: # Prepared, but not committed or aborted + enqueue_record = self.tpl.get_txn_map_record(xid)[0][1] + dequeue_record = qls.utils.create_record(qls.jrnl.DequeueRecord.MAGIC, \ + qls.jrnl.DequeueRecord.TXN_COMPLETE_COMMIT_FLAG, \ + self.tpl.current_journal_file, \ + self.high_rid_counter.get_next(), \ + enqueue_record.record_id, xid, None) + if txn_flag: + self.tpl.add_record(dequeue_record) + for queue_name in sorted(self.journals.keys()): + self.journals[queue_name].reconcile_transactions(prepared_list, txn_flag) + if len(prepared_list) > 0: + print 'Completing prepared transactions in prepared transaction list:' + for xid in prepared_list.keys(): + print ' ', qls.utils.format_xid(xid) + transaction_record = qls.utils.create_record(qls.jrnl.TransactionRecord.MAGIC_COMMIT, 0, \ + self.tpl.current_journal_file, \ + self.high_rid_counter.get_next(), None, xid, None) + if txn_flag: + self.tpl.add_record(transaction_record) + print + +class EnqueueMap(object): + """ + Map of enqueued records in a QLS journal + """ + def __init__(self, journal): + self.journal = journal + self.enq_map = {} + def add(self, journal_file, enq_record, locked_flag): + if enq_record.record_id in self.enq_map: + raise qls.err.DuplicateRecordIdError(self.journal.current_file_header, enq_record) + self.enq_map[enq_record.record_id] = [journal_file, enq_record, locked_flag] + def contains(self, rid): + """Return True if the map contains the given rid""" + return rid in self.enq_map + def delete(self, journal_file, deq_record): + if deq_record.dequeue_record_id in self.enq_map: + enq_list = self.enq_map[deq_record.dequeue_record_id] + del self.enq_map[deq_record.dequeue_record_id] + return enq_list + else: + raise qls.err.RecordIdNotFoundError(journal_file.file_header, deq_record) + def get(self, record_id): + if record_id in self.enq_map: + return self.enq_map[record_id] + return None + def lock(self, journal_file, dequeue_record): + if dequeue_record.dequeue_record_id not in self.enq_map: + raise qls.err.RecordIdNotFoundError(journal_file.file_header, dequeue_record) + self.enq_map[dequeue_record.dequeue_record_id][2] = True + def report_str(self, _, show_records): + """Return a string containing a text report for all records in the map""" + if len(self.enq_map) == 0: + return 'No enqueued records found.' + rstr = '%d enqueued records found' % len(self.enq_map) + if show_records: + rstr += ":" + rid_list = self.enq_map.keys() + rid_list.sort() + for rid in rid_list: + journal_file, record, locked_flag = self.enq_map[rid] + if locked_flag: + lock_str = '[LOCKED]' + else: + lock_str = '' + rstr += '\n 0x%x:%s %s' % (journal_file.file_header.file_num, record, lock_str) + else: + rstr += '.' + return rstr + def unlock(self, journal_file, dequeue_record): + """Set the transaction lock for a given record_id to False""" + if dequeue_record.dequeue_record_id in self.enq_map: + if self.enq_map[dequeue_record.dequeue_record_id][2]: + self.enq_map[dequeue_record.dequeue_record_id][2] = False + else: + raise qls.err.RecordNotLockedError(journal_file.file_header, dequeue_record) + else: + raise qls.err.RecordIdNotFoundError(journal_file.file_header, dequeue_record) + +class TransactionMap(object): + """ + Map of open transactions used while recovering a QLS journal + """ + def __init__(self, enq_map): + self.txn_map = {} + self.enq_map = enq_map + def abort(self, xid): + """Perform an abort operation for the given xid record""" + for journal_file, record, _ in self.txn_map[xid]: + if isinstance(record, qls.jrnl.DequeueRecord): + if self.enq_map.contains(record.dequeue_record_id): + self.enq_map.unlock(journal_file, record) + else: + journal_file.decr_enq_cnt(record) + del self.txn_map[xid] + def add(self, journal_file, record): + if record.xid is None: + raise qls.err.NonTransactionalRecordError(journal_file.file_header, record, 'TransactionMap.add()') + if isinstance(record, qls.jrnl.DequeueRecord): + try: + self.enq_map.lock(journal_file, record) + except qls.err.RecordIdNotFoundError: + # Not in emap, look for rid in tmap - should not happen in practice + txn_op = self._find_record_id(record.xid, record.dequeue_record_id) + if txn_op != None: + if txn_op[2]: + raise qls.err.AlreadyLockedError(journal_file.file_header, record) + txn_op[2] = True + if record.xid in self.txn_map: + self.txn_map[record.xid].append([journal_file, record, False]) # append to existing list + else: + self.txn_map[record.xid] = [[journal_file, record, False]] # create new list + def commit(self, xid): + """Perform a commit operation for the given xid record""" + mismatch_list = [] + for journal_file, record, lock in self.txn_map[xid]: + if isinstance(record, qls.jrnl.EnqueueRecord): + self.enq_map.add(journal_file, record, lock) # Transfer enq to emap + else: + if self.enq_map.contains(record.dequeue_record_id): + self.enq_map.unlock(journal_file, record) + self.enq_map.delete(journal_file, record)[0].decr_enq_cnt(record) + else: + mismatch_list.append('0x%x' % record.dequeue_record_id) + del self.txn_map[xid] + return mismatch_list + def contains(self, xid): + """Return True if the xid exists in the map; False otherwise""" + return xid in self.txn_map + def delete(self, journal_file, transaction_record): + """Remove a transaction record from the map using either a commit or abort header""" + if transaction_record.magic[-1] == 'c': + return self.commit(transaction_record.xid) + if transaction_record.magic[-1] == 'a': + self.abort(transaction_record.xid) + else: + raise qls.err.InvalidRecordTypeError(journal_file.file_header, transaction_record, + 'delete from Transaction Map') + def get(self, xid): + if xid in self.txn_map: + return self.txn_map[xid] + return None + def get_prepared_list(self): + """ + Prepared list is a map of xid(key) to one of None, True or False. These represent respectively: + None: prepared, but neither committed or aborted (interrupted before commit or abort) + False: prepared and aborted (interrupted before abort complete) + True: prepared and committed (interrupted before commit complete) + """ + prepared_list = {} + for xid in self.get_xid_list(): + for _, record, _ in self.txn_map[xid]: + if isinstance(record, qls.jrnl.EnqueueRecord): + prepared_list[xid] = None + else: + prepared_list[xid] = record.is_transaction_complete_commit() + return prepared_list + def get_xid_list(self): + return self.txn_map.keys() + def report_str(self, _, show_records): + """Return a string containing a text report for all records in the map""" + if len(self.txn_map) == 0: + return 'No outstanding transactions found.' + rstr = '%d outstanding transaction(s)' % len(self.txn_map) + if show_records: + rstr += ':' + for xid, op_list in self.txn_map.iteritems(): + rstr += '\n %s containing %d operations:' % (qls.utils.format_xid(xid), len(op_list)) + for journal_file, record, _ in op_list: + rstr += '\n 0x%x:%s' % (journal_file.file_header.file_num, record) + else: + rstr += '.' + return rstr + def _find_record_id(self, xid, record_id): + """ Search for and return map list with supplied rid.""" + if xid in self.txn_map: + for txn_op in self.txn_map[xid]: + if txn_op[1].record_id == record_id: + return txn_op + for this_xid in self.txn_map.iterkeys(): + for txn_op in self.txn_map[this_xid]: + if txn_op[1].record_id == record_id: + return txn_op + return None + +class JournalStatistics(object): + """Journal statistics""" + def __init__(self): + self.total_record_count = 0 + self.transient_record_count = 0 + self.filler_record_count = 0 + self.enqueue_count = 0 + self.dequeue_count = 0 + self.transaction_record_count = 0 + self.transaction_enqueue_count = 0 + self.transaction_dequeue_count = 0 + self.transaction_commit_count = 0 + self.transaction_abort_count = 0 + self.transaction_operation_count = 0 + def __str__(self): + fstr = 'Total record count: %d\n' + \ + 'Transient record count: %d\n' + \ + 'Filler_record_count: %d\n' + \ + 'Enqueue_count: %d\n' + \ + 'Dequeue_count: %d\n' + \ + 'Transaction_record_count: %d\n' + \ + 'Transaction_enqueue_count: %d\n' + \ + 'Transaction_dequeue_count: %d\n' + \ + 'Transaction_commit_count: %d\n' + \ + 'Transaction_abort_count: %d\n' + \ + 'Transaction_operation_count: %d\n' + return fstr % (self.total_record_count, + self.transient_record_count, + self.filler_record_count, + self.enqueue_count, + self.dequeue_count, + self.transaction_record_count, + self.transaction_enqueue_count, + self.transaction_dequeue_count, + self.transaction_commit_count, + self.transaction_abort_count, + self.transaction_operation_count) + +class Journal(object): + """ + Instance of a Qpid Linear Store (QLS) journal. + """ + def __init__(self, directory, xid_prepared_list, args): + self.directory = directory + self.queue_name = os.path.basename(directory) + self.files = {} + self.file_num_list = None + self.file_num_itr = None + self.enq_map = EnqueueMap(self) + self.txn_map = TransactionMap(self.enq_map) + self.current_journal_file = None + self.first_rec_flag = None + self.statistics = JournalStatistics() + self.xid_prepared_list = xid_prepared_list # This is None for the TPL instance only + self.args = args + self.last_record_offset = None # TODO: Move into JournalFile + self.num_filler_records_required = None # TODO: Move into JournalFile + self.fill_to_offset = None + def add_record(self, record): + if isinstance(record, qls.jrnl.EnqueueRecord) or isinstance(record, qls.jrnl.DequeueRecord): + if record.xid_size > 0: + self.txn_map.add(self.current_journal_file, record) + else: + self.enq_map.add(self.current_journal_file, record, False) + elif isinstance(record, qls.jrnl.TransactionRecord): + self.txn_map.delete(self.current_journal_file, record) + else: + raise qls.err.InvalidRecordTypeError(self.current_journal_file, record, 'add to Journal') + def get_enq_map_record(self, rid): + return self.enq_map.get(rid) + def get_txn_map_record(self, xid): + return self.txn_map.get(xid) + def get_outstanding_txn_list(self): + return self.txn_map.get_xid_list() + def get_queue_name(self): + return self.queue_name + def recover(self, high_rid_counter): + print 'Recovering', self.queue_name + self._analyze_files() + try: + while self._get_next_record(high_rid_counter): + pass + self._check_alignment() + except qls.err.NoMoreFilesInJournalError: + print 'No more files in journal' + except qls.err.FirstRecordOffsetMismatchError as err: + print '0x%08x: **** FRO ERROR: queue=\"%s\" fid=0x%x fro actual=0x%08x expected=0x%08x' % \ + (err.get_expected_fro(), err.get_queue_name(), err.get_file_number(), err.get_record_offset(), + err.get_expected_fro()) + def reconcile_transactions(self, prepared_list, txn_flag): + xid_list = self.txn_map.get_xid_list() + if len(xid_list) > 0: + print self.queue_name, 'contains', len(xid_list), 'open transaction(s):' + for xid in xid_list: + if xid in prepared_list.keys(): + commit_flag = prepared_list[xid] + if commit_flag is None: + print ' ', qls.utils.format_xid(xid), '- Assuming commit after prepare' + if txn_flag: + self.txn_map.commit(xid) + elif commit_flag: + print ' ', qls.utils.format_xid(xid), '- Completing interrupted commit operation' + if txn_flag: + self.txn_map.commit(xid) + else: + print ' ', qls.utils.format_xid(xid), '- Completing interrupted abort operation' + if txn_flag: + self.txn_map.abort(xid) + else: + print ' ', qls.utils.format_xid(xid), '- Ignoring, not in prepared transaction list' + if txn_flag: + self.txn_map.abort(xid) + def report(self, print_stats_flag): + print 'Journal "%s":' % self.queue_name + print '=' * (11 + len(self.queue_name)) + if print_stats_flag: + print str(self.statistics) + print self.enq_map.report_str(True, True) + print self.txn_map.report_str(True, True) + JournalFile.report_header() + for file_num in sorted(self.files.keys()): + self.files[file_num].report() + #TODO: move this to JournalFile, append to file info + if self.num_filler_records_required is not None and self.fill_to_offset is not None: + print '0x%x:0x%08x: %d filler records required for DBLK alignment to 0x%08x' % \ + (self.current_journal_file.file_header.file_num, self.last_record_offset, + self.num_filler_records_required, self.fill_to_offset) + print + #--- protected functions --- + def _analyze_files(self): + for dir_entry in os.listdir(self.directory): + dir_entry_bits = dir_entry.split('.') + if len(dir_entry_bits) == 2 and dir_entry_bits[1] == JournalRecoveryManager.JRNL_DIR_NAME: + fq_file_name = os.path.join(self.directory, dir_entry) + file_handle = open(fq_file_name) + args = qls.utils.load_args(file_handle, qls.jrnl.RecordHeader) + file_hdr = qls.jrnl.FileHeader(*args) + file_hdr.init(file_handle, *qls.utils.load_args(file_handle, qls.jrnl.FileHeader)) + if file_hdr.is_header_valid(file_hdr): + file_hdr.load(file_handle) + if file_hdr.is_valid(): + qls.utils.skip(file_handle, file_hdr.file_header_size_sblks * qls.utils.DEFAULT_SBLK_SIZE) + self.files[file_hdr.file_num] = JournalFile(file_hdr) + self.file_num_list = sorted(self.files.keys()) + self.file_num_itr = iter(self.file_num_list) + def _check_alignment(self): # TODO: Move into JournalFile + remaining_sblks = self.last_record_offset % qls.utils.DEFAULT_SBLK_SIZE + if remaining_sblks == 0: + self.num_filler_records_required = 0 + else: + self.num_filler_records_required = (qls.utils.DEFAULT_SBLK_SIZE - remaining_sblks) / \ + qls.utils.DEFAULT_DBLK_SIZE + self.fill_to_offset = self.last_record_offset + \ + (self.num_filler_records_required * qls.utils.DEFAULT_DBLK_SIZE) + if self.args.show_recs or self.args.show_all_recs: + print '0x%x:0x%08x: %d filler records required for DBLK alignment to 0x%08x' % \ + (self.current_journal_file.file_header.file_num, self.last_record_offset, + self.num_filler_records_required, self.fill_to_offset) + def _check_file(self): + if self.current_journal_file is not None: + if not self.current_journal_file.file_header.is_end_of_file(): + return True + if self.current_journal_file.file_header.is_end_of_file(): + self.last_record_offset = self.current_journal_file.file_header.file_handle.tell() + if not self._get_next_file(): + return False + fhdr = self.current_journal_file.file_header + fhdr.file_handle.seek(fhdr.first_record_offset) + return True + def _get_next_file(self): + if self.current_journal_file is not None: + file_handle = self.current_journal_file.file_header.file_handle + if not file_handle.closed: # sanity check, should not be necessary + file_handle.close() + file_num = 0 + try: + while file_num == 0: + file_num = self.file_num_itr.next() + except StopIteration: + pass + if file_num == 0: + return False + self.current_journal_file = self.files[file_num] + self.first_rec_flag = True + if self.args.show_recs or self.args.show_all_recs: + file_header = self.current_journal_file.file_header + print '0x%x:%s' % (file_header.file_num, file_header.to_string()) + return True + def _get_next_record(self, high_rid_counter): + if not self._check_file(): + return False + self.last_record_offset = self.current_journal_file.file_header.file_handle.tell() + this_record = qls.utils.load(self.current_journal_file.file_header.file_handle, qls.jrnl.RecordHeader) + if not this_record.is_header_valid(self.current_journal_file.file_header): + return False + if self.first_rec_flag: + if this_record.file_offset != self.current_journal_file.file_header.first_record_offset: + raise qls.err.FirstRecordOffsetMismatchError(self.current_journal_file.file_header, this_record) + self.first_rec_flag = False + self.statistics.total_record_count += 1 + start_journal_file = self.current_journal_file + if isinstance(this_record, qls.jrnl.EnqueueRecord): + ok_flag = self._handle_enqueue_record(this_record, start_journal_file) + high_rid_counter.check(this_record.record_id) + if self.args.show_recs or self.args.show_all_recs: + print '0x%x:%s' % (start_journal_file.file_header.file_num, \ + this_record.to_string(self.args.show_xids, self.args.show_data)) + elif isinstance(this_record, qls.jrnl.DequeueRecord): + ok_flag = self._handle_dequeue_record(this_record, start_journal_file) + high_rid_counter.check(this_record.record_id) + if self.args.show_recs or self.args.show_all_recs: + print '0x%x:%s' % (start_journal_file.file_header.file_num, this_record.to_string(self.args.show_xids)) + elif isinstance(this_record, qls.jrnl.TransactionRecord): + ok_flag = self._handle_transaction_record(this_record, start_journal_file) + high_rid_counter.check(this_record.record_id) + if self.args.show_recs or self.args.show_all_recs: + print '0x%x:%s' % (start_journal_file.file_header.file_num, this_record.to_string(self.args.show_xids)) + else: + self.statistics.filler_record_count += 1 + ok_flag = True + if self.args.show_all_recs: + print '0x%x:%s' % (start_journal_file.file_header.file_num, this_record) + qls.utils.skip(self.current_journal_file.file_header.file_handle, qls.utils.DEFAULT_DBLK_SIZE) + return ok_flag + def _handle_enqueue_record(self, enqueue_record, start_journal_file): + while enqueue_record.load(self.current_journal_file.file_header.file_handle): + if not self._get_next_file(): + enqueue_record.truncated_flag = True + return False + if not enqueue_record.is_valid(start_journal_file): + return False + if enqueue_record.is_external() and enqueue_record.data != None: + raise qls.err.ExternalDataError(self.current_journal_file.file_header, enqueue_record) + if enqueue_record.is_transient(): + self.statistics.transient_record_count += 1 + return True + if enqueue_record.xid_size > 0: + self.txn_map.add(start_journal_file, enqueue_record) + self.statistics.transaction_operation_count += 1 + self.statistics.transaction_record_count += 1 + self.statistics.transaction_enqueue_count += 1 + else: + self.enq_map.add(start_journal_file, enqueue_record, False) + start_journal_file.incr_enq_cnt() + self.statistics.enqueue_count += 1 + return True + def _handle_dequeue_record(self, dequeue_record, start_journal_file): + while dequeue_record.load(self.current_journal_file.file_header.file_handle): + if not self._get_next_file(): + dequeue_record.truncated_flag = True + return False + if not dequeue_record.is_valid(start_journal_file): + return False + if dequeue_record.xid_size > 0: + if self.xid_prepared_list is None: # ie this is the TPL + dequeue_record.transaction_prepared_list_flag = True + elif not self.enq_map.contains(dequeue_record.dequeue_record_id): + dequeue_record.warnings.append('NOT IN EMAP') # Only for non-TPL records + self.txn_map.add(start_journal_file, dequeue_record) + self.statistics.transaction_operation_count += 1 + self.statistics.transaction_record_count += 1 + self.statistics.transaction_dequeue_count += 1 + else: + try: + self.enq_map.delete(start_journal_file, dequeue_record)[0].decr_enq_cnt(dequeue_record) + except qls.err.RecordIdNotFoundError: + dequeue_record.warnings.append('NOT IN EMAP') + self.statistics.dequeue_count += 1 + return True + def _handle_transaction_record(self, transaction_record, start_journal_file): + while transaction_record.load(self.current_journal_file.file_header.file_handle): + if not self._get_next_file(): + transaction_record.truncated_flag = True + return False + if not transaction_record.is_valid(start_journal_file): + return False + if transaction_record.magic[-1] == 'a': + self.statistics.transaction_abort_count += 1 + else: + self.statistics.transaction_commit_count += 1 + if self.txn_map.contains(transaction_record.xid): + self.txn_map.delete(self.current_journal_file, transaction_record) + else: + transaction_record.warnings.append('NOT IN TMAP') +# if transaction_record.magic[-1] == 'c': # commits only +# self._txn_obj_list[hdr.xid] = hdr + self.statistics.transaction_record_count += 1 + return True + def _load_data(self, record): + while not record.is_complete: + record.load(self.current_journal_file.file_handle) + +class JournalFile(object): + def __init__(self, file_header): + self.file_header = file_header + self.enq_cnt = 0 + self.deq_cnt = 0 + self.num_filler_records_required = None + def incr_enq_cnt(self): + self.enq_cnt += 1 + def decr_enq_cnt(self, record): + if self.enq_cnt <= self.deq_cnt: + raise qls.err.EnqueueCountUnderflowError(self.file_header, record) + self.deq_cnt += 1 + def get_enq_cnt(self): + return self.enq_cnt - self.deq_cnt + def is_outstanding_enq(self): + return self.enq_cnt > self.deq_cnt + @staticmethod + def report_header(): + print 'file_num enq_cnt p_no efp journal_file' + print '-------- ------- ---- ----- ------------' + def report(self): + comment = '<uninitialized>' if self.file_header.file_num == 0 else '' + file_num_str = '0x%x' % self.file_header.file_num + print '%8s %7d %4d %4dk %s %s' % (file_num_str, self.get_enq_cnt(), self.file_header.partition_num, + self.file_header.efp_data_size_kb, + os.path.basename(self.file_header.file_handle.name), comment) diff --git a/qpid/tools/src/py/qls/efp.py b/qpid/tools/src/py/qls/efp.py index abf289dc12..93b77eea93 100644 --- a/qpid/tools/src/py/qls/efp.py +++ b/qpid/tools/src/py/qls/efp.py @@ -17,38 +17,137 @@ # under the License. # +""" +Module: qls.efp + +Contains empty file pool (EFP) classes. +""" + import os import os.path import qls.err +import shutil +import uuid class EfpManager(object): """ Top level class to analyze the Qpid Linear Store (QLS) directory for the partitions that make up the Empty File Pool (EFP). """ - def __init__(self, directory, args): + def __init__(self, directory, disk_space_required_kb): if not os.path.exists(directory): raise qls.err.InvalidQlsDirectoryNameError(directory) self.directory = directory - self.args = args - self.partitions = [] + self.disk_space_required_kb = disk_space_required_kb + self.efp_partitions = [] + self.efp_pools = {} + self.total_num_files = 0 + self.total_cum_file_size_kb = 0 + self.current_efp_partition = None + def add_file_pool(self, file_size_kb, num_files): + """ Add an EFP in the specified partition of the specified size containing the specified number of files """ + dir_name = EmptyFilePool.get_directory_name(file_size_kb) + print 'Adding pool \'%s\' to partition %s' % (dir_name, self.current_efp_partition.partition_number) + self.total_cum_file_size_kb += self.current_efp_partition.create_new_efp(file_size_kb, num_files) + self.total_num_files += num_files + def freshen_file_pool(self, file_size_kb, num_files): + """ Freshen an EFP in the specified partition and of the specified size to the specified number of files """ + if self.current_efp_partition is None: + partition_list = self.efp_partitions + partition_str = 'all partitions' + else: + partition_list = [self.current_efp_partition] + partition_str = 'partition %d' % self.current_efp_partition.partition_number + if file_size_kb is None: + pool_str = 'all pools' + else: + pool_str = 'pool \'%s\'' % EmptyFilePool.get_directory_name(int(file_size_kb)) + print 'Freshening %s in %s to %d files' % (pool_str, partition_str, num_files) + for self.current_efp_partition in partition_list: # Partition objects + if file_size_kb is None: + file_size_list = self.current_efp_partition.efp_pools.keys() + else: + file_size_list = ['%sk' % file_size_kb] + for file_size in file_size_list: + efp = self.current_efp_partition.efp_pools[file_size] + num_files_needed = num_files - efp.get_tot_file_count() + if num_files_needed > 0: + self.current_efp_partition.create_new_efp_files(qls.utils.efp_directory_size(file_size), + num_files_needed) + else: + print ' WARNING: Pool %s in partition %s already contains %d files: no action taken' % \ + (self.current_efp_partition.efp_pools[file_size].size_str, + self.current_efp_partition.partition_number, efp.get_num_files()) + def remove_file_pool(self, file_size_kb): + """ Remove an existing EFP from the specified partition and of the specified size """ + dir_name = EmptyFilePool.get_directory_name(file_size_kb) + print 'Removing pool \'%s\' from partition %s' % (dir_name, self.current_efp_partition.partition_number) + self.efp_partitions.remove(self.current_efp_partition) + shutil.rmtree(os.path.join(self.current_efp_partition.efp_directory, dir_name)) def report(self): - print 'Found', len(self.partitions), 'partition(s).' - if (len(self.partitions)) > 0: + print 'Empty File Pool (EFP) report:' + print '=============================' + print 'Found', len(self.efp_partitions), 'partition(s).' + if (len(self.efp_partitions)) > 0: EfpPartition.print_report_table_header() - for ptn in self.partitions: + for ptn in self.efp_partitions: ptn.print_report_table_line() print - for ptn in self.partitions: + for ptn in self.efp_partitions: ptn.report() - def run(self, _): + def run(self, arg_tup): + self._analyze_efp() + if arg_tup is not None: + _, arg_file_size, arg_num_files, arg_add, arg_remove, arg_freshen, arg_list = arg_tup + self._check_args(arg_tup) + if arg_add: + self.add_file_pool(int(arg_file_size), int(arg_num_files)) + if arg_remove: + self.remove_file_pool(int(arg_file_size)) + if arg_freshen: + self.freshen_file_pool(arg_file_size, int(arg_num_files)) + if arg_list: + self.report() + def _analyze_efp(self): for dir_entry in os.listdir(self.directory): try: - efpp = EfpPartition(os.path.join(self.directory, dir_entry)) - efpp.scan() - self.partitions.append(efpp) + efp_partition = EfpPartition(os.path.join(self.directory, dir_entry), self.disk_space_required_kb) + efp_partition.scan() + self.efp_partitions.append(efp_partition) + for efpl in efp_partition.efp_pools.iterkeys(): + if efpl not in self.efp_pools: + self.efp_pools[efpl] = [] + self.efp_pools[efpl].append(efp_partition.efp_pools[efpl]) + self.total_num_files += efp_partition.tot_file_count + self.total_cum_file_size_kb += efp_partition.tot_file_size_kb except qls.err.InvalidPartitionDirectoryNameError: pass + def _check_args(self, arg_tup): + """ Value check of args. The names of partitions and pools are validated against the discovered instances """ + arg_partition, arg_file_size, _, arg_add, arg_remove, arg_freshen, _ = arg_tup + if arg_partition is not None: + try: + if arg_partition[0] == 'p': # string partition name, eg 'p001' + partition_num = int(arg_partition[1:]) + else: # numeric partition, eg '1' + partition_num = int(arg_partition) + found = False + for partition in self.efp_partitions: + if partition.partition_number == partition_num: + self.current_efp_partition = partition + found = True + break + if not found: + raise qls.err.PartitionDoesNotExistError(arg_partition) + except ValueError: + raise qls.err.InvalidPartitionDirectoryNameError(arg_partition) + if self.current_efp_partition is not None: + pool_list = self.current_efp_partition.efp_pools.keys() + efp_directory_name = EmptyFilePool.get_directory_name(int(arg_file_size)) + if arg_add and efp_directory_name in pool_list: + raise qls.err.PoolDirectoryAlreadyExistsError(efp_directory_name) + if (arg_remove or arg_freshen) and efp_directory_name not in pool_list: + raise qls.err.PoolDirectoryDoesNotExistError(efp_directory_name) class EfpPartition(object): """ @@ -56,55 +155,59 @@ class EfpPartition(object): """ PTN_DIR_PREFIX = 'p' EFP_DIR_NAME = 'efp' - def __init__(self, directory): - self.base_dir = os.path.basename(directory) - if self.base_dir[0] is not EfpPartition.PTN_DIR_PREFIX: - raise qls.err.InvalidPartitionDirectoryNameError(directory) - try: - self.partition_number = int(self.base_dir[1:]) - except ValueError: - raise qls.err.InvalidPartitionDirectoryNameError(directory) + def __init__(self, directory, disk_space_required_kb): self.directory = directory - self.pools = [] - self.efp_count = 0 + self.partition_number = None + self.efp_pools = {} self.tot_file_count = 0 self.tot_file_size_kb = 0 - def get_directory(self): - return self.directory - def get_efp_count(self): - return self.efp_count - def get_name(self): - return self.base_dir - def get_number(self): - return self.partition_number - def get_number_pools(self): - return len(self.pools) - def get_tot_file_count(self): - return self.tot_file_count - def get_tot_file_size_kb(self): - return self.tot_file_size_kb + self._validate_partition_directory(disk_space_required_kb) + def create_new_efp_files(self, file_size_kb, num_files): + """ Create new EFP files in this partition """ + dir_name = EmptyFilePool.get_directory_name(file_size_kb) + if dir_name in self.efp_pools.keys(): + efp = self.efp_pools[dir_name] + else: + efp = EmptyFilePool(os.path.join(self.directory, EfpPartition.EFP_DIR_NAME), dir_name) + this_tot_file_size_kb = efp.create_new_efp_files(num_files) + self.tot_file_size_kb += this_tot_file_size_kb + self.tot_file_count += num_files + return this_tot_file_size_kb @staticmethod def print_report_table_header(): print 'p_no no_efp tot_files tot_size_kb directory' print '---- ------ --------- ----------- ---------' def print_report_table_line(self): - print '%4d %6d %9d %11d %s' % (self.get_number(), self.get_efp_count(), self.get_tot_file_count(), - self.get_tot_file_size_kb(), self.get_directory()) + print '%4d %6d %9d %11d %s' % (self.partition_number, len(self.efp_pools), self.tot_file_count, + self.tot_file_size_kb, self.directory) def report(self): - print 'Partition %s:' % self.base_dir + print 'Partition %s:' % os.path.basename(self.directory) EmptyFilePool.print_report_table_header() - for pool in self.pools: - pool.print_report_table_line() + for dir_name in self.efp_pools.keys(): + self.efp_pools[dir_name].print_report_table_line() print def scan(self): if os.path.exists(self.directory): efp_dir = os.path.join(self.directory, EfpPartition.EFP_DIR_NAME) for dir_entry in os.listdir(efp_dir): - efp = EmptyFilePool(os.path.join(efp_dir, dir_entry)) - self.efp_count += 1 + efp = EmptyFilePool(os.path.join(efp_dir, dir_entry), self.partition_number) + efp.scan() self.tot_file_count += efp.get_tot_file_count() self.tot_file_size_kb += efp.get_tot_file_size_kb() - self.pools.append(efp) + self.efp_pools[dir_entry] = efp + def _validate_partition_directory(self, disk_space_required_kb): + if os.path.basename(self.directory)[0] is not EfpPartition.PTN_DIR_PREFIX: + raise qls.err.InvalidPartitionDirectoryNameError(self.directory) + try: + self.partition_number = int(os.path.basename(self.directory)[1:]) + except ValueError: + raise qls.err.InvalidPartitionDirectoryNameError(self.directory) + if not qls.utils.has_write_permission(self.directory): + raise qls.err.WritePermissionError(self.directory) + if disk_space_required_kb is not None: + space_avail = qls.utils.get_avail_disk_space(self.directory) + if space_avail < (disk_space_required_kb * 1024): + raise qls.err.InsufficientSpaceOnDiskError(self.directory, space_avail, disk_space_required_kb * 1024) class EmptyFilePool(object): """ @@ -112,33 +215,89 @@ class EmptyFilePool(object): journal files (but it may also be empty). """ EFP_DIR_SUFFIX = 'k' - def __init__(self, directory): - self.base_dir = os.path.basename(directory) - if self.base_dir[-1] is not EmptyFilePool.EFP_DIR_SUFFIX: - raise qls.err.InvalidEfpDirectoryNameError(directory) - try: - self.data_size_kb = int(os.path.basename(self.base_dir)[:-1]) - except ValueError: - raise qls.err.InvalidEfpDirectoryNameError(directory) + EFP_JRNL_EXTENTION = '.jrnl' + def __init__(self, directory, partition_number): + self.base_dir_name = os.path.basename(directory) self.directory = directory - self.files = os.listdir(directory) + self.partition_number = partition_number + self.data_size_kb = None + self.files = [] + self.tot_file_size_kb = 0 + self._validate_efp_directory() + def create_new_efp_files(self, num_files): + """ Create one or more new empty journal files of the prescribed size for this EFP """ + this_total_file_size = 0 + for _ in range(num_files): + this_total_file_size += self._create_new_efp_file() + return this_total_file_size def get_directory(self): return self.directory - def get_file_data_size_kb(self): - return self.data_size_kb - def get_name(self): - return self.base_dir + @staticmethod + def get_directory_name(file_size_kb): + """ Static function to create an EFP directory name from the size of the files it contains """ + return '%dk' % file_size_kb def get_tot_file_count(self): return len(self.files) def get_tot_file_size_kb(self): return self.data_size_kb * len(self.files) @staticmethod def print_report_table_header(): - print 'file_size_kb file_count tot_file_size_kb efp_directory' + print 'data_size_kb file_count tot_file_size_kb efp_directory' print '------------ ---------- ---------------- -------------' def print_report_table_line(self): - print '%12d %10d %16d %s' % (self.get_file_data_size_kb(), self.get_tot_file_count(), + print '%12d %10d %16d %s' % (self.data_size_kb, self.get_tot_file_count(), self.get_tot_file_size_kb(), self.get_directory()) + def scan(self): + for efp_file in os.listdir(self.directory): + if self._validate_efp_file(os.path.join(self.directory, efp_file)): + self.files.append(efp_file) + def _add_efp_file(self, efp_file_name): + """ Add a single journal file of the appropriate size to this EFP. No file size check is made here. """ + self.files.append(efp_file_name) + self.tot_file_size_kb += os.path.getsize(efp_file_name) + def _create_new_efp_file(self): + """ Create a single new empty journal file of the prescribed size for this EFP """ + file_name = str(uuid.uuid4()) + EmptyFilePool.EFP_JRNL_EXTENTION + file_header = qls.jrnl.FileHeader(0, qls.jrnl.FileHeader.MAGIC, qls.utils.DEFAULT_RECORD_VERSION, 0, 0, 0) + file_header.init(None, None, qls.utils.DEFAULT_HEADER_SIZE_SBLKS, self.partition_number, self.data_size_kb, + 0, 0, 0, 0, 0) + efh = file_header.encode() + efh_bytes = len(efh) + file_handle = open(os.path.join(self.directory, file_name), 'wb') + file_handle.write(efh) + file_handle.write('\xff' * (qls.utils.DEFAULT_SBLK_SIZE - efh_bytes)) + file_handle.write('\x00' * (int(self.data_size_kb) * 1024)) + file_handle.close() + fqfn = os.path.join(self.directory, file_name) + self._add_efp_file(fqfn) + return os.path.getsize(fqfn) + def _validate_efp_directory(self): + if self.base_dir_name[-1] is not EmptyFilePool.EFP_DIR_SUFFIX: + raise qls.err.InvalidEfpDirectoryNameError(self.directory) + try: + self.data_size_kb = int(os.path.basename(self.base_dir_name)[:-1]) + except ValueError: + raise qls.err.InvalidEfpDirectoryNameError(self.directory) + def _validate_efp_file(self, efp_file): + file_size = os.path.getsize(efp_file) + expected_file_size = (self.data_size_kb * 1024) + qls.utils.DEFAULT_SBLK_SIZE + if file_size != expected_file_size: + print 'WARNING: File %s not of correct size (size=%d, expected=%d): Ignoring' % (efp_file, file_size, + expected_file_size) + return False + file_handle = open(efp_file) + args = qls.utils.load_args(file_handle, qls.jrnl.RecordHeader) + file_hdr = qls.jrnl.FileHeader(*args) + file_hdr.init(file_handle, *qls.utils.load_args(file_handle, qls.jrnl.FileHeader)) + if not file_hdr.is_header_valid(file_hdr): + file_handle.close() + return False + file_hdr.load(file_handle) + file_handle.close() + if not file_hdr.is_valid(): + return False + return True + # ============================================================================= diff --git a/qpid/tools/src/py/qls/err.py b/qpid/tools/src/py/qls/err.py index 702fbb9520..bceaf041c9 100644 --- a/qpid/tools/src/py/qls/err.py +++ b/qpid/tools/src/py/qls/err.py @@ -17,12 +17,20 @@ # under the License. # +""" +Module: qls.err + +Contains error classes. +""" + # --- Parent classes class QlsError(Exception): """Base error class for QLS errors and exceptions""" def __init__(self): Exception.__init__(self) + def __str__(self): + return '' class QlsRecordError(QlsError): """Base error class for individual records""" @@ -92,10 +100,22 @@ class FirstRecordOffsetMismatchError(QlsRecordError): def __str__(self): return 'First record offset mismatch: ' + QlsRecordError.__str__(self) + ' expected_offset=0x%x' % \ self.file_header.first_record_offset - + +class InsufficientSpaceOnDiskError(QlsError): + """Insufficient space on disk""" + def __init__(self, directory, space_avail, space_requried): + QlsError.__init__(self) + self.directory = directory + self.space_avail = space_avail + self.space_required = space_requried + def __str__(self): + return 'Insufficient space on disk: directory=%s; avail_space=%d required_space=%d' % \ + (self.directory, self.space_avail, self.space_required) + class InvalidClassError(QlsError): """Invalid class name or type""" def __init__(self, class_name): + QlsError.__init__(self) self.class_name = class_name def __str__(self): return 'Invalid class name "%s"' % self.class_name @@ -108,6 +128,14 @@ class InvalidEfpDirectoryNameError(QlsError): def __str__(self): return 'Invalid EFP directory name "%s"' % self.directory_name +#class InvalidFileSizeString(QlsError): +# """Invalid file size string""" +# def __init__(self, file_size_string): +# QlsError.__init__(self) +# self.file_size_string = file_size_string +# def __str__(self): +# return 'Invalid file size string "%s"' % self.file_size_string + class InvalidPartitionDirectoryNameError(QlsError): """Invalid EFP partition name - should be pNNN, where NNN is a 3-digit partition number""" def __init__(self, directory_name): @@ -158,6 +186,30 @@ class NonTransactionalRecordError(QlsRecordError): return 'Transactional operation on non-transactional record: ' + QlsRecordError.__str__() + \ ' operation=%s' % self.operation +class PartitionDoesNotExistError(QlsError): + """Partition name does not exist on disk""" + def __init__(self, partition_directory): + QlsError.__init__(self) + self.partition_directory = partition_directory + def __str__(self): + return 'Partition %s does not exist' % self.partition_directory + +class PoolDirectoryAlreadyExistsError(QlsError): + """Pool directory already exists""" + def __init__(self, pool_directory): + QlsError.__init__(self) + self.pool_directory = pool_directory + def __str__(self): + return 'Pool directory %s already exists' % self.pool_directory + +class PoolDirectoryDoesNotExistError(QlsError): + """Pool directory does not exist""" + def __init__(self, pool_directory): + QlsError.__init__(self) + self.pool_directory = pool_directory + def __str__(self): + return 'Pool directory %s does not exist' % self.pool_directory + class RecordIdNotFoundError(QlsRecordError): """Record Id not found in enqueue map""" def __init__(self, file_header, record): @@ -184,6 +236,14 @@ class UnexpectedEndOfFileError(QlsError): return 'Tried to read %d at offset %d in file "%s"; only read %d' % \ (self.size_read, self.file_offset, self.file_name, self.size_expected) +class WritePermissionError(QlsError): + """No write permission""" + def __init__(self, directory): + QlsError.__init__(self) + self.directory = directory + def __str__(self): + return 'No write permission in directory %s' % self.directory + class XidSizeError(QlsError): """Error class for Xid size mismatch""" def __init__(self, expected_size, actual_size, xid_str): diff --git a/qpid/tools/src/py/qls/jrnl.py b/qpid/tools/src/py/qls/jrnl.py index 5bce78bfad..f4fb16ef9f 100644 --- a/qpid/tools/src/py/qls/jrnl.py +++ b/qpid/tools/src/py/qls/jrnl.py @@ -17,567 +17,17 @@ # under the License. # -import os -import os.path +""" +Module: qls.jrnl + +Contains journal record classes. +""" + import qls.err +import qls.utils import string import struct -from time import gmtime, strftime -import zlib - -class HighCounter(object): - def __init__(self): - self.num = 0 - def check(self, num): - if self.num < num: - self.num = num - def get(self): - return self.num - def get_next(self): - self.num += 1 - return self.num - -class JournalRecoveryManager(object): - TPL_DIR_NAME = 'tpl' - JRNL_DIR_NAME = 'jrnl' - def __init__(self, directory, args): - if not os.path.exists(directory): - raise qls.err.InvalidQlsDirectoryNameError(directory) - self.directory = directory - self.args = args - self.tpl = None - self.journals = {} - self.high_rid_counter = HighCounter() - def report(self, print_stats_flag): - if self.tpl is not None: - self.tpl.report(print_stats_flag) - for queue_name in sorted(self.journals.keys()): - self.journals[queue_name].report(print_stats_flag) - def run(self, args): - tpl_dir = os.path.join(self.directory, JournalRecoveryManager.TPL_DIR_NAME) - if os.path.exists(tpl_dir): - self.tpl = Journal(tpl_dir, None, self.args) - self.tpl.recover(self.high_rid_counter) - print - jrnl_dir = os.path.join(self.directory, JournalRecoveryManager.JRNL_DIR_NAME) - prepared_list = self.tpl.txn_map.get_prepared_list() if self.tpl is not None else {} - if os.path.exists(jrnl_dir): - for dir_entry in sorted(os.listdir(jrnl_dir)): - jrnl = Journal(os.path.join(jrnl_dir, dir_entry), prepared_list, self.args) - jrnl.recover(self.high_rid_counter) - self.journals[jrnl.get_queue_name()] = jrnl - print - self._reconcile_transactions(prepared_list, args.txn) - def _reconcile_transactions(self, prepared_list, txn_flag): - print 'Transaction reconciliation report:' - print '==================================' - print len(prepared_list), 'open transaction(s) found in prepared transaction list:' - for xid in prepared_list.keys(): - commit_flag = prepared_list[xid] - if commit_flag is None: - status = '[Prepared, neither committed nor aborted - assuming commit]' - elif commit_flag: - status = '[Prepared, but interrupted during commit phase]' - else: - status = '[Prepared, but interrupted during abort phase]' - print ' ', Utils.format_xid(xid), status - if prepared_list[xid] is None: # Prepared, but not committed or aborted - enqueue_record = self.tpl.get_txn_map_record(xid)[0][1] - dequeue_record = Utils.create_record('QLSd', DequeueRecord.TXN_COMPLETE_COMMIT_FLAG, \ - self.tpl.current_journal_file, self.high_rid_counter.get_next(), \ - enqueue_record.record_id, xid, None) - if txn_flag: - self.tpl.add_record(dequeue_record) - for queue_name in sorted(self.journals.keys()): - self.journals[queue_name].reconcile_transactions(prepared_list, txn_flag) - if len(prepared_list) > 0: - print 'Completing prepared transactions in prepared transaction list:' - for xid in prepared_list.keys(): - print ' ', Utils.format_xid(xid) - transaction_record = Utils.create_record('QLSc', 0, self.tpl.current_journal_file, \ - self.high_rid_counter.get_next(), None, xid, None) - if txn_flag: - self.tpl.add_record(transaction_record) - print - -class EnqueueMap(object): - """ - Map of enqueued records in a QLS journal - """ - def __init__(self, journal): - self.journal = journal - self.enq_map = {} - def add(self, journal_file, enq_record, locked_flag): - if enq_record.record_id in self.enq_map: - raise qls.err.DuplicateRecordIdError(self.journal.current_file_header, enq_record) - self.enq_map[enq_record.record_id] = [journal_file, enq_record, locked_flag] - def contains(self, rid): - """Return True if the map contains the given rid""" - return rid in self.enq_map - def delete(self, journal_file, deq_record): - if deq_record.dequeue_record_id in self.enq_map: - enq_list = self.enq_map[deq_record.dequeue_record_id] - del self.enq_map[deq_record.dequeue_record_id] - return enq_list - else: - raise qls.err.RecordIdNotFoundError(journal_file.file_header, deq_record) - def get(self, record_id): - if record_id in self.enq_map: - return self.enq_map[record_id] - return None - def lock(self, journal_file, dequeue_record): - if dequeue_record.dequeue_record_id not in self.enq_map: - raise qls.err.RecordIdNotFoundError(journal_file.file_header, dequeue_record) - self.enq_map[dequeue_record.dequeue_record_id][2] = True - def report_str(self, _, show_records): - """Return a string containing a text report for all records in the map""" - if len(self.enq_map) == 0: - return 'No enqueued records found.' - rstr = '%d enqueued records found' % len(self.enq_map) - if show_records: - rstr += ":" - rid_list = self.enq_map.keys() - rid_list.sort() - for rid in rid_list: - journal_file, record, locked_flag = self.enq_map[rid] - if locked_flag: - lock_str = '[LOCKED]' - else: - lock_str = '' - rstr += '\n 0x%x:%s %s' % (journal_file.file_header.file_num, record, lock_str) - else: - rstr += '.' - return rstr - def unlock(self, journal_file, dequeue_record): - """Set the transaction lock for a given record_id to False""" - if dequeue_record.dequeue_record_id in self.enq_map: - if self.enq_map[dequeue_record.dequeue_record_id][2]: - self.enq_map[dequeue_record.dequeue_record_id][2] = False - else: - raise qls.err.RecordNotLockedError(journal_file.file_header, dequeue_record) - else: - raise qls.err.RecordIdNotFoundError(journal_file.file_header, dequeue_record) - -class TransactionMap(object): - """ - Map of open transactions used while recovering a QLS journal - """ - def __init__(self, enq_map): - self.txn_map = {} - self.enq_map = enq_map - def abort(self, xid): - """Perform an abort operation for the given xid record""" - for journal_file, record, _ in self.txn_map[xid]: - if isinstance(record, DequeueRecord): - if self.enq_map.contains(record.dequeue_record_id): - self.enq_map.unlock(journal_file, record) - else: - journal_file.decr_enq_cnt(record) - del self.txn_map[xid] - def add(self, journal_file, record): - if record.xid is None: - raise qls.err.NonTransactionalRecordError(journal_file.file_header, record, 'TransactionMap.add()') - if isinstance(record, DequeueRecord): - try: - self.enq_map.lock(journal_file, record) - except qls.err.RecordIdNotFoundError: - # Not in emap, look for rid in tmap - should not happen in practice - txn_op = self._find_record_id(record.xid, record.dequeue_record_id) - if txn_op != None: - if txn_op[2]: - raise qls.err.AlreadyLockedError(journal_file.file_header, record) - txn_op[2] = True - if record.xid in self.txn_map: - self.txn_map[record.xid].append([journal_file, record, False]) # append to existing list - else: - self.txn_map[record.xid] = [[journal_file, record, False]] # create new list - def commit(self, xid): - """Perform a commit operation for the given xid record""" - mismatch_list = [] - for journal_file, record, lock in self.txn_map[xid]: - if isinstance(record, EnqueueRecord): - self.enq_map.add(journal_file, record, lock) # Transfer enq to emap - else: - if self.enq_map.contains(record.dequeue_record_id): - self.enq_map.unlock(journal_file, record) - self.enq_map.delete(journal_file, record)[0].decr_enq_cnt(record) - else: - mismatch_list.append('0x%x' % record.dequeue_record_id) - del self.txn_map[xid] - return mismatch_list - def contains(self, xid): - """Return True if the xid exists in the map; False otherwise""" - return xid in self.txn_map - def delete(self, journal_file, transaction_record): - """Remove a transaction record from the map using either a commit or abort header""" - if transaction_record.magic[-1] == 'c': - return self.commit(transaction_record.xid) - if transaction_record.magic[-1] == 'a': - self.abort(transaction_record.xid) - else: - raise qls.err.InvalidRecordTypeError(journal_file.file_header, transaction_record, - 'delete from Transaction Map') - def get(self, xid): - if xid in self.txn_map: - return self.txn_map[xid] - return None - def get_prepared_list(self): - """ - Prepared list is a map of xid(key) to one of None, True or False. These represent respectively: - None: prepared, but neither committed or aborted (interrupted before commit or abort) - False: prepared and aborted (interrupted before abort complete) - True: prepared and committed (interrupted before commit complete) - """ - prepared_list = {} - for xid in self.get_xid_list(): - for _, record, _ in self.txn_map[xid]: - if isinstance(record, EnqueueRecord): - prepared_list[xid] = None - else: - prepared_list[xid] = record.is_transaction_complete_commit() - return prepared_list - def get_xid_list(self): - return self.txn_map.keys() - def report_str(self, _, show_records): - """Return a string containing a text report for all records in the map""" - if len(self.txn_map) == 0: - return 'No outstanding transactions found.' - rstr = '%d outstanding transaction(s)' % len(self.txn_map) - if show_records: - rstr += ':' - for xid, op_list in self.txn_map.iteritems(): - rstr += '\n %s containing %d operations:' % (Utils.format_xid(xid), len(op_list)) - for journal_file, record, _ in op_list: - rstr += '\n 0x%x:%s' % (journal_file.file_header.file_num, record) - else: - rstr += '.' - return rstr - def _find_record_id(self, xid, record_id): - """ Search for and return map list with supplied rid.""" - if xid in self.txn_map: - for txn_op in self.txn_map[xid]: - if txn_op[1].record_id == record_id: - return txn_op - for this_xid in self.txn_map.iterkeys(): - for txn_op in self.txn_map[this_xid]: - if txn_op[1].record_id == record_id: - return txn_op - return None - -class JournalStatistics(object): - """Journal statistics""" - def __init__(self): - self.total_record_count = 0 - self.transient_record_count = 0 - self.filler_record_count = 0 - self.enqueue_count = 0 - self.dequeue_count = 0 - self.transaction_record_count = 0 - self.transaction_enqueue_count = 0 - self.transaction_dequeue_count = 0 - self.transaction_commit_count = 0 - self.transaction_abort_count = 0 - self.transaction_operation_count = 0 - def __str__(self): - fstr = 'Total record count: %d\n' + \ - 'Transient record count: %d\n' + \ - 'Filler_record_count: %d\n' + \ - 'Enqueue_count: %d\n' + \ - 'Dequeue_count: %d\n' + \ - 'Transaction_record_count: %d\n' + \ - 'Transaction_enqueue_count: %d\n' + \ - 'Transaction_dequeue_count: %d\n' + \ - 'Transaction_commit_count: %d\n' + \ - 'Transaction_abort_count: %d\n' + \ - 'Transaction_operation_count: %d\n' - return fstr % (self.total_record_count, - self.transient_record_count, - self.filler_record_count, - self.enqueue_count, - self.dequeue_count, - self.transaction_record_count, - self.transaction_enqueue_count, - self.transaction_dequeue_count, - self.transaction_commit_count, - self.transaction_abort_count, - self.transaction_operation_count) - -class Journal(object): - """ - Instance of a Qpid Linear Store (QLS) journal. - """ - def __init__(self, directory, xid_prepared_list, args): - self.directory = directory - self.queue_name = os.path.basename(directory) - self.files = {} - self.file_num_list = None - self.file_num_itr = None - self.enq_map = EnqueueMap(self) - self.txn_map = TransactionMap(self.enq_map) - self.current_journal_file = None - self.first_rec_flag = None - self.statistics = JournalStatistics() - self.xid_prepared_list = xid_prepared_list # This is None for the TPL instance only - self.args = args - self.last_record_offset = None # TODO: Move into JournalFile - self.num_filler_records_required = None # TODO: Move into JournalFile - self.fill_to_offset = None - def add_record(self, record): - if isinstance(record, EnqueueRecord) or isinstance(record, DequeueRecord): - if record.xid_size > 0: - self.txn_map.add(self.current_journal_file, record) - else: - self.enq_map.add(self.current_journal_file, record, False) - elif isinstance(record, TransactionRecord): - self.txn_map.delete(self.current_journal_file, record) - else: - raise qls.err.InvalidRecordTypeError(self.current_journal_file, record, 'add to Journal') - def get_enq_map_record(self, rid): - return self.enq_map.get(rid) - def get_txn_map_record(self, xid): - return self.txn_map.get(xid) - def get_outstanding_txn_list(self): - return self.txn_map.get_xid_list() - def get_queue_name(self): - return self.queue_name - def recover(self, high_rid_counter): - print 'Recovering %s' % self.queue_name - self._analyze_files() - try: - while self._get_next_record(high_rid_counter): - pass - self._check_alignment() - except qls.err.NoMoreFilesInJournalError: - print 'No more files in journal' - except qls.err.FirstRecordOffsetMismatchError as err: - print '0x%08x: **** FRO ERROR: queue=\"%s\" fid=0x%x fro actual=0x%08x expected=0x%08x' % \ - (err.get_expected_fro(), err.get_queue_name(), err.get_file_number(), err.get_record_offset(), - err.get_expected_fro()) - def reconcile_transactions(self, prepared_list, txn_flag): - xid_list = self.txn_map.get_xid_list() - if len(xid_list) > 0: - print self.queue_name, 'contains', len(xid_list), 'open transaction(s):' - for xid in xid_list: - if xid in prepared_list.keys(): - commit_flag = prepared_list[xid] - if commit_flag is None: - print ' ', Utils.format_xid(xid), '- Assuming commit after prepare' - if txn_flag: - self.txn_map.commit(xid) - elif commit_flag: - print ' ', Utils.format_xid(xid), '- Completing interrupted commit operation' - if txn_flag: - self.txn_map.commit(xid) - else: - print ' ', Utils.format_xid(xid), '- Completing interrupted abort operation' - if txn_flag: - self.txn_map.abort(xid) - else: - print ' ', Utils.format_xid(xid), '- Ignoring, not in prepared transaction list' - if txn_flag: - self.txn_map.abort(xid) - def report(self, print_stats_flag): - print 'Journal "%s":' % self.queue_name - print '=' * (11 + len(self.queue_name)) - if print_stats_flag: - print str(self.statistics) - print self.enq_map.report_str(True, True) - print self.txn_map.report_str(True, True) - JournalFile.report_header() - for file_num in sorted(self.files.keys()): - self.files[file_num].report() - #TODO: move this to JournalFile, append to file info - if self.num_filler_records_required is not None and self.fill_to_offset is not None: - print '0x%x:0x%08x: %d filler records required for DBLK alignment to 0x%08x' % \ - (self.current_journal_file.file_header.file_num, self.last_record_offset, - self.num_filler_records_required, self.fill_to_offset) - print - #--- protected functions --- - def _analyze_files(self): - for dir_entry in os.listdir(self.directory): - dir_entry_bits = dir_entry.split('.') - if len(dir_entry_bits) == 2 and dir_entry_bits[1] == JournalRecoveryManager.JRNL_DIR_NAME: - fq_file_name = os.path.join(self.directory, dir_entry) - file_handle = open(fq_file_name) - args = Utils.load_args(file_handle, RecordHeader) - file_hdr = FileHeader(*args) - file_hdr.init(file_handle, *Utils.load_args(file_handle, FileHeader)) - if file_hdr.is_header_valid(file_hdr): - file_hdr.load(file_handle) - if file_hdr.is_valid(): - Utils.skip(file_handle, file_hdr.file_header_size_sblks * Utils.SBLK_SIZE) - self.files[file_hdr.file_num] = JournalFile(file_hdr) - self.file_num_list = sorted(self.files.keys()) - self.file_num_itr = iter(self.file_num_list) - def _check_alignment(self): # TODO: Move into JournalFile - remaining_sblks = self.last_record_offset % Utils.SBLK_SIZE - if remaining_sblks == 0: - self.num_filler_records_required = 0 - else: - self.num_filler_records_required = (Utils.SBLK_SIZE - remaining_sblks) / Utils.DBLK_SIZE - self.fill_to_offset = self.last_record_offset + (self.num_filler_records_required * Utils.DBLK_SIZE) - if self.args.show_recs or self.args.show_all_recs: - print '0x%x:0x%08x: %d filler records required for DBLK alignment to 0x%08x' % \ - (self.current_journal_file.file_header.file_num, self.last_record_offset, - self.num_filler_records_required, self.fill_to_offset) - def _check_file(self): - if self.current_journal_file is not None: - if not self.current_journal_file.file_header.is_end_of_file(): - return True - if self.current_journal_file.file_header.is_end_of_file(): - self.last_record_offset = self.current_journal_file.file_header.file_handle.tell() - if not self._get_next_file(): - return False - fhdr = self.current_journal_file.file_header - fhdr.file_handle.seek(fhdr.first_record_offset) - return True - def _get_next_file(self): - if self.current_journal_file is not None: - file_handle = self.current_journal_file.file_header.file_handle - if not file_handle.closed: # sanity check, should not be necessary - file_handle.close() - file_num = 0 - try: - while file_num == 0: - file_num = self.file_num_itr.next() - except StopIteration: - pass - if file_num == 0: - return False - self.current_journal_file = self.files[file_num] - self.first_rec_flag = True - if self.args.show_recs or self.args.show_all_recs: - print '0x%x:%s' % (self.current_journal_file.file_header.file_num, self.current_journal_file.file_header) - return True - def _get_next_record(self, high_rid_counter): - if not self._check_file(): - return False - self.last_record_offset = self.current_journal_file.file_header.file_handle.tell() - this_record = Utils.load(self.current_journal_file.file_header.file_handle, RecordHeader) - if not this_record.is_header_valid(self.current_journal_file.file_header): - return False - if self.first_rec_flag: - if this_record.file_offset != self.current_journal_file.file_header.first_record_offset: - raise qls.err.FirstRecordOffsetMismatchError(self.current_journal_file.file_header, this_record) - self.first_rec_flag = False - self.statistics.total_record_count += 1 - start_journal_file = self.current_journal_file - if isinstance(this_record, EnqueueRecord): - ok_flag = self._handle_enqueue_record(this_record, start_journal_file) - high_rid_counter.check(this_record.record_id) - if self.args.show_recs or self.args.show_all_recs: - print '0x%x:%s' % (start_journal_file.file_header.file_num, this_record) - elif isinstance(this_record, DequeueRecord): - ok_flag = self._handle_dequeue_record(this_record, start_journal_file) - high_rid_counter.check(this_record.record_id) - if self.args.show_recs or self.args.show_all_recs: - print '0x%x:%s' % (start_journal_file.file_header.file_num, this_record) - elif isinstance(this_record, TransactionRecord): - ok_flag = self._handle_transaction_record(this_record, start_journal_file) - high_rid_counter.check(this_record.record_id) - if self.args.show_recs or self.args.show_all_recs: - print '0x%x:%s' % (start_journal_file.file_header.file_num, this_record) - else: - self.statistics.filler_record_count += 1 - ok_flag = True - if self.args.show_all_recs: - print '0x%x:%s' % (start_journal_file.file_header.file_num, this_record) - Utils.skip(self.current_journal_file.file_header.file_handle, Utils.DBLK_SIZE) - return ok_flag - def _handle_enqueue_record(self, enqueue_record, start_journal_file): - while enqueue_record.load(self.current_journal_file.file_header.file_handle): - if not self._get_next_file(): - enqueue_record.truncated_flag = True - return False - if not enqueue_record.is_valid(start_journal_file): - return False - if enqueue_record.is_external() and enqueue_record.data != None: - raise qls.err.ExternalDataError(self.current_journal_file.file_header, enqueue_record) - if enqueue_record.is_transient(): - self.statistics.transient_record_count += 1 - return True - if enqueue_record.xid_size > 0: - self.txn_map.add(start_journal_file, enqueue_record) - self.statistics.transaction_operation_count += 1 - self.statistics.transaction_record_count += 1 - self.statistics.transaction_enqueue_count += 1 - else: - self.enq_map.add(start_journal_file, enqueue_record, False) - start_journal_file.incr_enq_cnt() - self.statistics.enqueue_count += 1 - return True - def _handle_dequeue_record(self, dequeue_record, start_journal_file): - while dequeue_record.load(self.current_journal_file.file_header.file_handle): - if not self._get_next_file(): - dequeue_record.truncated_flag = True - return False - if not dequeue_record.is_valid(start_journal_file): - return False - if dequeue_record.xid_size > 0: - if self.xid_prepared_list is None: # ie this is the TPL - dequeue_record.transaction_prepared_list_flag = True - elif not self.enq_map.contains(dequeue_record.dequeue_record_id): - dequeue_record.warnings.append('NOT IN EMAP') # Only for non-TPL records - self.txn_map.add(start_journal_file, dequeue_record) - self.statistics.transaction_operation_count += 1 - self.statistics.transaction_record_count += 1 - self.statistics.transaction_dequeue_count += 1 - else: - try: - self.enq_map.delete(start_journal_file, dequeue_record)[0].decr_enq_cnt(dequeue_record) - except qls.err.RecordIdNotFoundError: - dequeue_record.warnings.append('NOT IN EMAP') - self.statistics.dequeue_count += 1 - return True - def _handle_transaction_record(self, transaction_record, start_journal_file): - while transaction_record.load(self.current_journal_file.file_header.file_handle): - if not self._get_next_file(): - transaction_record.truncated_flag = True - return False - if not transaction_record.is_valid(start_journal_file): - return False - if transaction_record.magic[-1] == 'a': - self.statistics.transaction_abort_count += 1 - else: - self.statistics.transaction_commit_count += 1 - if self.txn_map.contains(transaction_record.xid): - self.txn_map.delete(self.current_journal_file, transaction_record) - else: - transaction_record.warnings.append('NOT IN TMAP') -# if transaction_record.magic[-1] == 'c': # commits only -# self._txn_obj_list[hdr.xid] = hdr - self.statistics.transaction_record_count += 1 - return True - def _load_data(self, record): - while not record.is_complete: - record.load(self.current_journal_file.file_handle) - -class JournalFile(object): - def __init__(self, file_header): - self.file_header = file_header - self.enq_cnt = 0 - self.deq_cnt = 0 - self.num_filler_records_required = None - def incr_enq_cnt(self): - self.enq_cnt += 1 - def decr_enq_cnt(self, record): - if self.enq_cnt <= self.deq_cnt: - raise qls.err.EnqueueCountUnderflowError(self.file_header, record) - self.deq_cnt += 1 - def get_enq_cnt(self): - return self.enq_cnt - self.deq_cnt - def is_outstanding_enq(self): - return self.enq_cnt > self.deq_cnt - @staticmethod - def report_header(): - print 'file_num enq_cnt p_no efp journal_file' - print '-------- ------- ---- ----- ------------' - def report(self): - comment = '<uninitialized>' if self.file_header.file_num == 0 else '' - file_num_str = '0x%x' % self.file_header.file_num - print '%8s %7d %4d %4dk %s %s' % (file_num_str, self.get_enq_cnt(), self.file_header.partition_num, - self.file_header.efp_data_size_kb, - os.path.basename(self.file_header.file_handle.name), comment) +import time class RecordHeader(object): FORMAT = '<4s2H2Q' @@ -590,14 +40,14 @@ class RecordHeader(object): self.record_id = record_id self.warnings = [] self.truncated_flag = False - def checksum_encode(self): + def encode(self): return struct.pack(RecordHeader.FORMAT, self.magic, self.version, self.user_flags, self.serial, self.record_id) def load(self, file_handle): pass @staticmethod def discriminate(args): """Use the last char in the header magic to determine the header type""" - return _CLASSES.get(args[1][-1], RecordHeader) + return CLASSES.get(args[1][-1], RecordHeader) def is_empty(self): """Return True if this record is empty (ie has a magic of 0x0000""" return self.magic == '\x00'*4 @@ -608,17 +58,12 @@ class RecordHeader(object): if self.magic[:3] != 'QLS' or self.magic[3] not in ['a', 'c', 'd', 'e', 'f', 'x']: return False if self.magic[-1] != 'x': - if self.version != Utils.RECORD_VERSION: - raise qls.err.InvalidRecordVersionError(file_header, self, Utils.RECORD_VERSION) + if self.version != qls.utils.DEFAULT_RECORD_VERSION: + raise qls.err.InvalidRecordVersionError(file_header, self, qls.utils.DEFAULT_RECORD_VERSION) if self.serial != file_header.serial: return False return True - def _get_warnings(self): - warn_str = '' - for warn in self.warnings: - warn_str += '<%s>' % warn - return warn_str - def __str__(self): + def to_string(self): """Return string representation of this header""" if self.is_empty(): return '0x%08x: <empty>' % (self.file_offset) @@ -628,6 +73,14 @@ class RecordHeader(object): return '0x%08x: [%c v=%d f=0x%04x rid=0x%x]' % \ (self.file_offset, self.magic[-1].upper(), self.version, self.user_flags, self.record_id) return '0x%08x: <error, unknown magic "%s" (possible overwrite boundary?)>' % (self.file_offset, self.magic) + def _get_warnings(self): + warn_str = '' + for warn in self.warnings: + warn_str += '<%s>' % warn + return warn_str + def __str__(self): + """Return string representation of this header""" + return RecordHeader.to_string(self) class RecordTail(object): FORMAT = '<4sL2Q' @@ -653,24 +106,28 @@ class RecordTail(object): if self.valid_flag is None: if not self.complete: return False - self.valid_flag = Utils.inv_str(self.xmagic) == record.magic and \ + self.valid_flag = qls.utils.inv_str(self.xmagic) == record.magic and \ self.serial == record.serial and \ self.record_id == record.record_id and \ - Utils.adler32(record.checksum_encode()) == self.checksum + qls.utils.adler32(record.checksum_encode()) == self.checksum return self.valid_flag - def __str__(self): + def to_string(self): """Return a string representation of the this RecordTail instance""" if self.valid_flag is not None: if not self.valid_flag: return '[INVALID RECORD TAIL]' - magic = Utils.inv_str(self.xmagic) + magic = qls.utils.inv_str(self.xmagic) magic_char = magic[-1].upper() if magic[-1] in string.printable else '?' return '[%c cs=0x%08x rid=0x%x]' % (magic_char, self.checksum, self.record_id) + def __str__(self): + """Return a string representation of the this RecordTail instance""" + return RecordTail.to_string(self) class FileHeader(RecordHeader): FORMAT = '<2H4x5QH' - def init(self, file_handle, _, file_header_size_sblks, partition_num, efp_data_size_kb, - first_record_offset, timestamp_sec, timestamp_ns, file_num, queue_name_len): + MAGIC = 'QLSf' + def init(self, file_handle, _, file_header_size_sblks, partition_num, efp_data_size_kb, first_record_offset, + timestamp_sec, timestamp_ns, file_num, queue_name_len): self.file_handle = file_handle self.file_header_size_sblks = file_header_size_sblks self.partition_num = partition_num @@ -681,11 +138,21 @@ class FileHeader(RecordHeader): self.file_num = file_num self.queue_name_len = queue_name_len self.queue_name = None - def load(self, file_handle): - self.queue_name = file_handle.read(self.queue_name_len) + def encode(self): + if self.queue_name is None: + return RecordHeader.encode(self) + struct.pack(self.FORMAT, self.file_header_size_sblks, \ + self.partition_num, self.efp_data_size_kb, \ + self.first_record_offset, self.timestamp_sec, \ + self.timestamp_ns, self.file_num, 0) + return RecordHeader.encode(self) + struct.pack(self.FORMAT, self.file_header_size_sblks, self.partition_num, \ + self.efp_data_size_kb, self.first_record_offset, \ + self.timestamp_sec, self.timestamp_ns, self.file_num, \ + self.queue_name_len) + self.queue_name def get_file_size(self): """Sum of file header size and data size""" - return (self.file_header_size_sblks * Utils.SBLK_SIZE) + (self.efp_data_size_kb * 1024) + return (self.file_header_size_sblks * qls.utils.DEFAULT_SBLK_SIZE) + (self.efp_data_size_kb * 1024) + def load(self, file_handle): + self.queue_name = file_handle.read(self.queue_name_len) def is_end_of_file(self): return self.file_handle.tell() >= self.get_file_size() def is_valid(self): @@ -704,18 +171,22 @@ class FileHeader(RecordHeader): return True def timestamp_str(self): """Get the timestamp of this record in string format""" - time = gmtime(self.timestamp_sec) + now = time.gmtime(self.timestamp_sec) fstr = '%%a %%b %%d %%H:%%M:%%S.%09d %%Y' % (self.timestamp_ns) - return strftime(fstr, time) - def __str__(self): + return time.strftime(fstr, now) + def to_string(self): """Return a string representation of the this FileHeader instance""" - return '%s fnum=0x%x fro=0x%08x p=%d s=%dk t=%s %s' % (RecordHeader.__str__(self), self.file_num, + return '%s fnum=0x%x fro=0x%08x p=%d s=%dk t=%s %s' % (RecordHeader.to_string(self), self.file_num, self.first_record_offset, self.partition_num, self.efp_data_size_kb, self.timestamp_str(), self._get_warnings()) + def __str__(self): + """Return a string representation of the this FileHeader instance""" + return FileHeader.to_string(self) class EnqueueRecord(RecordHeader): FORMAT = '<2Q' + MAGIC = 'QLSe' EXTERNAL_FLAG_MASK = 0x20 TRANSIENT_FLAG_MASK = 0x10 def init(self, _, xid_size, data_size): @@ -726,9 +197,13 @@ class EnqueueRecord(RecordHeader): self.data = None self.data_complete = False self.record_tail = None - def checksum_encode(self): - return RecordHeader.checksum_encode(self) + struct.pack(self.FORMAT, self.xid_size, self.data_size) + \ - self.xid + self.data + def checksum_encode(self): # encode excluding record tail + bytes = RecordHeader.encode(self) + struct.pack(self.FORMAT, self.xid_size, self.data_size) + if self.xid is not None: + bytes += self.xid + if self.data is not None: + bytes += self.data + return bytes def is_external(self): return self.user_flags & EnqueueRecord.EXTERNAL_FLAG_MASK > 0 def is_transient(self): @@ -750,13 +225,13 @@ class EnqueueRecord(RecordHeader): return True def load(self, file_handle): """Return True when load is incomplete and must be called again with new file handle""" - self.xid, self.xid_complete = Utils.load_data(file_handle, self.xid, self.xid_size) + self.xid, self.xid_complete = qls.utils.load_data(file_handle, self.xid, self.xid_size) if not self.xid_complete: return True if self.is_external(): self.data_complete = True else: - self.data, self.data_complete = Utils.load_data(file_handle, self.data, self.data_size) + self.data, self.data_complete = qls.utils.load_data(file_handle, self.data, self.data_size) if not self.data_complete: return True if self.xid_size > 0 or self.data_size > 0: @@ -769,6 +244,19 @@ class EnqueueRecord(RecordHeader): else: return True return False + def to_string(self, show_xid_flag, show_data_flag): + """Return a string representation of the this EnqueueRecord instance""" + if self.truncated_flag: + return '%s xid(%d) data(%d) [Truncated, no more files in journal]' % (RecordHeader.__str__(self), + self.xid_size, self.data_size) + if self.record_tail is None: + record_tail_str = '' + else: + record_tail_str = self.record_tail.to_string() + return '%s %s %s %s %s %s' % (RecordHeader.to_string(self), + qls.utils.format_xid(self.xid, self.xid_size, show_xid_flag), + qls.utils.format_data(self.data, self.data_size, show_data_flag), + record_tail_str, self._print_flags(), self._get_warnings()) def _print_flags(self): """Utility function to decode the flags field in the header and print a string representation""" fstr = '' @@ -784,19 +272,11 @@ class EnqueueRecord(RecordHeader): return fstr def __str__(self): """Return a string representation of the this EnqueueRecord instance""" - if self.truncated_flag: - return '%s xid(%d) data(%d) [Truncated, no more files in journal]' % (RecordHeader.__str__(self), - self.xid_size, self.data_size) - if self.record_tail is None: - record_tail_str = '' - else: - record_tail_str = str(self.record_tail) - return '%s %s %s %s %s %s' % (RecordHeader.__str__(self), Utils.format_xid(self.xid, self.xid_size), - Utils.format_data(self.data_size, self.data), record_tail_str, - self._print_flags(), self._get_warnings()) + return EnqueueRecord.to_string(self, False, False) class DequeueRecord(RecordHeader): FORMAT = '<2Q' + MAGIC = 'QLSd' TXN_COMPLETE_COMMIT_FLAG = 0x10 def init(self, _, dequeue_record_id, xid_size): self.dequeue_record_id = dequeue_record_id @@ -805,8 +285,8 @@ class DequeueRecord(RecordHeader): self.xid = None self.xid_complete = False self.record_tail = None - def checksum_encode(self): - return RecordHeader.checksum_encode(self) + struct.pack(self.FORMAT, self.dequeue_record_id, self.xid_size) + \ + def checksum_encode(self): # encode excluding record tail + return RecordHeader.encode(self) + struct.pack(self.FORMAT, self.dequeue_record_id, self.xid_size) + \ self.xid def is_transaction_complete_commit(self): return self.user_flags & DequeueRecord.TXN_COMPLETE_COMMIT_FLAG > 0 @@ -825,7 +305,7 @@ class DequeueRecord(RecordHeader): return True def load(self, file_handle): """Return True when load is incomplete and must be called again with new file handle""" - self.xid, self.xid_complete = Utils.load_data(file_handle, self.xid, self.xid_size) + self.xid, self.xid_complete = qls.utils.load_data(file_handle, self.xid, self.xid_size) if not self.xid_complete: return True if self.xid_size > 0: @@ -838,6 +318,19 @@ class DequeueRecord(RecordHeader): else: return True return False + def to_string(self, show_xid_flag): + """Return a string representation of the this DequeueRecord instance""" + if self.truncated_flag: + return '%s xid(%d) drid=0x%x [Truncated, no more files in journal]' % (RecordHeader.__str__(self), + self.xid_size, + self.dequeue_record_id) + if self.record_tail is None: + record_tail_str = '' + else: + record_tail_str = self.record_tail.to_string() + return '%s drid=0x%x %s %s %s %s' % (RecordHeader.to_string(self), self.dequeue_record_id, + qls.utils.format_xid(self.xid, self.xid_size, show_xid_flag), + record_tail_str, self._print_flags(), self._get_warnings()) def _print_flags(self): """Utility function to decode the flags field in the header and print a string representation""" if self.transaction_prepared_list_flag: @@ -848,27 +341,19 @@ class DequeueRecord(RecordHeader): return '' def __str__(self): """Return a string representation of the this DequeueRecord instance""" - if self.truncated_flag: - return '%s xid(%d) drid=0x%x [Truncated, no more files in journal]' % (RecordHeader.__str__(self), - self.xid_size, - self.dequeue_record_id) - if self.record_tail is None: - record_tail_str = '' - else: - record_tail_str = str(self.record_tail) - return '%s %s drid=0x%x %s %s %s' % (RecordHeader.__str__(self), Utils.format_xid(self.xid, self.xid_size), - self.dequeue_record_id, record_tail_str, self._print_flags(), - self._get_warnings()) + return DequeueRecord.to_string(self, False) class TransactionRecord(RecordHeader): FORMAT = '<Q' + MAGIC_ABORT = 'QLSa' + MAGIC_COMMIT = 'QLSc' def init(self, _, xid_size): self.xid_size = xid_size self.xid = None self.xid_complete = False self.record_tail = None - def checksum_encode(self): - return RecordHeader.checksum_encode(self) + struct.pack(self.FORMAT, self.xid_size) + self.xid + def checksum_encode(self): # encode excluding record tail + return RecordHeader.encode(self) + struct.pack(self.FORMAT, self.xid_size) + self.xid def is_valid(self, journal_file): if not RecordHeader.is_header_valid(self, journal_file.file_header): return False @@ -881,7 +366,7 @@ class TransactionRecord(RecordHeader): return True def load(self, file_handle): """Return True when load is incomplete and must be called again with new file handle""" - self.xid, self.xid_complete = Utils.load_data(file_handle, self.xid, self.xid_size) + self.xid, self.xid_complete = qls.utils.load_data(file_handle, self.xid, self.xid_size) if not self.xid_complete: return True if self.xid_size > 0: @@ -894,168 +379,24 @@ class TransactionRecord(RecordHeader): else: return True return False - def __str__(self): + def to_string(self, show_xid_flag): """Return a string representation of the this TransactionRecord instance""" if self.truncated_flag: return '%s xid(%d) [Truncated, no more files in journal]' % (RecordHeader.__str__(self), self.xid_size) if self.record_tail is None: record_tail_str = '' else: - record_tail_str = str(self.record_tail) - return '%s %s %s %s' % (RecordHeader.__str__(self), Utils.format_xid(self.xid, self.xid_size), record_tail_str, - self._get_warnings()) - -class Utils(object): - """Class containing utility functions for dealing with the journal""" - DBLK_SIZE = 128 - RECORD_VERSION = 2 - SBLK_SIZE = 4096 - @staticmethod - def adler32(data): - return zlib.adler32(data) & 0xffffffff - @staticmethod - def create_record(magic, uflags, journal_file, record_id, dequeue_record_id, xid, data): - record_class = _CLASSES.get(magic[-1]) - record = record_class(0, magic, Utils.RECORD_VERSION, uflags, journal_file.file_header.serial, record_id) - xid_length = len(xid) if xid is not None else 0 - if isinstance(record, EnqueueRecord): - data_length = len(data) if data is not None else 0 - record.init(None, xid_length, data_length) - elif isinstance(record, DequeueRecord): - record.init(None, dequeue_record_id, xid_length) - elif isinstance(record, TransactionRecord): - record.init(None, xid_length) - else: - raise qls.err.InvalidClassError(record.__class__.__name__) - if xid is not None: - record.xid = xid - record.xid_complete = True - if data is not None: - record.data = data - record.data_complete = True - record.record_tail = RecordTail(None) - record.record_tail.xmagic = Utils.inv_str(magic) - record.record_tail.checksum = Utils.adler32(record.checksum_encode()) - record.record_tail.serial = record.serial - record.record_tail.record_id = record.record_id - return record - @staticmethod - def format_data(dsize, data): - """Format binary data for printing""" - if data == None: - return '' - # << DEBUG >> - begin = data.find('msg') - end = data.find('\0', begin) - return 'data(%d)="%s"' % (dsize, data[begin:end]) - # << END DEBUG - if Utils._is_printable(data): - datastr = Utils._split_str(data) - else: - datastr = Utils._hex_split_str(data) - if dsize != len(data): - raise qls.err.DataSizeError(dsize, len(data), datastr) - return 'data(%d)="%s"' % (dsize, datastr) - @staticmethod - def format_xid(xid, xidsize=None): - """Format binary XID for printing""" - if xid == None and xidsize != None: - if xidsize > 0: - raise qls.err.XidSizeError(xidsize, 0, None) - return '' - if Utils._is_printable(xid): - xidstr = '"%s"' % Utils._split_str(xid) - else: - xidstr = '0x%s' % Utils._hex_split_str(xid) - if xidsize == None: - xidsize = len(xid) - elif xidsize != len(xid): - raise qls.err.XidSizeError(xidsize, len(xid), xidstr) - return 'xid(%d)=%s' % (xidsize, xidstr) - @staticmethod - def inv_str(in_string): - """Perform a binary 1's compliment (invert all bits) on a binary string""" - istr = '' - for index in range(0, len(in_string)): - istr += chr(~ord(in_string[index]) & 0xff) - return istr - @staticmethod - def load(file_handle, klass): - """Load a record of class klass from a file""" - args = Utils.load_args(file_handle, klass) - subclass = klass.discriminate(args) - result = subclass(*args) # create instance of record - if subclass != klass: - result.init(*Utils.load_args(file_handle, subclass)) - return result - @staticmethod - def load_args(file_handle, klass): - """Load the arguments from class klass""" - size = struct.calcsize(klass.FORMAT) - foffs = file_handle.tell(), - fbin = file_handle.read(size) - if len(fbin) != size: - raise qls.err.UnexpectedEndOfFileError(len(fbin), size, foffs, file_handle.name) - return foffs + struct.unpack(klass.FORMAT, fbin) - @staticmethod - def load_data(file_handle, element, element_size): - if element_size == 0: - return element, True - if element is None: - element = file_handle.read(element_size) - else: - read_size = element_size - len(element) - element += file_handle.read(read_size) - return element, len(element) == element_size - @staticmethod - def skip(file_handle, boundary): - """Read and discard disk bytes until the next multiple of boundary""" - if not file_handle.closed: - file_handle.read(Utils._rem_bytes_in_block(file_handle, boundary)) - #--- protected functions --- - @staticmethod - def _hex_str(in_str, begin, end): - """Return a binary string as a hex string""" - hstr = '' - for index in range(begin, end): - if Utils._is_printable(in_str[index]): - hstr += in_str[index] - else: - hstr += '\\%02x' % ord(in_str[index]) - return hstr - @staticmethod - def _hex_split_str(in_str):#, split_size = 50): - """Split a hex string into two parts separated by an ellipsis""" -# if len(in_str) <= split_size: -# return Utils._hex_str(in_str, 0, len(in_str)) -# return Utils._hex_str(in_str, 0, 10) + ' ... ' + Utils._hex_str(in_str, len(in_str)-10, len(in_str)) - return ''.join(x.encode('hex') for x in reversed(in_str)) - @staticmethod - def _is_printable(in_str): - """Return True if in_str in printable; False otherwise.""" - for this_char in in_str: - if this_char not in string.printable: - return False - return True - @staticmethod - def _rem_bytes_in_block(file_handle, block_size): - """Return the remaining bytes in a block""" - foffs = file_handle.tell() - return (Utils._size_in_blocks(foffs, block_size) * block_size) - foffs - @staticmethod - def _size_in_blocks(size, block_size): - """Return the size in terms of data blocks""" - return int((size + block_size - 1) / block_size) - @staticmethod - def _split_str(in_str, split_size = 50): - """Split a string into two parts separated by an ellipsis if it is longer than split_size""" - if len(in_str) < split_size: - return in_str - return in_str[:25] + ' ... ' + in_str[-25:] + record_tail_str = self.record_tail.to_string() + return '%s %s %s %s' % (RecordHeader.to_string(self), + qls.utils.format_xid(self.xid, self.xid_size, show_xid_flag), + record_tail_str, self._get_warnings()) + def __str__(self): + """Return a string representation of the this TransactionRecord instance""" + return TransactionRecord.to_string(self, False) # ============================================================================= -_CLASSES = { +CLASSES = { 'a': TransactionRecord, 'c': TransactionRecord, 'd': DequeueRecord, diff --git a/qpid/tools/src/py/qls/utils.py b/qpid/tools/src/py/qls/utils.py new file mode 100644 index 0000000000..758dc446c0 --- /dev/null +++ b/qpid/tools/src/py/qls/utils.py @@ -0,0 +1,206 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +""" +Module: qls.utils + +Contains helper functions for qpid_qls_analyze. +""" + +import os +import qls.jrnl +import stat +import string +import struct +import subprocess +import zlib + +DEFAULT_DBLK_SIZE = 128 +DEFAULT_SBLK_SIZE = 4096 # 32 dblks +DEFAULT_SBLK_SIZE_KB = DEFAULT_SBLK_SIZE / 1024 +DEFAULT_RECORD_VERSION = 2 +DEFAULT_HEADER_SIZE_SBLKS = 1 + +def adler32(data): + """return the adler32 checksum of data""" + return zlib.adler32(data) & 0xffffffff + +def create_record(magic, uflags, journal_file, record_id, dequeue_record_id, xid, data): + """Helper function to construct a record with xid, data (where applicable) and consistent tail with checksum""" + record_class = qls.jrnl.CLASSES.get(magic[-1]) + record = record_class(0, magic, DEFAULT_RECORD_VERSION, uflags, journal_file.file_header.serial, record_id) + xid_length = len(xid) if xid is not None else 0 + if isinstance(record, qls.jrnl.EnqueueRecord): + data_length = len(data) if data is not None else 0 + record.init(None, xid_length, data_length) + elif isinstance(record, qls.jrnl.DequeueRecord): + record.init(None, dequeue_record_id, xid_length) + elif isinstance(record, qls.jrnl.TransactionRecord): + record.init(None, xid_length) + else: + raise qls.err.InvalidClassError(record.__class__.__name__) + if xid is not None: + record.xid = xid + record.xid_complete = True + if data is not None: + record.data = data + record.data_complete = True + record.record_tail = _mk_record_tail(record) + return record + +def efp_directory_size(directory_name): + """"Decode the directory name in the format NNNk to a numeric size, where NNN is a number string""" + try: + if directory_name[-1] == 'k': + return int(directory_name[:-1]) + except ValueError: + pass + return 0 + +def format_data(data, data_size=None, show_data_flag=True): + """Format binary data for printing""" + return _format_binary(data, data_size, show_data_flag, 'data', qls.err.DataSizeError, False) + +def format_xid(xid, xid_size=None, show_xid_flag=True): + """Format binary XID for printing""" + return _format_binary(xid, xid_size, show_xid_flag, 'xid', qls.err.XidSizeError, True) + +def get_avail_disk_space(path): + df_proc = subprocess.Popen(["df", path], stdout=subprocess.PIPE) + output = df_proc.communicate()[0] + return int(output.split('\n')[1].split()[3]) + +def has_write_permission(path): + stat_info = os.stat(path) + return bool(stat_info.st_mode & stat.S_IRGRP) + +def inv_str(in_string): + """Perform a binary 1's compliment (invert all bits) on a binary string""" + istr = '' + for index in range(0, len(in_string)): + istr += chr(~ord(in_string[index]) & 0xff) + return istr + +def load(file_handle, klass): + """Load a record of class klass from a file""" + args = load_args(file_handle, klass) + subclass = klass.discriminate(args) + result = subclass(*args) # create instance of record + if subclass != klass: + result.init(*load_args(file_handle, subclass)) + return result + +def load_args(file_handle, klass): + """Load the arguments from class klass""" + size = struct.calcsize(klass.FORMAT) + foffs = file_handle.tell(), + fbin = file_handle.read(size) + if len(fbin) != size: + raise qls.err.UnexpectedEndOfFileError(len(fbin), size, foffs, file_handle.name) + return foffs + struct.unpack(klass.FORMAT, fbin) + +def load_data(file_handle, element, element_size): + """Read element_size bytes of binary data from file_handle into element""" + if element_size == 0: + return element, True + if element is None: + element = file_handle.read(element_size) + else: + read_size = element_size - len(element) + element += file_handle.read(read_size) + return element, len(element) == element_size + +def skip(file_handle, boundary): + """Read and discard disk bytes until the next multiple of boundary""" + if not file_handle.closed: + file_handle.read(_rem_bytes_in_block(file_handle, boundary)) + +#--- protected functions --- + +def _format_binary(bin_str, bin_size, show_bin_flag, prefix, err_class, hex_num_flag): + """Format binary XID for printing""" + if bin_str is None and bin_size is not None: + if bin_size > 0: + raise err_class(bin_size, len(bin_str), bin_str) + return '' + if bin_size is None: + bin_size = len(bin_str) + elif bin_size != len(bin_str): + raise err_class(bin_size, len(bin_str), bin_str) + out_str = '%s(%d)' % (prefix, bin_size) + if show_bin_flag: + if _is_printable(bin_str): + binstr = '"%s"' % _split_str(bin_str) + elif hex_num_flag: + binstr = '0x%s' % _str_to_hex_num(bin_str) + else: + binstr = _hex_split_str(bin_str) + out_str += '=%s' % binstr + return out_str + +def _hex_str(in_str, begin, end): + """Return a binary string as a hex string""" + hstr = '' + for index in range(begin, end): + if _is_printable(in_str[index]): + hstr += in_str[index] + else: + hstr += '\\%02x' % ord(in_str[index]) + return hstr + +def _hex_split_str(in_str, split_size = 50): + """Split a hex string into two parts separated by an ellipsis""" + if len(in_str) <= split_size: + return _hex_str(in_str, 0, len(in_str)) + return _hex_str(in_str, 0, 10) + ' ... ' + _hex_str(in_str, len(in_str)-10, len(in_str)) + #return ''.join(x.encode('hex') for x in reversed(in_str)) + +def _is_printable(in_str): + """Return True if in_str in printable; False otherwise.""" + for this_char in in_str: + if this_char not in string.letters and this_char not in string.digits and this_char not in string.punctuation: + return False + return True + +def _mk_record_tail(record): + record_tail = qls.jrnl.RecordTail(None) + record_tail.xmagic = inv_str(record.magic) + record_tail.checksum = adler32(record.checksum_encode()) + record_tail.serial = record.serial + record_tail.record_id = record.record_id + return record_tail + +def _rem_bytes_in_block(file_handle, block_size): + """Return the remaining bytes in a block""" + foffs = file_handle.tell() + return (_size_in_blocks(foffs, block_size) * block_size) - foffs + +def _size_in_blocks(size, block_size): + """Return the size in terms of data blocks""" + return int((size + block_size - 1) / block_size) + +def _split_str(in_str, split_size = 50): + """Split a string into two parts separated by an ellipsis if it is longer than split_size""" + if len(in_str) < split_size: + return in_str + return in_str[:25] + ' ... ' + in_str[-25:] + +def _str_to_hex_num(in_str): + """Turn a string into a hex number representation, little endian assumed (ie LSB is first, MSB is last)""" + return ''.join(x.encode('hex') for x in reversed(in_str)) diff --git a/qpid/tools/src/py/qpid-ha b/qpid/tools/src/py/qpid-ha index daa73d3312..8850a12adb 100755 --- a/qpid/tools/src/py/qpid-ha +++ b/qpid/tools/src/py/qpid-ha @@ -106,9 +106,9 @@ class StatusCmd(Command): if not ha_broker.status in ["active", "recovering"]: raise ExitStatus(1) if opts.expect: if opts.expect != ha_broker.status: raise ExitStatus(1) - if opts.all: + brokers = filter(None, re.sub(r'(^amqps?:)|(tcp:)', "", ha_broker.brokersUrl).split(",")) + if opts.all and brokers: opts.all=False - brokers = re.sub(r'(^amqp:)|(tcp:)', "", ha_broker.brokersUrl).split(",") for b in brokers: opts.broker = b try: diff --git a/qpid/tools/src/py/qpid-route b/qpid/tools/src/py/qpid-route index 21e5461664..a0db6bae93 100755 --- a/qpid/tools/src/py/qpid-route +++ b/qpid/tools/src/py/qpid-route @@ -239,7 +239,9 @@ class RouteManager: if url.name() not in self.brokerList: print " %s..." % url.name(), try: - b = self.qmf.addBroker(url, config._connTimeout) + url.authName = self.local.authName + url.authPass = self.local.authPass + b = self.qmf.addBroker(url, config._connTimeout, **config._conn_options) self.brokerList[url.name()] = b added = True print "Ok" diff --git a/qpid/tools/src/py/qpid_qls_analyze.py b/qpid/tools/src/py/qpid_qls_analyze.py index a540587547..1b2655896c 100755 --- a/qpid/tools/src/py/qpid_qls_analyze.py +++ b/qpid/tools/src/py/qpid_qls_analyze.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,13 +16,44 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# + +""" +qpid-qls-analyze + +Reads and analyzes a Qpid Linear Store (QLS) store directory. +""" import argparse import os import os.path -from qls import efp -from qls import jrnl +import qls.anal +import qls.efp + +class QlsAnalyzerArgParser(argparse.ArgumentParser): + def __init__(self): + argparse.ArgumentParser.__init__(self, description = 'Qpid Linear Store Analyzer', prog = 'qpid-qls-analyze') + self.add_argument('qls_dir', metavar='DIR', + help='Qpid Linear Store (QLS) directory to be analyzed') + self.add_argument('--efp', action='store_true', + help='Analyze the Emtpy File Pool (EFP) and show stats') + self.add_argument('--show-recs', action='store_true', + help='Show material records found during recovery') + self.add_argument('--show-all-recs', action='store_true', + help='Show all records (including fillers) found during recovery') + self.add_argument('--show-xids', action='store_true', + help='Show xid as hex number, otherwise show only xid length') + self.add_argument('--show-data', action='store_true', + help='Show data, otherwise show only data length') + self.add_argument('--stats', action='store_true', + help='Print journal record stats') + self.add_argument('--txn', action='store_true', + help='Reconcile incomplete transactions') + self.add_argument('--version', action='version', + version='%(prog)s ' + QqpdLinearStoreAnalyzer.QLS_ANALYZE_VERSION) + def parse_args(self, args=None, namespace=None): + args = argparse.ArgumentParser.parse_args(self, args, namespace) + # If required, perform additional validity checks here, raise errors if req'd + return args class QqpdLinearStoreAnalyzer(object): """ @@ -37,28 +68,10 @@ class QqpdLinearStoreAnalyzer(object): self.args = None self._process_args() self.qls_dir = os.path.abspath(self.args.qls_dir) - self.efp_manager = efp.EfpManager(self.qls_dir, self.args) - self.jrnl_recovery_mgr = jrnl.JournalRecoveryManager(self.qls_dir, self.args) - def _analyze_efp(self): - self.efp_manager.run(self.args) - def _analyze_journals(self): - self.jrnl_recovery_mgr.run(self.args) + self.efp_manager = qls.efp.EfpManager(self.qls_dir, None) + self.jrnl_recovery_mgr = qls.anal.JournalRecoveryManager(self.qls_dir, self.args) def _process_args(self): - parser = argparse.ArgumentParser(description = 'Qpid Linear Store Analyzer') - parser.add_argument('qls_dir', metavar='DIR', - help='Qpid Linear Store (QLS) directory to be analyzed') - parser.add_argument('--efp', action='store_true', - help='Analyze the Emtpy File Pool (EFP) and show stats') - parser.add_argument('--show-recs', action='store_true', - help='Show material records found during recovery') - parser.add_argument('--show-all-recs', action='store_true', - help='Show all records (including fillers) found during recovery') - parser.add_argument('--stats', action='store_true', - help='Print journal record stats') - parser.add_argument('--txn', action='store_true', - help='Reconcile incomplete transactions') - parser.add_argument('--version', action='version', version='%(prog)s ' + - QqpdLinearStoreAnalyzer.QLS_ANALYZE_VERSION) + parser = QlsAnalyzerArgParser() self.args = parser.parse_args() if not os.path.exists(self.args.qls_dir): parser.error('Journal path "%s" does not exist' % self.args.qls_dir) @@ -68,8 +81,8 @@ class QqpdLinearStoreAnalyzer(object): self.jrnl_recovery_mgr.report(self.args.stats) def run(self): if self.args.efp: - self._analyze_efp() - self._analyze_journals() + self.efp_manager.run(None) + self.jrnl_recovery_mgr.run() #============================================================================== # main program |