diff options
Diffstat (limited to 'java/perftests/bin')
| -rwxr-xr-x | java/perftests/bin/monitoring/monitor-broker.sh | 221 | ||||
| -rwxr-xr-x | java/perftests/bin/monitoring/runTests.sh | 148 | ||||
| -rwxr-xr-x | java/perftests/bin/monitoring/stop-monitored-broker.sh | 124 | ||||
| -rwxr-xr-x | java/perftests/bin/processing/process.sh | 388 | ||||
| -rwxr-xr-x | java/perftests/bin/processing/processAll.sh | 58 | ||||
| -rwxr-xr-x | java/perftests/bin/processing/processTests.py | 850 | ||||
| -rwxr-xr-x | java/perftests/bin/run_many.sh | 30 | ||||
| -rwxr-xr-x | java/perftests/bin/topicListener.sh | 32 | ||||
| -rwxr-xr-x | java/perftests/bin/topicPublisher.sh | 31 |
9 files changed, 0 insertions, 1882 deletions
diff --git a/java/perftests/bin/monitoring/monitor-broker.sh b/java/perftests/bin/monitoring/monitor-broker.sh deleted file mode 100755 index 27c8ff25f6..0000000000 --- a/java/perftests/bin/monitoring/monitor-broker.sh +++ /dev/null @@ -1,221 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -# -# This script starts a broker and then starts additional logging as required. -# *.pid files are generated in the LOG_DIR for later use by the stop-monitored-broker -# script. -# -# Currently this process starts: -# - The broker with additional QPID_OPTS for gc logging -# - Top to monitoring the CPU usage -# -# Additional processes can be started and as long as they write a PID into LOG_DIR/*.pid -# it will be shutdown with the stop script -# - -# -# Output the broker log file to aid problem diagnosis -# then exit. -# -brokerFailExit() -{ - echo "Broker failed to start up." - cat $BROKER_LOG - exit 1 -} - -showUsageExit() -{ - echo "Usage $0 <Path to Test Broker> <LOG DIR> <CPU Monitor Rate (s)> [Additional options to - pass to Qpid broker startup]" - exit 1 -} - -# -# Perform 3 attempts to get the broker PID via ps and grep -# if unable the output broker log and exit -# -getBrokerPID() -{ - attempts=3 - ready=0 - while [ $ready == 0 ] ; do - - PID=`ps auxwww| grep java | grep Xloggc | awk '{print $2}'` - - if [ ! $PID == 0 ] ; then - ready=1 - else - attempts=$[ $attempts - 1 ] - - if [ $attempts == 0 ] ; then - brokerFailExit - fi - - sleep 1 - fi - done - -} - - -# -# Additional Check to ensure that the broker process -# has correctly written 'Ready' to the log file. -# -checkBrokerStarted() -{ - attempts=3 - ready=0 - while [ $ready == 0 ] ; do - - grep Ready $BROKER_LOG > /dev/null - - if [ $? == 0 ] ; then - ready=1 - else - attempts=$[ $attempts - 1 ] - - if [ $attempts == 0 ] ; then - brokerFailExit - fi - - echo "Broker not ready sleeping 1s" - sleep 1 - fi - done -} - -# -# Command Line setup -# - -# Ensure we have minimum of three arguments -if [[ $# > 2 ]] ; then - BROKER_VERSION=$1 - LOG_DIR=$2 - CPU_MONITOR_RATE=$3 - # Remove these arguments from the $@ variable - shift - shift - shift -else - # If we have no arguments then use these as the default - CPU_MONITOR_RATE=0.5 - LOG_DIR=$QPID_WORK/logging - BROKER_VERSION=qpid-0.5 -fi - - -# -# Check the specified broker is reachable -# it it is not the log and show usage -# -if [ ! -d $BROKER_VERSION ] ; then - echo "Broker not available at: $BROKER_VERSION" - showUsageExit -fi - -# -# Check to see if we have an absolute path for logging -# -logStart=`echo $LOG_DIR|cut -c 1` - - -# -# If we don't have an absolute path then add the current -# directory path to the start. -# -if [[ $logStart != '/' ]] ; then - echo -n "$LOG_DIR is not absolute, using " - LOG_DIR=`pwd`/$LOG_DIR - echo $LOG_DIR -fi - -# -# Validate that the directory does not exist -# - this is so we can guarrantee a clean run. -# If it does exit then log and show usage -# -if [ -d $LOG_DIR ] ; then - echo "Log directory already exists : $LOG_DIR" - showUsageExit -fi - -# -# Create the logging directory -# -mkdir -p $LOG_DIR - -# -# Variable for broker log -# -BROKER_LOG=$LOG_DIR/broker.log - -# Variable to hold broker PID -PID=0 - -export QPID_OPTS="-Xloggc:$LOG_DIR/gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps" - -# -# Start Qpid Broker -# -echo "Starting Broker : $BROKER_VERSION" -pushd $BROKER_VERSION/bin > /dev/null -./qpid-server $@ 2> $BROKER_LOG >&2 & -popd > /dev/null - -# Wait and check startup was ok -echo "Waiting for broker startup" -getBrokerPID - -checkBrokerStarted - -echo $PID > $LOG_DIR/broker.pid - -# -# Start CPU Monitoring via TOP -# -echo "Starting CPU Monitor at RATE:$CPU_MONITOR_RATE on $SERVER1" -pushd $LOG_DIR > /dev/null - -echo $CPU_MONITOR_RATE > top.rate - -top -d $CPU_MONITOR_RATE -S -c -p $PID -b > broker_cpu.log & - -# -# Get top pid using $! -# -echo $! > $LOG_DIR/top.pid - -popd > /dev/null - - -# -# Generate Stat files -# -echo "Generating Stat data" -stat $BROKER_LOG > $BROKER_LOG.stat -stat $LOG_DIR/broker_cpu.log > $LOG_DIR/broker_cpu.log.stat -stat $LOG_DIR/gc.log > $LOG_DIR/gc.log.stat - diff --git a/java/perftests/bin/monitoring/runTests.sh b/java/perftests/bin/monitoring/runTests.sh deleted file mode 100755 index 676db0d4fa..0000000000 --- a/java/perftests/bin/monitoring/runTests.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -# -# Run specified performance tests and -# gather details about the test run -# - - -runTest() -{ - echo "$@" - echo "$@ --csv -o $LOG_DIR" >> $LOG_DIR/TestRun.log 2>&1 - ./$@ --csv -o $LOG_DIR >> $LOG_DIR/TestRun.log 2>&1 -} - -showUsageExit() -{ - echo "Usage $0 <Path to Test Pack> <LOG DIR> <TEST LIST FILE>" - exit 1 -} - -# Ensure we have minimum of three arguments -if [[ $# > 2 ]] ; then - TEST_VERSION=$1 - LOG_DIR=$2 - TEST_LIST=$3 - # Remove these arguments from the $@ variable - shift - shift - shift -else - showUsageExit -fi - -# -# Check the specified broker is reachable -# it it is not the log and show usage -# -if [ ! -d $TEST_VERSION ] ; then - echo "Tests not available at: $TEST_VERSION" - showUsageExit -fi - - -# -# Check to see if we have an absolute path for logging -# -logStart=`echo $LOG_DIR|cut -c 1` - -# -# If we don't have an absolute path then add the current -# directory path to the start. -# -if [[ $logStart != '/' ]] ; then - echo -n "$LOG_DIR is not absolute, using " - LOG_DIR=`pwd`/$LOG_DIR - echo $LOG_DIR -fi - -# -# Validate that the directory does not exist -# - this is so we can guarrantee a clean run. -# If it does exit then log and show usage -# -if [ -d $LOG_DIR ] ; then - echo "Log directory already exists : $LOG_DIR" - showUsageExit -fi - -# -# Check to see if we have an absolute path for test list -# -testListStart=`echo $TEST_LIST|cut -c 1` - -# -# If we don't have an absolute path then add the current -# directory path to the start. -# -if [[ $testListStart != '/' ]] ; then - echo -n "$TEST_LIST is not absolute, using " - TEST_LIST=`pwd`/$TEST_LIST - echo $TEST_LIST -fi - -# -# Validate that the directory does not exist -# - this is so we can guarrantee a clean run. -# If it does exit then log and show usage -# -# -r Check file exists and is readable -if [ ! -r $TEST_LIST ] ; then - echo "Test file is not readable : $TEST_LIST" - showUsageExit -fi - - - -# -# Create the logging directory -# -mkdir -p $LOG_DIR - - - -echo "Starting Test Run in : $TEST_VERSION" -pushd $TEST_VERSION/bin > /dev/null - -# -# Run tests -# - - -while read testCommand -do - runTest $testCommand -done < "$TEST_LIST" - - -popd > /dev/null - - -# -# Generate Stat files -# -echo "Generating Stat data" -for file in `find $LOG_DIR -name "*.csv"` ; do - stat $file > $file.stat -done diff --git a/java/perftests/bin/monitoring/stop-monitored-broker.sh b/java/perftests/bin/monitoring/stop-monitored-broker.sh deleted file mode 100755 index ad882b0bea..0000000000 --- a/java/perftests/bin/monitoring/stop-monitored-broker.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -# -# Script to stop the running of a monitored broker -# and the associated monitoring processes. -# -# Looks in the specifed directory for pid files and -# stops those proceses -# - - -usage() -{ - echo "Usage: $0 <LOG_DIR>" -} - -# -# Attempt to gracefully kill processs -# -stopRun() -{ - kill $PIDS -} - - -# -# Forcibly stop processes -# -forceStopRun() -{ - kill -9 $PIDS -} - -# -# Show usage if we are not started correctly -# -if [ $# != 1 ] ; then - usage - exit 1 -fi - -LOG_DIR=$1 - - -PIDS=`cat $LOG_DIR/*.pid` - -echo "Preparing to stop:" -# -# Escape the path so we can use sed to remove it. -# -path=`echo $LOG_DIR|sed -e s/\\\//\\\\\\\\\\\//g` - -for i in `ls $LOG_DIR/*.pid` ; do - # Remove path from pid item then remove any final '/' - echo $i|cut -d '.' -f 1| sed -e s/$path// |tr '/' ' ' -done - -status=`ps $PIDS |wc -l` - -if [ $status == 1 ] ; then - echo "Processes do not appear to be running." - echo "Have they already been stopped?" - exit 0 -fi - -attempts=3 - -while [ ! $status == 1 ] ; do - stopRun - sleep 1 - status=`ps $PIDS |wc -l` - - if [ $status == 1 ] ; then - echo "Done" - exit 0 - else - attempts=$[ $attempts - 1 ] - - if [ $attempts == 0 ] ; then - break - fi - - echo "Sleeping as processes not stopped" - sleep 2 - - fi -done - -# If we haven't been able to kill the processes then -# forcibly do it -if [ ! $status == 1 ] ; then - forceStopRun - sleep 1 - status=`ps $PIDS |wc -l` - - if [ $status == 1 ] ; then - echo "Done" - else - echo "Stop failed" - exit 1 - fi -else - echo "Done" -fi diff --git a/java/perftests/bin/processing/process.sh b/java/perftests/bin/processing/process.sh deleted file mode 100755 index 6dbf9b9ea8..0000000000 --- a/java/perftests/bin/processing/process.sh +++ /dev/null @@ -1,388 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - - -usage() -{ - echo "usage: process.sh <Qpid Version> <Test Type Queue/Topic/Latency..> <Volume of data %age> <broker details>" - echo "These parameters are used to title and name the graphs:" - echo 'Title = $<Qpid Version> $<Test Type> : $<Volume>% volume' - echo 'File = $<Qpid Version>-$<Broker Details>-$<Test Type>-$<Volume>' - exit 1 -} - -processCMSGCFile() -{ - # Extract all the ParNew timings: Ignore any lines corrupted by concurrent CMS logging. - grep -v CMS gc.log|grep ParNew > $work/parnew.gc.log - - # Calculate Full GC Data - grep failure -B 1 gc.log | sed -e :a -e '$!N;s/\n.(concurrent/ /;ta' -e 'P;D' |grep -v ^\- > $work/full.gc.log - cut -d ':' -f 1 $work/full.gc.log > $work/full.time.log - sed -e 's/failure)/\#/g' $work/full.gc.log |cut -d '#' -f 2- |awk '{print $3}' > $work/full.dur.log - - # Move to the working directory to further process the gathered data. - pushd $work &> /dev/null - - # Extract the Heap and timing data. - #8 is the full gc data - #5 is the paranew gc data - cat parnew.gc.log | awk '{print $8}' | cut -d '(' -f 2 | cut -d 'K' -f 1 > HEAP_MAX.log - cat parnew.gc.log | awk '{print $8}' | cut -d 'K' -f 1 > HEAP_PRE_GC.log - cat parnew.gc.log | awk '{print $8}' | cut -d 'K' -f 2 | cut -d '>' -f 2 > HEAP_POST_GC.log - cat parnew.gc.log | awk '{print $1}' | cut -d ':' -f 1 > GC_TIME.log - - calculateStats "MEM_PRE" HEAP_PRE_GC.log - calculateStats "MEM_POST" HEAP_POST_GC.log - calculateStatsBC "GC_FREQ" GC_TIME.log - - - # Calculate ParNew GC Cumulative total - cat parnew.gc.log |awk '{sum=sum+$6; print $6, sum}' > GC_DUR.log - # Give a count of GC occurances - parNewGCCount=`wc -l GC_DUR.log| awk '{print $1}'` - - # Create the Heap data file - paste GC_TIME.log HEAP_POST_GC.log HEAP_PRE_GC.log HEAP_MAX.log > GC.Heap.data - # Create ParNew GC Duration data file for graphing - paste GC_TIME.log GC_DUR.log > GC.Dur.data - # Create Full GC occurance log file for plotting - paste full.time.log full.dur.log > GC.Full.data - - # Calculate All GC Timing and give a count of their occurance - awk '{print $1}' GC_DUR.log > gc_dur.log - paste GC_TIME.log gc_dur.log > gc_all.log - cat GC.Full.data >> gc_all.log - sort -n gc_all.log | awk '{sum=sum+$2;print $1 , sum}' > GC.Dur.All.data - fullGCCount=`wc -l GC.Full.data| awk '{print $1}'` -} - -processG1GCFile() -{ - cat gc.log | grep -e \^[0-9]*.[0-9]*\: -e \^.*[0-9]*.\-\> > $work/g1.gc.log - - # Move to the working directory to further process the gathered data. - pushd $work &> /dev/null - - # Calculate Full GC Data - cat g1.gc.log | sed -e :a -e '$!N;s/\n.\ *\[/ \[/;ta' -e 'P;D' > full.gc.log - grep Full full.gc.log |awk '{print $1}'| tr ':' ' ' > full.time.log - grep Full full.gc.log |awk '{print $5}' > full.dur.log - fullGCCount=`wc -l GC.Full.data| awk '{print $1}'` - - # Create Full GC occurance log file for plotting - paste full.time.log full.dur.log > GC.Full.data - - # Extract the Heap and timing data. - # Create G1 Young Duration data file for graphing - grep "(young)," full.gc.log |awk '{print $1}' | tr ':' ' ' > GC_TIME.log - grep "(young)," full.gc.log |awk '{print $5}' > young.gc.time.log - # Calculate G1 young Cumulative total - cat young.gc.time.log |awk '{sum=sum+$1; print $1, sum}' > GC_DUR.log - # Create G1 Young Duration data file for graphing - paste GC_TIME.log GC_DUR.log > GC.Dur.data - - # Give a count of GC occurances - youngGCCount=`wc -l GC_DUR.log| awk '{print $1}'` - - # - # If we have no GCs then something is wrong - if [ $youngGCCount == 0 ] ; then - echo "Error no YoungGC log entries to proceses" - return - fi - - # Gather the Heap Size data - - grep "(young)," full.gc.log | awk '{print $8}' | cut -d '(' -f 2 | cut -d ')' -f 1 > HEAP_MAX.Sized.log - grep "(young)," full.gc.log | awk '{print $8}' | cut -d '-' -f 1 > HEAP_PRE_GC.Sized.log - grep "(young)," full.gc.log | awk '{print $8}' | cut -d '>' -f 2| cut -d '(' -f 1 > HEAP_POST_GC.Sized.log - - normaliseSizeMemFile HEAP_MAX.Sized.log HEAP_MAX.log - normaliseSizeMemFile HEAP_PRE_GC.Sized.log HEAP_PRE_GC.log - normaliseSizeMemFile HEAP_POST_GC.Sized.log HEAP_POST_GC.log - - - # Create the Heap data file - paste GC_TIME.log HEAP_POST_GC.log HEAP_PRE_GC.log HEAP_MAX.log > GC.Heap.data - - # Calculate All GC Timing and give a count of their occurance - awk '{print $1}' GC_DUR.log > gc_dur.log - paste GC_TIME.log gc_dur.log > gc_all.log - cat GC.Full.data >> gc_all.log - sort -n gc_all.log | awk '{sum=sum+$2;print $1 , sum}' > GC.Dur.All.data - -} - -calculateStatsBC() -{ -label=$1 -statFile=$2 -# Calculate GC Frequencies - prevFreq=0 - countFreq=0 - gcFreqTotal=0 - minFreq=99999999 - maxFreq=0 - - echo "" > GC_FREQ.log - for time in `cat $statFile` ; do - if [ $prevFreq == 0 ] ; then - prevFreq=$time - else - freq=`echo $time - $prevFreq | bc` - echo $freq >> GC_FREQ.log - - prevFreq=$time - gcFreqTotal=`echo $gcFreqTotal + $freq | bc` - countFreq=$[$countFreq + 1] - - if [ `echo "$freq > $maxFreq " | bc` == 1 ] ; then - maxFreq=$freq - fi - - if [ `echo "$freq < $minFreq " | bc ` == 1 ] ; then - minFreq=$freq - fi - fi - done - - # Extract Min/Max/Avg - echo "#type:min/max/avg" > $statFile.stats - - if [ $countFreq != 0 ] ; then - avgFreq=`echo $gcFreqTotal / $countFreq | bc -l 2&> /dev/null` - echo "$label:$minFreq/$maxFreq/$avgFreq" >> $statFile.stats - echo "Done GC Freq Stat generation for $statFile" - else - echo "$label:-/-/-" >> $statFile.stats - echo "Unable to calculate GC Freq stats as no freq entries found." - fi -} - -calculateStats() -{ -label=$1 -statFile=$2 - -# Calculate GC Frequencies - count=0 - gcTotal=0 - min=0 - max=0 - - - for item in `cat $statFile` ; do - if [ $min == 0 ] ; then - min=$item - fi - - gcTotal=$[$gcTotal + $item] - count=$[$count + 1] - - if [ $item -gt $max ] ; then - max=$item - fi - - if [ $item -lt $min ] ; then - min=$item - fi - done - - # Extract Min/Max/Avg - echo "#type:min/max/avg" > $statFile.stats - - if [ $count != 0 ] ; then - avg=`echo $gcTotal / $count | bc -l` - - echo "$label:$min/$max/$avg" >> $statFile.stats - echo "Done $label Stat generation for $statFile" - else - echo "$label:-/-/-" >> $statFile.stats - echo "Unable to calculate $label stats as no entries found." - fi - -} - -# -# Take an input file ($1) of lines -# <value><K|M> -# and output file $2 of <value> in whole M -# -normaliseSizeMemFile() -{ -rm -f $2 -for i in `cat $1` ; do - if [[ `echo $i | grep -c "K" ` == 1 ]] ; then - kb=`echo $i|cut -d 'K' -f 1` - echo $[ $kb / 1024 ] >> $2 - else - echo $i|cut -d 'M' -f 1 >> $2 - fi -done - - -} - - -# Parse command line -# TODO more advanced processing -# Check we have enough parameters -if [ $# != 4 ] ; then - # Take one arg to be a graph data file. - if [ $# == 1 ] ; then - textArray[0]="" # hold text - length=0 - # read whole file in loop - while read line - do - textArray[$length]=$line # store line - length=$(expr $length + 1) # increase length by 1 - done < $1 - - if [ $length != 2 ] ; then - usage - fi - - #Get Title and file name - title=${textArray[0]} - file=${textArray[1]} - - pushd `dirname $1` - - else - usage - fi -else - version=$1 - type=$2 - volume=$3 - brokerState=$4 - - - # Configure Graph Title and image file names - title="$version $type : $volume% volume" - file="$version-$brokerState-$type-$volume" -fi - -work=work - -mkdir -p $work - -echo -n "Processing GC Usage Data : " -ParNew=`grep -c ParNew gc.log` - -if [ $ParNew != 0 ] ; then - echo "CMS log file" - processCMSGCFile - PLOT="\"GC.Dur.data\" with lines axis x1y1 ti \"ParNew GC Time ($parNewGCCount)\", " -else - echo "G1 log file" - processG1GCFile - PLOT="\"GC.Dur.data\" with lines axis x1y1 ti \"G1 Young Time ($youngGCCount)\", " -fi - -# -# Colate Stats -# -echo -n "Colating stat Data : " -echo "#type:min/max/avg" > $file.statistics.txt -find .. -name "*stats" -exec grep -v "type" {} >> $file.statistics.txt \; - - -# Prepare the plot command -# If a Full GC occured during this test then plot those -if [[ $fullGCCount > 0 ]] ; then -PLOT=$PLOT"\"GC.Dur.data\" using 1:3 with lines axis x1y2 ti \"Cumulative Total Time(ParNew)\", \ - \"GC.Dur.All.data\" with lines axis x1y2 ti \"Cumulative Total Time(All)\", \ - \"GC.Full.data\" with points ti \"Full GCs Time ($fullGCCount)\" " -else -PLOT=$PLOT"\"GC.Dur.data\" using 1:3 with lines axis x1y2 ti \"Cumulative Total Time(ParNew)\", \ - \"GC.Dur.All.data\" with lines axis x1y2 ti \"Cumulative Total Time(All)\" " -fi - -if [ $ParNew != 0 ] ; then - gcs=$parNewGCCount -else - gcs=$youngGCCount -fi - -# Call out to gnuplot to generate graphs -# Only do this if we have data -if [ $gcs != 0 ] ; then - # Generate the Heap Graph and the GC Duration Graph - gnuplot << EOGNUPLOT -set xlabel "Run Time(s)" - -set title "$title : Heap Size" -set term png -set output "$file-Heap.png" -set ylabel "MB" +0.5,0 -plot "GC.Heap.data" using 1:2 with lines axis x1y1 ti "GC Size Post",\ - "GC.Heap.data" using 1:4 with lines axis x1y1 ti "GC Size Max ", \ - "GC.Heap.data" using 1:3 with lines axis x1y1 ti "GC Size Pre " - -set y2tics nomirror -set ytics nomirror -set key top left -set title "$title GC Time" -set ylabel "Time(s)" +0.5,0 -set y2label "Total Time(s)" -set output "$file-GCDuration.png" -plot $PLOT -EOGNUPLOT - -else - echo "Warning: No GC Data to graph" -fi - -# pop back to further process for CPU usage -popd &> /dev/null - -echo "Processing CPU Usage Data" - -# CPU.data is just TimeStamp + %age -cat broker_cpu.log |awk '{print $1 "T" $2 " " $3}' > $work/CPU.data - -# Move to work directory to run gnuplot -pushd $work &> /dev/null - -# Call out to gnuplot to generate graphs -# Generate the Heap Graph and the GC Duration Graph -gnuplot << EOGNUPLOT -set term png -set title "$title : CPU" -set output "$file-CPU.png" -unset key -set xlabel "Real Time(h:m)" -set ylabel "Usage(%)" - -set xdata time -set timefmt "%Y-%m-%dT%H:%M:%S" -set format x "%H:%M" -plot "CPU.data" using 1:2 with lines -EOGNUPLOT - -#" -# Pop out of $work directory. -# -popd &> /dev/null
\ No newline at end of file diff --git a/java/perftests/bin/processing/processAll.sh b/java/perftests/bin/processing/processAll.sh deleted file mode 100755 index 895d59c316..0000000000 --- a/java/perftests/bin/processing/processAll.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - - -usage() -{ - echo "processResults.sh <search dir>" - exit 1 -} - -root=`pwd` -echo $root -graphFile=graph.data - -if [ $# != 1 ] ; then - usage -fi - -mkdir -p results - -for file in `find $1 -name $graphFile` ; do - - dir=`dirname $file` - - echo Processing : $dir - pushd $dir &> /dev/null - - $root/process.sh $graphFile - - echo Copying Images - cp work/*png $root/results/ - - echo Copying Stats - cp work/*.statistics.txt $root/results/ - - - popd &> /dev/null - echo Done -done
\ No newline at end of file diff --git a/java/perftests/bin/processing/processTests.py b/java/perftests/bin/processing/processTests.py deleted file mode 100755 index 8db44eda79..0000000000 --- a/java/perftests/bin/processing/processTests.py +++ /dev/null @@ -1,850 +0,0 @@ -#!/usr/bin/env python -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os -import re -import datetime -import sys -import string -from optparse import OptionParser -from datetime import datetime, timedelta -import shutil - - -def showUsage(): - log("./processTests.py [-b|--broker-log-dir] <dir> [-t|--test-dir] <dir>") - - -ACCESS="Access" -MODIFY="Modify" - -BROKER_LOG="broker.log" -BROKER_PID="broker.pid" -BROKER_CPU="broker_cpu.log" -BROKER_CPU_DATED="broker_cpu.log.dated" -BROKER_STATS="broker.stats" -BROKER_GC="gc.log" - -GRAPH_DATA="graph.data" - -_verbose = False -_debug = False -_brokerLogs = "" - -def exitError(message): - log(message) - sys.exit(1) - -def main(): - global _log, _verbose, _debug, _brokerLogs - - # Load the - parser = OptionParser() - - parser.add_option("-v", "--verbose", dest="verbose", - action="store_true", default=False, help="enable verbose output") - - parser.add_option("-d", "--debug", dest="debug", - action="store_true", default=False, help="enable debug output") - - parser.add_option("-b", "--broker-log-dir", dest="brokerLogs", - action="store", default=True, help="Broker Logs") - - parser.add_option("-t", "--test-dir", dest="testDir", - action="store", default="", help="Test Results") - - - (options, args) = parser.parse_args() - - _verbose = options.verbose - _debug = options.debug - testDir = options.testDir - _brokerLogs = options.brokerLogs - - if testDir == "" or _brokerLogs == "" : - log("Broker Log Dir and Test Dir are both requried.") - showUsage() - - if not os.path.exists(testDir): - exitError("Test directory does not exist:" + testDir) - - if not os.path.exists(_brokerLogs): - exitError("Broker log directory does not exist:" + _brokerLogs) - - - # Standardize the format of the broker logs - preProcessBrokerLogs(_brokerLogs) - - # Get list of test results from test_dir - processTestResults(testDir) - -# -# Process the log files we know of -# -def preProcessBrokerLogs(resultDir): - - print "Pre Processing Broker Logs" - # Pre-Process GC - no pre processing required - - # Process Log4j - no processing required as file is already time stamped. - - # Pre-Process broker_cpu - processCPUUsage(resultDir) - -# -# Process the broker CPU log file and create an output file of format -# <Date Time> <CPU Usage> -# -# -def processCPUUsage(resultDir): - logfile=resultDir+os.sep+BROKER_CPU - datedFile=resultDir+os.sep+BROKER_CPU_DATED - - start = extractTime(ACCESS, logfile+".stat") - - pid = getPID(BROKER_PID) - - topRate = getFirstLine(_brokerLogs+os.sep+"top.rate") - - # - # Calulate addition required per process line output - # - if topRate.find(".") == -1: - seconds = topRate - millis = 0 - else: - split = topRate.split('.') - seconds = split[0] - # Convert - millis = float("0."+split[1]) * 1000 - - offset = timedelta(seconds=int(seconds),milliseconds=int(millis)) - - # - # Process the CPU log file and make a file of format: - # datetime <CPU% usage> <MEM% usage> - # - # Open log CPU file for reading - logfile = open(logfile, "r") - - # Open the output file, erasing any existing version - # Keep track of the min/max sum and entries,. - minCPU=float(sys.maxint) - maxCPU=0.0 - minMem=float(sys.maxint) - maxMem=0.0 - - entries=0 - sumCPU=0.0 - sumMem=0.0 - - output= open(datedFile, "w") - for line in logfile: - # - # Data format - # 0 1 2 3 4 5 6 7 8 9 10 11 - # PID USER PR NI %CPU TIME+ %MEM VIRT RES SHR S COMMAND - # PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND - # - # %CPU and %MEM are vary, probably based on os/version of top. - # lets auto-detect where it is. - # - # Index is 0 based for array usage. - index = 0 - if line.find("PID") != -1: - for key in line.split(" "): - strippedKey = key.lstrip() - if len(strippedKey) > 0: - # Record the key index - if (strippedKey == "%CPU"): - cpuIndex=index - if (strippedKey == "%MEM"): - memIndex=index - # Increase count for next key - index = index + 1 - - - # Find lines that contain our broker process - if line.find("QPBRKR") != -1: - - # Split line on whitespace - data = line.split() - - #Write out the date time (ISO-8601 format) - output.write(str(start)) - # Output the %CPU value - output.write(" "+str(data[cpuIndex])) - # Output the %MEM value - output.write(" "+str(data[memIndex])) - output.write('\n') - - # Add the offset based on the logging rate - start = start + offset - - # Record entires - entries = entries + 1 - - # Record Metrics - # Record CPU data - cpu = float(data[cpuIndex]) - if (cpu < minCPU): - minCPU = cpu - - if (cpu > maxCPU): - maxCPU = cpu - - sumCPU = sumCPU + cpu - - # Record Mem data - mem = float(data[memIndex]) - - if (mem < minMem): - minMem = mem - - if (mem > maxMem): - maxMem = mem - - sumMem = sumMem + mem - - - #end for - - # Close the files - logfile.close - output.close - - # Output stats file - statFile=resultDir+os.sep+BROKER_CPU+".stats" - output= open(statFile, "w") - output.write("#type:min/max/avg") - output.write('\n') - output.write("CPU:"+str(minCPU)+"/"+str(maxCPU)+"/"+str(float(sumCPU)/float(entries))) - output.write('\n') - output.write("MEM:"+str(minMem)+"/"+str(maxMem)+"/"+str(float(sumMem)/float(entries))) - output.write('\n') - output.close - - log("Pre Process of CPU Log file '"+BROKER_CPU+"' complete") - - -# -# Give an known process type get the recorded PID. -# -def getPID(process): - return getFirstLine(_brokerLogs+os.sep+process) - -# -# Get the first line of the file without EOL chars. -# NOTE: this will load the entire file into memory to do it. -# -def getFirstLine(fileName): - f = open(fileName,"r") - line = f.read().splitlines()[0] - f.close - return line - - -# -# Walk the directory given and process all csv test results -# -def processTestResults(resultDir): - for root, dirs, files in os.walk(resultDir, topdown=False): - if len(files) == 0: - exitError("Test result directory is empty:" + resultDir) - for file in files: - if file.endswith(".csv"): - processTestResult(root , file) - -def processTestResult(root, resultFile): - # Open stat file and extract test times, we determine: - # -start time based on the 'Access' value - # -end time based on the 'Modify' value 'Change' would also work - - statFile=root+os.sep+resultFile+".stat" - - if not os.path.exists(statFile): - log("Unable to process : Unable to open stat file:" + statFile) - return - - - createResultSetPackage(root, resultFile) - - -def extractTime(field, statFile): - stats = open(statFile, "r") - for line in stats: - if line.startswith(field): - if line.find("(") == -1: - dt = lineToDate(" ".join(line.split()[1:])) - - # - # TODO We need to handle time time zone issues as I'm sure we will have issues with the - # log4j matching. - - stats.close - return dt - -# -# Given a text line in ISO format convert it to a date object -# -def lineToDate(line): - #2009-06-22 17:04:44,320 - #2009-06-22 17:04:44.320 - pattern = re.compile(r'(?P<year>^[0-9][0-9][0-9][0-9])-(?P<month>[0-9][0-9])-(?P<day>[0-9][0-9]) (?P<hour>[0-9][0-9]):(?P<minute>[0-9][0-9]):(?P<seconds>[0-9][0-9])') - - - m = pattern.match(line) - if m: - year = int(m.group('year')) - month = int(m.group('month')) - day = int(m.group('day')) - hour = int(m.group('hour')) - minute = int(m.group('minute')) - seconds = int(m.group('seconds')) - - pattern = re.compile(r'(?P<year>^[0-9][0-9][0-9][0-9])-(?P<month>[0-9][0-9])-(?P<day>[0-9][0-9]) (?P<hour>[0-9][0-9]):(?P<minute>[0-9][0-9]):(?P<seconds>[0-9][0-9])[.|,](?P<micro>[0-9]+)') - m = pattern.match(line) - micro = None - if m: - micro = m.group('micro') - - if micro == None: - micro = 0 - - # Correct issue where micros are actually nanos - if int(micro) > 999999: - micro = int(micro) / 1000 - - return datetime(year,month,day,hour,minute,seconds,int(micro)) - else: - # Error we shouldn't get here - return null - -def createResultSetPackage(root, resultFile): - # Get the Name of the test to make a directory with said name - testName = resultFile.split(".csv")[0] - resultDir = root+ os.sep + testName - - log("Processing Result set for:"+ testName) - - mkdir(resultDir) - - # Move result file to new directory - shutil.move(root + os.sep + resultFile, resultDir) - - # Move stat file to new directory - shutil.move(root + os.sep + resultFile + ".stat", resultDir) - - statFile=resultDir + os.sep + resultFile + ".stat" - - # - # Get start and end time for test run - # - start = extractTime(ACCESS, statFile) - end = extractTime(MODIFY, statFile) - - sliceBrokerLogs(resultDir, start, end) - createGraphData(resultDir, testName) - createTestStatData(resultDir, testName) - - log("Created Result Package for:"+ testName) - -def sliceBrokerLogs(resultDir, start, end): - sliceCPULog(resultDir, start, end) - sliceLog4j(resultDir, start, end) - sliceGCLog(resultDir, start, end) - - -def sliceCPULog(resultDir, start, end): - global _brokerLogs - logfilePath=_brokerLogs+os.sep+BROKER_CPU_DATED - cpuSliceFile=resultDir+os.sep+BROKER_CPU - - # Process the CPU log file and make a file of format: - # datetime <CPU% usage> <MEM% usage> - # - # Open log CPU file for reading - logFile = open(logfilePath, "r") - - # Open the output file, erasing any existing version - # Keep track of the min/max sum and entries,. - minCPU=float(sys.maxint) - maxCPU=0.0 - minMem=float(sys.maxint) - maxMem=0.0 - - entries=0 - sumCPU=0.0 - sumMem=0.0 - - # - # Create outputfile - # - cpuslice = open(cpuSliceFile,"w") - for line in logFile: - data = line.split() - # - # //fixme remove tz addition. - # - lineTime = lineToDate(" ".join(data[0:2])+" +0000") - - if lineTime > start: - if lineTime < end: - # Write the data though to the new file - cpuslice.writelines(line) - - # Perform stat processing for the min/max/avg - - data = line.split() - - # - # Data format is - # <Date> <Time> <%CPU> <%MEM> - # 2010-02-19 10:16:17 157 28.1 - # - cpuIndex = 2 - memIndex = 3 - - # Record entires - entries = entries + 1 - - # Record Metrics - # Record CPU data - cpu = float(data[cpuIndex]) - if (cpu < minCPU): - minCPU = cpu - - if (cpu > maxCPU): - maxCPU = cpu - - sumCPU = sumCPU + cpu - - # Record Mem data - mem = float(data[memIndex]) - - if (mem < minMem): - minMem = mem - - if (mem > maxMem): - maxMem = mem - - sumMem = sumMem + mem - - logFile.close() - cpuslice.close() - log("Sliced CPU log") - - # Output stats file - statFile=cpuSliceFile+".stats" - output= open(statFile, "w") - output.write("#type:min/max/avg") - output.write('\n') - output.write("CPU:"+str(minCPU)+"/"+str(maxCPU)+"/"+str(float(sumCPU)/float(entries))) - output.write('\n') - output.write("MEM:"+str(minMem)+"/"+str(maxMem)+"/"+str(float(sumMem)/float(entries))) - output.write('\n') - output.close - - log("Generated stat data from CPU Log file") - - -def sliceGCLog(resultDir, start, end): - global _brokerLogs - logfilePath=_brokerLogs+os.sep+BROKER_GC - sliceFile=resultDir+os.sep+BROKER_GC - - gcstart = extractTime(ACCESS, logfilePath+".stat") - - # Open the output file, erasing any existing version - # Keep track of the min/max sum and entries,. - minGCDuration=float(sys.maxint) - maxGCDuration=0.0 - sumGCDuration=0.0 - entriesGCDuration = 0 - - - # Open log GC file for reading - logFile = open(logfilePath, "r") - - # Open the output file, erasing any existing version - output= open(sliceFile, "w") - - # Use a regular expression to pull out the Seconds.Millis values from the - # Start of the gc log line. - pattern = re.compile(r'(?P<seconds>^[0-9]+)\.(?P<millis>[0-9]+):') - - for line in logFile: - m = pattern.match(line) - - if m: - seconds = m.group('seconds'); - millis = m.group('millis'); - - offset = timedelta(seconds=int(seconds),milliseconds=int(millis)) - - lineTime = gcstart + offset - - if lineTime > start: - if lineTime < end: - output.writelines(line) - # Perform stat processing for the min/max/avg - - # Process GC Duration lines in ParNew gc , - # ensure we do not have CMS printed as that means the line line has been corrupted - if line.find("ParNew") != -1 & line.find("CMS") == -1: - - # - # Example data line - # 7.646: [GC 7.646: [ParNew: 14778K->461K(14784K), 0.0026610 secs] 49879K->36609K(73288K), 0.0027560 secs] [Times: user=0.01 sys=0.00, real=0.01 secs] - # - # So entry 5 is the ParNew time and 8 is the whole GC cycle. 14 entries total - - data = line.split() - - gcTime = 0 - # Check we have a valid ParNew Line - if (len(data) == 15): - - # Record entires - # Record GC Duration data - - entriesGCDuration = entriesGCDuration + 1 - gcTime = float(data[8]) - - if (gcTime < minGCDuration): - minGCDuration = gcTime - - if (gcTime > maxGCDuration): - maxGCDuration = gcTime - - sumGCDuration = sumGCDuration + gcTime - - - - # Close the files - logFile.close - output.close() - - log("Sliced gc log") - - # Output stats file - statFile=sliceFile+".stats" - output= open(statFile, "w") - output.write("#type:min/max/avg") - output.write('\n') - # - # Only provide GCDuration if it was processed - # - output.write("GC_DUR:%.14f/%.14f/%.14f" % (minGCDuration, maxGCDuration , (sumGCDuration/float(entriesGCDuration)))) - output.write('\n') - output.close - - log("Generated stat data from CPU Log file") - - -def sliceLog4j(resultDir, start, end): - global _brokerLogs - logfilePath=_brokerLogs+os.sep+BROKER_LOG - log4jSliceFile=resultDir+os.sep+BROKER_LOG - - log4jstart = extractTime(ACCESS, logfilePath+".stat") - - # - # Say that first line is the start of the file, - # This value will give a time value to the initial - # logging before Log4j kicks in. - # - lineTime = log4jstart - - # Process the broker log4j file - # Open log CPU file for reading - logFile = open(logfilePath, "r") - - # - # Create outputfile - # - log4jslice = open(log4jSliceFile,"w") - - for line in logFile: - data = line.split() - - # - # If the line has a time at the start then process it - # otherwise use the previous time. This means if there is - # a stack trace in the middle of the log file then it will - # be copied over to the split file as long as it is in the - # split time. - # - if (hasTime(data)): - # - # //fixme remove tz addition. - # - lineTime = lineToDate(" ".join(data[0:2])+" +0000") - - if lineTime > start: - if lineTime < end: - print line - log4jslice.writelines(line) - - logFile.close() - log4jslice.close() - log("Sliced broker log") - - -# -# Check the first two entries of data can make a datetime object -# -def hasTime(data): - date = data[0] - time = data[1] - - # Examples: - # 2009-06-22 17:04:44,246 - # 2009-06-22 17:04:44.2464 - # 2009-06-22 17:04:44 - - # ISO-8601 '-' format date - dateRE = re.compile('[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]') - - # - # Check for times with/out millis - # e.g. - # 10:00:00,000 - log4j - # 10:00:00.0000 - generated in script for cpu time - # - timeRE = re.compile('[0-9][0-9]:[0-9][0-9]:[0-9][0-9]?[0-9]*') - - return dateRE.match(date) and timeRE.match(time) - -def createGraphData(resultDir, testName): - # Create graph.data file for process.sh - # Format two lines : Title and filename - # $version $type : $volume% volume - # $version-$brokerState-$type-$volume - version=getBrokerVersion() - - test= extractTestValue("n",resultDir, testName) - volume = int(float(extractTestResult("Test * Size Throughput", resultDir, testName)) * 1000) - messageSize = extractTestValue("messageSize",resultDir, testName) - ackMode = ackModeToString(extractTestValue("consAckMode",resultDir, testName)) - - graphDataFile=resultDir+os.sep+GRAPH_DATA - - graphData = open(graphDataFile, "w") - - # - # Write Title - graphData.write(version+":"+test+":"+str(messageSize)+"kb x "+str(volume)+" msg/sec using "+ackMode) - graphData.write('\n') - - # - # Write FileName - graphData.writelines(version+"-"+testName) - graphData.write('\n') - graphData.close - log("Created graph.data") - - -def getBrokerVersion(): - global _brokerLogs - READY = "Qpid Broker Ready" - brokerLogFile = _brokerLogs + os.sep + BROKER_LOG - - log = open(brokerLogFile, "r") - - dataLine = "" - for line in log: - if line.find(READY) != -1: - dataLine = line - break - - # Log Entry - #2009-06-19 17:04:02,493 INFO [main] server.Main (Main.java:456) - Qpid Broker Ready :2.3.0.1 build: 727403M - # Split on READY - data = dataLine.split(READY) - - # So [1] should be - # :2.3.0.1 build: 727403M - readyEntries = data[1].split() - - # so spliting on white space should give us ':version' - # and a quick split on ':' will give us the version - version = readyEntries[0].split(':')[1] - - # Strip to ensure we have no whitespace - return version.strip() - - -def extractTestValue(property,resultDir,testName): - return extractTestData(property,resultDir,testName," =") - -def extractTestResult(property,resultDir,testName): - return extractTestData(property,resultDir,testName,":") - -def extractTestData(property,resultDir,testName,type): - resultFile = resultDir + os.sep + testName+".csv" - - results = open(resultFile, "r") - - dataLine = "" - for line in results: - if line.find("Total Tests:") == 0: - dataLine = line - - results.close() - - # Data is CSV - data = dataLine.split(',') - - found = False - result = "" - searchProperty = property+type - - for entry in data: - if found: - result = entry - break - if entry.strip() == searchProperty: - found=True - - return result.strip() - -def createTestStatData(resultDir, testName): - csvFilePath=resultDir + os.sep + testName + ".csv" - - # Open the output file, erasing any existing version - # Keep track of the min/max sum and entries,. - minLatency=float(sys.maxint) - maxLatency=0.0 - minThroughput=float(sys.maxint) - maxThroughput=0.0 - - entries=0 - sumLatency=0.0 - sumThroughput=0.0 - - # - # Open csv File - # - csvFile = open(csvFilePath,"r") - for line in csvFile: - - # The PingAsyncTestPerf test class outputs the latency and throughput data. - if line.find("PingAsyncTestPerf") != -1: - # - # Data format is - # <Test> <TestName> <Thread> <Status> <Time> <Latency> <Concurrency> <Thread> <TestSize> - #org.apache.qpid.ping.PingAsyncTestPerf, testAsyncPingOk, Dispatcher-Channel-1, Pass, 209.074, 219.706, 0, 1, 10 - LatencyIndex = 5 - ThroughputIndex = 4 - - # The PingLatencyTestPerf test class just outputs the latency data. - if line.find("PingLatencyTestPerf") != -1: - # - # Data format is - # <Test> <TestName> <Thread> <Status> <Time> <Latency> <Concurrency> <Thread> <TestSize> - # org.apache.qpid.ping.PingLatencyTestPerf, testPingLatency, Dispatcher-Channel-1, Pass, 397.05502, 0, 2, 1000 - LatencyIndex = 4 - ThroughputIndex = -1 - - - # Only process the test lines that have 'org.apache.qpid.ping', i.e. skip header and footer. - if line.find("org.apache.qpid.ping") != -1: - # Perform stat processing for the min/max/avg - data = line.split(",") - - # Record entires - entries = entries + 1 - - # Record Metrics - # Record Latency data - latency = float(data[LatencyIndex]) - if (latency < minLatency): - minLatency = latency - - if (latency > maxLatency): - maxLatency = latency - - sumLatency = sumLatency + latency - - if (ThroughputIndex != -1): - # Record Latency data - throughput = float(data[ThroughputIndex]) - if (throughput < minThroughput): - minThroughput = throughput - - if (throughput > maxThroughput): - maxThroughput = throughput - - sumThroughput = sumThroughput + throughput - - - csvFile.close() - - # Output stats file - statFile=resultDir + os.sep + testName+".stats" - output= open(statFile, "w") - output.write("#type:min/max/avg") - output.write('\n') - output.write("LATENCY:"+str(minLatency)+"/"+str(maxLatency)+"/"+str(float(sumLatency)/float(entries))) - output.write('\n') - - if (ThroughputIndex != -1): - # Output msgs/sec based on time for a batch of msgs - output.write("THROUGHPUT:"+str(float(1000)/maxThroughput)+"/"+str(float(1000)/minThroughput)+"/"+str(float(1000)/(float(sumThroughput)/float(entries)))) - output.write('\n') - - output.close - - log("Generated stat data from test "+testName+" CSV file") - - -def ackModeToString(ackMode): - if ackMode == '0': - return "Transacted" - elif ackMode == '1': - return "AutoAck" - elif ackMode == '2': - return "ClientAck" - elif ackMode == '3': - return "DupsOK" - elif ackMode == '257': - return "NoAck" - elif ackMode == '258': - return "PreAck" - else: - return str(ackMode) - - - -def debug(msg): - global _debug - if _debug: - log(msg) - -def log(msg): - print msg - -def mkdir(dir): - if not os.path.exists(dir): - os.mkdir(dir) - -if __name__ == "__main__": - main()
\ No newline at end of file diff --git a/java/perftests/bin/run_many.sh b/java/perftests/bin/run_many.sh deleted file mode 100755 index cca2ffec21..0000000000 --- a/java/perftests/bin/run_many.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - - -# args: -# <number of processes to start> -# <name of run> -# <command ro run> - -for i in `seq 1 $1`; do - $3 >$2.$i.out 2>>$2.err & - echo $! > $2.$i.pid -done; diff --git a/java/perftests/bin/topicListener.sh b/java/perftests/bin/topicListener.sh deleted file mode 100755 index 3a925910ad..0000000000 --- a/java/perftests/bin/topicListener.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - - -# XXX -Xmx512m -Xms512m -XX:NewSize=150m -QPID_LIBS=$QPID_HOME/lib/qpid-all.jar - -# Set other variables used by the qpid-run script before calling -export JAVA=java \ - JAVA_VM=-server \ - JAVA_MEM="-Xmx128m -Dlog4j.configuration=$HOME/log4j.properties" \ - JAVA_GC="-XX:-UseConcMarkSweepGC -XX:+HeapDumpOnOutOfMemoryError" \ - QPID_CLASSPATH=$QPID_LIBS - -. qpid-run -Damqj.logging.level="INFO" org.apache.qpid.oldtopic.Listener $* diff --git a/java/perftests/bin/topicPublisher.sh b/java/perftests/bin/topicPublisher.sh deleted file mode 100755 index e4e9981a75..0000000000 --- a/java/perftests/bin/topicPublisher.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# XXX -Xmx512m -Xms512m -XX:NewSize=150m -QPID_LIBS=$QPID_HOME/lib/qpid-all.jar - -# Set other variables used by the qpid-run script before calling -export JAVA=java \ - JAVA_VM=-server \ - JAVA_MEM=-Xmx128m \ - JAVA_GC="-XX:-UseConcMarkSweepGC -XX:+HeapDumpOnOutOfMemoryError" \ - QPID_CLASSPATH=$QPID_LIBS - -. qpid-run -Damqj.logging.level="INFO" org.apache.qpid.oldtopic.Publisher $* |