path: root/src/vstart.sh

#!/bin/sh

[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3
[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=1
[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3

extra_conf=""
new=0
standby=0
debug=0
start_all=1
start_mon=0
start_mds=0
start_osd=0
localhost=0
nodaemon=0
smallmds=0
overwrite_conf=1
cephx=0

MON_ADDR=""

conf="ceph.conf"

keyring_fn="keyring"
osdmap_fn="/tmp/ceph_osdmap.$$"
monmap_fn="/tmp/ceph_monmap.$$"

usage="usage: $0 [option]... [mon] [mds] [osd]\n"
usage=$usage"options:\n"
usage=$usage"\t-d, --debug\n"
usage=$usage"\t-s, --standby_mds: Generate standby-replay MDS for each active\n"
usage=$usage"\t-n, --new\n"
usage=$usage"\t--valgrind[_{osd,mds,mon}] 'toolname args...'\n"
usage=$usage"\t-m ip:port\t\tspecify monitor address\n"
usage=$usage"\t-k keep old configuration files\n"

usage_exit() {
	printf "$usage"
	exit
}

while [ $# -ge 1 ]; do
case $1 in
    -d | --debug )
	    debug=1
	    ;;
    -s | --standby_mds)
	    standby=1
	    ;;
    -l | --localhost )
	    localhost=1
	    ;;
    --new | -n )
	    new=1
	    ;;
    --valgrind )
	    [ -z "$2" ] && usage_exit
	    valgrind=$2
	    shift
	    ;;
    --valgrind_mds )
	    [ -z "$2" ] && usage_exit
	    valgrind_mds=$2
	    shift
	    ;;
    --valgrind_osd )
	    [ -z "$2" ] && usage_exit
	    valgrind_osd=$2
	    shift
	    ;;
    --valgrind_mon )
	    [ -z "$2" ] && usage_exit
	    valgrind_mon=$2
	    shift
	    ;;
    --nodaemon )
	    nodaemon=1
	    ;;
    --smallmds )
	    smallmds=1
	    ;;
    mon )
	    start_mon=1
	    start_all=0
	    ;;
    mds )
	    start_mds=1
	    start_all=0
	    ;;
    osd )
	    start_osd=1
	    start_all=0
	    ;;
    -m )
	    [ -z "$2" ] && usage_exit
	    MON_ADDR=$2
	    shift
	    ;;
    -x )
	    cephx=1
	    ;;
    -k )
	    overwrite_conf=0
	    ;;
    -o )
	    extra_conf="$extra_conf	$2
"
	    shift
	    ;;
    * )
	    usage_exit
esac
shift
done

if [ "$start_all" -eq 1 ]; then
	start_mon=1
	start_mds=1
	start_osd=1
fi

ARGS="-c $conf"

run() {
    type=$1
    shift
    eval "valg=\$valgrind_$type"
    [ -z "$valg" ] && valg="$valgrind"

    if [ -n "$valg" ]; then
	echo "valgrind --tool=$valg $* -f &"
	valgrind --tool=$valg $* -f &
	sleep 1
    else
	if [ "$nodaemon" -eq 0 ]; then
	    echo "$*"
	    $*
	else
	    echo "ceph-run $* -f &"
	    ./ceph-run $* -f &
	fi
    fi
}

if [ "$debug" -eq 0 ]; then
    CMONDEBUG='
	debug mon = 10
        debug ms = 1'
    COSDDEBUG='
        debug ms = 1'
    CMDSDEBUG='
        debug ms = 1'
else
    echo "** going verbose **"
    CMONDEBUG='
        lockdep = 1
	debug mon = 20
        debug paxos = 20
        debug auth = 20
        debug ms = 1'
    COSDDEBUG='
        lockdep = 1
        debug ms = 1
        debug osd = 25
        debug monc = 20
        debug journal = 20
        debug filestore = 20
        debug objclass = 20'
    CMDSDEBUG='
        lockdep = 1
        debug ms = 1
        debug mds = 20
        debug auth = 20
        debug monc = 20
        mds debug scatterstat = true
        mds verify scatter = true
        mds log max segments = 2'
fi

if [ -n "$MON_ADDR" ]; then
	CMON_ARGS=" -m "$MON_ADDR
	COSD_ARGS=" -m "$MON_ADDR
	CMDS_ARGS=" -m "$MON_ADDR
fi


# lockdep everywhere?
# export CEPH_ARGS="--lockdep 1"


# sudo if btrfs
test -d dev/osd0/. && test -e dev/sudo && SUDO="sudo"

if [ "$start_all" -eq 1 ]; then
    $SUDO ./init-ceph stop
fi
$SUDO rm -f core*

test -d out || mkdir out
$SUDO rm -rf out/*
test -d gmon && $SUDO rm -rf gmon/*

[ "$cephx" -eq 1 ] && test -e $keyring_fn && rm $keyring_fn


# figure machine's ip
HOSTNAME=`hostname`
if [ "$localhost" -eq 1 ]; then
    IP="127.0.0.1"
else
    echo hostname $HOSTNAME
    RAW_IP=`hostname --ip-address`
    # filter out IPv6 and localhost addresses
    IP="$(echo "$RAW_IP"|tr ' ' '\012'|grep -v :|grep -v '^127\.'|head -n1)"
    # if that left nothing, then try to use the raw thing, it might work
    if [ -z "IP" ]; then IP="$RAW_IP"; fi
    echo ip $IP
fi
echo "ip $IP"

[ -z "$CEPH_BIN" ] && CEPH_BIN=.
[ -z "$CEPH_PORT" ] && CEPH_PORT=6789



if [ "$cephx" -eq 1 ]; then
    CEPH_ADM="$CEPH_BIN/ceph -c $conf -k $keyring_fn"
else
    CEPH_ADM="$CEPH_BIN/ceph -c $conf"
fi


MONS=""
count=0
for f in a b c d e f g h i j k l m n o p q r s t u v w x y z
do
    if [ -z "$MONS" ];
    then
	MONS="$f"
    else
	MONS="$MONS $f"
    fi
    count=$(($count + 1))
    [ $count -eq $CEPH_NUM_MON ] && break;
done

DAEMONOPTS="
	log file = out/\$name.log
        admin socket = out/\$name.asok
	chdir = \"\"
	pid file = out/\$name.pid
        heartbeat file = out/\$name.heartbeat
"


if [ "$start_mon" -eq 1 ]; then

	if [ "$new" -eq 1 ]; then
		if [ $overwrite_conf -eq 1 ]; then
		        cat <<EOF > $conf
; generated by vstart.sh on `date`
[global]
        osd pg bits = 3
        osd pgp bits = 5  ; (invalid, but ceph should cope!)
$extra_conf
EOF
			[ "$cephx" -eq 1 ] && cat<<EOF >> $conf
        auth supported = cephx
EOF
			cat <<EOF >> $conf

[client]
        keyring = keyring

[mds]
$DAEMONOPTS
$CMDSDEBUG
        mds debug frag = true
        mds debug auth pins = true
        mds debug subtrees = true
        mds data = dev/mds.\$id
[osd]
$DAEMONOPTS
        osd data = dev/osd\$id
        osd journal = dev/osd\$id.journal
        osd journal size = 100
        osd class tmp = out
        osd class dir = .libs
        osd scrub load threshold = 5.0
        filestore xattr use omap = true
$COSDDEBUG
[mon]
$DAEMONOPTS
$CMONDEBUG
        mon cluster log file = out/cluster.mon.\$id.log
EOF
		fi

		if [ `echo $IP | grep '^127\\.'` ]
		then
			echo
			echo "WARNING: hostname resolves to loopback; remote hosts will not be able to"
			echo "  connect.  either adjust /etc/hosts, or edit this script to use your"
			echo "  machine's real IP."
			echo
		fi

	        $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mon. $keyring_fn
	        $SUDO $CEPH_BIN/ceph-authtool --gen-key --name=client.admin --set-uid=0 \
		    --cap mon 'allow *' \
		    --cap osd 'allow *' \
		    --cap mds allow \
		    $keyring_fn

		# build a fresh fs monmap, mon fs
		str="$CEPH_BIN/monmaptool --create --clobber"
		count=0
		for f in $MONS
		do
			str=$str" --add $f $IP:$(($CEPH_PORT+$count))"
			if [ $overwrite_conf -eq 1 ]; then
				cat <<EOF >> $conf
[mon.$f]
        host = $HOSTNAME
        mon data = dev/mon.$f
        mon addr = $IP:$(($CEPH_PORT+$count))
EOF
			fi
			count=$(($count + 1))
		done
		str=$str" --print $monmap_fn"
		echo $str
		$str

		for f in $MONS
		do
		    cmd="$CEPH_BIN/ceph-mon --mkfs -c $conf -i $f --monmap=$monmap_fn"
		    [ "$cephx" -eq 1 ] && cmd="$cmd --keyring=$keyring_fn"
		    echo $cmd
		    $cmd
		done

		rm $monmap_fn
	fi

	# start monitors
	if [ "$start_mon" -ne 0 ]; then
		for f in $MONS
		do
		    run 'mon' $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS
		done
	fi
fi

#osd
if [ "$start_osd" -eq 1 ]; then
    for osd in `seq 0 $((CEPH_NUM_OSD-1))`
    do
	if [ "$new" -eq 1 ]; then
	    if [ $overwrite_conf -eq 1 ]; then
		    cat <<EOF >> $conf
[osd.$osd]
        host = $HOSTNAME
EOF
		    rm -rf dev/osd$osd || true
		    for f in dev/osd$osd/* ; do btrfs sub delete $f || true ; done || true
		    mkdir -p dev/osd$osd
	    fi

	    uuid=`uuidgen`
	    echo "add osd$osd $uuid"
	    $SUDO $CEPH_ADM osd create $uuid
	    $SUDO $CEPH_ADM osd crush set $osd osd.$osd 1.0 host=localhost rack=localrack pool=default
	    $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS --mkfs --mkkey --osd-uuid $uuid

	    key_fn=dev/osd$osd/keyring
	    echo adding osd$osd key to auth repository
	    $SUDO $CEPH_ADM -i $key_fn auth add osd.$osd osd "allow *" mon "allow rwx"
	fi
	echo start osd$osd
	run 'osd' $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS $COSD_ARGS
    done
fi

# mds
if [ "$smallmds" -eq 1 ]; then
    cat <<EOF >> $conf
[mds]
	mds log max segments = 2
	mds cache size = 10000
EOF
fi

if [ "$start_mds" -eq 1 ]; then
    mds=0
    for name in a b c d e f g h i j k l m n o p
    do
	if [ "$new" -eq 1 ]; then
	    mkdir -p dev/mds.$name
	    key_fn=dev/mds.$name/keyring
	    if [ $overwrite_conf -eq 1 ]; then
	    	cat <<EOF >> $conf
[mds.$name]
        host = $HOSTNAME
EOF
		if [ "$standby" -eq 1 ]; then
		    mkdir -p dev/mds.${name}s
		    cat <<EOF >> $conf
       mds standby for rank = $mds
[mds.${name}s]
        mds standby replay = true
        mds standby for name = ${name}
EOF
		fi
	    fi
	    $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mds.$name $key_fn
	    $SUDO $CEPH_ADM -i $key_fn auth add mds.$name mon 'allow *' osd 'allow *' mds 'allow'
	    if [ "$standby" -eq 1 ]; then
		    $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mds.${name}s \
			dev/mds.${name}s/keyring
                    $SUDO $CEPH_ADM -i dev/mds.${name}s/keyring auth add mds.${name}s \
			mon 'allow *' osd 'allow *' mds 'allow'
	    fi
	fi
	
	run 'mds' $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS
	if [ "$standby" -eq 1 ]; then
	    run 'mds' $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS
	fi
	
	mds=$(($mds + 1))
	[ $mds -eq $CEPH_NUM_MDS ] && break

#valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m  #--debug_ms 20
#$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20
#$CEPH_ADM mds set_max_mds 2
    done
    cmd="$CEPH_ADM mds set_max_mds $CEPH_NUM_MDS"
    echo $cmd
    $cmd
fi

echo "started.  stop.sh to stop.  see out/* (e.g. 'tail -f out/????') for debug output."