1#!/usr/bin/env bash 2# create mon 3 4set -x 5set -e 6 7script_dir=$(readlink -f $(dirname $0)) 8 9base_dir=/var/tmp/ceph 10mon_ip=$1 11mon_dir=${base_dir}/mon.a 12pid_dir=${base_dir}/pid 13ceph_conf=${base_dir}/ceph.conf 14mnt_dir=${base_dir}/mnt 15image=/var/tmp/ceph_raw.img 16dev=/dev/loop200 17 18umount ${dev}p2 || true 19losetup -d $dev || true 20 21# partition osd 22if [ -d $base_dir ]; then 23 rm -rf $base_dir 24fi 25mkdir ${base_dir} 26cp ${script_dir}/ceph.conf $ceph_conf 27 28if [ ! -e $image ]; then 29 fallocate -l 4G $image 30fi 31 32mknod ${dev} b 7 200 || true 33losetup ${dev} ${image} || true 34 35PARTED="parted -s" 36SGDISK="sgdisk" 37 38echo "Partitioning ${dev}" 39${PARTED} ${dev} mktable gpt 40sleep 2 41 42${PARTED} ${dev} mkpart primary 0% 2GiB 43${PARTED} ${dev} mkpart primary 2GiB 100% 44 45partno=0 46echo "Setting name on ${dev}" 47${SGDISK} -c 1:osd-device-${partno}-journal ${dev} 48${SGDISK} -c 2:osd-device-${partno}-data ${dev} 49kpartx ${dev} 50 51# later versions of ceph-12 have a lot of changes, to compatible with the new 52# version of ceph-deploy. 53ceph_version=$(ceph -v | awk '{print $3}') 54ceph_maj=${ceph_version%%.*} 55if [ $ceph_maj -gt 12 ]; then 56 update_config=true 57 rm -f /var/log/ceph/ceph-mon.a.log || true 58 set_min_mon_release="--set-min-mon-release 14" 59 ceph_osd_extra_config="--check-needs-journal --no-mon-config" 60else 61 update_config=false 62 set_min_mon_release="" 63 ceph_osd_extra_config="" 64fi 65 66# prep osds 67 68mnt_pt=${mnt_dir}/osd-device-0-data 69mkdir -p ${mnt_pt} 70mkfs.xfs -f /dev/disk/by-partlabel/osd-device-0-data 71mount /dev/disk/by-partlabel/osd-device-0-data ${mnt_pt} 72cat << EOL >> $ceph_conf 73osd data = ${mnt_pt} 74osd journal = /dev/disk/by-partlabel/osd-device-0-journal 75 76# add mon address 77[mon.a] 78mon addr = ${mon_ip}:12046 79EOL 80 81# create mon 82rm -rf "${mon_dir:?}/"* 83mkdir -p ${mon_dir} 84mkdir -p ${pid_dir} 85rm -f /etc/ceph/ceph.client.admin.keyring 86 87ceph-authtool --create-keyring --gen-key --name=mon. ${base_dir}/keyring --cap mon 'allow *' 88ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' ${base_dir}/keyring 89 90monmaptool --create --clobber --add a ${mon_ip}:12046 --print ${base_dir}/monmap $set_min_mon_release 91 92sh -c "ulimit -c unlimited && exec ceph-mon --mkfs -c ${ceph_conf} -i a --monmap=${base_dir}/monmap --keyring=${base_dir}/keyring --mon-data=${mon_dir}" 93 94if [ $update_config = true ]; then 95 sed -i 's/mon addr = /mon addr = v2:/g' $ceph_conf 96fi 97 98cp ${base_dir}/keyring ${mon_dir}/keyring 99 100cp $ceph_conf /etc/ceph/ceph.conf 101 102cp ${base_dir}/keyring /etc/ceph/keyring 103cp ${base_dir}/keyring /etc/ceph/ceph.client.admin.keyring 104chmod a+r /etc/ceph/ceph.client.admin.keyring 105 106ceph-run sh -c "ulimit -n 16384 && ulimit -c unlimited && exec ceph-mon -c ${ceph_conf} -i a --keyring=${base_dir}/keyring --pid-file=${base_dir}/pid/root@$(hostname).pid --mon-data=${mon_dir}" || true 107 108# after ceph-mon creation, ceph -s should work. 109if [ $update_config = true ]; then 110 # start to get whole log. 111 ceph-conf --name mon.a --show-config-value log_file 112 113 # add fsid to ceph config file. 114 fsid=$(ceph -s | grep id | awk '{print $2}') 115 sed -i 's/perf = true/perf = true\n\tfsid = '$fsid' \n/g' $ceph_conf 116 117 # unify the filesystem with the old versions. 118 sed -i 's/perf = true/perf = true\n\tosd objectstore = filestore\n/g' $ceph_conf 119 cat ${ceph_conf} 120fi 121 122# create osd 123 124i=0 125 126mkdir -p ${mnt_dir} 127 128uuid=$(uuidgen) 129ceph -c ${ceph_conf} osd create ${uuid} $i 130ceph-osd -c ${ceph_conf} -i $i --mkfs --mkkey --osd-uuid ${uuid} ${ceph_osd_extra_config} 131ceph -c ${ceph_conf} osd crush add osd.${i} 1.0 host=$(hostname) root=default 132ceph -c ${ceph_conf} -i ${mnt_dir}/osd-device-${i}-data/keyring auth add osd.${i} osd "allow *" mon "allow profile osd" mgr "allow *" 133 134# start osd 135pkill -9 ceph-osd || true 136sleep 2 137 138mkdir -p ${pid_dir} 139env -i TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728 ceph-osd -c ${ceph_conf} -i 0 --pid-file=${pid_dir}/ceph-osd.0.pid 140