xref: /onnv-gate/usr/src/cmd/avs/sdbc/etc/dscfg_reconfigure.cluster.sh (revision 7836:4e95154b5b7a)

#!/usr/bin/ksh
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# NWS DataServices within SunCluster reconfiguration script.
#
# Description:
#
# This script is called from /usr/cluster/lib/sc/run_reserve at
# appropriate times to start and stop the NWS DataServices as SunCluster
# disk device groups are brought online or taken offline.
#
# SNDR configuration requires that a resource group to be configured.
# 1. The resource group name should be same as device group name with -stor-rg
#    added. e.g. if device group name is abc-dg then resource group name
#    would be abc-dg-stor-rg.
# 2. It should have 2 resources in it, unless one of the resource types is the
#    SUNW.GeoCtlAVS. One of type SUNW.LogicalHostname and either SUNW.HAStorage
#    or SUNW.HAStoragePlus types. Resource type versioning is ignored.
#    HAStorage type resource, should have ServicePaths property set to
#    device group name. HAStoragePlus type resource, should have either the
#    FilesystemMountPoints pointing to a files system associated with the
#    device group name, or GlobalDevicePaths property set to device group name.
#    LogicalHostname type resource should have a failoverIP address in it and
#    it will be used by SNDR to communicate with the secondary side.
#
# As SNDR requires that the LogicalHost (failover) IP address which is a
# part of resource group for SNDR, to be hosted on the same node where the
# device group is, it tries to move the resource group also alongwith the
# device group, in become_primary case of run_reserve script. While
# in primary_to_secondary case, it will try to kill the switchover function
# if it is still running in background, after stopping NWS data services.
#
# Usage:
#
# /usr/cluster/sbin/dscfg_reconfigure { start | stop } diskgroup
#
# Configuration:
#
# Scripts to be run should have been symlinked into $NWS_START_DIR and
# $NWS_STOP_DIR.  Note that the scripts are processed in lexical order,
# and that unlike /etc/rc?.d/ there is no leading S or K character.
#
# Exit status:
#
# 0 - success
# 1 - error
#

#
# Global variables
#

# this program
typeset -r ARGV0=$(basename $0)

# directory full of start scripts
typeset -r NWS_START_DIR=/usr/cluster/lib/dscfg/start

# directory full of stop scripts
typeset -r NWS_STOP_DIR=/usr/cluster/lib/dscfg/stop

# the syslog facility to use.
# - conceptually this should be based on the output of
#   "scha_cluster_get -O SYSLOG_FACILITY", but that won't work early
#   during boot.
typeset -r SYSLOG_FACILITY=daemon

PATH=$PATH:/usr/cluster/bin:/etc

# Variables for retrying scswitch of Resource group for SNDR
retry_num=12
retry_interval=10
rgname=
rgstat=
skip_resource=0
count_LogicalHostname=0
count_HAStoragePlus=0

# Since the switchover of the resource group is called in background,
# the stop action of the reconfig script will kill the background switchover
# if it is running. Since we are stopping the NWS services on the node, there
# is no need to switch the resource group, so  it is killed.
# The pid of the process is kept in file /var/run/scnws/$dg.pid.
# Input:  dg - device group
# Output: Nothing, kills the process

function kill_scswitch
{
        dg=$1
        if [ -f /var/run/scnws/$dg.pid ]
        then
                for i in `cat /var/run/scnws/$dg.pid`
                do
                        pid=$i
                        kill -9 $pid
                done
                rm -f /var/run/scnws/$dg.pid
        fi
}

# Get the status of the resource group on this node, using scha commands.
# Input: resource group - $1
# Output: Status

function get_rgstat
{
	rg=$1
	rgstat=`scha_resourcegroup_get -O RG_STATE -G $rg`
}

# This function is called in background from do_scswitch function, to
# switch the resource group to this node, which is becoming primary for
# the diskgroup. If the status of resource group is Offline, it will use
# scswitch command to switch the resource group to this node. If it has
# become Online, cleanup pid file. If it is Pending, the resource group
# is in the state of becoming online, so wait for sometime to become Online..
# scswitch may fail, so the function retries $retry_num times, waiting for
# $retry_interval seconds.
# Input: resource group - $1, Diskgroup/Diskset - $2
# Output: 0 - success, 1 - failure

function switchfunc
{
        rg=$1
        dg=$2
	how_many=0
	sleep 2
	while [ $how_many != $retry_num ]
	do
		get_rgstat $rg
		case "$rgstat" in
		"ONLINE")
		 	rm -f /var/run/scnws/$dg.pid
			return 0
			;;

		"OFFLINE")
			logger -p ${SYSLOG_FACILITY}.notice \
			-t "NWS.[$ARGV0]" `gettext "scswitch of resource group"` "$rg"

			scswitch -z -g $rg -h $(hostname)
			retval=$?
			if [ $retval != 0 ]
			then
				sleep $retry_interval
				how_many=$(($how_many + 1))
			fi
			;;

		"PENDING_ONLINE")
			logger -p ${SYSLOG_FACILITY}.notice \
			-t "NWS.[$ARGV0]" `gettext "pending online of resource group"` "$rg"
			sleep $retry_interval
			how_many=$(($how_many + 1))
			;;

		*)
			logger -p ${SYSLOG_FACILITY}.notice \
			-t "NWS.[$ARGV0]" `gettext "Improper resource group status for Remote Mirror"` "$rgstat"
		 	rm -f /var/run/scnws/$dg.pid
			return 1
			;;
		esac
	done
	logger -p ${SYSLOG_FACILITY}.err \
	-t "NWS.[$ARGV0]" "Did not switch resource group for Remote Mirror. System Administrator intervention required"
 	rm -f /var/run/scnws/$dg.pid
	return 1
}


# This function calls switchfunc function in background, to switch the
# resource group for SNDR. It validates the diskgroup/diskset is configured
# for SNDR, checks if the resource group is in Managed state etc.
# If it detects a mis-configuration, it will disable SNDR for the
# device group being processed. This is to prevent cluster hangs and panics.
#
# The ServicePaths extension property of HAStorage type resource or the
# GlobalDevicePaths extension property of HAStoragePlus, both of which
# specify the device group, serve as a link or mapping to retrieve the
# resource group associated with the SNDR configured device group.
# Switchfunc is called in the background to avoid the deadlock situation arising
# out of switchover of resource group from within device group switchover.
#
# In run_reserve context, we are doing the device group switchover, trying to
# bring it online on the node. Device group is not completely switched online,
# until the calling script run_reserve returns. In the process, we are calling
# the associated SNDR resource group switchover using scswitch command.
# Resource group switchover will trigger the switchover of device group also.
#
# If resource group switchover is called in foreground, before the device
# group has become online, then it will result in switching the device group
# again, resulting in deadlock. Resource group can not become online until
# the device group is online and the device group can not become online until the
# script returns, causing this circular dependency resulting in deadlock.
#
# Calling the resource group switch in background allows current run_reserve
# script to return immediately, allowing device group to become online.
# If the device group is already online on the node, then the resource group
# does not cause the device group switchover again.
#
# Input: Device group dg - $1
# Output: 0 - success
#	  1 - either dg not applicable for SNDR or error
#	  2 - SNDR mis-configuration

function do_scswitch
{
	dg=$1

        if [ ! -x /usr/cluster/bin/scha_resource_get \
		-o ! -x /usr/cluster/bin/scha_resourcegroup_get ]
        then
                return 1
        fi

# hard coded rg name from dg
	rgname="$dg-stor-rg"
	scha_resourcegroup_get -O rg_description -G $rgname > /dev/null
	if [ $? != 0 ]
	then
# There is no device group configured in cluster for SNDR with this cluster tag
		return 1
	fi

# Check the state of resource group

	get_rgstat $rgname
	if [ -z "$rgstat" \
		-o "$rgstat" = "UNMANAGED" -o "$rgstat" = "ERROR_STOP_FAILED" ]
	then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" \
		`gettext "Improper Remote Mirror resource group state"` "$rgstat"
        	return 2
	fi

# Check whether resources are of proper type and they are enabled

	rs_list=`scha_resourcegroup_get -O resource_list -G $rgname`
	if [ -z "$rs_list" ]
	then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" \
		`gettext "No resources in Remote Mirror resource group <$rgname>"`
		return 2
	fi
	for rs in $rs_list
	do
		rs_type=`scha_resource_get -O type -R $rs -G $rgname  | cut -d':' -f1`
		case "$rs_type" in
		SUNW.LogicalHostname)
			rs_enb=`scha_resource_get -O ON_OFF_SWITCH -R $rs -G $rgname`
			if [ "$rs_enb" = "ENABLED" ]
			then
			count_LogicalHostname=$(($count_LogicalHostname + 1))
			fi
			;;
		SUNW.HAStoragePlus)
			rs_enb=`scha_resource_get -O ON_OFF_SWITCH -R $rs -G $rgname`
			if [ "$rs_enb" = "ENABLED" ]
			then
			count_HAStoragePlus=$(($count_HAStoragePlus + 1))
			fi
			;;
		esac
	done
	if [ $count_LogicalHostname -lt 1 ]
	then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" `gettext "Missing Enabled Logical Host in resource group <$rgname> for Remote Mirror"`
		return 2
	elif [ $count_LogicalHostname -gt 1 ]
        then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" `gettext "Too Many Enabled Logical Host in resource group <$rgname> for Remote Mirror"`
		return 2
	fi

	if [ $count_HAStoragePlus -lt 1 ]
	then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" `gettext "Missing Enabled HAStoragePlus in resource group <$rgname> for Remote Mirror"`
		return 2
	elif [ $count_HAStoragePlus -gt 1 ]
	then
		logger -p ${SYSLOG_FACILITY}.notice \
		-t "NWS.[$ARGV0]" `gettext "Too Many Enabled HAStoragePlus in resource group <$rgname> for Remote Mirror"`
		return 2
	fi

# Invoke switchfunc to switch the resource group.

	switchfunc $rgname $dg &
	pid=$!
	mkdir -p /var/run/scnws/
	rm -f /var/run/scnws/$dg.pid
	echo $pid > /var/run/scnws/$dg.pid

	return 0
}


#
# Functions
#

usage()
{
	logger -p ${SYSLOG_FACILITY}.err \
	    -t "NWS.[$ARGV0]" "usage: $ARGV0 { start | stop } diskgroup"
	exit 1
}


# Input: arg1) $NWS_START_DIR - location of NWS scripts
#	 arg2) start / stop
#	 arg3 ) device group - $2
#	 arg4) sndr_ena / sndr_dis
# Output: Nothing. Log error if seen

process_dir()
{
	typeset dir=$1
	typeset arg1=$2
	typeset dg=$3
	typeset arg2=$4
	typeset RDC=$dir/10rdc

	if [[ -d $dir ]]
	then
		for f in $dir/*
		do
			# process scripts in the directories in lexical order
			# note - no leading S or K unlike /etc/rc?.d/

			if [ -s $f ] && [ $arg2 != "sndr_dis" ]
			then
				# run script and pipe output through
				# logger into syslog

				/usr/bin/ksh $f $arg1 $dg 2>&1 |
				    logger -p ${SYSLOG_FACILITY}.notice \
					-t "NWS.[${ARGV0}:$(basename $f)]"
			else
			# SNDR misconfigured - prevent start
                            if [ -s $f ] && [ $f != $RDC ]
                            then
                                # run script and pipe output through
                                # logger into syslog
                                /usr/bin/ksh $f $arg1 $dg 2>&1 |
                                    logger -p ${SYSLOG_FACILITY}.notice \
                                        -t "NWS.[${ARGV0}:$(basename $f)]"
			    fi
			fi
		done
	else
		logger -p ${SYSLOG_FACILITY}.err \
		    -t "NWS.[$ARGV0]" "no directory: $dir"
	fi
}


#
# main
#

if [ $# -ne 2 ]
then
	usage
	# not reached
fi


case "$1" in
start)
	logger -p ${SYSLOG_FACILITY}.notice -t "NWS.[$ARGV0]" "starting: $ARGV0 $*"
	do_scswitch $2
	retval=$?
	if [ $retval == 2 ]
	then
		logger -p ${SYSLOG_FACILITY}.err \
		    -t "NWS.[$ARGV0]" "**FATAL ERROR** Remote Mirror is mis-configured and DISABLED for devicegroup <"$2"> "
		# Disable SNDR
		process_dir $NWS_START_DIR start "$2" sndr_dis
	else
		process_dir $NWS_START_DIR start "$2" sndr_ena
	fi
	;;
stop)
	logger -p ${SYSLOG_FACILITY}.notice -t "NWS.[$ARGV0]" "stopping: $ARGV0 $*"
	process_dir $NWS_STOP_DIR stop "$2" sndr_ena
	kill_scswitch $2
	;;

*)
	usage
	# not reached
	;;
esac

logger -p ${SYSLOG_FACILITY}.notice -t "NWS.[$ARGV0]" "completed: $ARGV0 $*"

exit 0