Home | History | Annotate | Download | only in ebs
      1 #
      2 # CDDL HEADER START
      3 #
      4 # The contents of this file are subject to the terms of the
      5 # Common Development and Distribution License (the License).
      6 # You may not use this file except in compliance with the License.
      7 #
      8 # You can obtain a copy of the license at usr/src/CDDL.txt
      9 # or http://www.opensolaris.org/os/licensing.
     10 # See the License for the specific language governing permissions
     11 # and limitations under the License.
     12 #
     13 # When distributing Covered Code, include this CDDL HEADER in each
     14 # file and include the License file at usr/src/CDDL.txt.
     15 # If applicable, add the following below this CDDL HEADER, with the
     16 # fields enclosed by brackets [] replaced with your own identifying
     17 # information: Portions Copyright [yyyy] [name of copyright owner]
     18 #
     19 # CDDL HEADER END
     20 #
     21 
     22 #
     23 # Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
     24 # Use is subject to license terms.
     25 #
     26 #ident	"@(#)functions.ksh	1.10	08/08/06 SMI"
     27 #
     28 
     29 PKG=SUNWscebs
     30 MYNAME=`basename $0`
     31 MYDIR=`dirname $0`
     32 MYTMPDIR=/var/tmp
     33 LOGFILE=/var/tmp/${RESOURCE}_logfile
     34 TMPFILE=/var/tmp/${RESOURCE}_sqlout
     35 SCLOGGER=/usr/cluster/lib/sc/scds_syslog
     36 
     37 syslog_tag()
     38 {
     39 	${SET_DEBUG}
     40 	print "SC[${PKG:-??}.${COMPONENT:-??}.${METHOD:-??}]:${RESOURCEGROUP:-??}:${RESOURCE:-??}"
     41 }
     42 
     43 scds_syslog()
     44 {
     45 	${SCLOGGER} "$@" &
     46 }
     47 
     48 debug_message()
     49 {
     50 	if [ "${DEBUG}" = "${RESOURCE}" -o "${DEBUG}" = "ALL" ]
     51 	then
     52 	   SET_DEBUG="set -x"
     53 	   DEBUG_TEXT=${1}
     54 
     55 	   scds_syslog -p daemon.debug -t $(syslog_tag) -m \
     56 	      "%s" "$DEBUG_TEXT"
     57         else
     58 	   SET_DEBUG=
     59         fi
     60 }
     61 
     62 log_message()
     63 {
     64 	debug_message "Function: log_message - Begin"
     65 	${SET_DEBUG}
     66 
     67 	if [ -s "${LOGFILE}" ]
     68 	then
     69 	   PRIORITY=$1
     70 	   HEADER=$2
     71 
     72 	   strings ${LOGFILE} > ${LOGFILE}.copy
     73 
     74 	   while read MSG_TXT
     75 	   do
     76 		scds_syslog -p daemon.${PRIORITY} -t $(syslog_tag) -m \
     77 		   "%s - %s" \
     78 		   "${HEADER}" "${MSG_TXT}"
     79 	   done < ${LOGFILE}.copy
     80 
     81 	   /usr/bin/cat /dev/null > ${LOGFILE}
     82 	fi
     83 
     84         debug_message "Function: log_message - End"
     85 }
     86 
     87 set_redirection()
     88 {
     89 	debug_message "Function: set_redirection - Begin"
     90 	${SET_DEBUG}
     91 
     92 	if /usr/bin/getent passwd ${APPSUSER} | /usr/bin/awk -F: '{print $7}' | /usr/bin/grep csh > /dev/null
     93 	then
     94 	   OUTPUT=">& ${LOGFILE}"
     95 	else
     96 	   OUTPUT="> ${LOGFILE} 2>&1"
     97 	fi
     98 
     99 	debug_message "Function: set_redirection - End"
    100 }
    101 
    102 query_pids()
    103 {
    104 	debug_message "Function: query_pids - Begin"
    105 	${SET_DEBUG}
    106 
    107 	# Usage: query_pids "parameter"
    108 	#
    109 	# Currently, query_pids() is used as follows,
    110 	#
    111 	# query_pids "-w FND"
    112 	# query_pids "-c -w FND"
    113 	# query_pids "-w FNDFSFNDFS"
    114 	# query_pids "-w CPMGR"
    115 	# query_pids "-E 'FND|POXCON|RCVOLTM|INCTM'"
    116 	#
    117 	# Output: rc=0 if the query is TRUE, else rc=1
    118 	#         ${pids} contains the queried process ids
    119 
    120 	rc=1
    121 
    122 	PROCESS=${1}
    123 
    124 	if [ -x /sbin/zonename ]
    125 	then
    126 	   pids=`/usr/bin/ps -u ${APPSUSER} -o pid,args,zone | /usr/bin/grep " ${ZONENAME}$" | \
    127 		eval /usr/xpg4/bin/grep ${PROCESS} | /usr/bin/grep -v grep | /usr/bin/awk '{print $1}'`
    128 	else
    129 	   pids=`/usr/bin/ps -u ${APPSUSER} -o pid,args | \
    130 		eval /usr/xpg4/bin/grep ${PROCESS} | /usr/bin/grep -v grep | /usr/bin/awk '{print $1}'`
    131 	fi
    132 
    133 	[ "${pids}" ] && rc=0
    134 
    135 	debug_message "Function: query_pids - End"
    136 	return ${rc}
    137 }
    138 
    139 validate()
    140 {
    141 	debug_message "Function: validate - Begin"
    142 	${SET_DEBUG}
    143 
    144 	rc=0
    145 
    146 	if [ ! -d "${COMNTOP}" ]
    147 	then
    148 	   # SCMSGS
    149 	   # @explanation
    150 	   # The Oracle E-Business Suite applications directory does not
    151 	   # exist.
    152 	   # @user_action
    153 	   # Check that the correct pathname was entered for the applications
    154 	   # directory when registering the resource and that the directory
    155 	   # exists.
    156 	   scds_syslog -p daemon.err -t $(syslog_tag) -m \
    157 		"Validate - Applications directory %s does not exist" \
    158 		"${COMNTOP}"
    159 	   rc=1
    160 	else
    161 	   debug_message "Validate - Applications directory ${COMNTOP} exists"
    162 	fi
    163 
    164 	if [ ! -d "${ADSCRIPT_PATH}" ]
    165 	then
    166 	   scds_syslog -p daemon.err -t $(syslog_tag) -m \
    167 		"Validate - Applications directory %s does not exist" \
    168 		"${ADSCRIPT_PATH}"
    169 	   rc=1
    170 	else
    171 	   debug_message "Validate - Applications directory ${ADSCRIPT_PATH} exists"
    172 	fi
    173 
    174 	if /usr/bin/cat /etc/passwd | /usr/bin/awk -F: '{print $1}' | /usr/bin/grep "^${APPSUSER}$" > /dev/null
    175 	then
    176 	   debug_message "Validate - Application user ${APPSUSER} exists"
    177 	else
    178 	   # SCMSGS
    179 	   # @explanation
    180 	   # The Oracle E-Business Suite applications userid was not found in
    181 	   # /etc/passwd.
    182 	   # @user_action
    183 	   # Ensure that a local applications userid is defined on all nodes
    184 	   # within the cluster.
    185 	   scds_syslog -p daemon.warning -t $(syslog_tag) -m \
    186 		"Validate - Application user <%s> does not exist" \
    187 		"${APPSUSER}"
    188 	fi
    189 
    190 	if [ "${COMPONENT}" = "cmg" ]
    191 	then
    192 	   if [ -f /opt/SUNWscebs/.${SID}_passwd ]
    193 	   then
    194 		FILEPERM="$(/usr/bin/ls -l /opt/SUNWscebs/.${SID}_passwd | /usr/bin/awk '{print $1,$3}')"
    195 
    196 		if [ "${FILEPERM}" != "-r-------- root" ]
    197 		then
    198 		   # SCMSGS
    199 		   # @explanation
    200 		   # The restricted passwd file used to start and stop the Concurrent Manager 
    201 		   # is not restricted to be read only by root.
    202 		   # @user_action
    203 		   # Change the owner and permissions so that the restricted file
    204 		   # is only readable by owner root.
    205 		   scds_syslog -p daemon.warning -t $(syslog_tag) -m \
    206 			"Validate - /opt/SUNWscebs/.%s_passwd is not restricted to be read only by root" \
    207 			"${SID}"
    208 		fi
    209 	   fi
    210 
    211 	   if [ ${#APPS_PASSWD} -eq 0 ]
    212 	   then
    213 		# SCMSGS
    214 		# @explanation
    215 		# A password used to start and stop the Concurrent Manager was not set.
    216 		# @user_action
    217 		# Either define the password within /opt/SUNWscebs/cmg/cmg_config as the value
    218 		# to the APPS_PASSWD keyword or define the password within the restricted file
    219 		# /opt/SUNWscebs/.${APP_SID}_passwd that is only readable by root.
    220 		scds_syslog -p daemon.err -t $(syslog_tag) -m \
    221 		   "Validate - Concurrent Manager password not set" 
    222 	 	rc=1
    223 	   fi
    224 
    225 	   if [ ! -d "${ORACLE_HOME}" ]
    226      	   then
    227 	      scds_syslog -p daemon.err -t $(syslog_tag) -m \
    228 		"Validate - ORACLE_HOME directory %s does not exist" \
    229 		"${ORACLE_HOME}"
    230 	      rc=1
    231 	   else
    232 	      debug_message "Validate - ORACLE_HOME directory ${ORACLE_HOME} exists"
    233 	   fi
    234 
    235 	   if [ "${CON_LIMIT}" -gt "100" ]
    236 	   then
    237 	      # SCMSGS
    238 	      # @explanation
    239 	      # The value specified for CON_LIMIT is invalid.
    240 	      # @user_action
    241 	      # Either accept the default value of CON_LIMIT=50 or change the
    242 	      # value of CON_LIMIT to be less than or equal to 100 when
    243 	      # registering the resource. CON_LIMIT=50 will now be used.
    244 	      scds_syslog -p daemon.info -t $(syslog_tag) -m \
    245 		"Validate - CON_LIMIT=%s is incorrect, default CON_LIMIT=50 is being used" \
    246 		"${CON_LIMIT}"
    247 
    248 	      CON_LIMIT=50
    249 	   else
    250 	      debug_message "Validate - CON_LIMIT=${CON_LIMIT}"
    251 	   fi
    252 	fi
    253 
    254         debug_message "Function: validate - End"
    255 	return ${rc}
    256 }
    257 
    258 orderly_stop()
    259 {
    260 	COMPONENT=${1}
    261 	SECONDS=0
    262 
    263 	debug_message "Function: orderly_stop_${COMPONENT} - Begin"
    264 	${SET_DEBUG}
    265 
    266 	case ${COMPONENT} in
    267 	   	cmg)	PATTERN="FND"
    268 			;;
    269 		cmglsr)	PATTERN="tnslsnr"
    270 			;;
    271 	   	frm) 	PATTERN="-w f60webmx"
    272 			;;
    273 	   	rep) 	PATTERN="-w rwmts60"
    274 			;;
    275 		opmn)   PATTERN="-w opmn"
    276 			;;
    277 	esac
    278 
    279 	for func in stopebs cleanup 
    280 	do
    281 	   case ${func} in
    282 		stopebs) 	((FORCE_TIME=${STOP_TIMEOUT} *75/100));;
    283 		cleanup)	((FORCE_TIME=${STOP_TIMEOUT} *5/100));;
    284 	   esac
    285 
    286 	   ${func} ${COMPONENT} "${PATTERN}"
    287 
    288 	   while [ ${SECONDS} -lt ${FORCE_TIME} ]
    289 	   do
    290 		query_pids "${PATTERN}"
    291 		rc=$?
    292 
    293 		if [ "${rc}" -eq 0 ]
    294 		then
    295 		   if [ "${COMPONENT}" = "cmg" ]
    296 		   then
    297 			query_pids "-c FND"
    298 			if [ "${pids}" -le 1 ]
    299 			then
    300 			   # If we are down to the last FND and that
    301 			   # process is CPMGR then it's safe to remove it
    302 			   query_pids "-w CPMGR"
    303 			   /usr/bin/kill ${pids}
    304 			fi
    305 		   fi
    306 
    307 		   sleep 2
    308 		else
    309 		   break
    310 		fi
    311 	   done
    312 
    313 	   # Test if we broke out of the loop or the time
    314 	   # allocated for the function has expired.
    315 
    316 	   query_pids "${PATTERN}"
    317 	   rc=$?
    318 
    319 	   if [ "${rc}" -ne 0 ]
    320 	   then
    321 		# Break out of running all the functions
    322 		break
    323 	   fi
    324 	done
    325 
    326 	debug_message "Function: orderly_stop_${COMPONENT} - End"
    327 	return 0
    328 }
    329 	
    330 cleanup()
    331 {
    332 	COMPONENT=${1}
    333 	PATTERN=${2}
    334 	
    335 	debug_message "Function: cleanup_${COMPONENT} - Begin"
    336 	${SET_DEBUG}
    337 
    338 	typeset TWOTASK_SID=
    339 
    340 	query_pids "${PATTERN}"
    341 
    342 	for pid in ${pids}
    343 	do
    344 	   TWOTASK_SID=$(/usr/bin/pargs -e ${pid} | /usr/bin/grep TWO_TASK | /usr/bin/awk -F= '{print $2}')
    345 	   [ "${TWOTASK_SID}"  = "${SID}" ] && /usr/bin/kill -9 ${pid}
    346 	done
    347 
    348 	if [ "${COMPONENT}" = "cmg" ]
    349 	then
    350 	   query_pids "-E 'FND|POXCON|RCVOLTM|INCTM'"
    351 
    352 	   for pid in ${pids}
    353 	   do
    354 		TWOTASK_SID=$(/usr/bin/pargs -e ${pid} | /usr/bin/grep TWO_TASK | /usr/bin/awk -F= '{print $2}')
    355 		[ "${TWOTASK_SID}"  = "${SID}" ] && /usr/bin/kill -9 ${pid}
    356 	   done
    357 	fi
    358 
    359 	debug_message "Function: cleanup_${COMPONENT} - End"
    360 }
    361 
    362 startebs()
    363 {
    364 	COMPONENT=${1}
    365 
    366 	debug_message "Function: startebs ${COMPONENT} - Begin"
    367 	${SET_DEBUG}
    368 	
    369 	/usr/bin/rm ${LOGFILE}
    370 
    371 	# Turn off PMF restart if the Component has been manually started
    372 	checkebs ${COMPONENT} 
    373 	rc=$?
    374 
    375 	if [ "${rc}" -eq 0 ]
    376 	then
    377 	   /usr/bin/sleep 120 &
    378 	   /usr/cluster/bin/pmfadm -s ${RESOURCEGROUP},${RESOURCE},0.svc
    379 
    380 	   # SCMSGS
    381 	   # @explanation
    382 	   # The specified Oracle E-Business Suite component has been started
    383 	   # manually.
    384 	   # @user_action
    385 	   # None required. Informational message.
    386 	   scds_syslog -p daemon.notice -t $(syslog_tag) -m \
    387 		"startebs - %s was manually started" \
    388 		"${COMPONENT}" 
    389 	
    390 	   return 0
    391 	fi
    392 		
    393 	# LHOSTNAME and LDPRELOAD are set within control_ebs
    394 
    395 	case ${COMPONENT} in
    396 		cmg)	su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} ${ADSCRIPT_PATH}/adcmctl.sh start apps/${APPS_PASSWD} ${OUTPUT}" > /dev/null
    397 			rc=$?
    398 			;;
    399 	   	cmglsr)	su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} ${ADSCRIPT_PATH}/adalnctl.sh start ${OUTPUT}" > /dev/null
    400 			rc=$?
    401 			;;
    402 	   	frm)	su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} ${ADSCRIPT_PATH}/adfrmctl.sh start ${OUTPUT}" > /dev/null
    403 			rc=$?
    404 			;;
    405 	   	rep)	su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} ${ADSCRIPT_PATH}/adrepctl.sh start ${OUTPUT}" > /dev/null
    406 			rc=$?
    407 			;;
    408 		opmn)	# New for v12 as most services are stared under Oracle Process Manager (OPMN)
    409 			# Note that the Concurrent Manager Listener and Concurrent Manager remain the 
    410 			# same and are not started under OPMN. 
    411 			#
    412 			# Note that we always the OPMN service.
    413 
    414 			su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} \
    415 			   ${ADSCRIPT_PATH}/adopmnctl.sh start ${OUTPUT}" > /dev/null
    416 			rc=$?
    417 
    418 			OPMN_COMPONENTS=`/usr/bin/echo ${OPMN_COMPONENTS} | /usr/bin/tr "/" " " | /usr/bin/tr '[:upper:]' '[:lower:]'`
    419 
    420 			if [ "${OPMN_COMPONENTS}" = "all" ]
    421 			then
    422 				OPMN_COMPONENTS="opmn http_server oacore forms oafm"
    423 			fi
    424 
    425 			for component in ${OPMN_COMPONENTS}
    426 			do
    427 			   case ${component} in
    428 				opmn)   # Oracle Process Manager is mandatory and was started earlier
    429 					;;
    430 				http_server) # Oracle HTTP Server
    431 					su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} \
    432 					   ${ADSCRIPT_PATH}/adapcctl.sh start ${OUTPUT}" > /dev/null
    433 					;;
    434 				oacore) # OACORE OC4J Instance
    435 					su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} \
    436 					   ${ADSCRIPT_PATH}/adoacorectl.sh start ${OUTPUT}" > /dev/null
    437 					;;
    438 				forms)  # FORMS OC4J Instance
    439 					su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} \
    440 					   ${ADSCRIPT_PATH}/adformsctl.sh start ${OUTPUT}" > /dev/null
    441 					;;
    442 				oafm)   # OAFM OC4J Instance
    443 					su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} \
    444 					   ${ADSCRIPT_PATH}/adoafmctl.sh start ${OUTPUT}" > /dev/null
    445 					;;
    446 			   esac
    447 			done
    448 
    449 			;;
    450 	esac
    451 
    452 	[ "${rc}" -ne 0 ] && log_message err startebs ${COMPONENT}
    453 
    454 	debug_message "Function: startebs ${COMPONENT} - End"
    455 	return ${rc}
    456 }
    457 
    458 stopebs()
    459 {
    460 	COMPONENT=${1}
    461 
    462 	debug_message "Function: stopebs ${COMPONENT} - Begin"
    463 	${SET_DEBUG}
    464 	
    465 	typeset TWOTASK_SID=
    466 
    467 	# LHOSTNAME and LDPRELOAD are set within control_ebs
    468 
    469 	case ${COMPONENT} in
    470 		cmg)	# Manually Stop any FNDFSFNDFS processes
    471 
    472 			query_pids "-w FNDFSFNDFS"
    473 
    474 			for pid in ${pids}
    475 			do
    476 			   TWOTASK_SID=$(/usr/bin/pargs -e ${pid} | /usr/bin/grep TWO_TASK | /usr/bin/awk -F= '{print $2}')
    477 			   [ "${TWOTASK_SID}"  = "${SID}" ] && /usr/bin/kill -9 ${pid}
    478 			done 
    479 		
    480 			su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} ${ADSCRIPT_PATH}/adcmctl.sh stop apps/${APPS_PASSWD} ${OUTPUT}" > /dev/null
    481 			rc=$?
    482 			;;
    483 	   	cmglsr)	su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} ${ADSCRIPT_PATH}/adalnctl.sh stop ${OUTPUT}" > /dev/null
    484 			rc=$?
    485 			;;
    486 		frm)	su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} ${ADSCRIPT_PATH}/adfrmctl.sh stop ${OUTPUT}" > /dev/null
    487 			rc=$?
    488 			;;
    489 		rep)	su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} ${ADSCRIPT_PATH}/adrepctl.sh stop ${OUTPUT}" > /dev/null
    490 			rc=$?
    491 			;;
    492 		opmn)   su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} ${ADSCRIPT_PATH}/adopmnctl.sh stopall ${OUTPUT}" > /dev/null
    493 			rc=$?
    494 			;;
    495 	esac
    496 
    497 	[ "${rc}" -ne 0 ] && log_message err stopebs ${COMPONENT}
    498 
    499 	debug_message "Function: stopebs ${COMPONENT} - End"
    500 }
    501 
    502 checkebs()
    503 {
    504 	COMPONENT=${1}
    505 
    506 	debug_message "Function: checkebs ${COMPONENT} - Begin"
    507 	${SET_DEBUG}
    508 
    509 	rc=0
    510 
    511 	case ${COMPONENT} in
    512 	   	cmg)	#
    513 			# As wait_for_online processing also calls the probe
    514 			#  we only try the full probe if there are a few 
    515 			#  concurrent manager processes running
    516 			#
    517 	
    518 			query_pids "-c -w FND"
    519 
    520 			if [ "${pids}" -le 1 ]
    521 			then
    522 			   return 100
    523 			fi
    524 
    525 			#
    526 			# Check for the Internal Manager pid
    527 			#  (Ensures a quick bail out if the IM goes down)
    528 			#
    529 
    530 			query_pids "-w CPMGR"
    531 			rc=$?
    532 
    533 			if [ "${rc}" -eq 0 ]
    534 			then
    535 			   debug_message "check_cmg - Internal Manager found"
    536 			else
    537 			   return 100
    538 			fi
    539 
    540 			# 
    541 			# Test if we can still connect to the database
    542 			#  otherwise we request a restart after  
    543 			#  two failures within the probe_interval
    544 			#
    545 
    546 			case "${VERSION}" in
    547 			   12.*)        ORA_ENVFILE=`/usr/bin/grep ORA_ENVFILE= ${ADSCRIPT_PATH}/adalnctl.sh | \
    548 					   /usr/bin/awk -F= '{print $2}' | /usr/bin/tr -d '"'`
    549 		
    550 					TNSADMIN=`/usr/bin/grep TNS_ADMIN= ${ORA_ENVFILE} | \
    551 					   /usr/bin/awk -F= '{print $2}' | /usr/bin/tr -d '"'`
    552 
    553 					ORAENV='env ORACLE_HOME=${ORACLE_HOME} env TNS_ADMIN=${TNSADMIN}'
    554 					;;
    555 			   *)           . ${ORACLE_HOME}/${APP_SID}.env
    556 					ORAENV=
    557 					;;
    558 			esac
    559 
    560 			/usr/bin/echo exit | eval ${ORAENV} \
    561        		 	   ${ORACLE_HOME}/bin/sqlplus -s apps/${APPS_PASSWD}@${SID} > ${TMPFILE}
    562 
    563 			if /usr/bin/grep ORA- $TMPFILE >/dev/null
    564 			then
    565 			   # SCMSGS
    566 			   # @explanation
    567 			   # While probing the Oracle E-Business Suite
    568 			   # concurrent manager, a test to connect to the
    569 			   # database failed.
    570 			   # @user_action
    571 			   # None, if two successive failures occur the
    572 			   # concurrent manager resource will be restarted.
    573 			   scds_syslog -p daemon.err -t $(syslog_tag) -m \
    574 				"check_cmg - Database connect to %s failed" \
    575 				"${SID}"
    576 
    577 			   return 50
    578 			else
    579 			   debug_message "check_cmg - Database connect ${SID} OK"
    580 			fi
    581 	
    582 			#
    583 			# Calculate the number of concurrent processes running as a percentage
    584 			# 	the maximum number of concurrent processes allowed
    585 			#
    586 	
    587 			output="set echo off verify off termout off feedback off head off"
    588 
    589 			sqlrun="select count(*) Actual from apps.fnd_v\$process, apps.fnd_concurrent_processes \
    590 			where Pid = Oracle_Process_ID And Spid = Os_Process_ID \
    591 			And (Process_Status_Code in ('A', 'C', 'T'));"
    592 
    593 			sqlmax="select sum(MAX_PROCESSES) Target from apps.Fnd_Concurrent_Queues ;"
    594 
    595 			# Find number of concurrent processes running
    596 
    597 			eval ${ORAENV} ${ORACLE_HOME}/bin/sqlplus -s apps/${APPS_PASSWD}@${SID} <<-END > ${TMPFILE}
    598 			${output}
    599 			${sqlrun}
    600 			END
    601 
    602 			integer FNDRUN=`/usr/bin/tail -1 ${TMPFILE} | /usr/bin/awk '{print $1}'`
    603 
    604 			# Find maximum number of concurrent processes
    605 
    606 			eval ${ORAENV} ${ORACLE_HOME}/bin/sqlplus -s apps/${APPS_PASSWD}@${SID} <<-END > ${TMPFILE}
    607 			${output}
    608 			${sqlmax}
    609 			END
    610 
    611 			integer FNDMAX=`/usr/bin/tail -1 ${TMPFILE} | /usr/bin/awk '{print $1}'`
    612 
    613 			[ ${FNDMAX} -eq 0 ] && return 100
    614 
    615 			# Calculate the percentage
    616 
    617 			integer ACTUAL=`/usr/bin/echo "${FNDRUN} / ${FNDMAX} * 100" | bc -l`
    618 
    619 			# Test if actual percentage is less than the user defined limit
    620 
    621 			if [ "${FNDRUN}" -lt "${FNDMAX}" ] 
    622 			then
    623 			   if [ ${ACTUAL} -lt ${CON_LIMIT} ]
    624 			   then
    625        		 		# SCMSGS
    626        		 		# @explanation
    627        		 		# While probing the Oracle E-Business Suite
    628        		 		# concurrent manager, the actual percentage of
    629        		 		# processes running is below the user defined
    630        		 		# acceptable limit set by CON_LIMIT when the
    631        		 		# resource was registered.
    632        		 		# @user_action
    633        		 		# Determine why the number of actual processes
    634        		 		# for the concurrent manager is below the
    635        		 		# limit set by CON_LIMIT. The concurrent
    636        		 		# manager resource will be restarted.
    637        		 		scds_syslog -p daemon.err -t $(syslog_tag) -m \
    638        		 		   "check_cmg - Actual (%s) FND processes running is below limit (%s)" \
    639 				   "${ACTUAL}%" "${CON_LIMIT}%"
    640 
    641        		 		# SCMSGS
    642        		 		# @explanation
    643        		 		# While probing the Oracle E-Business Suite
    644        		 		# concurrent manager, the actual percentage of
    645        		 		# processes running is below the user defined
    646        		 		# acceptable limit set by CON_LIMIT when the
    647        		 		# resource was registered.
    648        		 		# @user_action
    649        		 		# Determine why the number of actual processes
    650        		 		# for the concurrent manager is below the
    651        		 		# limit set by CON_LIMIT. The concurrent
    652        		 		# manager resource will be restarted.
    653        		 		scds_syslog -p daemon.err -t $(syslog_tag) -m \
    654        		 		   "check_cmg - FUNDRUN =  %s, FNDMAX = %s" \
    655 				   "${FNDRUN}" "${FNDMAX}"
    656 
    657 				rc=50
    658 		  	   else
    659 				debug_message "check_cmg - FNDRUN = ${FNDRUN}, FNDMAX = ${FNDMAX}"
    660 		  	   fi
    661 			else
    662 			   debug_message "check_cmg - FNDRUN = ${FNDRUN}, FNDMAX = ${FNDMAX}"
    663 			fi
    664 			;;
    665 	   	cmglsr)	ORA_ENVFILE=$(/usr/bin/grep ORA_ENVFILE= ${ADSCRIPT_PATH}/adalnctl.sh |\
    666 				/usr/bin/awk -F= '{print $2}' | /usr/bin/tr -d '"')
    667 
    668 			listener_name=$(/usr/bin/grep listener_name= ${ADSCRIPT_PATH}/adalnctl.sh |\
    669 				 /usr/bin/awk -F= '{print $2}' | /usr/bin/tr -d '"')
    670 
    671 			. ${ORA_ENVFILE}
    672 
    673 			lsnrctl status $listener_name > ${TMPFILE}
    674 			rc=$?
    675 
    676 			if [ "${rc}" -ne 0 ]
    677 			then
    678 			   if /usr/bin/grep TNS-01169: ${TMPFILE}
    679 			   then
    680 				rc=0
    681 			   else
    682 				rc=100
    683 			   fi
    684 			fi
    685 			
    686 			;;
    687 		frm)	query_pids "-w f60srvm ${USESID}"
    688 			rc=$?
    689 	
    690 			if [ "${rc}" -eq 0 ]
    691 			then
    692 			   query_pids "-w f60webmx ${USESID}"
    693 			   rc=$?
    694 
    695 			   [ "${rc}" -ne 0 ] && rc=50
    696 			else
    697 			   rc=100
    698 		 	fi
    699 			;;
    700 		rep)	query_pids "-w rwmts60 ${USESID}"
    701 			rc=$?
    702 	
    703 			[ "${rc}" -ne 0 ] && rc=100
    704 			;;
    705 		opmn)   # Here we need to determine if the OPMN resource is currently starting or
    706 			# if it has successfully started. This is required so that we can probe
    707 			# against different process type states, i.e.
    708 			#
    709 			# When starting, process type state = "Alive"
    710 			#
    711 			# After successful start, process type states = "Init Alive"
    712 			#
    713 			# We tolerate "Init" because OPMN will attempt to restart a failed process
    714 			# type after it has been successfully started. In this regard we let OPMN
    715 			# attempt a restart and hence tolerate "Init".
    716 
    717 			status_check=
    718 
    719 			/usr/bin/pgrep -u root -f "start_opmn .*-R ${RESOURCE} " >/dev/null 2>&1
    720 			rc=$?
    721 
    722 			if [ "${rc}" -eq 0 ]
    723 			then
    724 			   # If we are starting then we should allow OPMN some time to start the 
    725 			   # process types as opposed to keep checking the status. In this regard 
    726 			   # we'll wait 10 seconds between each "wait-for-online" probe.
    727 			   sleep 10
    728 			   status_check="Alive"
    729 			else
    730 			   status_check="Init|Alive"
    731 			fi
    732 
    733 			OPMN_COMPONENTS=`/usr/bin/echo ${OPMN_COMPONENTS} | /usr/bin/tr "/" " " | /usr/bin/tr '[:upper:]' '[:lower:]'`
    734 
    735 			if [ "${OPMN_COMPONENTS}" = "all" ]
    736 			then
    737 			   OPMN_COMPONENTS="opmn http_server oacore forms oafm"
    738 			fi
    739 
    740 			processtype=
    741 
    742 			for component in ${OPMN_COMPONENTS}
    743 			do
    744 			   case ${component} in
    745 				opmn)           # Don't use opmn as a process type
    746 						;;
    747 				http_server)    processtype="${processtype} HTTP_Server";;
    748 				*)              processtype="${processtype} ${component}";;
    749 			   esac
    750 			done
    751 
    752 			# Check if a ${processtype} is "Init" or "Alive", if not then 25 is
    753 			# added to ${rc} which then represents any cumulative failure. So,
    754 			# if only one ${processtype} is not "Init" or "Alive" then rc=25.
    755 			#
    756 			# This allows OPMN upto "4 probes" to recover a failed process
    757 			# type, i.e. If the failure persists after each probe, then rc=25
    758 			# is added to the failure history, therefore after 4 successive
    759 			# failures the failure history will sum 100.
    760 			#
    761 			# Also, if all ${processtype}'s are not "Init" or "Alive", then rc=100
    762 			# will be returned which will trigger a restart of the OPMN resource.
    763 			#
    764 			# This also ensures that the OPMN resource comes "Online" only
    765 			# after all OPMN components are "Alive" when starting.
    766 			#
    767 			# The following sample OPMN status output may help understand the process
    768 			# type states.
    769 			# -------------------+--------------------+---------+---------
    770 			# ias-component      | process-type       |     pid | status
    771 			# -------------------+--------------------+---------+---------
    772 			# OC4J               | oafm               |   12112 | Alive
    773 			# OC4J               | forms              |   11231 | Alive
    774 			# OC4J               | oacore             |   10063 | Alive
    775 			# HTTP_Server        | HTTP_Server        |    9517 | Alive
    776 			# ASG                | ASG                |     N/A | Down
    777 
    778 			su ${APPSUSER} -c "${TASK_COMMAND} env ${LHOSTNAME} env ${LDPRELOAD} \
    779 			   ${ADSCRIPT_PATH}/adopmnctl.sh status ${OUTPUT}" > /dev/null
    780 
    781 			rc=0
    782 
    783 			for pt in ${processtype}
    784 			do
    785 			   # ${LOGFILE} is encapsulated within ${OUTPUT}, refer to set_redirection().
    786 			   # ${status_check} is set earlier on within this function.
    787 
    788 			   status=`/usr/bin/grep -w ${pt} ${LOGFILE} | /usr/bin/awk -F"|" '{print $4}' | /usr/bin/tr -d ' '`
    789 
    790 			   [ "${status}" ] || status=Unknown
    791 
    792 			   if /usr/bin/echo "${status}" | /usr/xpg4/bin/grep -qE ${status_check}
    793 			   then
    794 				# Setting rc=1 simply triggers a degraded status for the fault monitor. In
    795 				# this scenario, if "Init" is an acceptable state (because the process type
    796 				# has been successfully started before) then if "Init" is found we return
    797 				# a non-zero return code which will cause the fault monitor to be degraded.
    798 				#
    799 				# Either OPMN will restart the process type or not. However, during this
    800 				# period the OPMN resource will show as degraded until either all the
    801 				# process types are successfully restarted or a restart/failover is
    802 				# successful. 
    803 				# 
    804 				# In this regard cumulative rc=1's is more than enough time for OPMN to 
    805 				# restart a resource, i.e. "Init" will either succeed or an internal OPMN
    806 				# timeout will occur causing the process type state to be down, which in
    807 				# turn will trigger a OPMN resource restart.
    808 
    809 				[ "${status}" = "Init" ] && rc=1
    810 				rc=`expr ${rc} + 0`
    811 			   else
    812 				rc=`expr ${rc} + 25`
    813 			   fi
    814 
    815 			   debug_message "Function: checkebs opmn - ${pt} is ${status}"
    816 
    817 			done
    818 
    819 			/usr/bin/cat /dev/null > ${LOGFILE}
    820 			;;
    821 	esac
    822 
    823 	debug_message "Function: checkebs ${COMPONENT} - End"
    824 	return ${rc}
    825 }
    826