ha-services/apache/apache.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the License).
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/CDDL.txt
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/CDDL.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets [] replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * apache.c - Common utilities for highly available apache
 */

#pragma ident	"@(#)apache.c	1.66	07/06/06 SMI"

#include <stdio.h>
#include <stdlib.h>
#include <strings.h>
#include <unistd.h>
#include <libintl.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <rgm/libdsdev.h>
#include <ds_common.h>
#include "apache.h"

static int get_apachectl_value(const char *ctlfile, const char *var,
    char *value);
static void remove_pidfile(scds_handle_t scds_handle);
static int validate_monitor_uri_list(scds_handle_t scds_handle,
    scds_net_resource_list_t *snrlp, boolean_t print_messages);
static int validate_server_url(char *uri, scds_net_resource_list_t *snrlp,
    boolean_t print_messages);

typedef struct run_cmd_return_struct {
	/* B_TRUE if system() returns -1, meaning fork or exec failed */
	boolean_t system_failed;
	/*
	 * B_TRUE if cmd was interrupted by a signal, i.e. if
	 * WIFSIGNALED() is true.
	 */
	boolean_t cmd_did_not_finish;

	/* Return code from system() */
	int rc_system;
	/*
	 * Return code from cmd, i.e. WEXITSTATUS(system(run_cmd))
	 * Value not valid if system_failed == B_TRUE or
	 * if cmd_did_not_finish == B_TRUE
	 */
	int rc_cmd;
} run_cmd_return_t;

/*
 * run_cmd() runs the command requested, redirecting stdout and stderr to the
 * requested place(s). A debug message is issued, if requested before
 * attempting to run the command.  See typedef definitions of run_cmd_args_t
 * and run_cmd_return_t for explanation of args to run_cmd() and return
 * information from run_cmd().
 *
 * Parameters:
 *   arg_cmd_to_run - Command run_cmd() will run
 *   arg_log_error  - whether or not to issue scds_syslog () messages
 */

void run_cmd(char *arg_cmd_to_run, boolean_t arg_log_error,
	run_cmd_return_t *argp_rc, boolean_t print_messages)
{
	/* Declare return code for internal functions */
	int rc;

	/* Initialize return info */
	argp_rc->system_failed = B_FALSE;

	argp_rc->cmd_did_not_finish = B_FALSE;
	argp_rc->rc_system = 0;
	argp_rc->rc_cmd = 0;

	/* Run the command */
	rc = system(arg_cmd_to_run);
	/* Set return value. */
	argp_rc->rc_system = rc;

	/* If system() has an error fork-ing or exec-ing... */
	if (rc == -1) {
		/* ...set return value and... */
		argp_rc->system_failed = B_TRUE;
		/* ...say so, if we're logging messages */
		rc = errno; /*lint !e746 */
		if (arg_log_error == B_TRUE) {
			/*
			 * SCMSGS
			 * @explanation
			 * Failure in executing the command.
			 * @user_action
			 * Check the syslog message for the command
			 * description. Check whether the system is low in
			 * memory or the process table is full and take
			 * appropriate action. Make sure that the executable
			 * exists.
			 */
			scds_syslog(LOG_ERR,
				"Cannot Execute %s: %s.",
				arg_cmd_to_run,
				strerror(rc));
		}
		if (print_messages) {
			(void) fprintf(stderr, gettext("Cannot Execute %s: "
				"%s.\n"), arg_cmd_to_run,
				gettext(strerror(rc)));
		}
	/* If system() ran OK... */
	} else {
		/* If we didn't complete due to being signaled... */
		if (WIFSIGNALED((uint_t)rc)) {
			/* ...set return value and... */
			argp_rc->cmd_did_not_finish = B_TRUE;
			/* ...say so, if we're logging messages */
			if (arg_log_error == B_TRUE) {
				/*
				 * SCMSGS
				 * @explanation
				 * Need explanation of this message!
				 * @user_action
				 * Need a user action for this message.
				 */
				scds_syslog(LOG_ERR,
					"%s failed to complete.",
					arg_cmd_to_run);
			}
			if (print_messages) {
				(void) fprintf(stderr, gettext("%s failed to "
					"complete.\n"), arg_cmd_to_run);
			}
		/* If we completed... */
		} else {
			/* ...get arg_cmd_to_run's exit status */
			argp_rc->rc_cmd = WEXITSTATUS((uint_t)rc);
		}
	}
}


/*
 * The initial timeout allowed  for the apache dataservice to
 * be fully up and running. We will wait for for 3 % (SVC_WAIT_PCT)
 * of the start_timeout time before probing the service.
 */
#define	SVC_WAIT_PCT		2

/*
 * We need to use 95% of probe_timeout to connect to the port and the
 * remaining time is used to disconnect from port in the svc_probe function.
 */
#define	SVC_CONNECT_TIMEOUT_PCT		95

/*
 * We need to wait for SVC_WAIT_TIME ( 5 secs) for pmf
 * to send the failure message before probing the service
 */

#define	SVC_WAIT_TIME		5

/*
 * This value will be used as disconnect timeout, if there is no
 * time left from the probe_timeout.
 */

#define	SVC_DISCONNECT_TIMEOUT_SECONDS		2

/*
 * This variable will be set by svc_validate and will also be
 * used by svc_start.
 */

static char binpath[SCDS_CMD_SIZE] = "";

/*
 * svc_validate():
 * Do apache specific validation of the resource configration.
 * Called by start/validate/update/monitor methods.
 * Return 0 on success, > 0 on failures.
 *
 * svc_validate will check for the following
 * 1. Bin_dir
 * 2. Executable permissions, if filesystem is mounted on this node
 * 3. Parse httpd.conf file
 * 4. Port_list
 * 5. Logical hostname resources
 * 6. Extension properties
 */

int
svc_validate(scds_handle_t scds_handle, boolean_t print_messages)
{
	struct stat statbuf;
	char apache_cmd[SCDS_ARRAY_SIZE];
	char cmd_buffer[SCDS_ARRAY_SIZE];
	run_cmd_return_t rcrc;
	int err = 0, rc = 0, i;
	scds_net_resource_list_t *snrlp = NULL;
	scds_port_list_t *portlist = NULL;
	scha_extprop_value_t *bindir = NULL;

	scds_hasp_status_t	hasp_status;
	/* default is to perform all fs related checks */
	boolean_t do_file_checks = B_TRUE;

	/*
	 * apachectl is the control file for "regular" apache and
	 * for mod_ssl+apache. apache-ssl uses httpsdctl instead.
	 * these are the *ctls we can work with, NULL marks end of list
	 */
	char *xctl[] = {"httpsdctl", "apachectl", NULL};

	(void) scds_get_ext_property(scds_handle, "Bin_dir",
		SCHA_PTYPE_STRING, &bindir);
	/* Check that the bindir or bindir string is not NULL */
	if (bindir == NULL || bindir->val.val_str == NULL) {
		/*
		 * SCMSGS
		 * @explanation
		 * The property has not been set by the user and must be.
		 * @user_action
		 * Reissue the scrgadm command with the required property and
		 * value.
		 */
		scds_syslog(LOG_ERR, "Property %s is not set.", "Bin_dir");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is not "
				"set.\n"), "Bin_dir");
		}
		return (1);
	} else {
		/* Copy Bin_dir path to static global buffer */
		if (strlcpy(binpath, bindir->val.val_str, sizeof (binpath))
			>= sizeof (binpath)) {
			/*
			 * SCMSGS
			 * @explanation
			 * An internal error has occurred.
			 * @user_action
			 * Save a copy of the /var/adm/messages files on all
			 * nodes. Contact your authorized Sun service provider
			 * for assistance in diagnosing the problem.
			 */
			scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
				"Insufficient space in buffer");
			if (print_messages) {
				(void) fprintf(stderr, gettext("INTERNAL "
					"ERROR: %s.\n"), gettext("Insufficient "
					"space in buffer"));
			}
			return (1);
		}
	}

	/* check for HAStoragePlus resources */
	rc = scds_hasp_check(scds_handle, &hasp_status);
	if (rc != SCHA_ERR_NOERR) {
		/* scha_hasp_check() logs everytime it fails */
		if (print_messages) {
			(void) fprintf(stderr, gettext("INTERNAL ERROR: %s.\n"),
				gettext("scds_hasp_check failed"));
		}
		return (1);
	}

	if (hasp_status == SCDS_HASP_NOT_ONLINE) {
		/*
		 * SCMSGS
		 * @explanation
		 * The resource depends on a SUNW.HAStoragePlus resource that
		 * is not online on any cluster node.
		 * @user_action
		 * Bring all SUNW.HAStoragePlus resources, that this HA-NFS
		 * resource depends on, online before performing the operation
		 * that caused this error.
		 */
		scds_syslog(LOG_ERR, "Resource depends on a "
			"SUNW.HAStoragePlus type resource that is "
			"not online anywhere.");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Resource depends on a "
				"SUNW.HAStoragePlus type resource that is "
				"not online anywhere.\n"));
		}
		return (1);
	} else if (hasp_status == SCDS_HASP_ERR_CONFIG) {
		/* problem syslogged by scds_hasp_check */
		if (print_messages) {
			(void) fprintf(stderr, gettext("This resource depends "
				"on a HAStoragePlus resouce that is in a "
				"different Resource Group. This configuration "
				"is not supported.\n"));
		}
		return (1);
	}

	/* zero out the contents of statbuf, helps avoid lint lint 644 */
	bzero(&statbuf, sizeof (statbuf));

	/*
	 * We need to work with either httpsdctl or apachectl,
	 * cant call webserver_type() to decide because it might
	 * falsely return REGULAR for some HAStoragePlus configs
	 */
	for (i = 0; xctl[i] != NULL; i++) {
		(void) snprintf(apache_cmd, sizeof (apache_cmd), "%s/%s",
				binpath, xctl[i]);
		if (stat(apache_cmd, &statbuf) != 0) {
			if (errno == ENOENT) {
				/* no cause for alarm, check next in list */
				continue;
			} else {
				/*
				 * failed for this ctl, and *not* with
				 * ENOENT. This cant be good...
				 */
				rc = errno;
				/*
				 * SCMSGS
				 * @explanation
				 * The start script is not accessible and
				 * executable. This may be due to the script
				 * not existing or the permissions not being
				 * set properly.
				 * @user_action
				 * Make sure the script exists, is in the
				 * proper directory, and has read nd execute
				 * permissions set appropriately.
				 */
				scds_syslog(LOG_ERR,
					"Cannot access start script %s: %s",
					apache_cmd, strerror(rc));
				if (print_messages) {
					(void) fprintf(stderr,
						gettext("Cannot access start "
						"script %s: %s\n"), apache_cmd,
						gettext(strerror(rc)));
				}
				return (1);
			}
		} else {
			/* stat worked: found a ctl, no need to look further */
			do_file_checks = B_TRUE;
			break;
		}
	}

	/* if xtl[i] is NULL then all stats failed with ENOENT */
	if (xctl[i] == NULL) {
		if (hasp_status == SCDS_HASP_ONLINE_NOT_LOCAL) {
			/* Bin_dir is on a failover fs thats not here */
			do_file_checks = B_FALSE;
		} else {
			/*
			 * all ENOENTs when there should have been one
			 * ctl here!! Print an error message for the
			 * last one: that should be apachectl
			 */
			scds_syslog(LOG_ERR,
					"Cannot access start script %s: %s",
					apache_cmd, strerror(ENOENT));
			if (print_messages) {
				(void) fprintf(stderr, gettext("Cannot access "
					"start script %s: %s\n"), apache_cmd,
					gettext(strerror(ENOENT)));
			}
			return (1);
		}
	}

	/*
	 * apache_cmd contains the right *ctl, statbuf has the stat()
	 * results for that ctl and do_file_checks has been (re)set,
	 * everything is fine at this point.
	 */
	if (do_file_checks) {
		/* check that the binary is executable */
		if (!(statbuf.st_mode & S_IXUSR)) {
			/*
			 * SCMSGS
			 * @explanation
			 * This file does not have the expected default
			 * execute permissions.
			 * @user_action
			 * Reset the permissions to allow execute permissions
			 * using the chmod command.
			 */
			scds_syslog(LOG_ERR,
				"Incorrect permissions set for %s.",
				apache_cmd);
			if (print_messages) {
				(void) fprintf(stderr, gettext("Incorrect "
					"permissions set for %s.\n"),
					apache_cmd);
			}
			return (1);
		}

		/* Run apachectl configtest if everything is available here */
		if ((hasp_status == SCDS_HASP_ONLINE_LOCAL) ||
			(hasp_status == SCDS_HASP_NO_RESOURCE)) {
			/* Assemble the command. */
			rc = snprintf(cmd_buffer, sizeof (cmd_buffer),
				"%s configtest >/dev/null 2>&1", apache_cmd);

			/* If the snprintf has an error... */
			if (rc < 0) {
				/* ...syslog an error message */
				char *internal_err_str = "String handling "
					"error creating apachectl configtest "
					"command";

				scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
					internal_err_str);
				if (print_messages) {
					(void) fprintf(stderr,
						gettext("INTERNAL ERROR: "
						"%s.\n"),
						gettext(internal_err_str));
				}
				return (1);
			}

			/* Issue debug message before starting. */
			scds_syslog_debug(DBG_LEVEL_LOW, "Starting server to "
				"check config file with apachectl configtest "
				"command.");

			/* Run "apachectl configtest" */
			run_cmd(cmd_buffer, B_TRUE, &rcrc, print_messages);

			/* If config file doesn't validate, return with error */
			if (rcrc.system_failed || rcrc.cmd_did_not_finish) {
				/*
				 * If system() failed or command did not finish
				 * (was interrupted), error message was already
				 * issued
				 */
				return (1);
			}
			if (rcrc.rc_cmd != 0) {
				/* non-zero return code means error */
				/*
				 * SCMSGS
				 * @explanation
				 * The command noted did not return the
				 * expected value. Additional information may
				 * be found in the error message after the
				 * ":", or in subsequent messages in syslog.
				 * @user_action
				 * This message is issued from a general
				 * purpose routine. Appropriate action may be
				 * indicated by the additional information in
				 * the message or in syslog.
				 */
				scds_syslog(LOG_ERR, "Command {%s} failed: %s.",
					cmd_buffer, "httpd cannot parse "
					"httpd.conf");
				if (print_messages) {
					(void) fprintf(stderr,
						gettext("Command {%s} failed: "
						"%s.\n"), cmd_buffer,
						gettext("httpd cannot parse "
						"httpd.conf"));
				}
				return (1);
			}
		}
	} else {
		/*
		 * SCMSGS
		 * @explanation
		 * This resource will not perform some filesystem specific
		 * checks (during VALIDATE or MONITOR_CHECK) on this node
		 * because atleast one SUNW.HAStoragePlus resource that it
		 * depends on is online on some other node.
		 * @user_action
		 * None.
		 */
		scds_syslog(LOG_INFO, "Skipping checks dependant on "
			"HAStoragePlus resources on this node.");
	} /* fs specific checks */

	/* Network aware service should have at least one port specified */

	err = scds_get_port_list(scds_handle, &portlist);
	if (err != SCHA_ERR_NOERR) {
		/*
		 * SCMSGS
		 * @explanation
		 * API operation has failed in retrieving the cluster
		 * property.
		 * @user_action
		 * For property name, check the syslog message. For more
		 * details about API call failure, check the syslog messages
		 * from other components.
		 */
		scds_syslog(LOG_ERR,
			"Failed to retrieve the property %s: %s.",
			"Port_list", scds_error_string(err));
		if (print_messages) {
			(void) fprintf(stderr, gettext("Failed to retrieve "
				"the property %s: %s.\n"), "Port_list",
				gettext(scds_error_string(err)));
		}
		goto finished_validate;
	}

	if (portlist == NULL || portlist->num_ports < 1) {
		scds_syslog(LOG_ERR, "Property %s is not set.", "Port_list");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is not "
				"set.\n"), "Port_list");
		}
		err = 1;
		goto finished_validate;
	}

	/*
	 * Return an error if there is an error when trying to get the
	 * available network address resources for this resource
	 */
	if ((err = scds_get_rs_hostnames(scds_handle, &snrlp))
		!= SCHA_ERR_NOERR) {
		/*
		 * SCMSGS
		 * @explanation
		 * Error trying to retrieve network address associated with a
		 * resource.
		 * @user_action
		 * For a failover data service, add a network address resource
		 * to the resource group. For a scalable data service, add a
		 * network resource to the resource group referenced by the
		 * RG_dependencies property.
		 */
		scds_syslog(LOG_ERR,
			"Error in trying to access the configured network "
			"resources : %s.", scds_error_string(err));
		if (print_messages) {
			(void) fprintf(stderr, gettext("Error in trying to "
				"access the configured network resources "
				": %s.\n"), gettext(scds_error_string(err)));
		}
		goto finished_validate;
	}

	/* Return an error if there are no network address resources */
	if (snrlp == NULL || snrlp->num_netresources == 0) {
		/*
		 * SCMSGS
		 * @explanation
		 * A resource has no associated network address.
		 * @user_action
		 * For a failover data service, add a network address resource
		 * to the resource group. For a scalable data service, add a
		 * network resource to the resource group referenced by the
		 * RG_dependencies property.
		 */
		scds_syslog(LOG_ERR,
			"No network address resource in resource group.");
		if (print_messages) {
			(void) fprintf(stderr, gettext("No network address "
				"resource in resource group.\n"));
		}
		err = 1;
		goto finished_validate;
	}

	/* Check to make sure other important extension props are set */
	if (scds_get_ext_monitor_retry_count(scds_handle) <= 0) {
		scds_syslog(LOG_ERR,
			"Property %s is not set.",
			"Monitor_retry_count");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is "
				"not set.\n"), "Monitor_retry_count");
		}
		err = 1; /* Validation Failure */
		goto finished_validate;
	}
	if (scds_get_ext_monitor_retry_interval(scds_handle) <= 0) {
		scds_syslog(LOG_ERR,
			"Property %s is not set.",
			"Monitor_retry_interval");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is "
				"not set.\n"), "Monitor_retry_interval");
		}
		err = 1; /* Validation Failure */
		goto finished_validate;
	}
	if (scds_get_ext_probe_timeout(scds_handle) <= 0) {
		scds_syslog(LOG_ERR,
			"Property %s is not set.",
			"Probe_timeout");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is "
				"not set.\n"), "Probe_timeout");
		}
		err = 1; /* Validation Failure */
		goto finished_validate;
	}

	/*
	 * Make sure that URIs provided (if any) look OK. Also makes sure
	 * that all the Uris have hostnames that are in the list of network
	 * resources used by the resource.
	 */
	if (validate_monitor_uri_list(scds_handle, snrlp, print_messages) !=
	    0) {
		err = 1;
		goto finished_validate;
	}

	/* All validation checks were successful */
	/*
	 * SCMSGS
	 * @explanation
	 * The validation of the configuration for the data service was
	 * successful.
	 * @user_action
	 * None. This is only an informational message.
	 */
	scds_syslog(LOG_INFO, "Successful validation.");

finished_validate:
	if (snrlp)
		scds_free_net_list(snrlp);
	if (portlist)
		scds_free_port_list(portlist);

	return (err);
}


/*
 * svc_start():
 */

int
svc_start(scds_handle_t scds_handle)
{
	char	cmd[SCDS_CMD_SIZE];
	char	*rsname = NULL, *rgname = NULL;
	int	rc = SCHA_ERR_NOERR;

	/* Get resource and resource group names */
	rsname = (char *)scds_get_resource_name(scds_handle);
	rgname = (char *)scds_get_resource_group_name(scds_handle);

	switch (webserver_type(scds_handle)) {
	case MOD_SSL: (void) snprintf(cmd, sizeof (cmd),
			"%s/apachectl startssl", binpath);
		break;
	case APACHE_SSL: (void) snprintf(cmd, sizeof (cmd),
			"%s/httpsdctl start", binpath);
		break;
	case REGULAR: (void) snprintf(cmd, sizeof (cmd),
			"%s/apachectl start", binpath);
		break;
	case ST_ERROR: /* error already logged */
	default: return (1);
	}

	/*
	 * SCMSGS
	 * @explanation
	 * Sun Cluster is starting the application with the specified command.
	 * @user_action
	 * This is an informational message, no user action is needed.
	 */
	scds_syslog(LOG_NOTICE,
		"Starting server with command %s.", cmd);

	remove_pidfile(scds_handle);

	rc = scds_pmf_start(scds_handle, SCDS_PMF_TYPE_SVC, 0, cmd, -1);

	if (rc == SCHA_ERR_NOERR) {
		/*
		 * SCMSGS
		 * @explanation
		 * The resource successfully started the application.
		 * @user_action
		 * This message is informational; no user action is needed.
		 */
		scds_syslog(LOG_NOTICE,
			"Start of %s completed successfully.", cmd);
		(void) scha_resource_setstatus(rsname, rgname,
			SCHA_RSSTATUS_OK,
			"Completed successfully.");
	} else {
		char msg[SCDS_ARRAY_SIZE];

		/*
		 * SCMSGS
		 * @explanation
		 * Sun Cluster could not start the application. It would
		 * attempt to start the service on another node if possible.
		 * @user_action
		 * 1) Check prior syslog messages for specific problems and
		 * correct them.
		 *
		 * 2) This problem may occur when the cluster is under load
		 * and Sun Cluster cannot start the application within the
		 * timeout period specified. You may consider increasing the
		 * Start_timeout property.
		 *
		 * 3) If the resource was unable to start on any node,
		 * resource would be in START_FAILED state. In this case, use
		 * scswitch to bring the resource ONLINE on this node.
		 *
		 * 4) If the service was successfully started on another node,
		 * attempt to restart the service on this node using scswitch.
		 *
		 * 5) If the above steps do not help, disable the resource
		 * using scswitch. Check to see that the application can run
		 * outside of the Sun Cluster framework. If it cannot, fix any
		 * problems specific to the application, until the application
		 * can run outside of the Sun Cluster framework. Enable the
		 * resource using scswitch. If the application runs outside of
		 * the Sun Cluster framework but not in response to starting
		 * the data service, contact your authorized Sun service
		 * provider for assistance in diagnosing the problem.
		 */
		scds_syslog(LOG_ERR,
			"Failed to start %s.", cmd);

		(void) sprintf(msg, "Failed to start %s.", APP_NAME);
		(void) scha_resource_setstatus(rsname, rgname,
			SCHA_RSSTATUS_FAULTED, msg);
	}

	return (rc);
}


/*
 * svc_wait():
 *
 * wait for the data service to start up fully and make sure it is running
 * healthy
 */

int
svc_wait(scds_handle_t scds_handle)
{
	int		svc_start_timeout, probe_timeout,
			probe_result, i;
	scha_err_t	err = SCHA_ERR_NOERR;
	scds_netaddr_list_t *netaddr = NULL;
	scha_extprop_value_t *urilist = NULL;
	scha_str_array_t *uris = NULL;

	/* obtain the network resource to use for probing */
	err = scds_get_netaddr_list(scds_handle, &netaddr);
	if (err != SCHA_ERR_NOERR) {
		scds_syslog(LOG_ERR,
			"Error in trying to access the "
			"configured network resources : %s.",
			scds_error_string(err));
		return (1);
	}

	/* Return an error if there are no network resources */
	if (netaddr == NULL || netaddr->num_netaddrs == 0) {
		scds_syslog(LOG_ERR,
			"No network address resource in resource group.");
		return (1);
	}

	/* Retrieve the list of uris that we were told to probe */
	err = scds_get_ext_property(scds_handle, "Monitor_Uri_List",
			SCHA_PTYPE_STRINGARRAY, &urilist);
	if ((err != SCHA_ERR_NOERR) && (err != SCHA_ERR_PROP)) {
		/* failed with something other than SCHA_ERR_PROP */
		scds_syslog(LOG_ERR,
				"Failed to retrieve the "
				"property %s: %s.",
				"Monitor_Uri_List", scds_error_string(err));
		scds_free_netaddr_list(netaddr);
		return (1);
	}
	if (urilist != NULL) {
		uris = urilist->val.val_strarray;
	}

	/* Get the Start method timeout and the Probe timeout value */
	svc_start_timeout = scds_get_rs_start_timeout(scds_handle);
	probe_timeout = scds_get_ext_probe_timeout(scds_handle);

	/*
	 * sleep for SVC_WAIT_PCT percentage of start_timeout time
	 * before actually probing the dataservice. This is to allow
	 * the dataservice to be fully up inorder to reply to the
	 * probe.
	 */
	if (scds_svc_wait(scds_handle,
		(svc_start_timeout * SVC_WAIT_PCT)/100)
		!= SCHA_ERR_NOERR) {
		scds_syslog(LOG_ERR, "Failed to start %s.", APP_NAME);
		scds_free_ext_property(urilist);
		scds_free_netaddr_list(netaddr);
		return (1);
	}

	do {
		scds_syslog_debug(DBG_LEVEL_HIGH,
			"Probing service, t = %d\n", time(NULL));

		/*
		 * Probe the data service on the logicalhostnames
		 * and the ports.
		 */
		probe_result = 0;
		for (i = 0; (i < netaddr->num_netaddrs) && (probe_result == 0);
		    i++) {
			probe_result = svc_probe(scds_handle,
				netaddr->netaddrs[i].hostname,
				netaddr->netaddrs[i].port_proto.port,
				probe_timeout,
				B_FALSE);
		}
		/* probe all uris (if any) that were supplied */
		if (uris) {
			for (i = 0; (i < (int)uris->array_cnt) &&
			    (probe_result == 0); i++) {
				probe_result = probe_uri(scds_handle,
				    uris->str_array[i], probe_timeout, B_TRUE);
			}
		}

		if (probe_result == SCHA_ERR_NOERR) {
			/* Success. Free up resources and return */
			scds_free_ext_property(urilist);
			scds_free_netaddr_list(netaddr);
			return (SCHA_ERR_NOERR);
		}

		/*
		 * Dataservice is still trying to come up. Sleep for a while
		 * before probing again.
		 */
		if (scds_svc_wait(scds_handle, SVC_WAIT_TIME)
			!= SCHA_ERR_NOERR) {
			scds_syslog(LOG_ERR,
				"Failed to start %s.", APP_NAME);
			scds_free_ext_property(urilist);
			scds_free_netaddr_list(netaddr);
			return (1);
		}

	/* We rely on RGM to timeout and terminate the program */
	} while (1);
}


/*
 * Stop the apache server
 * Return 0 on success, > 0 on failures.
 */

int
svc_stop(scds_handle_t scds_handle)
{
	char	*rsname = NULL, *rgname = NULL;
	char	msg[SCDS_ARRAY_SIZE];
	int	rc;

	/* Get resource and resource group names */
	rsname = (char *)scds_get_resource_name(scds_handle);
	rgname = (char *)scds_get_resource_group_name(scds_handle);

	/*
	 * SCMSGS
	 * @explanation
	 * Sun Cluster is stopping the specified application.
	 * @user_action
	 * This is an informational message, no user action is needed.
	 */
	scds_syslog(LOG_NOTICE, "Stopping %s.", APP_NAME);

	/*
	 * Give the whole timeout to scds_pmf_stop.  scds_pmf_stop()
	 * will 80% of that time to sending the SIGTERM then 15% to
	 * sending SIGKILL.
	 */

	if (scds_pmf_stop(scds_handle, SCDS_PMF_TYPE_SVC,
		SCDS_PMF_SINGLE_INSTANCE, SIGTERM,
		scds_get_rs_stop_timeout(scds_handle))
		!= SCHA_ERR_NOERR) {
		/*
		 * SCMSGS
		 * @explanation
		 * Sun Cluster failed to stop the application.
		 * @user_action
		 * Use process monitor facility (pmfadm (1M)) with -L option
		 * to retrieve all the tags that are running on the server.
		 * Identify the tag name for the application in this resource.
		 * This can be easily identified as the tag ends in the string
		 * ".svc" and contains the resource group name and the
		 * resource name. Then use pmfadm (1M) with -s option to stop
		 * the application.
		 *
		 * This problem may occur when the cluster is under load and
		 * Sun Cluster cannot stop the application within the timeout
		 * period specified. You may consider increasing the
		 * Stop_timeout property.
		 *
		 * If the error still persists, then reboot the node.
		 */
		scds_syslog(LOG_ERR, "Failed to stop %s.", APP_NAME);
	}

	/*
	 * Regardless of whether the SIGTERM succeeded send SIGKILL to
	 * the pmf tag. This will ensure that the process tree goes
	 * away if it still exists. If it doesn't exist by then, we
	 * return NOERR.
	 *
	 * Notice that this call will return with success, even if the
	 * tag does not exist by now.
	 *
	 * Timeout of -1 will wait until PMF succeeds or we are timed
	 * out by RGM.
	 */
	if ((rc = scds_pmf_stop(scds_handle, SCDS_PMF_TYPE_SVC, 0,
		SIGKILL, -1)) != SCHA_ERR_NOERR) {
		/*
		 * Failed to stop the application even with SIGKILL.
		 */
		/*
		 * SCMSGS
		 * @explanation
		 * The STOP method was unable to stop the application by
		 * sending it a SIGKILL.
		 * @user_action
		 * Contact your authorized Sun service provider to determine
		 * whether a workaround or patch is available.
		 */
		scds_syslog(LOG_ERR,
			"Failed to stop the application with SIGKILL. "
			"Returning with failure from stop method.");
		(void) sprintf(msg, "Failed to stop %s.", APP_NAME);
		(void) scha_resource_setstatus(rsname, rgname,
			SCHA_RSSTATUS_FAULTED, msg);

	} else {	/* rc == SCHA_ERR_NOERR */
		/*
		 * SCMSGS
		 * @explanation
		 * The STOP method successfully stopped the resource.
		 * @user_action
		 * This message is informational; no user action is needed.
		 */
		scds_syslog(LOG_NOTICE,
			"Successfully stopped the application");
		(void) sprintf(msg, "Successfully stopped %s.", APP_NAME);
		(void) scha_resource_setstatus(rsname, rgname,
			SCHA_RSSTATUS_OFFLINE, msg);
	}

	return (rc);
}


/*
 * This function starts the fault monitor for a apache resource.
 * This is done by starting the probe under PMF. The PMF tag
 * is derived as <RG-name,RS-name,instance_number.mon>. The restart
 * option of PMF is used but not the "infinite restart". Instead
 * interval/retry_time is obtained from the RTR file.
 */

int
mon_start(scds_handle_t scds_handle)
{
	scds_syslog_debug(DBG_LEVEL_HIGH,
		"Calling MONITOR_START method for resource %s.",
		scds_get_resource_name(scds_handle));

	/*
	 * The probe apache_probe is assumed to be available in the same
	 * subdirectory where the other callback methods for the RT are
	 * installed. The last parameter to scds_pmf_start denotes the
	 * child monitor level. Since we are starting the probe under PMF
	 * we need to monitor the probe process only and hence we are using
	 * a value of 0.
	 */
	if (scds_pmf_start(scds_handle, SCDS_PMF_TYPE_MON,
		SCDS_PMF_SINGLE_INSTANCE, "apache_probe", 0)
		!= SCHA_ERR_NOERR) {
		/*
		 * SCMSGS
		 * @explanation
		 * The fault monitor for this data service was not started.
		 * There may be prior messages in syslog indicating specific
		 * problems.
		 * @user_action
		 * The user should correct the problems specified in prior
		 * syslog messages.
		 *
		 * This problem may occur when the cluster is under load and
		 * Sun Cluster cannot start the application within the timeout
		 * period specified. You may consider increasing the
		 * Monitor_Start_timeout property.
		 *
		 * Try switching the resource group to another node using
		 * scswitch (1M).
		 */
		scds_syslog(LOG_ERR,
			"Failed to start fault monitor.");
		return (1);
	}

	/*
	 * SCMSGS
	 * @explanation
	 * The fault monitor for this data service was started successfully.
	 * @user_action
	 * No action needed.
	 */
	scds_syslog(LOG_INFO, "Started the fault monitor.");

	return (SCHA_ERR_NOERR);
}


/*
 * This function stops the fault monitor for a apache resource.
 * This is done via PMF. The PMF tag for the fault monitor is
 * constructed based on <RG-name_RS-name,instance_number.mon>.
 */

int
mon_stop(scds_handle_t scds_handle)
{
	scds_syslog_debug(DBG_LEVEL_HIGH,
		"Calling MONITOR_STOP method for resource %s.",
		scds_get_resource_name(scds_handle));

	if (scds_pmf_stop(scds_handle, SCDS_PMF_TYPE_MON,
		SCDS_PMF_SINGLE_INSTANCE, SIGKILL, -1)
		!= SCHA_ERR_NOERR) {
		/*
		 * SCMSGS
		 * @explanation
		 * An attempt was made to stop the fault monitor and it
		 * failed. There may be prior messages in syslog indicating
		 * specific problems.
		 * @user_action
		 * If there are prior messages in syslog indicating specific
		 * problems, these should be corrected. If that doesn't
		 * resolve the issue, the user can try the following.
		 *
		 * Use process monitor facility (pmfadm (1M)) with -L option
		 * to retrieve all the tags that are running on the server.
		 * Identify the tag name for the fault monitor of this
		 * resource. This can be easily identified as the tag ends in
		 * string ".mon" and contains the resource group name and the
		 * resource name. Then use pmfadm (1M) with -s option to stop
		 * the fault monitor.
		 *
		 * This problem may occur when the cluster is under load and
		 * Sun Cluster cannot stop the fault monitor within the
		 * timeout period specified. You may consider increasing the
		 * Monitor_Stop_timeout property.
		 *
		 * If the error still persists, then reboot the node.
		 */
		scds_syslog(LOG_ERR,
			"Failed to stop fault monitor.");
		return (1);
	}

	/*
	 * SCMSGS
	 * @explanation
	 * The fault monitor for this data service was stopped successfully.
	 * @user_action
	 * No action needed.
	 */
	scds_syslog(LOG_INFO, "Stopped the fault monitor.");

	return (SCHA_ERR_NOERR);
}


/*
 * svc_probe(): Do data service specific probing. Return a value
 * between 0 (success) and 100(complete failure).
 *
 * The probe does a simple socket connection to the apache server on the
 * specified port which is configured as the resource extension property
 * (Port_list) and pings the dataservice. If the probe fails to connect to
 * the port, we return a value of 100 indicating that there is a total
 * failure. If the connection goes through and the disconnect to the port
 * fails, then a value of 50 is returned indicating a partial failure.
 */

int
svc_probe(scds_handle_t scds_handle, char *hostname,
	int port, int timeout, boolean_t arg_syslog_msgs)
{
	ulong_t	t1, t2;
	int	sock, rc = 0, retval = 0, time_used, time_remaining;
	size_t	size = 0;
	char	buf[SCDS_ARRAY_SIZE];
	long	connect_timeout;
	server_type st;


	/*
	 * Probe the dataservice by doing a socket connection to the port
	 * specified in the port_list property to the host that is
	 * serving the apache dataservice. If the apache service which is
	 * configured to listen on the specified port, replies to the
	 * connection, then the probe is successfull. Else we will wait for
	 * a time period set in probe_timeout property before concluding
	 * that the probe failed.
	 */

	/*
	 * Use the SVC_CONNECT_TIMEOUT_PCT percentage of timeout
	 * to connect to the port
	 */
	connect_timeout = (SVC_CONNECT_TIMEOUT_PCT * timeout)/100;
	t1 = (ulong_t)(gethrtime()/1E9);

	/*
	 * the probe makes a connection to the specified hostname and port.
	 * The connection is timed for 95% of the actual probe_timeout.
	 */
	rc = scds_fm_tcp_connect(scds_handle, &sock, hostname, port,
		connect_timeout);
	if (rc) {
		/*
		 * SCMSGS
		 * @explanation
		 * An error occurred while fault monitor attempted to probe
		 * the health of the data service.
		 * @user_action
		 * Wait for the fault monitor to correct this by doing restart
		 * or failover. For more error description, look at the syslog
		 * messages.
		 */
		scds_syslog(LOG_ERR,
			"Failed to connect to host %s and "
			"port %d: %s.",
			hostname, port, strerror(errno));
		/* this is a complete failure */
		return (SCDS_PROBE_COMPLETE_FAILURE);
	} else {
		scds_syslog_debug(DBG_LEVEL_LOW,
			"Successful connection to server %s "
			"port %d for resource %s.",
			hostname, port,
			scds_get_resource_name(scds_handle));
	}

	t2 = (ulong_t)(gethrtime()/1E9);

	/*
	 * Compute the actual time it took to connect. This should be less than
	 * or equal to connect_timeout, the time allocated to connect.
	 * If the connect uses all the time that is allocated for it,
	 * then the remaining value from the probe_timeout that is passed to
	 * this function will be used as disconnect timeout. Otherwise, the
	 * the remaining time from the connect call will also be added to
	 * the disconnect timeout.
	 *
	 */

	time_used = (int)(t2 - t1);

	/*
	 * Use the remaining time(timeout - time_took_to_connect) to disconnect
	 */

	time_remaining = timeout - time_used;

	/* what type of server is it? */
	st = webserver_type(scds_handle);
	if (st == ST_ERROR) {
		/* couldnt even figure out what the server is */
		retval = SCDS_PROBE_COMPLETE_FAILURE;
		goto finished;
	}

	/*
	 * If all the time is used up, use a small hardcoded timeout
	 * to still try to disconnect. This will avoid the fd leak.
	 */
	if (time_remaining <= 0) {
		scds_syslog_debug(DBG_LEVEL_LOW,
			"svc_probe used entire timeout of "
			"%d seconds during connect operation "
			"and exceeded the timeout by %d seconds. "
			"Attempting disconnect with timeout %d",
			connect_timeout,
			abs(time_remaining),
			SVC_DISCONNECT_TIMEOUT_SECONDS);

		time_remaining = SVC_DISCONNECT_TIMEOUT_SECONDS;
	} else if (st == REGULAR) { /* dont bother secure servers */

		/* Generic HTML/1.0 HEAD check */
		(void) strcpy(buf, "HEAD / HTTP/1.0\n\n");

		size = strlen(buf);
		if (scds_fm_tcp_write(scds_handle, sock,
			buf, &size, time_remaining) < 0) {
			/*
			 * write()s should never fail unless the server
			 * (apache) has closed its end of the socket.
			 * That sounds like a serious problem. Hence 1
			 * as probe result.
			 */
			retval = SCDS_PROBE_COMPLETE_FAILURE;
			if (arg_syslog_msgs) {
				/*
				 * SCMSGS
				 * @explanation
				 * The data service fault monitor probe was
				 * trying to read from or write to the service
				 * specified and failed. Sun Cluster will
				 * attempt to correct the situation by either
				 * doing a restart or a failover of the data
				 * service. The problem may be due to an
				 * overloaded system or other problems,
				 * causing a timeout to occur before
				 * communications could be completed.
				 * @user_action
				 * If this problem is due to an overloaded
				 * system, you may consider increasing the
				 * Probe_timeout property.
				 */
				scds_syslog(LOG_ERR,
					"Failed to communicate with "
					"server %s port %d: %s.",
					hostname, port, strerror(errno));
			}

			goto finished;
		}

		/*
		 * Data sent to us by server may span several packets,
		 * hence must do things in a loop().
		 */
		do {
			t2 = (ulong_t)(gethrtime()/1E9);
			time_used = (int)(t2 - t1);
			time_remaining = timeout - time_used;
			if (time_remaining &l