ha-services/dns/dns.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the License).
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/CDDL.txt
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/CDDL.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets [] replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * dns.c - Common utilities for highly available DNS
 *
 */
#pragma ident	"@(#)dns.c	1.37	07/06/06 SMI"

#include <strings.h>
#include <sys/wait.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <errno.h>
#include <stdio.h>
#include <libintl.h>
#include <ds_common.h>
#include "dns.h"

char 	*svcs[] = {"/network/dns/server:default", NULL};

/*
 * The initial timeout allowed  for the dataservice to
 * be fully up and running.
 */
#define	SVC_WAIT_PCT		0

/*
 * SVC_WAIT_TIME is used only during starting in svc_wait().
 * In svc_wait() we need to be sure that the service is up
 * before returning, thus we need to call svc_probe() to
 * monitor the service. SVC_WAIT_TIME is the time between
 * such probes.
 */
#define		SVC_WAIT_TIME		2

/*
 * svc_validate():
 * Do DNS specific validation of the resource configration.
 * Called by start/validate/update/monitor methods.
 *
 * If print_messages is true, it will also print any messages
 * to stderr (we assume the locale has been set for us).
 * This is in addition to syslogging.
 *
 * Return 0 on success, > 0 on failures.
 */

int
svc_validate(scds_handle_t handle, char *mode, boolean_t print_messages)
{
	char	dns_named_dir[SCDS_ARRAY_SIZE];
	char	dns_config[SCDS_ARRAY_SIZE];
	int 	rc;
	scha_str_array_t *config_dir;
	scds_net_resource_list_t *snrlp;
	struct stat statbuf;
	scds_port_list_t	*port_list;
	scds_hasp_status_t	hasp_status;


	/*
	 * Just in case! Actually the caller should
	 * make sure mode is not NULL
	 */
	if (mode == NULL) {
		scds_syslog(LOG_ERR, "Property %s is not set.", DNS_MODE_USED);
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is not "
				"set.\n"), DNS_MODE_USED);
		}
		return (1);
	}


	if (os_newer_than_s10()) {
		rc = check_disabled_smf_services(svcs, print_messages);
		if (rc != 0)
			return (1);
	}


	/* There should be only 1 port for HA DNS */
	rc = scds_get_port_list(handle, &port_list);
	if (rc != SCHA_ERR_NOERR) {
		/*
		 * SCMSGS
		 * @explanation
		 * An API operation has failed while retrieving the resource
		 * property. Low memory or API call failure might be the
		 * reasons.
		 * @user_action
		 * In case of low memory, the problem will probably cured by
		 * rebooting. If the problem reoccurs, you might need to
		 * increase swap space by configuring additional swap devices.
		 * Otherwise, if it is API call failure, check the syslog
		 * messages from other components. For the resource name and
		 * property name, check the current syslog message.
		 */
		scds_syslog(LOG_ERR, "Failed to retrieve the resource "
			"property %s: %s.", SCHA_PORT_LIST,
			scds_error_string(rc));
		if (print_messages) {
			(void) fprintf(stderr, gettext("Failed to retrieve the "
				"resource property %s: %s.\n"), SCHA_PORT_LIST,
				gettext(scds_error_string(rc)));
		}
		return (1);
	}
	if (port_list->num_ports != 1) {
		/*
		 * SCMSGS
		 * @explanation
		 * A multi-valued (comma-separated) list was provided to the
		 * scrgadm command for the property, while the implementation
		 * supports only one value for this property.
		 * @user_action
		 * Specify a single value for the property on the scrgadm
		 * command.
		 */
		scds_syslog(LOG_ERR, "Property %s should have only one value.",
		    SCHA_PORT_LIST);
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s should "
				"have only one value.\n"), SCHA_PORT_LIST);
		}
		return (1);
	}
	scds_free_port_list(port_list);

	/*
	 * Return an error if unable to get the Logical host resources
	 * to use for this resource
	 */
	rc = scds_get_rs_hostnames(handle, &snrlp);
	if (rc != SCHA_ERR_NOERR) {
		scds_syslog(LOG_ERR, "Failed to retrieve the resource "
			"property %s: %s.", SCHA_NETWORK_RESOURCES_USED,
			scds_error_string(rc));
		if (print_messages) {
			(void) fprintf(stderr, gettext("Failed to retrieve the "
				"resource property %s: %s.\n"),
				SCHA_NETWORK_RESOURCES_USED,
				gettext(scds_error_string(rc)));
		}
		return (1);
	}
	/* Return error if there are no Logicalhost resources configured */
	if ((snrlp == NULL) || (snrlp->num_netresources == 0)) {
		/*
		 * SCMSGS
		 * @explanation
		 * The probe method for this data service could not find a
		 * LogicalHostname resource in the same resource group as the
		 * data service.
		 * @user_action
		 * Use scrgadm to configure the resource group to hold both
		 * the data service and the LogicalHostname.
		 */
		scds_syslog(LOG_ERR, "No LogicalHostname resource in "
			"resource group.");
		if (print_messages) {
			(void) fprintf(stderr, gettext("No LogicalHostname "
				"resource in resource group.\n"));
		}
		return (1);
	}
	scds_free_net_list(snrlp);

	if (scds_get_ext_monitor_retry_count(handle) <= 0) {
		scds_syslog(LOG_ERR, "Property %s is not set.",
		    "Monitor_retry_count");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is not "
				"set.\n"), "Monitor_retry_count");
		}
		return (1);
	}

	if (scds_get_ext_monitor_retry_interval(handle) <= 0) {
		scds_syslog(LOG_ERR, "Property %s is not set.",
		    "Monitor_retry_interval");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is not "
				"set.\n"), "Monitor_retry_interval");
		}
		return (1);
	}

	if (scds_get_ext_probe_timeout(handle) <= 0) {
		scds_syslog(LOG_ERR, "Property %s is not set.",
		    "Probe_timeout");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is not "
				"set.\n"), "Probe_timeout");
		}
		return (1);
	}

	/* check for HAStoragePlus resources */
	rc = scds_hasp_check(handle, &hasp_status);
	if (rc != SCHA_ERR_NOERR) {
		/* scds_hasp_check() logs everytime it fails */
		if (print_messages) {
			(void) fprintf(stderr, gettext("INTERNAL ERROR: %s.\n"),
				gettext("scds_hasp_check failed"));
		}
		return (1);
	}

	if (hasp_status == SCDS_HASP_NOT_ONLINE) {
		scds_syslog(LOG_ERR, "Resource depends on a "
			"SUNW.HAStoragePlus type resource that is "
			"not online anywhere.");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Resource depends on a "
				"SUNW.HAStoragePlus type resource that is "
				"not online anywhere.\n"));
		}
		return (1);
	} else if (hasp_status == SCDS_HASP_ERR_CONFIG) {
		/* problem syslogged by scds_hasp_check */
		if (print_messages) {
			(void) fprintf(stderr, gettext("This resource depends "
				"on a HAStoragePlus resouce that is in a "
				"different Resource Group. This configuration "
				"is not supported.\n"));
		}
		return (1);
	}

	config_dir = scds_get_ext_confdir_list(handle);
	if ((config_dir == NULL) || (config_dir->array_cnt == 0)) {
		scds_syslog(LOG_ERR, "Property %s is not set.",
			"Confdir_list");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Property %s is not "
				"set.\n"), "Confdir_list");
		}
		return (1);
	}

	if (config_dir->array_cnt > 1) {
		/*
		* SCMSGS
		* @explanation
		* Failover data service must have one and only one
		* value for Confdir_list.
		* @user_action
		* Create a failover resource group for each
		* configuration file.
		*/
		scds_syslog(LOG_ERR,
			"Failover %s data service must have exactly "
			"one value for extension property %s.",
			APP_NAME, "Config_dir");
		if (print_messages) {
			(void) fprintf(stderr, gettext("Failover %s data "
					"service must have exactly "
					"one value for extension "
					"property %s.\n"),
				APP_NAME, "Config_dir");
		}
		return (1);
	}

	scds_syslog_debug(DBG_LEVEL_LOW, "The DNS config mode is %s.", mode);
	if (strcmp(mode, "conf") == 0) {
		rc = snprintf(dns_config, sizeof (dns_config),
		    "%s/named.conf", config_dir->str_array[0]);
		if (rc == -1) {
			scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
			    "String handling error creating path to "
			    "configuration file: named.conf. "
			    "The path may be too long");
			if (print_messages) {
				(void) fprintf(stderr, gettext("INTERNAL "
					"ERROR: %s.\n"),
					gettext("String handling error "
					"creating path to configuration file: "
					"named.conf. The path may be too "
					"long"));
			}
			return (1);
		}
	} else if (strcmp(mode, "boot") == 0) {
		rc = snprintf(dns_config, sizeof (dns_config),
		    "%s/named.boot", config_dir->str_array[0]);
		if (rc == -1) {
			scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
			    "String handling error creating path to "
			    "configuration file: named.boot. "
			    "The path may be too long");
			if (print_messages) {
				(void) fprintf(stderr, gettext("INTERNAL "
					"ERROR: %s.\n"),
					gettext("String handling error "
					"creating path to configuration file: "
					"named.boot. The path may be too "
					"long"));
			}
			return (1);
		}
	} else {
		/*
		 * SCMSGS
		 * @explanation
		 * An invalid value was supplied for the property.
		 * @user_action
		 * Supply "conf" or "boot" as the value for DNS_mode property.
		 */
		scds_syslog(LOG_ERR, "Invalid value %s for "
			"property %s.", mode, DNS_MODE_USED);
		if (print_messages) {
			(void) fprintf(stderr, gettext("Invalid value %s for "
				"property %s.\n"), mode, DNS_MODE_USED);
		}
		return (1);
	}

	if (stat(dns_config, &statbuf) != 0) {
		/*
		 * if hasp_status is not SCDS_HASP_ONLINE_NOT_LOCAL and the
		 * stat above fails, we are in trouble. Also, if the stat
		 * fails for anything else other than an ENOENT (when
		 * hasp_status is SCDS_HASP_ONLINE_NOT_LOCAL), thats also
		 * an error.
		 */
		if ((hasp_status != SCDS_HASP_ONLINE_NOT_LOCAL) ||
			(errno != ENOENT)) {			/*lint !e746 */
			rc = errno;
			scds_syslog(LOG_ERR,
			    "File %s is not readable: %s.",
			    dns_config, strerror(rc)); 	/*lint !e746 */
			if (print_messages) {
				(void) fprintf(stderr, gettext("File %s is not "
					"readable: %s.\n"), dns_config,
					gettext(strerror(rc)));
			}
			return (1);
		}
	}

	/*
	 * Check to see if the named directory which houses
	 * the database files for the Zones in DNS is accessible
	 */
	rc = snprintf(dns_named_dir, sizeof (dns_named_dir), "%s/named",
	    config_dir->str_array[0]);
	if (rc == -1) {
		scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
		    "String handling error creating path to "
		    "database directory. The path may be too long");
		if (print_messages) {
			(void) fprintf(stderr, gettext("INTERNAL ERROR: %s.\n"),
				gettext("String handling error creating path "
				"to database directory. The path may be too "
				"long"));
		}
		return (1);
	}


	if (stat(dns_named_dir, &statbuf) != 0) {
		if ((hasp_status != SCDS_HASP_ONLINE_NOT_LOCAL) ||
			(errno != ENOENT)) {
			rc = errno;
			/*
			 * SCMSGS
			 * @explanation
			 * The DNS database directory is not readable. This
			 * may be due to the directory not existing or the
			 * permissions not being set properly.
			 * @user_action
			 * Make sure the directory exists and has read
			 * permission set appropriately. Look at the prior
			 * syslog messages for any specific problems and
			 * correct them.
			 */
			scds_syslog(LOG_ERR, "DNS database directory %s is "
				"not readable: %s", dns_named_dir,
				strerror(rc));
			if (print_messages) {
				(void) fprintf(stderr, gettext("DNS database "
					"directory %s is not readable: %s\n"),
					dns_named_dir,
					gettext(strerror(rc)));
			}
			return (1);
		}
	}


	/* check that the binary is accessible */
	if (stat(DNS_BINARY, &statbuf) != 0) {
		rc = errno;
		scds_syslog(LOG_ERR, "File %s is not readable: %s.",
			DNS_BINARY, strerror(rc));
		if (print_messages) {
			(void) fprintf(stderr, gettext("File %s is not "
				"readable: %s.\n"), DNS_BINARY,
				gettext(strerror(rc)));
		}
		return (1);
	}
	/* check that the binary is executable */
	if ((statbuf.st_mode & S_IXUSR) != S_IXUSR) {
		scds_syslog(LOG_ERR, "Incorrect permissions set for %s.",
			DNS_BINARY);
		if (print_messages) {
			(void) fprintf(stderr, gettext("Incorrect permissions "
				"set for %s.\n"), DNS_BINARY);
		}
		return (1);
	}

	return (SCHA_ERR_NOERR);
}

int
svc_start(scds_handle_t handle, char *mode)
{
	char	dns_config[SCDS_ARRAY_SIZE];
	char	dns_named[SCDS_ARRAY_SIZE];
	char	cmd[SCDS_ARRAY_SIZE];
	int 	rc;
	scha_str_array_t *config_dir;

	scds_syslog_debug(DBG_LEVEL_HIGH, "Calling START method for "
		"resource %s.", scds_get_resource_name(handle));

	/* Make sure everything looks OK */
	if (svc_validate(handle, mode, B_FALSE) != 0) {
		scds_syslog(LOG_ERR, "Failed to validate configuration.");
		return (1); /* Bail out, no point in continuing with this res */
	}

	scds_syslog_debug(DBG_LEVEL_LOW, "DNS_mode property is set to %s.",
		mode);

	/* If Solaris 10 or later, disable SMF service */
	if (os_newer_than_s10()) {
		rc = disable_smf_services(svcs);
		if (rc != 0)
			return (1);
	}

	config_dir = scds_get_ext_confdir_list(handle);
	if (config_dir == NULL) {
		scds_syslog(LOG_ERR, "Property %s is not set.",
			"Confdir_list");
		return (1);
	}

	if (strcmp(mode, "conf") == 0) {
		rc = snprintf(dns_config, sizeof (dns_config), "%s/named.conf",
			config_dir->str_array[0]);
		if (rc == -1) {
			scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
				"String handling error creating path to "
				"configuration file: named.conf. "
				"The path may be too long");
			return (1);
		}
	} else if (strcmp(mode, "boot") == 0) {
			rc = snprintf(dns_config, sizeof (dns_config),
			    "%s/named.boot", config_dir->str_array[0]);
			if (rc == -1) {
				scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
					"String handling error creating path "
					"to configuration file: named.boot. "
					"The path may be too long");
				return (1);
			}
	} else {
		scds_syslog(LOG_ERR, "Invalid value %s for property %s.",
			mode, DNS_MODE_USED);
		return (1);
	}

	rc = snprintf(dns_named, sizeof (dns_named), DNS_BINARY);
	if (rc == -1) {
		scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
			"String handling error creating path to "
			"in.named. The path may be too long");
		return (1);
	}


	/* Now construct the command to start DNS */
	rc = snprintf(cmd, sizeof (cmd), "%s -c %s ", dns_named, dns_config);
	if (rc == -1) {
		scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
			"String handling error creating "
			"start command. The path may be too long");
		return (1);
	}

	/*
	 * SCMSGS
	 * @explanation
	 * Sun Cluster is starting the specified application with the
	 * specified command.
	 * @user_action
	 * This is an informational message, no user action is needed.
	 */
	scds_syslog(LOG_NOTICE, "Starting %s with command %s.", APP_NAME, cmd);

	/* Start DNS */
	if (scds_pmf_start(handle, SCDS_PMF_TYPE_SVC, 0, cmd, -1) !=
		SCHA_ERR_NOERR) {
		char msg[SCDS_ARRAY_SIZE];

		(void) snprintf(msg, sizeof (msg), "Failed to start %s.",
			APP_NAME);

		/* cant use scds syslog(LOG_ERR, msg); make scmsgs complains */
		scds_syslog(LOG_ERR, "Failed to start %s.", APP_NAME);

		(void) scha_resource_setstatus(scds_get_resource_name(handle),
			scds_get_resource_group_name(handle),
			SCHA_RSSTATUS_FAULTED, msg);
		exit(1);
	}

	scds_syslog(LOG_INFO, "Completed successfully.");
	(void) scha_resource_setstatus(scds_get_resource_name(handle),
		scds_get_resource_group_name(handle), SCHA_RSSTATUS_OK,
		"Completed successfully.");

	return (SCHA_ERR_NOERR);
}


/*
 * dns_svc_start() calls svc_wait() just after it calls svc_start()
 * and before it returns.  svc_start() starts up the application (DNS
 * server), but does not wait for the application to complete coming up
 * before returning.
 *
 * The RGM framework specifies that the START method should not return until
 * the application is up.  svc_wait() verifies that the application is
 * up before it returns.  It does this by probing the application across all
 * its port/ip combinations.  When the probing is successful, svc_wait()
 * returns immediately indicating success.  When the probing is unsuccessful
 * (i.e., svc_wait() probed without success for it's allotted time,
 * SVC_TIMEOUT_PCT/100 * start method timeout value), svc_wait() returns
 * an error.
 *
 * Since we don't want the START method to timeout, we don't try to probe
 * for 100% of the start method timeout value, but only SVC_TIMEOUT_PCT/100
 * of it.  Also, since probing too early crashes some applications, we wait
 * a percentage of the start method timeout value before starting to probe,
 * SVC_WAIT_PCT.
 *
 * Returns: 0=probing succeeded, application is up
 *          1=time ran out without a successful probe, application wasn't
 *            determined to be up.
 */
int
svc_wait(scds_handle_t handle)
{
	int err = 0, svc_start_timeout, probe_result;

	svc_start_timeout = scds_get_rs_start_timeout(handle);

	/*
	 * sleep for SVC_WAIT_PCT percentage of start_timeout time
	 * before actually probing the dataservice. This is to allow
	 * the dataservice to be fully up inorder to reply to the
	 * probe. NOTE: the value for SVC_WAIT_PCT could be different
	 * for different dataservices.
	 * Instead of calling sleep(),
	 * call scds_svc_wait() so that if service fails too
	 * many times, we give up and return early.
	 */
	err = scds_svc_wait(handle, (svc_start_timeout * SVC_WAIT_PCT / 100));

	if (err != SCHA_ERR_NOERR) {
		scds_syslog_debug(DBG_LEVEL_LOW, "INTERNAL ERROR: %s.",
			"Failed to wait before probing service");
	}

	while (1) {
		/* probe the data service */
		probe_result = svc_probe(handle, B_FALSE);
		if (probe_result == 0) {
			/* everything looks good */
			/*
			 * SCMSGS
			 * @explanation
			 * While attempting to check the health of the data
			 * service, probe detected that the resource status is
			 * fine and it is online.
			 * @user_action
			 * This is informational message. No user action is
			 * needed.
			 */
			scds_syslog(LOG_NOTICE, "Service is online.");
			return (0);
		}

		/*
		 * SCMSGS
		 * @explanation
		 * The specific service or process is not yet up.
		 * @user_action
		 * This is an informative message. Suitable action may be
		 * taken if the specified service or process does not come up
		 * within a configured time limit.
		 */
		scds_syslog(LOG_NOTICE, "Waiting for %s to come up.", APP_NAME);

		/*
		 * Dataservice is still trying to come up. Sleep for a while
		 * before probing again. Instead of calling sleep(),
		 * call scds_svc_wait() so that if service fails too
		 * many times, we give up and return early.
		 */
		err = scds_svc_wait(handle, SVC_WAIT_TIME);
		if (err != SCHA_ERR_NOERR)
			return (err);

	/* We rely on RGM to timeout and terminate the program */
	}
}


int
svc_stop(scds_handle_t handle)
{
	int svc_stop_timeout;
	char msg[SCDS_ARRAY_SIZE];

	scds_syslog_debug(DBG_LEVEL_HIGH, "Calling STOP method for "
		"resource %s.", scds_get_resource_name(handle));

	svc_stop_timeout = scds_get_rs_stop_timeout(handle);

	scds_syslog(LOG_NOTICE, "Stopping %s.", APP_NAME);

	if (scds_pmf_stop(handle, SCDS_PMF_TYPE_SVC, 0, SIGTERM,
	    svc_stop_timeout) != SCHA_ERR_NOERR) {
		scds_syslog(LOG_ERR, "Failed to stop %s.",
		    APP_NAME" . Retrying.."); /* strings concatenated */
	}

	/*
	 * Do a stop again, this time with infinite timeout and with SIGKILL.
	 * Even if the tag is gone by now, scds_pmf_stop will not complain
	 * so we are fine even if we do this unconditionally
	 */
	if (scds_pmf_stop(handle, SCDS_PMF_TYPE_SVC, 0, SIGKILL, -1) !=
	    SCHA_ERR_NOERR) {
		scds_syslog(LOG_ERR, "Failed to stop %s.", APP_NAME);
		(void) snprintf(msg, sizeof (msg), "Failed to stop %s.",
			APP_NAME);
		(void) scha_resource_setstatus(scds_get_resource_name(handle),
			scds_get_resource_group_name(handle),
			SCHA_RSSTATUS_FAULTED, msg);
		return (1);
	} else {
		/*
		 * SCMSGS
		 * @explanation
		 * The resource was successfully stopped by Sun Cluster.
		 * @user_action
		 * No user action is required.
		 */
		scds_syslog(LOG_NOTICE, "Successfully stopped %s.", APP_NAME);
		(void) snprintf(msg, sizeof (msg), "Successfully stopped %s.",
		    APP_NAME);
		(void) scha_resource_setstatus(scds_get_resource_name(handle),
		    scds_get_resource_group_name(handle), SCHA_RSSTATUS_OFFLINE,
		    msg);
		return (SCHA_ERR_NOERR);
	}
}


/*
 * This function starts the fault monitor for a HA-DNS resource.
 * This is done by starting the probe under PMF. The PMF tag
 * is derived as RG-name,RS-name.mon. The restart option of PMF
 * is used but not the "infinite restart". Instead
 * interval/retry_time is obtained from the RTR file.
 */

int
svc_fm_start(scds_handle_t handle)
{

	scds_syslog_debug(DBG_LEVEL_HIGH, "Calling MONITOR_START method "
		"for resource %s.", scds_get_resource_name(handle));

	if (scds_pmf_start(handle, SCDS_PMF_TYPE_MON, 0, "dns_probe", 0)
		!= SCHA_ERR_NOERR) {
		scds_syslog(LOG_ERR, "Failed to start fault monitor.");
		return (1);
	}

	scds_syslog(LOG_INFO, "Started the fault monitor.");

	return (SCHA_ERR_NOERR);
}


/*
 * This function stops the fault monitor for a HA-DNS resource.
 * This is done via PMF. The PMF tag for the fault monitor is
 * constructed based on RG-name_RS-name.mon.
 */

int
svc_fm_stop(scds_handle_t handle)
{
	scds_syslog_debug(DBG_LEVEL_HIGH, "Calling MONITOR_STOP method for "
		"resource %s.", scds_get_resource_name(handle));

	if (scds_pmf_stop(handle, SCDS_PMF_TYPE_MON, 0, SIGKILL, -1) !=
	    SCHA_ERR_NOERR) {
		scds_syslog(LOG_ERR, "Failed to stop fault monitor.");
		return (1);
	}

	scds_syslog(LOG_INFO, "Stopped the fault monitor.");

	return (SCHA_ERR_NOERR);
}

/*
 * svc_probe(): Do data service specific probing. Return a value
 * between 0 (success) and 100(complete failure).
 */
int
svc_probe(scds_handle_t handle, boolean_t arg_syslog_msgs)
{
	scds_netaddr_list_t *snrlp;
	int rc, probe_remaining_time, retval, probe_timeout;
	int exit_code;
	hrtime_t probe_start_time;
	char cmd[SCDS_ARRAY_SIZE];

	probe_start_time = gethrtime();
	probe_timeout = scds_get_ext_probe_timeout(handle);

	rc = scds_get_netaddr_list(handle, &snrlp);
	if (rc != SCHA_ERR_NOERR) {
		scds_syslog(LOG_ERR, "Failed to retrieve the resource "
			"property %s: %s.", SCHA_NETWORK_RESOURCES_USED,
			scds_error_string(rc));
	}

	if ((snrlp == NULL) || (snrlp->num_netaddrs == 0)) {
		if (arg_syslog_msgs) {
			scds_syslog(LOG_ERR, "No LogicalHostname resource "
				"in resource group.");
		}

		exit(1);
	}

	probe_remaining_time = probe_timeout - (int)((gethrtime() -
		probe_start_time) / 1e9);
	if (probe_remaining_time < 1) {
		if (arg_syslog_msgs) {
			scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
				"No time left for service probe");
		}

		retval = SCDS_PROBE_COMPLETE_FAILURE / 2;
		goto finished;
	}

	/*
	 * By using timeout=0, we actually get a timeout of 1 sec (but
	 * with no exponential backoff). So effectively, nslookup runs
	 * for retry * 1 = probe_remaining_time seconds.
	 */
	rc = snprintf(cmd, sizeof (cmd), "/usr/sbin/nslookup -retry=%d "
		"-timeout=0 %s %s >/dev/null 2>&1", probe_remaining_time,
		snrlp->netaddrs[0].hostname, snrlp->netaddrs[0].hostname);
	if (rc == -1) {
		if (arg_syslog_msgs) {
			scds_syslog(LOG_ERR, "INTERNAL ERROR: %s.",
				"String handling error creating "
				"the nslookup command for probe");
		}
		retval = SCDS_PROBE_COMPLETE_FAILURE / 2;
		goto finished;
	}

	/* run nslookup */
	rc = scds_timerun(handle, cmd, probe_remaining_time, SIGKILL,
		&exit_code);
	if (rc != 0) {
		if (arg_syslog_msgs) {
			/*
			 * SCMSGS
			 * @explanation
			 * The command could not be run successfully.
			 * @user_action
			 * The error message specifies both - the exact
			 * command that failed, and the reason why it failed.
			 * Try the command manually and see if it works.
			 * Consider increasing the timeout if the failure is
			 * due to lack of time. For other failures, contact
			 * your authorized Sun service provider.
			 */
			scds_syslog(LOG_ERR, "Command [%s] failed: %s.",
				cmd, scds_error_string(rc));
		}

		if ((rc == SCHA_ERR_TIMEOUT) || (rc == SCHA_ERR_INTERNAL)) {
			retval = SCDS_PROBE_COMPLETE_FAILURE / 2;
		} else {
			retval = SCDS_PROBE_COMPLETE_FAILURE;
		}

		goto finished;
	}

	/* check the return code from nslookup */
	if (exit_code != 0) {
		if (arg_syslog_msgs) {
			/*
			 * SCMSGS
			 * @explanation
			 * Fault monitor was unable to perform complete health
			 * check of the service.
			 * @user_action
			 * 1) Fault monitor would take appropiate action (by
			 * restarting or failing over the service.).
			 *
			 * 2) Data service could be under load, try increasing
			 * the values for Probe_timeout and
			 * Thororugh_probe_interval properties.
			 *
			 * 3) If this problem continues to occur, look at
			 * other messages in syslog to determine the root
			 * cause of the problem. If all else fails reboot
			 * node.
			 */
			scds_syslog(LOG_ERR, "Probe failed.");
		}
		retval = SCDS_PROBE_COMPLETE_FAILURE;
		goto finished;
	}

	/* All OK */
	retval = 0;

finished:
	scds_free_netaddr_list(snrlp);
	return (retval);

}