pxfs/mount/mount_client_impl.cc

//
// CDDL HEADER START
//
// The contents of this file are subject to the terms of the
// Common Development and Distribution License (the License).
// You may not use this file except in compliance with the License.
//
// You can obtain a copy of the license at usr/src/CDDL.txt
// or http://www.opensolaris.org/os/licensing.
// See the License for the specific language governing permissions
// and limitations under the License.
//
// When distributing Covered Code, include this CDDL HEADER in each
// file and include the License file at usr/src/CDDL.txt.
// If applicable, add the following below this CDDL HEADER, with the
// fields enclosed by brackets [] replaced with your own identifying
// information: Portions Copyright [yyyy] [name of copyright owner]
//
// CDDL HEADER END
//

//
// Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
// Use is subject to license terms.
//

#pragma ident	"@(#)mount_client_impl.cc	1.106	08/05/20 SMI"

#include <sys/errno.h>
#include <sys/strerror.h>

#include <sys/types.h>
#include <sys/thread.h>
#include <sys/file.h>
#include <sys/pathname.h>
#include <sys/sysmacros.h>
#include <sys/vfs.h>
#include <sys/mount.h>
#include <sys/dnlc.h>

#include <h/naming.h>
#include <sys/cladm_int.h>
#include <sys/cladm_debug.h>
#include <sys/sol_conv.h>
#include <solobj/solobj_impl.h>
#include <nslib/ns.h>
#include <sys/vm_util.h>

#include <h/repl_pxfs.h>
#include <pxfs/common/pxfslib.h>
#include <pxfs/mount/mount_client_impl.h>
#include <pxfs/mount/mount_debug.h>
#include <pxfs/device/device_service_mgr.h>
#include <pxfs/server/fs_impl.h>
#include <pxfs/server/repl_pxfs_server.h>
#include <pxfs/client/pxvfs.h>

#ifndef VXFS_DISABLED
#include <pxfs/server/vxfs_dependent_impl.h>
#endif

//
// For update of the mnttab modification time. The function
// vfs_mnttab_modtimeupd is declared static in vfs.c for
// Solaris 8 and 9. For Solaris 10, it is global.
//
#include <sys/sol_version.h>
#if SOL_VERSION >= __s10
#define	GLOBAL_MNTTAB_MODTIME_INTERFACE
#else
extern timespec_t vfs_mnttab_mtime;
#endif

//lint -e1512
//
// Warning(1512) destructor for base class is not virtual -- In a
// final pass through all the classes, we have found a class that is
// the base class of a derivation and has a destructor but the
// destructor is not virtual. It is conventional for inherited classes
// to have virtual destructors so that is it safe to 'delete' a
// pointer to a base class.
//
// Ths classes prov_common_iter and prov_common_setin the file
// prov_common.h have to be changed to have virtual destructors.
//


// Static data member initialization.
mount_client_impl	*mount_client_impl::this_mount_client = NULL;
os::mutex_t		mount_client_impl::mount_client_lock;

mount_client_impl::mount_client_impl() :
	mntvp(NULL),
	unmount_pxvfs_v1_p(NULL)
{
}

mount_client_impl::~mount_client_impl()
{
	if (mntvp != NULL) {
		VN_RELE(mntvp);
		mntvp = NULL; // for lint
	}
	if (unmount_pxvfs_v1_p != NULL) {
		VFS_RELE(unmount_pxvfs_v1_p->get_vfsp());
		unmount_pxvfs_v1_p = NULL; // for lint
	}
}

//
// This is called when all CORBA references are released.
//
void
mount_client_impl::_unreferenced(unref_t arg)
{
	if (!_last_unref(arg)) {
		// _last_unref() should always be true since we don't use 0->1.
		ASSERT(0);
		return;
	}

	//
	// If _unreferenced() is called because activate() failed
	// then delete this object. If for some reason the mount server
	// crashes and we get unreferenced, we don't panic when referencing
	// "this_mount_client".
	//
	if (CORBA::is_nil(keepalive)) {
		delete this;
	}
}

//
// This is called  to get a new reference.  Doing get_objref() here would
// get the highest reference version that was compiled.  We want the
// highest reference which is currently committed so we use this indirect
// way.
//
void
mount_client_impl::_generic_method(CORBA::octet_seq_t &,
    CORBA::object_seq_t &objs, Environment &e)
{
	objs[0] = get_objref();
}

//
// After a rolling upgrade commit, new mount server references should
// be used.  Currently, this is only called by pxvfs::unmount for
// the new interface which supports forced unmount.
//
void
mount_client_impl::update_mount_server_ref()
{
	Environment e;
	CORBA::Object_var obj;

	ASSERT(this_mount_client != NULL);

	mount_client_lock.lock();
	replica::service_admin_var	sa = pxfslib::get_service_admin_ref(
	    "mount_client_impl::startup", "mount", e);
	if (e.exception()) {
		mount_client_lock.unlock();
		e.clear();
		return;
	}
	obj = sa->get_root_obj(e);
	if (e.exception()) {
		mount_client_lock.unlock();
		e.clear();
		return;
	}
	this_mount_client->server = fs::mount_server::_narrow(obj);
	mount_client_lock.unlock();
}

//
// upgrade_mount_client - use the new version of the mount_server
//
void
mount_client_impl::upgrade_mount_client(fs::mount_server_ptr
    mountserver_p, Environment &)
{
	server = fs::mount_server::_duplicate(mountserver_p);
}

//
// After a rolling upgrade commit, this mount_server reference will
// will be released (in update_mount_server_ref() above).  So we need
// to duplicate here to prevent the caller from using a bogus reference
// after the replacement done above.
//
fs::mount_server_ptr
mount_client_impl::get_server()
{
	fs::mount_server_ptr	mount_server_p;

	ASSERT(this_mount_client != NULL);

	mount_server_p = (fs::mount_server::_duplicate(
	    this_mount_client->server));

	return (mount_server_p);
}

//
// Return a new reference to the local mount client.
//
fs::mount_client_ptr
mount_client_impl::get_client_ref()
{
	ASSERT(this_mount_client != NULL);
	return (this_mount_client->get_objref());
}

//
// Return a pointer to the local mount client.
//
mount_client_impl *
mount_client_impl::get_client()
{
	ASSERT(this_mount_client != NULL);
	return (this_mount_client);
}

//
// Lock the mount point and perform standard mount point checks.
// Return an exception if the lock fails.
// XXX should we locally lock vfs_t if this is for a remount?
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::lock_mountpoint(const char *mountpoint, int32_t mntflags,
    Environment &_environment)
{
	mount_lock.lock();
	if (mntvp != NULL || _environment.is_orphan()) {
		//
		// The mount point is already locked by a call to us.
		// This call should be a retry from a new primary or
		// the global mount code has allowed two mounts (or remounts)
		// to happen at the same time which shouldn't happen.
		// If we modify the mount server to allow more than one
		// mount point to be locked at the same time, we will
		// need to use the mount point string to hash into a table.
		// XXX We could save the mountpoint string to verify that
		// it is the same as the previous call.
		//
		mount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_AMBER,
		    ("client:lock_mountpoint retry %s\n",
		    mountpoint));
		return;
	}

	// Lookup the mount point path name to get a vnode pointer.
	vnode_t *vp;
	int	error = lookupname((char *)mountpoint, UIO_SYSSPACE, FOLLOW,
	    NULL, &vp);
	if (error != 0) {
		mount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:lock_mountpoint lookup %s error %d\n",
		    mountpoint, error));
		os::sc_syslog_msg msg(SC_SYSLOG_GLOBAL_MOUNT_TAG, NULL, NULL);
		//
		// SCMSGS
		// @explanation
		// While mounting a Cluster file system, the directory on
		// which the mount is to take place could not be opened.
		// @user_action
		// Fix the reported error and retry. The most likely problem
		// is that the directory does not exist - in that case, create
		// it with the appropriate permissions and retry.
		//
		(void) msg.log(SC_SYSLOG_WARNING, MESSAGE,
		    "Could not mount '%s' because there was an error (%d) in "
		    "opening the directory.", mountpoint, error);
		pxfslib::throw_exception(_environment, error);
		return;
	}

	if (vn_vfswlock(vp) != 0) {
		mount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:lock_mountpoint vn_vfswlock %s\n",
		    mountpoint));
		pxfslib::throw_exception(_environment, EBUSY);
		VN_RELE(vp);
		return;
	}

	if (vp->v_flag & VNOMOUNT) {
		mount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:lock_mountpoint VNOMOUNT %s\n",
		    mountpoint));
		pxfslib::throw_exception(_environment, EINVAL);
		vn_vfsunlock(vp);
		VN_RELE(vp);
		return;
	}

	// Make sure we are the only holder of the mount point.
	dnlc_purge_vp(vp);

	if (vn_ismntpt(vp) ||
	    ((mntflags & (MS_REMOUNT | MS_OVERLAY)) == 0 &&
	    (vp->v_count != 1 || (vp->v_flag & VROOT) != 0))) {
		mount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:lock_mountpoint %s mounted %d count %d\n",
		    mountpoint, vn_ismntpt(vp), vp->v_count));
		pxfslib::throw_exception(_environment, EBUSY);
		vn_vfsunlock(vp);
		VN_RELE(vp);
		return;
	}

	//
	// XXX Note that we don't check that vp->v_type == VDIR.
	// This should be checked for most file systems but not "namefs".
	// Also, the "busy" check above isn't quite right for "namefs".
	//

	mntvp = vp;
	mount_lock.unlock();
}

//
// Unlock the mount point.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::unlock_mountpoint(const char *, Environment &_environment)
{
	//
	// If the mount point vnode is not locked, this is a retry from a new
	// primary.
	//
	mount_lock.lock();
	if (mntvp == NULL || _environment.is_orphan()) {
		mount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_AMBER,
		    ("client:unlock_mountpoint retry\n"));
		return;
	}

	vnode_t *vp = mntvp;
	mntvp = NULL;
	mount_lock.unlock();

	vn_vfsunlock(vp);
	VN_RELE(vp);
}

//
// This is called by the mount server to lock the mount
// point and prepare to unmount the file system.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::prepare_unmount(fs::filesystem_ptr fsptr,
    solobj::cred_ptr credobj, Environment &_environment)
{
	CL_PANIC(0);
}

//
// Forced unmount support version
//
// This is called by the mount server in preparation for the unmount.
// The method pxvfsp->purge_caches provides the needed preparation:
// (1)purge the dnlc; (2)release the cached root vnode; (3) empty the
// inactive vnode list; and (4)lock the filesystem (vfs). The unmount_vfsp
// is initialized - this is used when the mount server calls
// ::remove_notify after the underlying filesystem has been unmounted.
// Note: unmounting of filesystems is done serially.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::prepare_unmount_1(fs::filesystem_ptr fsptr, int32_t flags,
    solobj::cred_ptr credobj, bool skip_purge, Environment &_environment)
{
	CL_PANIC(0);
}

//
// prepare_unmount_v1
//
// This is called by the mount server in preparation for the unmount.
// The method pxvfsp->purge_caches provides the needed preparation:
// (1)purge the dnlc; (2)release the cached root vnode; (3) empty the
// inactive vnode list; and (4)lock the filesystem (vfs).
//
// The unmount_vfsp is initialized - this is used when the mount server calls
// ::remove_notify after the underlying filesystem has been unmounted.
//
// Unmounting of filesystems is done serially.
//
// This supports forced unmount.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::prepare_unmount_v1(pxfs_v1::filesystem_ptr fsptr,
    int32_t flags, solobj::cred_ptr credobj, bool skip_purge,
    Environment &_environment)
{
	unmount_lock.lock();
	if (unmount_pxvfs_v1_p != NULL || _environment.is_orphan()) {
		unmount_lock.unlock();
		//
		// A second call to prepare_unmount has been made without
		// calling unmount_failed() or remove_notify().
		// If the vfs's match, this call is a retry from a new primary.
		// If the vfs's don't match, the global unmount code has
		// allowed two unmounts to happen at the same time which
		// shouldn't happen.
		//
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_AMBER,
		    ("client:prepare_unmount_v1 retry\n"));
		return;
	}

	//
	// Obtain the local vfs struct acting as a proxy for fs.
	// Note that we can't be sure to find the pxvfs if the mount point
	// is locked by a mount in progress, mount_server_impl::mount has
	// returned but pxvfs::mount() hasn't entered the pxvfs yet.
	//
	pxvfs	*pxvfsp = pxvfs::find_pxvfs(fsptr, NULL);
	if (pxvfsp == NULL) {
		unmount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:prepare_unmount_v1 couldn't find fs\n"));
		pxfslib::throw_exception(_environment, EINVAL);
		return;
	}
	vfs_t	*vfsp = pxvfsp->get_vfsp();
	ASSERT(vfsp != NULL);

	if (skip_purge) {
		unmount_pxvfs_v1_p = pxvfsp;
		unmount_lock.unlock();

		// Release the hold from find_pxvfs().
		VFS_RELE(vfsp);
		return;
	}

	//
	// The vfs_t keeps the hold on the vnode, we are just using the
	// pointer and thus shouldn't release it.
	//
	vnode_t		*vp = vfsp->vfs_vnodecovered;

	if (vn_vfswlock(vp) != 0) {
		unmount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:prepare_unmount_v1 vn_vfswlock pxvfs %p\n",
		    pxvfsp));
		// Release the hold we got from find_pxvfs().
		VFS_RELE(vfsp);
		pxfslib::throw_exception(_environment, EBUSY);
		return;
	}

	cred_t	*credp = solobj_impl::conv(credobj);
	if (pxvfsp->purge_caches(flags & MS_FORCE ? true : false, credp)) {
		unmount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:prepare_unmount_v1 busy\n"));
		vn_vfsunlock(vp);

		// Release the hold we got from find_pxvfs().
		VFS_RELE(vfsp);
		pxfslib::throw_exception(_environment, EBUSY);
		return;
	}

	// Release the hold from find_pxvfs().
	VFS_RELE(vfsp);
	unmount_pxvfs_v1_p = pxvfsp;
	unmount_lock.unlock();
}

//
// Undo the locking done by prepare_unmount().
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
// This method supports only mount versions before 1.1
//
void
mount_client_impl::unmount_failed(Environment &_environment)
{
	//
	// If unmount is not in progress, this is a retry from a new primary.
	//
	unmount_lock.lock();
	if ((unmount_pxvfs_v1_p == NULL) || _environment.is_orphan()) {
		unmount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_AMBER,
		    ("client:unmount_failed retry\n"));
		return;
	}

}

//
// Forced unmount support version
//
// Undo the locking done by prepare_unmount().
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::unmount_failed_1(bool skip, Environment &_environment)
{
	//
	// If unmount is not in progress, this is a retry from a new primary.
	//
	unmount_lock.lock();
	if ((unmount_pxvfs_v1_p == NULL) || _environment.is_orphan()) {
		unmount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_AMBER,
		    ("client:unmount_failed_1 retry\n"));
		return;
	}

	unmount_failed_v1(skip);
}

//
// unmount_failed_v1
//
// Supports forced unmount.
//
// Undo the locking done by prepare_unmount().
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::unmount_failed_v1(bool skip)
{
	//
	// Note: if the mount server calls us and then fails before sending
	// a checkpoint to its secondary, the new primary could call us
	// "at the same time". In order to avoid using a mutex here,
	// we set unmount_pxvfsp now to make the race window small.
	//
	pxvfs	*pxvfsp = unmount_pxvfs_v1_p;
	unmount_pxvfs_v1_p = NULL;
	unmount_lock.unlock();

	if (!skip) {
		vfs_t	*vfsp = pxvfsp->get_vfsp();
		vfs_unlock(vfsp);
		vn_vfsunlock(vfsp->vfs_vnodecovered);

		pxvfsp->unmount_failed();
	}
}

//
// Instantiate a non-HA file system on this node.
// The mount point should be locked before calling this.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
// XXX This code will need to change when new file system types
// are supported by PXFS.
//
void
mount_client_impl::instantiate(const sol::mounta &ma, sol::uintptr_t mvp,
    solobj::cred_ptr credobj, fs::filesystem_out fs_obj, fs::fs_info &fsinfo,
    CORBA::String_out mntoptions, Environment &_environment)
{
	CL_PANIC(0);
}

//
// Instantiate a non-HA file system on this node.
// The mount point should be locked before calling this.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
// XXX This code will need to change when new file system types
// are supported by PXFS.
//
void
mount_client_impl::instantiate_v1(const sol::mounta &ma, sol::uintptr_t mvp,
    solobj::cred_ptr credobj, pxfs_v1::filesystem_out fs_obj,
    pxfs_v1::fs_info &fsinfo,
    CORBA::String_out mntoptions, Environment &_environment)
{
	ASSERT(ma.flags & MS_SYSSPACE);

	//
	// If we are on the node doing the mount system call, 'mvp'
	// will be the address of the locked mount point vnode.
	// Otherwise, 'mntvp' should be non-NULL from lock_mountpoint().
	//
	vnode_t		*vp;
	if (mvp != NULL) {
		vp = (vnode_t *)mvp;
	} else {
		vp = mntvp;
		ASSERT(vp != NULL);
	}

	int datalen;

#ifndef VXFS_DISABLED
	if (strcmp(ma.fstype, "vxfs") == 0) {
		datalen =
		    vxfs_dependent_impl::vxfs_fixup_args(ma,
			vxfs_dependent_impl::VX_MOUNT);
		if (datalen == -1) {
			pxfslib::throw_exception(_environment, ENOENT);
			return;
		}
	} else {
		datalen = (int)ma.data.length();
	}
#else
	datalen = (int)ma.data.length();
#endif

	//
	// We set MS_NOSPLICE so that the underlying file system isn't
	// linked into the file system name space.
	//
	// We turn off MS_GLOBAL, as we are mounting the underlying filesystem
	// locally. With Solaris 9 build 58, Solaris disables mount in progress
	// checks if MS_GLOBAL is specified. We have to make sure that
	// MS_GLOBAL is turned off here, as we want these checks to be made.
	// These checks make sure that if a global mount and a local mount
	// happen concurrently, and are trying to mount the same device,
	// on different mount-points, only one of them succeeds.
	//
	char		*options;
	struct mounta	mnta;
	mnta.spec = ((sol::mounta &)ma).spec;
	mnta.dir = ((sol::mounta &)ma).dir;
	mnta.flags = ma.flags | MS_NOSPLICE;
	mnta.flags &= ~MS_GLOBAL;
	mnta.fstype = ((sol::mounta &)ma).fstype;
	mnta.dataptr = (char *)ma.data.buffer();
	mnta.datalen = datalen;
	int	len;
	if (mnta.flags & MS_OPTIONSTR) {
		len = (int)ma.options.length();
		options = new char[(size_t)len];	//lint !e571
		mnta.optptr = os::strcpy(options,
		    (const char *)ma.options.buffer());
		mnta.optlen = len;
	} else {
		len = MAX_MNTOPT_STR;
		options = new char [(size_t)len];	//lint !e571
		mnta.optptr = NULL;
		mnta.optlen = 0;
	}

	//
	// XXX Need a way to detect if this is a retry.
	// We could lookup ma.spec and use vfs_devsearch()
	// except this won't work for some file system types.
	// It also doesn't work if the device really is busy
	// (i.e., already mounted somewhere else) and this call
	// is not a retry.
	//

	//
	// Call the wrapped filesystem's mount routine, producing
	// a vfs structure.
	//
	cred_t	*credp = solobj_impl::conv(credobj);
	vfs	*vfsp = NULL;
	int	error = domount(NULL, &mnta, vp, credp, &vfsp);

	if (error == 0 && (mnta.flags & MS_OPTIONSTR) == 0) {
		error = vfs_buildoptionstr(&vfsp->vfs_mntopts, options, len);
	}
	if (error) {
		delete [] options;
		pxfslib::throw_exception(_environment, error);
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:instantiate_v1 %s error %d\n",
		    (const char *)ma.dir, error));
		return;
	}

	ASSERT(vfsp != NULL);

	//
	// Create a filesystem object.
	//
	fs_norm_impl	*fsp =
	    new fs_norm_impl(vfsp, mnta.fstype, mnta.spec, options);
	fs_obj = fsp->get_objref();

	// Fill in the fs_info structure.
	(void) os::strcpy(fsinfo.fstype, vfssw[vfsp->vfs_fstype].vsw_name);
	fsinfo.fsbsize = vfsp->vfs_bsize;
	fsinfo.fsdev = vfsp->vfs_dev;
	fsinfo.fsflag = vfsp->vfs_flag;
	// XXX: should be...    fsinfo.fsid = vfs->vfs_fsid;
	fsinfo.fsid.val[0] = vfsp->vfs_fsid.val[0];
	fsinfo.fsid.val[1] = vfsp->vfs_fsid.val[1];

	mntoptions = options;
}

//
// Instantiate a HA file system on this node.
// This method is called by mount_server_impl::notify_change(). The
// purpose of this code is similar to instantiate_ha() except that
// a filesystem pointer is passed instead of a vnode. Here we
// extract the vnode from the filesystem pointer.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
// XXX This code will need to change when new file system types
// are supported by PXFS.
//
void
mount_client_impl::reinstantiate_ha(const sol::mounta &ma,
    fs::filesystem_ptr fsptr, solobj::cred_ptr credobj,
    const char *dev_name, Environment &_environment)
{
	CL_PANIC(0);
}

//
// Instantiate a HA file system on this node.
// This method is called by mount_server_impl::notify_change(). The
// purpose of this code is similar to instantiate_ha() except that
// a filesystem pointer is passed instead of a vnode. Here we
// extract the vnode from the filesystem pointer.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
// XXX This code will need to change when new file system types
// are supported by PXFS.
//
void
mount_client_impl::reinstantiate_ha_v1(const sol::mounta &ma,
    pxfs_v1::filesystem_ptr fsptr, solobj::cred_ptr credobj,
    const char *dev_name, Environment &_environment)
{
	pxvfs	*pxvfsp = pxvfs::find_pxvfs(fsptr, NULL);
	// XXX Same bug possible as prepare_unmount().
	ASSERT(pxvfsp != NULL);

	vfs_t	*vfsp = pxvfsp->get_vfsp();
	ASSERT(vfsp != NULL);

	vnode_t	*vp = vfsp->vfs_vnodecovered;
	ASSERT(vp != NULL);

	VFS_RELE(pxvfsp->get_vfsp());

	instantiate_ha_common(ma, vp, credobj, dev_name,
	    VERSION_1, _environment);
}

//
// Instantiate a HA file system on this node.
// The mount point should be locked before calling this.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
// XXX This code will need to change when new file system types
// are supported by PXFS.
//
void
mount_client_impl::instantiate_ha(const sol::mounta &ma, sol::uintptr_t mvp,
    solobj::cred_ptr credobj, const char *dev_name, Environment &_environment)
{
	//
	// If we are on the node doing the mount system call, 'mvp'
	// will be the address of the locked mount point vnode.
	// Otherwise, 'mntvp' should be non-NULL from lock_mountpoint().
	//
	vnode_t	*vp;
	if (mvp != NULL) {
		vp = (vnode_t *)mvp;
	} else {
		vp = mntvp;
		ASSERT(vp != NULL);
	}

	instantiate_ha_common(ma, vp, credobj, dev_name,
	    VERSION_0, _environment);
}

//
// Instantiate a HA file system on this node.
// The mount point should be locked before calling this.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
// XXX This code will need to change when new file system types
// are supported by PXFS.
//
void
mount_client_impl::instantiate_ha_v1(const sol::mounta &ma, sol::uintptr_t mvp,
    solobj::cred_ptr credobj, const char *dev_name, Environment &_environment)
{
	//
	// If we are on the node doing the mount system call, 'mvp'
	// will be the address of the locked mount point vnode.
	// Otherwise, 'mntvp' should be non-NULL from lock_mountpoint().
	//
	vnode_t		*vp;
	if (mvp != NULL) {
		vp = (vnode_t *)mvp;
	} else {
		vp = mntvp;
		ASSERT(vp != NULL);
	}

	instantiate_ha_common(ma, vp, credobj, dev_name,
	    VERSION_1, _environment);
}

//
// Common code for both instantiate_ha() and reinstantiate_ha().
//
// Create a file system replica and register with the
// replica manager. Also register with the cluster version manager for
// callbacks in support of rolling upgrade.
//
void
mount_client_impl::instantiate_ha_common(const sol::mounta &ma, vnode_t *vp,
    solobj::cred_ptr credobj, const char *dev_name, mount_ver_t mount_ver,
    Environment &)
{
	// Nested invocations need their own Environment
	Environment	e;

	char	id[20];
	os::sprintf(id, "%u", orb_conf::node_number());

	// Create a file system replica for this node
	repl_pxfs_server	*repl_srvr_v1 = NULL;
	ASSERT(mount_ver == VERSION_1);
	repl_srvr_v1 = new repl_pxfs_server(vp, ma,
	    solobj_impl::conv(credobj), id);

	//
	// Have the filesystem replica register with the Version Manager
	// for upgrade callbacks.
	//
	repl_srvr_v1->upgrade_callback_register(ma);

	replica::service_dependencies	serv_deps(1, 1);
	serv_deps[0] = dev_name;
	replica::prov_dependencies	prov_deps(1, 1);
	prov_deps[0].service = dev_name;
	//
	// Note that we assume the "provider ID" is the same for both
	// the file system and the device service.
	// Also note that the (const char *) is needed so a copy of
	// the string is made instead of just a pointer assignment.
	//
	prov_deps[0].repl_prov_desc = (const char *)id;

	repl_srvr_v1->register_with_rm(1, &serv_deps, &prov_deps, true, e);
	if (e.exception()) {
		//
		// Clean up callback registration
		//
		repl_srvr_v1->upgrade_callback_unregister();

		//
		// Check for service or provider already registered.
		// If it is, this is either a retry after a failover
		// or an attempt to mount a mounted file system.
		//
		if (!(replica::service_already_exists::_exnarrow(
		    e.exception()) != NULL ||
		    replica::repl_prov_already_exists::_exnarrow(
		    e.exception()) != NULL)) {
			MOUNT_DBPRINTF(
			    MOUNT_TRACE_CLIENT,
			    MOUNT_RED,
			    ("client:instantiate_ha - Error FS replica %s\n",
			    dev_name));
		}
		e.clear();
	}
}

//
// Respond to notification from the VFS list server that there's a new file
// system to be added to this node's list.
// The moint point should have previously been locked with lock_mountpoint().
//
// The basic work to be done is to construct a proxy for the filesystem object
// given as argument, initialize it, set its flags field, and cross-link it
// with a local mountpoint vnode.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::add_notify_locked(const sol::mounta &ma,
    const char *mntoptions, fs::filesystem_ptr fsptr, const fs::fs_info &fsinfo,
    Environment &_environment)
{
	CL_PANIC(0);
}

//
// Respond to notification from the VFS list server that there's a new file
// system to be added to this node's list.
// The moint point should have previously been locked with lock_mountpoint().
//
// The basic work to be done is to construct a proxy for the filesystem object
// given as argument, initialize it, set its flags field, and cross-link it
// with a local mountpoint vnode.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::add_notify_locked_v1(const sol::mounta &ma,
    const char *mntoptions,
    pxfs_v1::filesystem_ptr fsptr, const pxfs_v1::fs_info &fsinfo,
    Environment &_environment)
{
	//
	// If the mount point vnode is not set, this is a retry from a new
	// primary.
	//
	mount_lock.lock();
	if (mntvp == NULL || _environment.is_orphan()) {
		mount_lock.unlock();
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_AMBER,
		    ("client:add_notify_locked_v1 %s retry\n",
		    (const char *)ma.dir));
		return;
	}

	vnode_t		*vp = mntvp;
	mntvp = NULL;
	mount_lock.unlock();

	ASSERT(!vn_ismntpt(vp));

	//
	// Now that we've obtained the mount point vnode,
	// the rest of the work is common.
	// Note that we transfer our hold on mntvp to the vfs_t.
	//
	add_notify_common_v1(ma, mntoptions, fsptr, fsinfo, vp);
}

//
// Respond to notification from the VFS list server that there's a new file
// system to be added to this node's list.
//
// The basic work to be done is to lock the mount point, construct a proxy
// for the filesystem object, initialize it, set its flags field,
// and link it into the file system name space.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation. The required
// idempotence is problematic because of a retry racing with an initial
// attempt.  This is resolved by serializing calls using the add_notify_lock
// and checking for an already existing mount.
//
void
mount_client_impl::add_notify(const sol::mounta &ma, const char *mntoptions,
    bool is_ha_repl, const char *dev_name, fs::filesystem_ptr fsptr,
    const fs::fs_info &fsinfo, Environment &_environment)
{
	CL_PANIC(0);
}

//
// Respond to notification from the VFS list server that there's a new file
// system to be added to this node's list.
//
// The basic work to be done is to lock the mount point, construct a proxy
// for the filesystem object, initialize it, set its flags field,
// and link it into the file system name space.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::add_notify_v1(const sol::mounta &ma, const char *mntoptions,
    bool is_ha_repl, const char *dev_name, pxfs_v1::filesystem_ptr fsptr,
    const pxfs_v1::fs_info &fsinfo, Environment &_environment)
{
	ASSERT(ma.flags & MS_SYSSPACE);

	vnode_t		*vp;
	int		error = lookupname(((sol::mounta &)ma).dir,
	    UIO_SYSSPACE, FOLLOW, NULL, &vp);
	if (error != 0) {
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:add_notify_v1 lookup %s error %d\n",
		    (const char *)ma.dir, error));
		os::sc_syslog_msg msg(SC_SYSLOG_GLOBAL_MOUNT_TAG, NULL, NULL);
		(void) msg.log(SC_SYSLOG_WARNING, MESSAGE,
		    "Could not mount '%s' because there was an error (%d) in "
		    "opening the directory.", (const char *)ma.dir,
		    error);
		pxfslib::throw_exception(_environment, error);
		return;
	}

	//
	// If the mount point is a PXFS vnode for the file system we
	// are trying to add, then this is a retry after a failover.
	// XXX This condition might not be met for a while, the race
	// mentioned in lock_mountpoint() for mntvp could apply here.
	//
	vfs_t	*vfsp = vp->v_vfsp;
	if ((vfsp->vfs_flag & VFS_PXFS) &&
	    fsptr->_equiv(pxvfs::VFSTOPXFS(vfsp)->get_fsobj())) {
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_GREEN,
		    ("client:add_notify_v1 %s retry\n",
		    (const char *)ma.dir));
		VN_RELE(vp);
		return;
	}

	if (vn_vfswlock(vp) != 0) {
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:add_notify_v1 %s vn_vfswlock\n",
		    (const char *)ma.dir));
		pxfslib::throw_exception(_environment, EBUSY);
		VN_RELE(vp);
		return;
	}

	if (vp->v_flag & VNOMOUNT) {
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:add_notify_v1 %s NVOMOUNT\n",
		    (const char *)ma.dir));
		pxfslib::throw_exception(_environment, EINVAL);
		vn_vfsunlock(vp);
		VN_RELE(vp);
		return;
	}

	// Make sure we are the only holder of the mount point.
	dnlc_purge_vp(vp);

	if (vn_ismntpt(vp) ||
	    (ma.flags & (MS_REMOUNT | MS_OVERLAY)) == 0 &&
	    (vp->v_count != 1 || (vp->v_flag & VROOT) != 0)) {
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:add_notify_v1 %s mounted %d count %d\n",
		    (const char *)ma.dir,
		    vn_ismntpt(vp), vp->v_count));
		pxfslib::throw_exception(_environment, EBUSY);
		vn_vfsunlock(vp);
		VN_RELE(vp);
		return;
	}

	//
	// XXX Note that we don't check that vp->v_type == VDIR.
	// This should be checked for most file systems but not "namefs".
	// Also, the "busy" check above isn't quite right for "namefs".
	//

	//
	// Check to see if we need to start a file system replica.
	//
	if (is_ha_repl) {
		//
		// Create the file system service and register with the
		// replica manager.
		// XXX kcred.
		//
		char	id[20];
		os::sprintf(id, "%u", orb_conf::node_number());
		repl_pxfs_server	*repl_srvr =
		    new repl_pxfs_server(vp, ma, kcred, id);
		//
		// Have the filesystem replica register with the Version Manager
		// for upgrade callbacks.
		//
		repl_srvr->upgrade_callback_register(ma);

		replica::service_dependencies	serv_deps(1, 1);
		serv_deps[0] = dev_name;
		replica::prov_dependencies	prov_deps(1, 1);
		prov_deps[0].service = dev_name;
		//
		// Note that we assume the "provider ID" is the same for both
		// the file system and the device service.
		// Also note that the (const char *) is needed so a copy of
		// the string is made instead of just a pointer assignment.
		//
		prov_deps[0].repl_prov_desc = (const char *)id;

		// Nest invocation requires own Environment
		Environment	e;

#ifdef _FAULT_INJECTION
	//
	// When this fault is triggered, repl_srvr->register_with_rm is
	// called with a bad paramter.  The result of this is:
	// UserException: replica::invalid_dependency.
	//
	if (fault_triggered(FAULTNUM_PXFS_ADD_NOTIFY, NULL, NULL)) {
		repl_srvr->register_with_rm(1, NULL, &prov_deps, true, e);
	} else
#endif
		repl_srvr->register_with_rm(1, &serv_deps, &prov_deps, true, e);
		if (e.exception()) {
			//
			// Clean up callback registration
			//
			repl_srvr->upgrade_callback_unregister();

			//
			// Rather than impact node startup by throwing an
			// exception and returning, we just continue on
			// with mounting the filesystem.  Just because
			// a server replica for this filesystem didn't
			// start here is no reason to interfere with the
			// global mount.
			//
#ifdef DEBUG
			e.exception()->print_exception(
			    "failed to register:");
#endif
			e.clear();

			MOUNT_DBPRINTF(
			    MOUNT_TRACE_CLIENT,
			    MOUNT_RED,
			    ("client:add_notify_v1 failed RM reg"
			    " %s mount point %s nodeid %s\n",
			    (const char *)ma.spec, (const char *)ma.dir, id));

			os::sc_syslog_msg msg(SC_SYSLOG_GLOBAL_MOUNT_TAG,
			    NULL, NULL);
			//
			// SCMSGS
			// @explanation
			// Filesystem availability may be lessened due to
			// reduced component redundancy.
			// @user_action
			// Check the device.
			//
			(void) msg.log(SC_SYSLOG_WARNING, MESSAGE,
			    "mount_client_impl::add_notify() "
			    "failed to start filesystem replica for "
			    "%s at mount point %s    nodeid %s",
			    (const char *)ma.spec, (const char *)ma.dir, id);
		}
	}

	//
	// Now that we have the locked mount point vnode,
	// the rest of the work is common.
	// Note that we transfer our hold on vp to the vfs_t.
	//
	add_notify_common_v1(ma, mntoptions, fsptr, fsinfo, vp);
}

//
// Respond to a request from the VFS list server to remove a file system from
// this node's list.
// Note: we assume that the caller has globally locked the relevant mountpoint.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::remove_notify(const char *mountpoint,
    const char *, bool, Environment &_environment)
{
	CL_PANIC(0);
}

//
// Respond to a request from the VFS list server to remove a file system from
// this node's list.
// Note: we assume that the caller has globally locked the relevant mountpoint.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
// This supports forced unmount.
//
void
mount_client_impl::remove_notify_1(const char *mountpoint, bool unlink_vfs,
    Environment &_environment)
{
	unmount_lock.lock();
	if ((unmount_pxvfs_v1_p == NULL) || _environment.is_orphan()) {
		unmount_lock.unlock();
		//
		// This should be a retry after the mount server failed
		// over to another node.
		//
		return;
	}

	remove_notify_v1(mountpoint, unlink_vfs);
}

//
// remove_notify_v1
//
// Respond to a request from the VFS list server to remove a file system from
// this node's list.
//
// Note: we assume that the caller has globally locked the relevant mountpoint.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::remove_notify_v1(const char *mountpoint, bool unlink_vfs)
{
	//
	// Note: if the mount server calls us and then fails before sending
	// a checkpoint to its secondary, the new primary could call us
	// "at the same time". In order to avoid using a mutex here,
	// we set unmount_pxvfsp now to make the race window small.
	//
	pxvfs	*pxvfsp = unmount_pxvfs_v1_p;
	unmount_pxvfs_v1_p = NULL;
	unmount_lock.unlock();

	vfs_t	*vfsp = pxvfsp->get_vfsp();
	ASSERT(vfsp != NULL);

	MOUNT_DBPRINTF(
	    MOUNT_TRACE_CLIENT,
	    MOUNT_AMBER,
	    ("client:remove_notify_v1 %s pxvfsp %p\n",
	    mountpoint, pxvfsp));

	//
	// Release the unmount lock and wake up any sleepers waiting to create
	// pxfobjs.
	//
	pxvfsp->unmount_succeeded();

	//
	// Remove vfsp from the vfs list and release it.
	//
	if (unlink_vfs) {
		vnode_t		*coveredvp = vfsp->vfs_vnodecovered;
		ASSERT(coveredvp != NULL);
		VN_HOLD(coveredvp);
		vfs_remove(vfsp);
		vn_vfsunlock(coveredvp);
		VN_RELE(coveredvp);
	}
}

//
// Remove a file system from the name space.
// This should be called if this node is not the server for the file
// system being removed or it is not the last replica (is_ha_repl is true).
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::remove_client(const char *, const char *spec,
    bool is_ha_repl, const char *, fs::filesystem_ptr fsptr,
    solobj::cred_ptr credobj, Environment &_environment)
{
	CL_PANIC(0);
}

//
// Remove a file system from the name space.
// This should be called if this node is not the server for the file
// system being removed or it is not the last replica (is_ha_repl is true).
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::remove_client_v1(const char *, const char *spec,
    bool is_ha_repl, const char *, pxfs_v1::filesystem_ptr fsptr,
    solobj::cred_ptr credobj, Environment &_environment)
{
	//
	// Obtain the proxy vfs struct for fs.
	//
	pxvfs	*pxvfsp =
	    pxvfs::find_pxvfs(fsptr, NULL);
	if (pxvfsp == NULL) {
		// Must be a retry or was unmounted before we got here.
		return;
	}

	vfs_t	*vfsp = pxvfsp->get_vfsp();
	ASSERT(vfsp != NULL);

	if (vn_vfswlock(vfsp->vfs_vnodecovered)) {
		// Release the hold we got from find_pxvfs().
		VFS_RELE(vfsp);

		// Couldn't get the covered mount point lock.
		pxfslib::throw_exception(_environment, EBUSY);
		return;
	}

	cred_t	*credp = solobj_impl::conv(credobj);
	if (pxvfsp->purge_caches(true, credp)) {
		vn_vfsunlock(vfsp->vfs_vnodecovered);

		// Release the hold we got from find_pxvfs().
		VFS_RELE(vfsp);

		pxfslib::throw_exception(_environment, EBUSY);
		return;
	}

	//
	// Release the fsmgr_client/pxvfs binding.
	//
	pxvfsp->unmount_succeeded();

	//
	// Remove vfsp from the vfs list and release it.
	//
	vnode_t		*coveredvp = vfsp->vfs_vnodecovered;
	ASSERT(coveredvp != NULL);
	VN_HOLD(coveredvp);
	vfs_remove(vfsp);
	vn_vfsunlock(coveredvp);
	VN_RELE(coveredvp);

	// Release the hold for unmount_pxvfsp we got from find_pxvfs().
	VFS_RELE(vfsp);

	char name[20];

	//
	// Shutdown replica if needed.
	//
	if (is_ha_repl) {
		replica::service_admin_var	sa =
		    pxfslib::get_service_admin_ref(
			"mount_client_impl::remove_client", spec,
			_environment);
		if (_environment.exception()) {
			//
			// Need to shut down this replica but
			// can't get the service_admin object to do it.
			//
			// The only reason get_service_admin_ref()
			// should fail is if service
			// registration failed/never occurred,
			// or if the RM is in the process
			// of shutting down the service.
			//
#ifdef DEBUG
			_environment.exception()->print_exception(
			    "mount_client_impl::remove_client "
			    "get_service_admin_ref()"); // XXX
#endif
			    MOUNT_DBPRINTF(
			    MOUNT_TRACE_CLIENT,
			    MOUNT_AMBER,
			    ("client:remove_client_v1 failed get_service_admin "
			    "spec %s\n",
			    spec));
			_environment.clear();
		} else {
			os::sprintf(name, "%d", orb_conf::node_number());
			sa->change_repl_prov_status(name,
			    replica::SC_REMOVE_REPL_PROV, true, _environment);
			if (_environment.exception()) {
				os::sc_syslog_msg msg(
				    SC_SYSLOG_GLOBAL_MOUNT_TAG,
				    NULL, NULL);
				//
				// SCMSGS
				// @explanation
				// The system was unable to remove a PXFS
				// replica on the node that this message was
				// seen.
				// @user_action
				// Contact your authorized Sun service
				// provider to determine whether a workaround
				// or patch is available.
				//
				(void) msg.log(SC_SYSLOG_WARNING, MESSAGE,
				    "mount_client_impl::remove_client()"
				    " failed attempted"
				    " RM change_repl_prov_status() to"
				    " remove client,"
				    " spec %s, name %s",
				    spec, name);
				_environment.clear();
			}
		}
	}
}

//
// Receive notification from server that proxy file system
// flags need to be set; used to implement remounts.
// It also unlocks the mount point and updates /etc/mnttab.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::set_flags(const sol::mounta &, const char *mntoptions,
    fs::filesystem_ptr fsptr, uint32_t vfsflags, Environment &_environment)
{
	CL_PANIC(0);
}

//
// Receive notification from server that proxy file system
// flags need to be set; used to implement remounts.
// It also unlocks the mount point and updates /etc/mnttab.
//
// Note: this routine can be called multiple times if the primary
// fails over and the new primary retries this operation so
// this operation must be idempotent.
//
void
mount_client_impl::set_flags_v1(const sol::mounta &, const char *mntoptions,
    pxfs_v1::filesystem_ptr fsptr, uint32_t vfsflags, Environment &_environment)
{
	//
	// If the mount point vnode is not set, this is a retry from a new
	// primary.
	//
	mount_lock.lock();
	if (mntvp == NULL || _environment.is_orphan()) {
		mount_lock.unlock();
		return;
	}

	vnode_t		*vp = mntvp;
	mntvp = NULL;
	mount_lock.unlock();

	pxvfs	*pxvfsp = pxvfs::find_pxvfs(fsptr, NULL);
	// XXX Same bug possible as prepare_unmount().
	ASSERT(pxvfsp != NULL);
	pxvfsp->set_mntoptions(mntoptions);
	vfs_t	*vfsp = pxvfsp->get_vfsp();

	// Release the hold we got from find_pxvfs()
	VFS_RELE(vfsp);
	ASSERT(vfsp != NULL);

	vfsp->vfs_flag = vfsflags;
	vfsp->vfs_mtime = ddi_get_time();

	//
	// Free the old mount options table and add a new one - then
	// fill the new table with the options.
	// Update the mnttab modification time. The function
	// vfs_mnttab_modtimeupd is declared static in vfs.c for
	// Solaris 8 and 9. For Solaris 10, it is global.
	//
#ifdef GLOBAL_MNTTAB_MODTIME_INTERFACE
	vfs_list_lock();
	vfs_createopttbl(&vfsp->vfs_mntopts, mntoptions);
	vfs_parsemntopts(&vfsp->vfs_mntopts, (char *)mntoptions, 1);
	vfs_mnttab_modtimeupd();
	vfs_list_unlock();
#else
	vfs_createopttbl(&vfsp->vfs_mntopts, mntoptions);
	vfs_parsemntopts(&vfsp->vfs_mntopts, (char *)mntoptions, 1);
	gethrestime(&vfs_mnttab_mtime);
#endif

	// Unlock the mount point.
	vn_vfsunlock(vp);
	VN_RELE(vp);
}

//
// Register the mount client with the mount server.
// This is called by _cladm(CL_INITIALIZE, CL_GBLMNT_ENABLE).
// The return value is the errno value for the system call.
//
int
mount_client_impl::activate()
{
	//
	// Check to make sure clexecd is running.
	//
	char				name[20];
	Environment			e;

	repl_pxfs::ha_mounter_var	mounter;
	naming::naming_context_var	ctxp = ns::root_nameserver();

	os::sprintf(name, "ha_mounter.%d", orb_conf::node_number());

	CORBA::Object_var		obj = ctxp->resolve(name, e);
	if (e.exception() == NULL) {
		mounter = repl_pxfs::ha_mounter::_narrow(obj);
	}
	if ((e.exception() != NULL) || CORBA::is_nil(mounter)) {
		if (e.exception() != NULL) {
			CLEXEC_EXCEPTION(e, "mount_client_impl::activate",
			    "ha_mounter");
			e.clear();
		} else {
			CLADM_DBPRINTF(CLADM_TRACE_CLEXEC, CLADM_RED,
			    ("mount_client_impl::activate:%s"
			    " CORBA::is_nil(mounter)\n", name));
		}
		MOUNT_DBPRINTF(
		    MOUNT_TRACE_CLIENT,
		    MOUNT_RED,
		    ("client:activate: resolve %s failed\n",
		    name));
		return (EAGAIN);
	}

	bool alive = mounter->is_alive(e);

	CLADM_DBPRINTF(CLADM_TRACE_CLEXEC, CLADM_GREEN,
	    ("mount_client_impl::activate:%s alive %d except %p\n",
	    name, (int)alive, e.exception()));

	if (e.exception() != NULL || !alive) {
		//
		// XXX There is currently a bug where low memory can return
		// EAGAIN from the xdoor upcall. We attempt recovery here
		// for now.
		//
		CORBA::SystemException *exp =
		    CORBA::SystemException::_exnarrow(e.exception());
		CLEXEC_EXCEPTION(e, "mount_client_impl::activate", "is_alive");
		if (exp != NULL && exp->_minor() == EAGAIN) {
			e.clear();
			alive = mounter->is_alive(e);
			CLADM_DBPRINTF(CLADM_TRACE_CLEXEC, CLADM_AMBER,
			    ("mount_client_impl::activate:%s alive %d exp %p\n",
			    name, (int)alive, e.exception()));
			if (e.exception() != NULL || !alive) {
				CLEXEC_EXCEPTION(e,
				    "mount_client_impl::activate", "is_alive");
				e.clear();
				MOUNT_DBPRINTF(
				    MOUNT_TRACE_CLIENT,
				    MOUNT_RED,
				    ("client:activate: 1: "
				    "clexecd does not appear to be running\n"));
				return (EAGAIN);
			}
		} else {
			if (e.exception() != NULL) {
#ifdef DEBUG
				// XXX
				e.exception()->print_exception(
				    "mount_client_impl::activate: ");
#endif
				MOUNT_DBPRINTF(
				    MOUNT_TRACE_CLIENT,
				    MOUNT_RED,
				    ("client:activate "
				    "failed mounter::is_alive\n"));
			}
			e.clear();
			MOUNT_DBPRINTF(
			    MOUNT_TRACE_CLIENT,
			    MOUNT_RED,
			    ("client:activate: "
			    "clexecd does not appear to be running\n"));
			CLADM_DBPRINTF(CLADM_TRACE_CLEXEC, CLADM_RED,
			    ("mount_client_impl::activate %s"
			    " clexecd does not appear to be running\n",
			    name));
			return (EAGAIN);
		}
	}

	//
	// Ideally, there should be checks here that verify that the
	// prerequisites for establishing the global name space have been
	// satisfied.  However, this would amount to checking that the mount
	// point(s) into which the base global mount(s) would be done exist,
	// which is infeasible to do here.
	//

	//
	// Ensure idempotence.
	//
	mount_client_lock.lock();
	if (this_mount_client != NULL) {
		mount_client_lock.unlock();
		return (0);
	}

	//
	// Get the global mount server object.
	//
	replica::service_admin_var	sa =
	    pxfslib::get_service_admin_ref(
	    "mount_client_impl::startup", "mount", e);
	if (e.exception()) {
		mount_client_lock.unlock();
		e.clear();
		return (EIO);
	}
	obj = sa->get_root_obj(e);
	if (e.exception()) {
		mount_client_lock.unlock();
		e.clear();
		return (EIO);
	}

	mount_client_impl	*mcp = new mount_client_impl();
	mcp->server = fs::mount_server::_narrow(obj);
	ASSERT(!CORBA::is_nil(mcp->server));

	//
	// Register with mount_server.
	//
	fs::mount_client_var	clientv = mcp->get_objref();
	mcp->server->add_client(clientv, orb_conf::node_number(),
	    mcp->keepalive, e);
	CORBA::Exception	*ex = e.exception();
	if (ex != NULL) {
		int	error;

		//
		// Note: mcp will be deleted when
		// mount_client_impl::_unreferenced() is called.
		//
		mount_client_lock.unlock();

		fs::mount_server::mount_err	*merrp =
		    fs::mount_server::mount_err::_exnarrow(ex);
		if (merrp != NULL) {
			error = merrp->error;
			MOUNT_DBPRINTF(
			    MOUNT_TRACE_CLIENT,
			    MOUNT_RED,
			    ("client:activate add_client failed %s\n",
			    (const char *)merrp->mntpnt));
		} else {
			error = pxfslib::get_err(e);
			MOUNT_DBPRINTF(
			    MOUNT_TRACE_CLIENT,
			    MOUNT_RED,
			    ("client:activate add_client failed\n"));
			e.exception()->print_exception(
			    "mount_client_impl::activate"); // XXX
		}
		e.clear();
		return (error);
	}

	this_mount_client = mcp;
	mount_client_lock.unlock();

	return (0);
}


//
// Return true if activate() has been called.
//
bool
mount_client_impl::is_activated()
{
	return (this_mount_client != NULL);
}

//
// Common code for add_notify_locked_v1() and add_notify_v1().
//
void
mount_client_impl::add_notify_common_v1(const sol::mounta &ma,
    const char *mntoptions,
    pxfs_v1::filesystem_ptr fsptr, const pxfs_v1::fs_info &fsinfo,
    vnode_t *coveredvp)
{
	//
	// Find or create a local pxvfs struct to act as a proxy for fs.
	//
	pxvfs	*pxvfsp = pxvfs::find_pxvfs(fsptr, &fsinfo);
	ASSERT(pxvfsp != NULL);
	pxvfsp->set_mntoptions(mntoptions);
	vfs_t	*vfsp = pxvfsp->get_vfsp();
	ASSERT(vfsp != NULL);

	MOUNT_DBPRINTF(
	    MOUNT_TRACE_CLIENT,
	    MOUNT_GREEN,
	    ("client:add_notify_common_v1 add %s pxvfsp %p\n",
	    (const char *)ma.dir, pxvfsp));

	//
	// The vfs-specific fields should be initialized unless we
	// switch to a lazy propagation of vfs list changes scheme.
	// Our current algorithm immediately ("eagerly") pushes vfs list
	// changes to the client, so we require
	// find_pxvfs() to fully initialize the vfs struct.
	//
	ASSERT(vfsp->vfs_fstype != 0);

	//
	// Lock the vfs_t similar to domount().
	// We should always get the lock since this vfs_t is newly created.
	//
	int	error = vfs_lock(vfsp);
	ASSERT(error == 0);

	// Check if this mount should not be visible via /etc/mnttab
	if (ma.flags & MS_NOMNTTAB) {
		vfsp->vfs_flag |= VFS_NOMNTTAB;
	} else {
		vfsp->vfs_flag &= ~VFS_NOMNTTAB;
	}

	//
	// Initialize values for /etc/mnttab on this node.
	//
	if ((const char *)ma.spec == NULL || *(const char *)ma.spec == '\0') {
		vfs_setresource(vfsp, VFS_NORESOURCE);
	} else {
		vfs_setresource(vfsp, (const char *)ma.spec);
	}
	vfs_setmntpoint(vfsp, (const char *)ma.dir);
	// XXX Use global time?
	vfsp->vfs_mtime = ddi_get_time();
	vfs_createopttbl(&vfsp->vfs_mntopts, mntoptions);
	vfs_parsemntopts(&vfsp->vfs_mntopts, (char *)mntoptions, 1);
	//
	// Hook vfsp into the local vfs list.
	// Note: we don't need to have a cluster-wide lock held
	// on the vfs list because we expect the global mount point
	// locking to prevent any mount races on this mount point.
	//
	VFS_HOLD(vfsp);
	vfs_list_add(vfsp);

	//
	// Splice the mount into the name space by setting v_vfsmountedhere
	// in the covered vnode.
	// Note that we transfer the hold on coveredvp to v_vfsmountedhere.
	//
	ASSERT(!vn_ismntpt(coveredvp));
	coveredvp->v_vfsmountedhere = vfsp;
	vfsp->vfs_vnodecovered = coveredvp;

	// Release the hold we got from find_pxvfs().
	VFS_RELE(vfsp);

	vfs_unlock(vfsp);
	vn_vfsunlock(coveredvp);
}

//
// Notify intent to mount a device.
// Device is "locked" until it is either mounted or the requesting node dies.
//
int
mount_client_impl::devlock(int cmd, struct pathname *devpnp)
{
	//
	// Verify that the mount client is already active, returning
	// failure if it isn't.
	//
	if (!is_activated()) {
		return (ENODEV);
	}

	Environment	e;

	//
	// If this is a request to unlock the device, unlock it.
	//
	if (cmd == CL_GBLMNT_UNLOCK) {
		get_server()->devunlock(devpnp->pn_path, e);
		return (pxfslib::get_err(e));
	}

	//
	// We have to save a copy of the path name since lookuppn() will
	// clobber it.
	//
	char	*spec = os::strcpy(new char [devpnp->pn_pathlen + 1],
	    devpnp->pn_path);

	//
	// We try to determine if this node has a local connection to the device
	// (i.e., it can be a device replica). If we determine its not
	// local, return an error without trying to get the lock since
	// there is no point in attempting to start the service until one
	// of the nodes that can be a replica boots.
	//
	vnode_t		*vp;
	int		error = lookuppn(devpnp, NULL, FOLLOW, NULL, &vp);
	if (error == 0) {
		// Check for a PXFS special file.
		if (vp->v_flag & VPXFS) {
			//
			// Contact DCS to get the list of nodes that this
			// device is attached to and whether or not its an
			// HA device.
			//
			bool			dev_is_ha;
			CORBA::String_var	dev_name;
			sol::nodeid_seq_t_var	dev_nids;

			error = dcs_get_configured_nodes(vp->v_rdev,
			    fs::dc_callback::_nil(),
			    dev_is_ha, dev_name, dev_nids);
			if (error == 0) {
				uint32_t i, n = dev_nids->length();
				for (i = 0; i < n; i++) {
					if (dev_nids[i] ==
					    orb_conf::node_number()) {
						break;
					}
				}
				if (i == n) {
					//
					// We didn't find a local connection.
					//
					VN_RELE(vp);
					delete [] spec;
					return (ENXIO);
				}
			}
		}
		VN_RELE(vp);
	}

	// Try to get the lock.
	fs::mount_client_var	clientv = get_client_ref();

	bool log_message = B_TRUE;
	sol::nodeid_t lock_owner = 0;
	os::sc_syslog_msg msg(SC_SYSLOG_GLOBAL_MOUNT_TAG, NULL, NULL);
	do {
		get_server()->devlock(clientv, orb_conf::node_number(),
		    spec, e);
		error = pxfslib::get_err(e);
		e.clear();

		if (error == ETIMEDOUT && log_message) {
			//
			// Try to get the id of the node holding the lock
			// If we don't find it, it would be because it was
			// released. We ignore this.
			//
			get_server()->get_devlock_owner(spec, lock_owner, e);
			if (lock_owner) {
				// Log an error message once.
				char nodename[CL_MAX_LEN + 1];
				clconf_get_nodename(lock_owner, nodename);
				//
				// SCMSGS
				// @explanation
				// Sun Cluster boot is waiting for the
				// mentioned node to complete fsck/mount of a
				// global filesystem and release the lock on a
				// device.
				// @user_action
				// Check the console of the specified cluster
				// node to see if any of the nodes are waiting
				// for a manual fsck to be done. If this is
				// so, exiting the shell after performing the
				// fsck will allow the boot of the other nodes
				// to continue.
				//
				(void) msg.log(SC_SYSLOG_NOTICE, MESSAGE,
				    "Sun Cluster is waiting for lock on device "
				    "%s. Lock is currently held by %s for "
				    "fsck/mount.",
				    spec, nodename);
				log_message = B_FALSE;
			}
		}
	} while (error == EAGAIN || error == ETIMEDOUT);

	if (log_message == B_FALSE) {
		//
		// Log a message that the wait is over.
		//
		if (error == 0) {
			//
			// SCMSGS
			// @explanation
			// Sun Cluster successfully obtained a lock on a
			// device to perform fsck/mount.
			// @user_action
			// This is an informational message, no user action is
			// needed.
			//
			(void) msg.log(SC_SYSLOG_NOTICE, MESSAGE,
			    "Lock on device %s obtained. Proceeding.",
			    spec);
		} else {
			//
			// SCMSGS
			// @explanation
			// Sun Cluster was unable to lock a device.
			// @user_action
			// Check the error returned for why this happened. In
			// cases like an interrupted system call, no user
			// action is required.
			//
			(void) msg.log(SC_SYSLOG_NOTICE, MESSAGE,
			    "Unable to lock device %s. Error (%s).",
			    spec, strerror(error));
		}
	}

	delete [] spec;
	return (error);
}

//
// Called from cladmin() to import all global mounts
// (i.e., "/usr/cluster/lib/sc/clconfig -g").
//
extern "C" int
pxfs_mount_client_enable(int cmd, int onoff)
{
	int	error;

	switch (cmd) {
	case CL_GBLMNT_ENABLE:
		if (onoff) {
			error = device_service_mgr::activate();
			if (error == 0)
				error = mount_client_impl::activate();
			return (error);
		} else {
			// cladmin returns ENOTSUP
			ASSERT(0);
			return (0);
		}
	case CL_SWITCHBACK_ENABLE:
		return (device_service_mgr::do_switchbacks());

	default:
		return (EINVAL);
	}
}

//
// Called from cladmin() to lock devices so they aren't fsck'ed by different
// nodes at the same time (i.e., "/usr/cluster/lib/sc/clconfig -m devname").
//
extern "C" int
pxfs_mount_client_lock(int cmd, struct pathname *devpnp)
{
	return (mount_client_impl::devlock(cmd, devpnp));
}

//
// Called after the pxfs loadable module has been loaded.
//
sol::error_t
pxfs_mount_client_startup()
{
	extern int (*pxfs_mount_client_enable_ptr)(int cmd, int onoff);
	extern int (*pxfs_mount_client_lock_ptr)(int cmd,
	    struct pathname *devpnp);
	pxfs_mount_client_enable_ptr = pxfs_mount_client_enable;
	pxfs_mount_client_lock_ptr = pxfs_mount_client_lock;
	return (0);
}

//
// Called before the pxfs loadable module is unloaded.
//
sol::error_t
pxfs_mount_client_shutdown()
{
	return (mount_client_impl::is_activated() ? EBUSY : 0);
}