pxfs/mount/mount_server_impl.h

//
// CDDL HEADER START
//
// The contents of this file are subject to the terms of the
// Common Development and Distribution License (the License).
// You may not use this file except in compliance with the License.
//
// You can obtain a copy of the license at usr/src/CDDL.txt
// or http://www.opensolaris.org/os/licensing.
// See the License for the specific language governing permissions
// and limitations under the License.
//
// When distributing Covered Code, include this CDDL HEADER in each
// file and include the License file at usr/src/CDDL.txt.
// If applicable, add the following below this CDDL HEADER, with the
// fields enclosed by brackets [] replaced with your own identifying
// information: Portions Copyright [yyyy] [name of copyright owner]
//
// CDDL HEADER END
//

//
// Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
// Use is subject to license terms.
//

#ifndef _MOUNT_SERVER_IMPL_H
#define	_MOUNT_SERVER_IMPL_H

#pragma ident	"@(#)mount_server_impl.h	1.48	08/05/20 SMI"

#ifdef __cplusplus
extern "C" {
#endif

#ifdef __cplusplus
}
#endif

#include <h/sol.h>
#include <h/pxfs.h>
#include <h/pxfs_v1.h>
#include <h/repl_pxfs.h>
#include <sys/list_def.h>	// List template
#include <sys/threadpool.h>
#include <pxfs/common/pxfslib.h>
#include <pxfs/mount/mount_replica_impl.h>
#include <pxfs/mount/mount_debug.h>
#include <pxfs/mount/bitmap.h>

// Forward declarations.
class mount_server_impl;
class mount_client_elem;

//
// The mount HA service supports multiple versions of pxfs file system
// software. The pxfs file system software does not change instantaneously.
// The version of pxfs file system software is chosen at the time the
// system mounts the file system. This means that different file systems
// can concurrently be using different pxfs software versions.
//
// A pxfs file system can be mounted on a mount point within another
// pxfs file system. Obviously, one should not unmount a file system
// while there are still file systems mounted on mount points within this
// file system. Thus there can be an unmount dependency order between
// pxfs file systems. The mount service does not keep track of the true
// dependency order. Instead the mount service simply unmounts file systems
// in the reverse order in which the file systems were mounted. This is
// cautious, but safe. A file system using new pxfs version software can
// be mounted on a file system using old pxfs version software,
// and vice versa under unusual cases. So file systems using different
// version pxfs software can have dependencies between them.
//
// The above two facts led to the decision to jointly manage pxfs file
// systems using different software versions of pxfs. The convention is that
// the string "_v1" means that the entity supports pxfs version 1 file
// systems.
//

//
// Class used to store one file system mount element in a list of all mounts.
//
class fs_elem : public _DList::ListElem {
public:
	fs_elem(pxfs_v1::filesystem_ptr fsp, const pxfs_v1::fs_info &finfo,
	    const sol::mounta &md, const char *options, bool is_ha,
	    const char *name, const sol::nodeid_seq_t &nids);

	~fs_elem();

	enum	fs_elem_ver_t { VERSION_0, VERSION_1 };

	//
	// Public data members.
	//

	//
	// An fs_elem can support either version 0 or 1,
	// but not both simultaneously.
	// A union does not allow members requiring
	// initialization or destruction.
	//
	pxfs_v1::filesystem_var	fs_v1_ptr;

	union {
		pxfs_v1::fs_info	fs_v1_info;
	};

	sol::mounta		ma;
	CORBA::String_var	mntoptions;
	bool			dev_is_ha;
	CORBA::String_var	dev_name;
	sol::nodeid_seq_t	dev_nids;
	fs_elem_ver_t		fs_elem_ver;

private:
	// Disallowed operations.
	fs_elem();
	fs_elem & operator = (const fs_elem &);
	fs_elem(const fs_elem &);
};

//
// Class used to record device locks (see mount_server_impl::devlock()).
//
class devlock_elem : public _SList::ListElem {
public:
	devlock_elem(fs::mount_client_ptr client_p, sol::nodeid_t nodeid,
	    const char *name);
	~devlock_elem();

	//
	// Public data members.
	//
	fs::mount_client_var	owner;		// node which owns the lock
	char			*spec;		// device path name
	uint_t			nwaiters;	// non 0 if someone is waiting
	sol::nodeid_t		ownerid;	// nodeid which owns the lock
	bitmap<NODEID_MAX>	waiters;	// for deadlock detection
	bool			unlocked;	// true if unlock was signaled
	os::mutex_t		waiter_lock;	// signal device lock waiters
	os::condvar_t		waiter_cv;	// signal device lock waiters

private:
	// Disallowed operations.
	devlock_elem & operator = (const devlock_elem &);
	devlock_elem(const devlock_elem &);
};

//
// Class used to store mount_client elements.
//
class mount_client_elem :
    public mc_replica_of<fs::mount_client_died>,
    public _SList::ListElem
{
public:
	// Primary constructor.
	mount_client_elem(mount_server_impl *server,
		fs::mount_client_ptr client, sol::nodeid_t nid,
		mount_replica_impl *replp);

	// Secondary constructor.
	mount_client_elem(mount_server_impl *server,
		fs::mount_client_ptr client, sol::nodeid_t nid,
		bool is_shutdown, fs::mount_client_died_ptr obj);

	~mount_client_elem();
	void _unreferenced(unref_t arg);

	//
	// Public data members.
	//
	mount_server_impl	*serverp;	// pointer to mount server
	fs::mount_client_var	clientptr;	// mount_client reference
	sol::nodeid_t		nodeid;		// nodeid of the client
	bool			shutdown;	// true if being shut down

private:
	// Disallowed operations.
	mount_client_elem & operator = (const mount_client_elem &);
	mount_client_elem(const mount_client_elem &);
};

//
// This is used by mount_server::mount() to detect when the client
// crashes and there is a failover of the server. In this case,
// we need to unlock the mount point on all nodes.
//
class mount_state : public transaction_state {
public:
	enum	mount_ver_t { VERSION_0, VERSION_1, VERSION_UNKNOWN };

	mount_state(const sol::mounta &md, fs::mount_client_ptr client_p,
	    mount_server_impl &srvr, mount_ver_t ver);

	virtual ~mount_state();

	//
	// Routines to implement the transaction_state template.
	//
	virtual void orphaned(Environment &e);
	virtual void committed();

	// Public data members.
	fs::mount_client_var	mountpoint_lock_c;
	sol::mounta		ma;
	sol::error_t		error;
	bool			is_remount;

	//
	// A mount_state can support either version 0 or 1,
	// but not both simultaneously.
	// A union does not allow members requiring
	// initialization or destruction.
	//
	pxfs_v1::filesystem_var	fs_v1_ptr;

	union {
		pxfs_v1::fs_info	fs_v1_info;
	};

	CORBA::String_var	mntoptions;
	mount_ver_t		mount_ver;

private:
	// Reference to server which created this object.
	mount_server_impl	&server;

	// Disallowed operations.
	mount_state();
	mount_state & operator = (const mount_state &);
	mount_state(const mount_state &);
};

//
// This is used by mount_server::remove_client() to detect when the client
// crashes and there is a failover of the server. In this case,
// we need to unlock the mount point on all nodes.
//
class unmount_state : public transaction_state {
public:
	unmount_state(pxfs_v1::filesystem_ptr fsptr, int32_t flags,
	    solobj::cred_ptr crptr, mount_server_impl &srvr,
	    fs::mount_client_ptr clptr);

	virtual ~unmount_state();

	enum	unmount_ver_t { VERSION_0, VERSION_1 };

	//
	// Routines to implement the transaction_state template.
	//
	virtual void orphaned(Environment &e);
	virtual void committed();

	enum state_t {
		START, UNMOUNTED, NOTIFIED, COMMITTED
	};

	//
	// Public data members.
	//
	fs::mount_client_var	skip;		// skip prepare_unmount()

	int32_t			flags;		// umount system call flags
	//
	// An unmount_state can support either version 0 or 1,
	// but not both simultaneously.
	// A union does not allow members requiring
	// initialization or destruction.
	//
	// file system being unmounted
	//
	pxfs_v1::filesystem_var	fs_v1_obj;

	solobj::cred_var	credobj;
	state_t			state;		// checkpoint state for unmount
	sol::error_t		error;
	CORBA::String_var	service_name;	// HA FS service to shut down
	unmount_ver_t		unmount_ver;

private:
	// Reference to server which created this object.
	mount_server_impl	&server;

	// Disallowed operations.
	unmount_state();
	unmount_state & operator = (const unmount_state &);
	unmount_state(const unmount_state &);
};

//
// This is used by DCS to notify the mount server of device configuration
// changes.
//
class dc_callback_impl : public mc_replica_of<fs::dc_callback> {
public:
	// Primary constructor.
	dc_callback_impl(mount_server_impl &srvr, mount_replica_impl *serverp);

	// Secondary constructor.
	dc_callback_impl(mount_server_impl &srvr, fs::dc_callback_ptr obj);

	~dc_callback_impl();
	void _unreferenced(unref_t arg);

protected:
	// fs::dc_callback::*
	/* dc_callback */
	void notify_change(sol::dev_t gdev, const sol::nodeid_seq_t &nodes,
	    Environment &_environment);

	bool still_active(sol::dev_t gdev, Environment &_environment);
	//

private:
	// Reference to server which created this object.
	mount_server_impl	&server;
};

//
// Note: Only the HA version of fs::mount_server interface is
// needed since there is only one mount server per cluster.
// All nodes need to be able to become the primary so that the list
// of mounted files is never lost (also enables rolling upgrade).
//
class mount_server_impl : public mc_replica_of<fs::mount_server> {
public:
	// Primary constructor.
	mount_server_impl(mount_replica_impl *serverp);

	// Secondary constructor.
	mount_server_impl(mount_replica_impl *serverp,
	    fs::mount_server_ptr obj);

	~mount_server_impl();

	void _generic_method(CORBA::octet_seq_t &,
	    CORBA::object_seq_t &, Environment &);

	void	_unreferenced(unref_t arg);

	//
	// Called from mount_replica_impl when switching to primary,
	// secondary, or spare.
	//
	void	convert_to_primary();
	void	convert_to_secondary();
	void	convert_to_spare();
	void	freeze_primary();
	void	unfreeze_primary();

	//
	// This is called to upgrade mount_client references
	// during Rolling Upgrade commit.
	//
	void    upgrade_client_reference(Environment &_environment);

	//
	// This is called if a mount client dies
	// (mount_client_elem::_unreferenced()).
	//
	void	client_died(mount_client_elem *cep);

	// Helper routine for handling mount clients dying. See client_died().
	void	do_unref(mount_client_elem *unref_clientp);

	//
	// Helper routine for mount_state::orphaned() to clean up mount().
	// Return true if the operation is completed.
	//
	bool	mount_orphaned(mount_state *statep, bool orph,
	    Environment &env);

	//
	// Common code for mount(), remount() and mount_orphaned().
	//
	void	mount_end_v1(fs::mount_client_ptr skip, const sol::mounta &ma,
	    const char *mntoptions, pxfs_v1::filesystem_ptr fs,
	    const pxfs_v1::fs_info &fsinfo, bool is_remount, Environment &env);

	// Helper routine for unmount_state::orphaned() to clean up unmount().
	void	unmount_orphaned(unmount_state *statep, bool ret_err,
	    Environment &env);

	// Helper routine to update device configuration changes.
	void	notify_change(sol::dev_t gdev, const sol::nodeid_seq_t &nodes,
	    Environment &env);

	// Helper routine to check if device is in use.
	bool	still_active(sol::dev_t gdev);

	//
	// Helper functions for checkpointing state on a secondary.
	// These are similar to the IDL operations above but don't have
	// the _environment.
	//
	void	ckpt_add_client(fs::mount_client_died_ptr clobj,
	    fs::mount_client_ptr client_p, sol::nodeid_t nodeid, bool shutdown);

	void	ckpt_remove_client(fs::mount_client_ptr client_p);

	void	ckpt_mount_start_v1(const sol::mounta &ma,
	    fs::mount_client_ptr client, Environment &env);

	void	ckpt_mount_err(sol::error_t error, Environment &env);

	void	ckpt_mount_v1(
	    pxfs_v1::filesystem_ptr	fs,
	    const pxfs_v1::fs_info	&fsinfo,
	    const sol::mounta		&ma,
	    const char			*mntoptions,
	    bool			dev_is_ha,
	    const char			*dev_name,
	    const sol::nodeid_seq_t	&dev_nids);

	void	ckpt_mount_middle_v1(
	    pxfs_v1::filesystem_ptr	fs,
	    const pxfs_v1::fs_info	&fsinfo,
	    const char			*mntoptions,
	    bool			dev_is_ha,
	    const char			*dev_name,
	    const sol::nodeid_seq_t	&dev_nids,
	    Environment			&env);

	void	ckpt_remount_middle_v1(
	    pxfs_v1::filesystem_ptr	fs,
	    uint32_t			vfsflags,
	    const char			*mntoptions,
	    Environment			&env);

	void	ckpt_unmount_start_v1(
	    pxfs_v1::filesystem_ptr	fs,
	    int32_t			flags,
	    solobj::cred_ptr		credobj,
	    fs::mount_client_ptr	client,
	    bool			is_shutdown,
	    Environment			&env);

	void	ckpt_unmount_middle(sol::error_t error, Environment &env);

	void	ckpt_unmount_notified(Environment &env);

	void	ckpt_unmount_end(Environment &env);

	void	ckpt_unmount_shutdown(fs::mount_client_ptr client);

	void	ckpt_devlock(fs::mount_client_ptr client, sol::nodeid_t nodeid,
	    const char *dev_name);

	void	ckpt_devunlock(const char *dev_name);

	void	ckpt_get_dc_callback(fs::dc_callback_ptr cb);

	void	ckpt_notify_change(sol::dev_t gdev,
	    const sol::nodeid_seq_t &dev_nids);

	void	ckpt_upgrade_client_list(fs::mount_client_ptr client_p,
	    sol::nodeid_t nodeid);

	void	ckpt_upgrade_devlock_list(const char *dev_name,
	    fs::mount_client_ptr client_p);

	void	dump_state(repl_pxfs::mount_replica_ptr ckptp,
	    Environment &env);

	// Checkpoint accessor function
	repl_pxfs::mount_replica_ptr	get_checkpoint();

protected:
	// fs::mount_server::*
	/* mount_server */
	void	add_client(fs::mount_client_ptr client_p, sol::nodeid_t nodeid,
	    fs::mount_client_died_out clobj, Environment &_environment);

	void	remove_client(fs::mount_client_ptr client_p,
	    solobj::cred_ptr credobj, Environment &_environment);

	void	mount(const sol::mounta &ma, sol::uintptr_t mvp,
	    solobj::cred_ptr credobj, fs::mount_client_ptr client_p,
	    bool dev_is_ha, const char *dev_name,
	    const sol::nodeid_seq_t &dev_nids, fs::filesystem_out fs,
	    fs::fs_info &fsinfo, CORBA::String_out mntoptions,
	    Environment &_environment);

	void	mount_v1(const sol::mounta &ma, sol::uintptr_t mvp,
	    solobj::cred_ptr credobj, fs::mount_client_ptr client_p,
	    bool dev_is_ha, const char *dev_name,
	    const sol::nodeid_seq_t &dev_nids, pxfs_v1::filesystem_out fs,
	    pxfs_v1::fs_info &fsinfo, CORBA::String_out mntoptions,
	    Environment &_environment);

	void	remount(fs::filesystem_ptr fs, fs::fobj_ptr mntpnt,
	    const sol::mounta &ma, solobj::cred_ptr credobj,
	    fs::mount_client_ptr client_p, uint32_t &vfsflags,
	    CORBA::String_out mntoptions, Environment &_environment);

	void	remount_v1(pxfs_v1::filesystem_ptr fs, pxfs_v1::fobj_ptr mntpnt,
	    const sol::mounta &ma, solobj::cred_ptr credobj,
	    fs::mount_client_ptr client_p, uint32_t &vfsflags,
	    CORBA::String_out mntoptions, Environment &_environment);

	void	unmount(fs::filesystem_ptr fs, solobj::cred_ptr credobj,
	    fs::mount_client_ptr client_p, sol::nodeid_t nodeid,
	    bool is_shutdown, Environment &_environment);

	void	unmount_1(fs::filesystem_ptr fs, int32_t flags,
	    solobj::cred_ptr credobj,
	    fs::mount_client_ptr c, sol::nodeid_t nodeid,
	    bool is_shutdown, Environment &_environment);

	void	unmount_v1(pxfs_v1::filesystem_ptr fs, int32_t flags,
	    solobj::cred_ptr credobj,
	    fs::mount_client_ptr client_p, sol::nodeid_t nodeid,
	    bool is_shutdown, Environment &_environment);

	void	devlock(fs::mount_client_ptr client_p, sol::nodeid_t nodeid,
	    const char *dev_name, Environment &_environment);

	void	devunlock(const char *dev_name, Environment &_environment);

	void	get_devlock_owner(const char *dev_name,
	    sol::nodeid_t &lock_owner, Environment &_environment);

	fs::dc_callback_ptr	get_dc_callback(Environment &_environment);
	//

private:
	enum fs_status_t {
		AVAILABLE, NOT_AVAILABLE, ERROR
	};

	// Helper function to test for simultaneous remounts
	void		check_multiple_remounts(const char *pathp,
	    Environment &_environment);

	// Version for forced unmount rolling upgrade support
	sol::error_t	unmount_common_1(fs::mount_client_ptr skip,
	    fs_elem *fep, int32_t flags,
	    solobj::cred_ptr credobj, unmount_state::state_t state,
	    sol::error_t error, const char *service_name, Environment &env);

	// Helper routine to lock mount points.
	sol::error_t	lock_mntpnt(fs::mount_client_ptr skip,
	    const char *mountpoint, int32_t mntflags, Environment &env);

	// Helper routine to unlock mount points already locked.
	void		unlock_mntpnt(fs::mount_client_ptr skip,
	    mount_client_elem *endp,
	    const char *mountpoint, Environment &env);

	// Helper routine to find a mount_client_elem given a node number.
	mount_client_elem	*find_client(nodeid_t nid);
	mount_client_elem	*find_client(fs::mount_client_ptr client_p);

	// Return the fs_elem for fs or NULL if not in fs_list.
	fs_elem			*find_fs(pxfs_v1::filesystem_ptr fs);
	fs_elem			*find_fs(const char *spec);

	// Return the devlock_elem or NULL if not in devlock_list.
	devlock_elem		*find_devlock(const char *spec);

	// Return true if nodeid is waiting for a lock.
	bool			find_devlock_waiter(sol::nodeid_t nodeid);

	// Find out if the given filesystem is available or not.
	fs_status_t		get_fs_status(fs_elem *fep);

	//
	// Lists of mount clients and file systems.
	//
	// lock ordering:
	//	client_list_lock.lock();
	//	fs_list_lock.lock();
	//	devlock_list_lock.lock();
	//

	typedef IntrList<mount_client_elem, _SList>	client_list_t;
	typedef IntrList<fs_elem, _DList>		fs_list_t;
	typedef IntrList<devlock_elem, _SList>		devlock_list_t;

	// List of mount clients
	client_list_t	client_list;

	// Lock to protect the list of clients.
	os::rwlock_t	client_list_lock;

	// List of globally mounted file systems.
	fs_list_t	fs_list;

	// Lock to protect the list of filesystems.
	os::rwlock_t	fs_list_lock;

	// List of device locks to prevent multiple simultaneous fsck's.
	devlock_list_t	devlock_list;

	// Lock to protect the list of device locks.
	os::rwlock_t	devlock_list_lock;

	// Lock protecting current the currentmnt string
	os::mutex_t	current_mount_lock;

	// CV to broadcast change of currentmnt state back to NULL
	os::condvar_t	currentmnt_cv;

	//
	// IDL pointer to the dc_callback object.
	// This is protected by fs_list_lock
	// (could be a separate lock though).
	//
	fs::dc_callback_var	dc_callback_obj;

	// Pointer to our replica server object.
	mount_replica_impl	*repl_serverp;

	// True if this replica is the primary.
	bool		primary;

	// True if this service is frozen. Protected by fs_list_lock.
	bool		frozen;

	// Current mountpoint that is being processed.
	char	*currentmnt;
};

#ifdef __cplusplus
extern "C" {
#endif

#ifdef __cplusplus
}
#endif

#endif	// _MOUNT_SERVER_IMPL_H