pxfs/server/repl_pxfs_server.h

//
// CDDL HEADER START
//
// The contents of this file are subject to the terms of the
// Common Development and Distribution License (the License).
// You may not use this file except in compliance with the License.
//
// You can obtain a copy of the license at usr/src/CDDL.txt
// or http://www.opensolaris.org/os/licensing.
// See the License for the specific language governing permissions
// and limitations under the License.
//
// When distributing Covered Code, include this CDDL HEADER in each
// file and include the License file at usr/src/CDDL.txt.
// If applicable, add the following below this CDDL HEADER, with the
// fields enclosed by brackets [] replaced with your own identifying
// information: Portions Copyright [yyyy] [name of copyright owner]
//
// CDDL HEADER END
//

//
// Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
// Use is subject to license terms.
//

#ifndef	REPL_PXFS_SERVER_H
#define	REPL_PXFS_SERVER_H

#pragma ident	"@(#)repl_pxfs_server.h	1.10	08/05/20 SMI"

#include <sys/vfs.h>

#include <sys/os.h>
#include <repl/service/replica_tmpl.h>

#include <pxfs/common/pxfslib.h>
#include "../version.h"
#include PXFS_IDL(pxfs)
#include PXFS_IDL(repl_pxfs)

// Forward declarations.
class fs_repl_impl;
class fobj_ii;

//
// HA service class for a single mounted pxfs file system.
// There is one of these for each mounted file system so that we can
// fail over file systems independently of eachother.
//
class repl_pxfs_server : public repl_server<REPL_PXFS_VER::fs_replica> {
public:
	repl_pxfs_server(vnode_t *mvp, const sol::mounta &ma, cred_t *cr,
	    const char *id);
	~repl_pxfs_server();

	// Helper function to get the mount error (if any).
	int get_mount_error() const;

	// Update the mount arguments after a remount (see fs_ii::remount()).
	void set_mountargs(const sol::mounta &ma);

	//
	// Helper function to mark this file system as unmounted. This
	// method is used to manipulate the current replica only. In
	// secondaries the check point function will set the flag.
	//
	void mark_fs_unmounted();

	//
	// Tell whether this file system has been unmounted.
	//
	bool is_fs_unmounted();

	// Handle new invocations after service freeze has started.
	bool	check_freeze(Environment &env);

	// Required functions for replica framework.
	// replica::repl_prov::=
	void become_secondary(Environment &_environment);

	void add_secondary(replica::checkpoint_ptr sec_chkpt,
	    const char *secondary_name, Environment &_environment);

	void remove_secondary(const char *secondary_name,
	    Environment &_environment);

	void freeze_primary_prepare(Environment &_environment);

	void freeze_primary(Environment &_environment);

	void unfreeze_primary(Environment &_environment);

	void become_primary(const replica::repl_name_seq &secondary_names,
	    Environment &_environment);

	void become_spare(Environment &_environment);

	void shutdown(Environment &_environment);

	uint32_t forced_shutdown(Environment &_environment);

	void shutdown_spare(replica::repl_prov_shutdown_type,
		Environment &_environment);

	CORBA::Object_ptr get_root_obj(Environment &_environment);
	//

	void ckpt_blocks_allocated(
	    const repl_pxfs_v1::blocks_allocated_t &current_allocations,
	    PXFS_VER::blkcnt_t blocks_free, Environment &_environment);

	void ckpt_server_status(PXFS_VER::server_status_t status,
	    Environment &_environment);

	// fs_replica::=
	void ckpt_locks(PXFS_VER::fobj_ptr obj,
	    const REPL_PXFS_VER::lock_info_seq_t &locks,
	    Environment &_environment);

	void ckpt_new_fsobj(PXFS_VER::filesystem_ptr fs_obj,
	    const char *mntoptions, Environment &_environment);

	void ckpt_mnt_error(sol::error_t error, Environment &_environment);

	void ckpt_entry_state(bool exists, Environment &_environment);

	void ckpt_target(PXFS_VER::fobj_ptr obj,
	    const PXFS_VER::fobj_info &fobjinfo,
	    Environment &_environment);

	void ckpt_target_remove(PXFS_VER::fobj_ptr obj,
	    const PXFS_VER::fobj_info &fobjinfo,
	    uint64_t delete_id, Environment &_environment);

	void ckpt_fobj_return(PXFS_VER::fobj_ptr ret_obj,
	    const PXFS_VER::fobj_info &ret_info, Environment &_environment);

	void ckpt_error_return(sol::error_t error, Environment &_environment);

	void ckpt_new_fobj(PXFS_VER::fobj_ptr obj,
	    const PXFS_VER::fobjid_t &fid,
	    PXFS_VER::fobj_type_t type,
	    Environment &_environment);

	void ckpt_delete_fobj(uint64_t delete_id, Environment &_environment);

	void ckpt_deletecnt(uint64_t delete_id, Environment &_environment);

	void ckpt_fobj_state(PXFS_VER::fobj_ptr obj, uint64_t delete_id,
	    Environment &_environment);

	void ckpt_new_fsmgr(PXFS_VER::fsmgr_server_ptr servermgr,
	    PXFS_VER::fsmgr_client_ptr clientmgr, sol::nodeid_t nodeid,
	    Environment &_environment);

	void ckpt_remount(const sol::mounta &ma, const char *mntoptions,
	    Environment &_environment);

	void ckpt_lockfs_info(uint64_t lf_lock, uint64_t lf_flags,
	    uint64_t lf_key, const char *lf_comment,
	    Environment &_environment);

	void ckpt_lockfs_start(uint64_t lf_lock, uint64_t lf_flags,
	    uint64_t lf_key, const char *lf_comment,
	    Environment &_environment);

	void ckpt_lockfs_failure(sol::error_t err, Environment &_environment);

	void ckpt_cachedata_flag(PXFS_VER::file_ptr obj, bool flag,
	    Environment &_environment);

	void ckpt_vx_tunefs(const REPL_PXFS_VER::vx_tunefs_t &tunefs,
	    Environment &_environment);

	void ckpt_remove_file_locks_by_sysid(int32_t sysid,
	    Environment &_environment);

	void ckpt_remove_file_locks_by_nlmid(int32_t nlmid,
	    Environment &_environment);

	void ckpt_service_version(unsigned short new_major,
	    unsigned short new_minor, Environment &);

	void ckpt_fs_is_unmounted(Environment &_environment);
	//
	// Checkpoint accessor function.
	REPL_PXFS_VER::fs_replica_ptr	get_checkpoint_fs_replica();

	//
	// Register with the Version Manager for upgrade callbacks.
	// Called by the mount_client when instantiating a filesytem replica.
	//
	void upgrade_callback_register(const sol::mounta &);

	// Unregister with the Version Manager
	void upgrade_callback_unregister();

	//
	// fs_version_callback_impl::do_callback is called by the
	// version manager.  That method calls here to pass on the version
	// update.
	//
	void upgrade_callback(const version_manager::vp_version_t &,
	    Environment &);

	// Set the initial protocol version number.
	void set_version(const version_manager::vp_version_t &);

	version_manager::upgrade_callback_var callback_object_v;

	//
	// These are both protected by version_lock.
	// current_version is the versioned protocol number the version
	// manager has told us we should be running as.
	// pending_version is set when a secondary gets a callback before
	// the primary sends a checkpoint so a failover during upgrade
	// commit is processed correctly.
	//
	version_manager::vp_version_t current_version;
	version_manager::vp_version_t pending_version;

	//
	// This lock protects 'current_version', _ckpt_proxy and
	// provides locking between upgrade callbacks and become_primary().
	// It needs to be a rwlock since we make checkpoint calls while
	// holding the lock.
	//
	os::rwlock_t version_lock;

	// Helper function to track outstanding invocations.
	void	decrement_invo_count();

private:
	//
	// Pointer to the fs that is being replicated.
	// If the mount fails, mnt_error is set to the errno.
	// We hold a reference to the file system so that _unreferenced()
	// doesn't delete the file system object while we have a pointer
	// to it.
	//
	PXFS_VER::filesystem_var	fs_v;
	fs_repl_impl	*fsp;
	int		mnt_error;	// saved errno from VFS_MOUNT()

	//
	// Our copy of the mountdata and mount point vnode.
	// This is used to mount/remount the underlying file system.
	//
	sol::mounta	mountdata;
	vnode_t		*mnt_vp;
	cred_t		*crp;
	bool		fs_is_unmounted;

	// Checkpoint proxy.
	REPL_PXFS_VER::fs_replica_ptr	_ckpt_proxy;

	//
	// used for creating unique ucc names when registering with the
	// version manager for callbacks to a filesystem replica.
	//
	static int	unique_integer;

	//
	// A PxFS client issuing cascaded invocations results in the server
	// issuing further invocations to other PxFS clients. Those clients
	// in turn may need to issue further invocations to service the
	// request. If the relevant PxFS service is frozen right after the
	// server checked for service freeze and before a client issued an
	// invocation to service the cascaded invocation, the client's new
	// invocation will be frozen. The cascaded invocation will not
	// complete until the client can complete it's invocation. That
	// will not complete until the service is unfrozen. We have a
	// deadlock.
	//
	// To address this deadlock we use check_freeze() method of PxFS
	// replicated service. check_freeze() decided how to handle new
	// requests depending on the state of the service. If the state is
	// 'PRIMARY' new invocations will be allowed and the invocation
	// count in incremented. If the current state is 'FREEZING' the
	// thread blocks until all active invocations from the server have
	// been serviced and the state of the this file system replicated
	// service changes to 'FROZEN'. At this point blocked invocations
	// are un-blocked and they return a 'PRIMARY_FROZEN' exception. The
	// invocation will be retried by the client after the service is
	// unfrozen. If current state is 'FROZEN' the thread will return a
	// 'PRIMARY_FROZEN' exception immediately.
	//
	enum primary_state_t {
		NOT_PRIMARY,	// this is not the primary
		PRIMARY,	// Primary active and not frozen
		FREEZING,	// Primary blocks just deeply nested invos
		FROZEN		// Invos to primary return SERVICE_FROZEN
	};

	primary_state_t	replica_state;	// Current state of provider

	//
	// Variables to track active invocations from this service.
	//
	int64_t		active_invo_count;
	os::mutex_t	active_invo_lock;
	os::condvar_t	active_invo_cv;
};

class fs_version_callback_impl :
    public McServerof<version_manager::upgrade_callback> {
public:
	fs_version_callback_impl(replica::repl_prov_ptr replica_p);
	~fs_version_callback_impl();

	void _unreferenced(unref_t);

	// IDL methods.
	void do_callback(const char *ucc_name,
	    const version_manager::vp_version_t &new_version,
	    Environment &_environment);

private:
	replica::repl_prov_var prov_v;
};

#include <pxfs/server/repl_pxfs_server_in.h>

#endif	// REPL_PXFS_SERVER_H