pxfs/lock/nlm_pxfs.cc

//
// CDDL HEADER START
//
// The contents of this file are subject to the terms of the
// Common Development and Distribution License (the License).
// You may not use this file except in compliance with the License.
//
// You can obtain a copy of the license at usr/src/CDDL.txt
// or http://www.opensolaris.org/os/licensing.
// See the License for the specific language governing permissions
// and limitations under the License.
//
// When distributing Covered Code, include this CDDL HEADER in each
// file and include the License file at usr/src/CDDL.txt.
// If applicable, add the following below this CDDL HEADER, with the
// fields enclosed by brackets [] replaced with your own identifying
// information: Portions Copyright [yyyy] [name of copyright owner]
//
// CDDL HEADER END
//

//
// Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
// Use is subject to license terms.
//

#pragma ident	"@(#)nlm_pxfs.cc	1.42	08/05/20 SMI"

#include <sys/flock.h>
#include <sys/fcntl.h>

#include <h/naming.h>
#include <nslib/ns.h>
#include <orb/infrastructure/orbthreads.h>

#include <pxfs/lock/fs_collection_impl.h>
#include <pxfs/lock/lock_debug.h>
#include <pxfs/client/pxvfs.h>

enum {
	MAX_ITEMS = 1000
};

const char		fscollection_name[] = "fscollection";
const unsigned long	fscollection_name_size = 12;

//
// This new class is used to create a deferred task in the ORB that
// will clean up the NLM's lm_sysid cache for sysids (identifies remote
// clients) that aren't holding any locks.  The "execute" method will
// call back into the NLM base Solaris code to execute lm_free_sysids(),
// which does the actual work.
//
// In the old implementation, the constructor for this object was
// called everytime the Solaris hook lm_free_nlm_sysid_table()
// was invoked by the cache reaper thread. If memory was low, then the
// the constructor would block, waiting for memory to free up, but this
// also blocked the single-threaded reaper thread from proceeding.
//
// There are two solutions:
// 1.  Qualify the new statement with (os::NO_SLEEP), so that the
//	allocation won't block if there's no memory.  This is the simplest
//	fix because it won't block the reaper thread.  A deferred
//	task object will be created for every invocation.
//
// 2.  In this solution, we observe that
//	if a deferred task object has already been created to free sysid
//	storage, there's no need to create another one if the first still
//	exists (which means it hasn't yet been executed).  The first will do
//	the same thing as all the others.  This has two benefits: saving
//	on storage at a critical time and saving the processing time of
//	allocating storage.  The approach is to allocate exactly one
//	nlm_sysid_cleanup object at the time the PXFS server kernel module
//	is loaded and exists for all time until the module is unloaded.
//	The object synchronizes concurrrent access between the reaper
//	thread and the deferred task's thread executing method "execute."
//	If the object is "in use" it means it's already on the queue, so
//	there's nothing to do but return.  If not "in use" then flag
//	it as "in use" and enqueue on the deferred task queue.
//
//	We chose solution 2 to implement.
//
class nlm_sysid_cleanup : public defer_task {
public:
	nlm_sysid_cleanup() { obj_in_use = false; }
	~nlm_sysid_cleanup() {}
	//
	// Need this method to override base method's habit of deleting
	// object.
	//
	void task_done() {}

	void execute();

	void set_lock() { cleanup_lock.lock(); }
	void unlock() { cleanup_lock.unlock(); }

	void signal() { nlm_cv.signal(); }
	void wait_for_signal() { nlm_cv.wait(&cleanup_lock); }

	void set_inuse(bool in_use) { obj_in_use = in_use; }
	bool is_inuse() { return (obj_in_use); }

private:
	//
	// The reason to define a mutex lock here is to synchronize access
	// from a thread putting the clean up object on the deferred
	// task queue and a thread executing the deferred task.
	//
	os::mutex_t	cleanup_lock;
	bool		obj_in_use;

	// Condition variable used by 'execute' and 'nlm_fini'
	os::condvar_t	nlm_cv;
};

//
// Allocate deferred task object.  Exists for all time.  The reason
// we allocate from the heap instead of statically is to avoid a
// possible race condition.  Sun Cluster does
// not control the time when a statically constructed object
// actually gets created. It is theoretically possible that
// immediately after the module gets loaded and before the statically
// constructed object gets created, the system might try to
// exercise this code.
//
static nlm_sysid_cleanup *nlm_cleanup_task = NULL;

extern "C" {
	extern void (*lm_set_nlm_status)(int, flk_nlm_status_t);
	extern void (*lm_remove_file_locks)(int);
	extern int  (*lm_has_file_locks)(int, int);
	extern void (*lm_free_nlm_sysid_table)(void);
	extern void lm_free_sysids(void *);
}

//
// Effects:  This method is the workhorse of the deferred task.  It calls
// back into the NLM base Solaris code itself and executes a routine that
// does the actual work.
//
void
nlm_sysid_cleanup::execute()
{
	//
	// The following invocation is a call directly into the bowels
	// of the NLM itself.  The routine iterates over the list of
	// sysids, checks to see whether a sysid holds locks anywhere,
	// and if not, frees the sysid storage; if it does, skips.
	//
	// Note that the actual argument to this base Solaris NLM routine
	// doesn't matter because that argument is never used in the
	// body of the routine.
	//

	lm_free_sysids(NULL);

	//
	// Change the "in use" flag to false under protection of the
	// mutex.  False means the task object is no longer in use.
	// The nlm_free_sysid_table() method is free to test the "in use"
	// flag.
	//
	nlm_cleanup_task->set_lock();
	nlm_cleanup_task->set_inuse(false);

	//
	// We must tell the thread calling nlm_fini() to unload the
	// PXFS server module that it's okay to destroy this object.
	//
	nlm_cleanup_task->signal();

	nlm_cleanup_task->unlock();

	//
	// Note that the deferred task object is never deleted. It exists
	// for all eternity.
	//
}

//
// There is one Network Lock Manager (NLM) per cluster node.
// It is implemented by the lockd daemon.  Clients talking to
// an NLM on some node can acquire locks only for those file
// systems that were NFS-exported from that node. Since the
// primary servers for NFS-exported PXFS file systems could
// reside on any node in the cluster, a client could have
// file locks anywhere. The locks acquired through this node's
// NLM are uniquely identifed by the node id. Note that the NLM
// is a PXFS client.
//
// There is one Local Lock Manager (LLM) per cluster node.  The
// primary server of each PXFS file system uses the LLM at its
// node to store all locks acquired, whether those locks were
// acquired via an NLM for NFS clients or for local clients
// on the cluster. Thus, there may be two kinds of PXFS file
// locks at this LLM--for NFS clients (NLM) and local cluster
// clients.
//
// The Local Lock Manager (LLM) records locks acquired for local file
// systems like UFS and VxFS that are NFS-exported. Permitting local
// file system NLM locks to coexist with PXFS NLM locks was NOT
// part of the original design of the NLM for Sun Cluster, and
// is therefore, a new requirement. It was envisioned that clients
// of the cluster would only use PXFS. And this design decision
// can be seen in the Solaris NLM code, which assumes that when
// a cluster is booted, PXFS is the only file system being used.
//
// It turns out that we can accommodate this new requirement fairly
// easily in the PXFS code (in this file), which is plugged into the
// Solaris NLM hooks at the time the PXFS server kernel module is
// loaded (loaded at every node).  These routines typically go
// out to all PXFS primary servers in the cluster to do the work
// at the LLMs. In additon, we need to consider the NLM locks for
// the local file system at the PXFS client node (NLM is a PXFS
// client), and so we fiddle with either the sysid or the nlmid
// (setting it to 0) and call the LLM routines.
//

//
// Effects: Returns 1 (true) if file locks are held by
//   a remote client identified by "sysid" at any
//   fs_collection object. Returns 0 (false) otherwise.
//   Note that this routine is callable from C.
// Parameters:
//   sysid   (IN):  uniquely identifies lock owner
//   chklck  (IN):
//
static int
nlm_has_file_locks(int sysid, int chklck)
{
	// 1. Get the name server
	// 2. For each node in the cluster, fetch the fs collection object
	//    a. invoke has_file_locks() method on fs collection object.
	//	- if any exception is raised, then clear it and continue
	//	- if a result is true, then return true immediately.
	//	- if all results are false, then return false

	Environment			e;
	naming::naming_context_var	ctxp;

	ctxp = ns::root_nameserver();	// get root name server

	naming::binding_list_var bl;
	naming::binding_iterator_var binditer;

	ctxp->list(MAX_ITEMS, bl, binditer, e);
	//
	// The name server should never fail, as long as one node of the
	// cluster is up.
	//
	if (e.exception()) {
		e.clear();
		return (0);
	}

	uint_t len = bl->length();

	bool result = false;
	int retres = 0;
	for (uint_t index = 0; index < len; index++) {
		char *bindname = bl[index].binding_name;

		//
		// Compare just the first 12 characters of the name, those
		// characters being "fscollection".
		//
		if (os::strncmp(bindname, fscollection_name,
			fscollection_name_size) == 0) {
			//
			// If the name server cannot resolve the name, then
			// this node must have died.  We continue with next
			// iteration as if the node is dead, and no locks
			// are held.
			//
			CORBA::Object_var obj = ctxp->resolve(bindname, e);

			if (e.exception()) {
				e.clear();
				LOCK_DBPRINTF(
				    LOCK_TRACE_NLM,
				    LOCK_RED,
				    ("nlm_has_file_locks: can't resolve "
				    "name\n"));
				continue;
			}
			ASSERT(is_not_nil(obj));

			// coerce corba object to fs_collection object
			fs::fs_collection_var fscoll =
				fs::fs_collection::_narrow(obj);
			ASSERT(is_not_nil(fscoll));

			result = fscoll->has_file_locks(sysid, chklck, e);

			//
			// If any exceptions were raised, then the method
			// invocation has failed.  We aossume that the remote
			// node failed, and therefore the locks are gone.
			// We just continue with the next iteration.
			//
			if (e.exception()) {
				e.clear();
				continue;
			}

			//
			// If any method invocation returns true, then
			// the client still holds locks somewhere; we
			// don't care where. Return immediately.
			//
			if (result) {
				retres = 1;
				break;
				//				return (1);
			}
		} // end if
	} // end for

	//
	// Now that we've checked for clients of PXFS files, we must check
	// for clients of local file systems and ask whether those
	// clients have locks anywhere.
	//
	// Since the Solaris NLM code set the upper 2 bytes to indicate
	// we're in a cluster using PXFS, we need to reset those bytes
	// to 0 here so that we can ask about locks for the local file
	// system.  Calling this macro extracts just the sysid (lower
	// two bytes) and sets it to a new variable; this effectively
	// sets the upper two bytes to 0.
	//
	int local_sysid = GETSYSID(sysid);

	//
	// Check to see if the client holds NLM locks on the local
	// file system. Ask the LLM in a direct call to the Solaris code.
	//
	if (flk_sysid_has_locks(local_sysid, chklck) == 1) {
		retres = 1;
	} // end if

	return (retres);
}
// nlm_has_file_locks


//
// Effects:  This routine sets the status of the local lock manager
//   to the argument "status" for the given NLM identified by argument
//   "nlmid." It also sets the status for NLM locks obtained for the
//   local file system at the NLM's node. Returns nothing.
//   Note that this routine is callable from C.
// Parameters:
//	nlmid	(IN): node id of NLM server
//	status	(IN): state of NLM server
//			legal values are NLM_UP, NLM_DOWN,
//			and NLM_SHUTTING_DOWN.
//
static void
nlm_set_nlm_status(int32_t nlmid, flk_nlm_status_t status)
{
	fs::nlm_status new_status = fs::nlm_up;
	flk_nlm_status_t new_state = FLK_NLM_UP;

	switch (status) {
	case FLK_NLM_UP:
		new_status = fs::nlm_up;
		new_state = FLK_NLM_UP;
		break;
	case FLK_NLM_DOWN:
		new_status = fs::nlm_down;
		new_state = FLK_NLM_DOWN;
		break;
	case FLK_NLM_SHUTTING_DOWN:
		new_status = fs::nlm_shutting_down;
		new_state = FLK_NLM_SHUTTING_DOWN;
		break;
	default:
		ASSERT(0);
		// NOTREACHED
	} // end switch

	//
	// When we change the state of the NLM "nlmid" we must consider
	// the local file system's NLM locks. The following call talks
	// directly to the LLM on node "nlmid." To identify the local
	// file system's NLM locks, we set nlmid to 0.
	//
	// The reason we can merely set the nlmid to 0 is that we
	// are taking advantage of a loophole in the called routine.
	// Note that in the Solaris routine "cl_flk_set_nlm_status(nlmid,
	// state)" in usr/src/uts/common/os/flock.c, there's a "Requires"
	// clause in the comment, which says the following:
	//    "nlmid" must be >= 1 and <= clconf_maximum_nodeid()"
	// It says "Requires" because it cannot enforce it. The routine
	// returns void, that is, nothing, so we cannot report an error.
	// We leave it to the caller to ensure the constraint. Since
	// the routine doesn't enforce, that leaves a loophole.
	// This solution exploits that loophole. We can consider
	// UFS locks in this routine even though we're booted as
	// a cluster, by setting the first argument to 0. Someday
	// we should update that comment in Solaris.
	//
	cl_flk_set_nlm_status(0, new_state);

	//
	// The following call considers NLM locks acquired for
	// PXFS files only. It sets the status for the NLM locks
	// acquired via the NLM residing at node "nlmid." This may have
	// the effect of talking to all PXFS file systems in the cluster
	// and setting the status at the LLMs serving those file systems.
	//
	// Note that the old and new enum's for nlm_status are identical.
	// Different enum definitions are used to prevent a circular dependency
	// due to rolling upgrade.
	//
	pxvfs::set_nlm_status(nlmid, (pxfs_v1::nlm_status)new_status);
}

//
// Requires: "sysid" is a pair [nlmid, sysid].  The lower half is 16-bit
//	quantity, the real sysid generated by the NLM server; the upper half
//	identifies the node of the cluster where the NLM server ran.
// Effects: Delete all the advisory file locks held by a remote client
//    at all fs_collection objects.  The client is identified by "sysid."
//    It also remove NLM locks for all local file systems.
//   Note that this routine is callable from C.
// Parameters:
//   sysid   (IN):  uniquely identifies lock owner
//
static void
nlm_remove_file_locks(int32_t sysid)
{
	//
	// Remove the NLM locks on a local file system.  Set the upper
	// bytes of the sysid to 0, which indicates local locks.
	// The following macro extracts the low-order two bytes and
	// assigns them to a new variable; this effectively makes the
	// high-order two bytes to be 0.  The reason we do this instead
	// of something more straightforward like the in-line routine
	//   pxfslib::set_nodeid(sysid, 0)
	// is that that that routine does NOT work correctly when the
	// nodeid is set to 0; in fact, it's a no-op.
	//
	int new_sysid = GETSYSID(sysid);
	cl_flk_remove_locks_by_sysid(new_sysid);

	//
	// Remove the NLM file locks on PXFS files only. The
	// lock owner is identified by the sysid [nlmid, sysid]
	// pair on all PXFS file systems.
	//
	pxvfs::remove_file_locks(sysid);
}

//
// Effects:  This routine creates a deferred task from the common
// threadpool.  The task will call back into the NLM base Solaris code
// to the actual work of freeing any sysids not holding locks.
//
// Note:  This routine will be plugged into the lm_free_nlm_sysid_table
// hook in the base Solaris code by "nlm_init" when the PXFS server kernel
// module is loaded.
//
static void
nlm_free_sysid_table()
{
	//
	// If this callback is called before the kernel module is completely
	// loaded, then the deferred task object could be NULL.  We bail
	// out if this is the case.
	//
	if (nlm_cleanup_task == NULL) {
		return;
	}

	nlm_cleanup_task->set_lock();

	//
	// If the object is not in use, then flag it as "in use" and
	// then queue it on the deferred task queue for eventual
	// execution.
	//
	if (!nlm_cleanup_task->is_inuse()) {
		nlm_cleanup_task->set_inuse(true);
		nlm_cleanup_task->unlock();

		//
		// Enqueue nlm_cleanup_task object on the deferred task
		// queue.
		//
		common_threadpool::the().defer_processing(nlm_cleanup_task);

		return;
	} // end if

	nlm_cleanup_task->unlock();
}

void
nlm_init()
{
	//
	// Allocate the deferred task here when the PXFS server module
	// loaded.
	//
	if (nlm_cleanup_task == NULL) {
		nlm_cleanup_task = new nlm_sysid_cleanup();
	}

	lm_set_nlm_status = nlm_set_nlm_status;
	lm_remove_file_locks = nlm_remove_file_locks;
	lm_has_file_locks = nlm_has_file_locks;
	lm_free_nlm_sysid_table = nlm_free_sysid_table;
}

void
nlm_fini()
{
	if (nlm_cleanup_task != NULL) {
		//
		// Before destroying the deferred task object, wait
		// until it's no longer in use.  We'll be signalled by
		// the thread executing 'execute' method.
		//
		nlm_cleanup_task->set_lock();
		while (true) {
			nlm_cleanup_task->wait_for_signal();

			if (!nlm_cleanup_task->is_inuse()) {
				nlm_cleanup_task->unlock();

				// Delete the task object.
				delete nlm_cleanup_task;
			} // end if
		} // end while
	} // end if

	lm_set_nlm_status = NULL;
	lm_remove_file_locks = NULL;
	lm_has_file_locks = NULL;
	lm_free_nlm_sysid_table = NULL;
}