pxfs/server/cl_flock.cc

//
// CDDL HEADER START
//
// The contents of this file are subject to the terms of the
// Common Development and Distribution License (the License).
// You may not use this file except in compliance with the License.
//
// You can obtain a copy of the license at usr/src/CDDL.txt
// or http://www.opensolaris.org/os/licensing.
// See the License for the specific language governing permissions
// and limitations under the License.
//
// When distributing Covered Code, include this CDDL HEADER in each
// file and include the License file at usr/src/CDDL.txt.
// If applicable, add the following below this CDDL HEADER, with the
// fields enclosed by brackets [] replaced with your own identifying
// information: Portions Copyright [yyyy] [name of copyright owner]
//
// CDDL HEADER END
//

//
// Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
// Use is subject to license terms.
//

#pragma ident	"@(#)cl_flock.cc	1.8	08/05/20 SMI"

#include <sys/flock_impl.h>
#include <sys/flock.h>
#include <sys/disp.h>

#include <sys/list_def.h>

#include <orb/infrastructure/clusterproc.h>

#include "../version.h"
#include PXFS_IDL(pxfs)
#include <pxfs/lib/pxfs_debug.h>
#include <pxfs/server/cl_flock.h>
#include <pxfs/server/fobj_impl.h>
#include <pxfs/server/frange_lock.h>

//lint -e1512
//
// Warning(1512) destructor for base class is not virtual -- In a
// final pass through all the classes, we have found a class that is
// the base class of a derivation and has a destructor but the
// destructor is not virtual. It is conventional for inherited classes
// to have virtual destructors so that is it safe to 'delete' a
// pointer to a base class.
//

static void cl_flk_handle_waking_locks(void *);
static void cl_flk_state_notify_lock_woken(lock_descriptor_t *);
static fobj_ii *get_fobj_ii(lock_descriptor_t *);

typedef SList<lock_descriptor_t> locks_list;

//
// This list stores the previously blocked lock requests that have now been
// granted and await the client being notified.  After this notification is
// delivered, the lock is pushed to the processed_list
//
static locks_list granted_list;

// Protects the 'granted_list'
static os::mutex_t granted_list_lock;

//
// The locking order is: graph_lock, granted_list_lock, processed_list_lock
//

//
// The thread that delivers client notifications sleeps on this cv. Whenever
// a lock is added to the 'granted_list', this cv is prodded
//
static os::condvar_t sleep_cond;

//
// This procedure is run by a separate thread, and processes all the
// locks that come in the 'granted_list'.  Processing a lock consists of
// notifying its client that it has been woken up, and placing it on the
// 'processed_list'.  When the 'granted_list' is empty, the thread goes
// to sleep, and is signalled awake when a new entry is placed in the
// 'granted_list'.
//
void
cl_flk_handle_waking_locks(void *)
{
	lock_descriptor_t		*lock;
	PXFS_VER::pxfs_llm_callback_ptr	lock_cb;
	PXFS_VER::pxfs_llm_callback_ptr	lock_cb_dup;
	Environment			e;
	lock_context_t			*lock_contextp;
	frange_lock			*fr_lockp;

	granted_list_lock.lock();
	for (;;) {

		sleep_cond.wait(&granted_list_lock);

#ifdef _FAULT_INJECTION
		if (fault_triggered(FAULTNUM_PXFS_GRANTEDLOCK_S_O, NULL,
		    NULL)) {
			granted_list_lock.unlock();
			FAULTPT_PXFS(FAULTNUM_PXFS_GRANTEDLOCK_S_O,
			    FaultFunctions::generic);
			granted_list_lock.lock();
		}
#endif

		while ((lock = granted_list.reapfirst()) != NULL) {

			// get the callback object
			lock_contextp = CL_FLK_GET_CONTEXT(lock);
			lock_cb = lock_contextp->cb_obj_v1;
			ASSERT(!CORBA::is_nil(lock_cb));

			fr_lockp = ((fobj_ii *)(lock_contextp->fobj_abstract_p))
			    ->get_frange_lock();
			ASSERT(fr_lockp != NULL);

			//
			// Grab the 'processed_list_lock' before dropping the
			// 'granted_list_lock', to handle the race between this
			// procedure and cl_flk_cancel_blocked_request().
			//
			fr_lockp->lock_processed_list();
			granted_list_lock.unlock();

			fr_lockp->add_to_processed_list(lock);

			// Duplicate the callback object, and then drop the
			// `processed_list_lock'
			lock_cb_dup =
			    PXFS_VER::pxfs_llm_callback::_duplicate(lock_cb);

			fr_lockp->unlock_processed_list();

			// invoke the callback function.
			lock_cb_dup->wakeup(e);

			// Release the duplicated reference.
			CORBA::release(lock_cb_dup);

			// We ignore exceptions since there is nothing to do
			// if the client crashed
			e.clear();

			granted_list_lock.lock();
		}
	}
}

//
// Remove from the granted list and the LLM all locks associated with
// 'fobj_iip'.
//
void
remove_granted_locks(fobj_ii *fobj_iip)
{
	lock_descriptor_t	*lock;

	frange_lock		*fr_lockp = fobj_iip->get_frange_lock();
	ASSERT(fr_lockp != NULL);

	granted_list_lock.lock();
	granted_list.atfirst();
	while ((lock = granted_list.get_current()) != NULL) {
		if (get_fobj_ii(lock) != fobj_iip) {
			granted_list.advance();
			continue;
		}
		(void) granted_list.erase(lock);
		granted_list_lock.unlock();
		PXFS_DBPRINTF(
		    PXFS_TRACE_FLK,
		    PXFS_GREEN,
		    ("(%p) Removing granted lock (%p)\n",
		    fobj_iip, lock));
		(void) fr_lockp->process_pxfs_lock(lock, true);
		granted_list_lock.lock();
		granted_list.atfirst();
	}
	granted_list_lock.unlock();
}

//
// This function adds the lock provided to the 'granted_list' of locks.
// This routine is called (indirectly) by the local lock manager when a
// blocked PXFS thread is granted.
//
void
cl_flk_state_notify_lock_woken(lock_descriptor_t *lock)
{
	granted_list_lock.lock();

	// Add the lock to the list
	granted_list.prepend(lock);

	// Signal awake the thread that makes the callbacks
	sleep_cond.signal();

	granted_list_lock.unlock();
}

//
// This routine is invoked when a sleeping PXFS lock is going to be
// deleted.  It checks to see if the lock is already on one of the
// sleeping PXFS lists, and if it is, it wakes it up.
//
void
cl_flk_state_notify_woken_lock_deleted(lock_descriptor_t *lock)
{
	PXFS_VER::pxfs_llm_callback_ptr	lock_cb;
	bool				found;
	lock_context_t			*lock_contextp;

	frange_lock			*fr_lockp = get_fobj_ii(lock)->
					    get_frange_lock();
	ASSERT(fr_lockp != NULL);

	//
	// The lock has beed woken up previously - this means that it is on one
	// of the PXFS lists unless this call is made as a result of
	// fobj_ii::remove_all_locks().
	// The order of searching the lists has to be 'granted_list',
	// 'processed_list', because of the implementation of
	// cl_flk_handle_waking_locks()
	//
	granted_list_lock.lock();
	found = granted_list.erase(lock);
	granted_list_lock.unlock();
	if (!found) {
		(void) fr_lockp->remove_from_processed_list(lock);
	}

	// Relase the CORBA reference
	lock_contextp = CL_FLK_GET_CONTEXT(lock);
	lock_cb = lock_contextp->cb_obj_v1;

	// A woken-up lock must have a valid callback object associated with it
	ASSERT(!CORBA::is_nil(lock_cb));

	fr_lockp->remove_cb_object(lock_cb);

	// Drop the reference to the callback object.
	CORBA::release(lock_cb);
}

//
// Helper function that returns the fobj that the given PXFS lock belongs to.
//
fobj_ii *
get_fobj_ii(lock_descriptor_t *lock)
{
	lock_context_t *fobj_lock_infop = CL_FLK_GET_CONTEXT(lock);

	return ((fobj_ii *)fobj_lock_infop->fobj_abstract_p);
}

//
// This function is called by the local lock manager whenever the
// state of a PXFS lock changes. It dispatches the notifications to the
// appropriate function.
//
void
flk_state_transition_notify(lock_descriptor_t *lock, int old_state,
    int new_state)
{
	ASSERT(IS_PXFS(lock));

	frange_lock	*fr_lockp;

	PXFS_DBPRINTF(
	    PXFS_TRACE_FLK,
	    PXFS_GREEN,
	    ("(%p) state transition (%d - %d): (%d) (%d, %d)\n",
	    CL_FLK_GET_CONTEXT(lock)->fobj_abstract_p, old_state, new_state,
	    (int)lock->l_type, (int)lock->l_start, (int)lock->l_end));

	switch (new_state) {
	case FLK_ACTIVE_STATE:
		fr_lockp = get_fobj_ii(lock)->get_frange_lock();
		ASSERT(fr_lockp != NULL);

		// Lock has become active - notify the fobj
		fr_lockp->lock_active(lock);
		return;

	case FLK_GRANTED_STATE:
	case FLK_INTERRUPTED_STATE:
	case FLK_CANCELLED_STATE:
		// Lock has been woken up.
		if ((old_state == FLK_GRANTED_STATE) ||
		    (old_state == FLK_CANCELLED_STATE) ||
		    (old_state == FLK_INTERRUPTED_STATE)) {
			// Lock has been woken up before - it is already in the
			// PXFS lists.
			return;
		}
		cl_flk_state_notify_lock_woken(lock);
		return;

	case FLK_DEAD_STATE:

		if ((old_state == FLK_GRANTED_STATE) ||
		    (old_state == FLK_CANCELLED_STATE) ||
		    (old_state == FLK_INTERRUPTED_STATE)) {
			// A woken-up lock is being deleted
			cl_flk_state_notify_woken_lock_deleted(lock);

		} else if (old_state == FLK_ACTIVE_STATE) {
			fr_lockp = get_fobj_ii(lock)->get_frange_lock();
			ASSERT(fr_lockp != NULL);

			// Active lock has been deleted
			fr_lockp->lock_deleted(lock);
		}

		// The other cases of a lock transitioning to the 'DEAD' state
		// are not of interest to PXFS

		return;

	default:
		break;
	}
}

//
// Creates the thread that notifies sleeping clients.
//
int
cl_flk_init()
{
	//
	// Create a lwp in the scheduling class SYS.
	//
	if ((clnewlwp(cl_flk_handle_waking_locks, NULL, MINCLSYSPRI,
	    NULL, NULL)) != 0) {
		cmn_err(CE_WARN, "cl_flk_init: cannot create thread");
		return (-1);
	}
	return (0);
}