libmeta/common/meta_mirror_resync.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

/*
 * mirror operations
 */

#include <meta.h>
#include <sys/lvm/md_mirror.h>
#include <thread.h>

extern	int	md_in_daemon;
extern md_mn_client_list_t *mdmn_clients;

/*
 * chain of mirrors
 */
typedef struct mm_unit_list {
	struct mm_unit_list	*next;	/* next in chain */
	mdname_t		*namep;	/* mirror name */
	mm_pass_num_t		pass;	/* pass number */
	uint_t			done;	/* resync done */
} mm_unit_list_t;

/*
 * resync mirror
 * meta_lock for this set should be held on entry.
 */
int
meta_mirror_resync(
	mdsetname_t		*sp,
	mdname_t		*mirnp,
	daddr_t			size,
	md_error_t		*ep,
	md_resync_cmd_t		cmd	/* Start/Block/Unblock/Kill */
)
{
	char			*miscname;
	md_resync_ioctl_t	ri;

	/* should have a set */
	assert(sp != NULL);
	assert(sp->setno == MD_MIN2SET(meta_getminor(mirnp->dev)));

	/* make sure we have a mirror */
	if ((miscname = metagetmiscname(mirnp, ep)) == NULL)
		return (-1);
	if (strcmp(miscname, MD_MIRROR) != 0) {
		return (mdmderror(ep, MDE_NOT_MM, meta_getminor(mirnp->dev),
		    mirnp->cname));
	}

	/* start resync */
	(void) memset(&ri, 0, sizeof (ri));
	MD_SETDRIVERNAME(&ri, MD_MIRROR, sp->setno);
	ri.ri_mnum = meta_getminor(mirnp->dev);
	ri.ri_copysize = size;
	switch (cmd) {
	case MD_RESYNC_FORCE_MNSTART:
		ri.ri_flags |= MD_RI_RESYNC_FORCE_MNSTART;
		break;
	case MD_RESYNC_START:
		ri.ri_flags = 0;
		break;
	case MD_RESYNC_BLOCK:
		ri.ri_flags = MD_RI_BLOCK;
		break;
	case MD_RESYNC_UNBLOCK:
		ri.ri_flags = MD_RI_UNBLOCK;
		break;
	case MD_RESYNC_KILL:
		ri.ri_flags = MD_RI_KILL;
		break;
	case MD_RESYNC_KILL_NO_WAIT:
		ri.ri_flags = MD_RI_KILL | MD_RI_NO_WAIT;
		break;
	default:
		/* TODO: Add new error MDE_BAD_RESYNC_FLAGS */
		return (mderror(ep, MDE_BAD_RESYNC_OPT, mirnp->cname));
	}

	if (metaioctl(MD_IOCSETSYNC, &ri, &ri.mde, mirnp->cname) != 0)
		return (mdstealerror(ep, &ri.mde));

	/* return success */
	return (0);
}

/*
 * free units
 */
static void
free_units(
	mm_unit_list_t	*mirrors[MD_PASS_MAX + 1]
)
{
	uint_t		i;

	for (i = 0; (i < (MD_PASS_MAX + 1)); ++i) {
		mm_unit_list_t	*p, *n;

		for (p = mirrors[i], n = NULL; (p != NULL); p = n) {
			n = p->next;
			Free(p);
		}
		mirrors[i] = NULL;
	}
}

/*
 * setup_units:	build lists of units for each pass
 */
static int
setup_units(
	mdsetname_t	*sp,
	mm_unit_list_t	*mirrors[MD_PASS_MAX + 1],
	md_error_t	*ep
)
{
	mdnamelist_t	*mirrornlp = NULL;
	mdnamelist_t	*p;
	int		rval = 0;

	/* should have a set */
	assert(sp != NULL);

	/* for each mirror */
	if (meta_get_mirror_names(sp, &mirrornlp, 0, ep) < 0)
		return (-1);
	for (p = mirrornlp; (p != NULL); p = p->next) {
		md_mirror_t	*mirrorp;
		mm_unit_list_t	*lp;

		/* get unit structure */
		if ((mirrorp = meta_get_mirror(sp, p->namep, ep)) == NULL) {
			rval = -1;	/* record, but ignore errors */
			continue;
		}

		/* save info */
		lp = Zalloc(sizeof (*lp));
		lp->namep = p->namep;
		lp->pass = mirrorp->pass_num;
		if ((lp->pass < 0) || (lp->pass > MD_PASS_MAX))
			lp->pass = MD_PASS_MAX;

		/* put on list */
		lp->next = mirrors[lp->pass];
		mirrors[lp->pass] = lp;
	}

	/* cleanup, return error */
	metafreenamelist(mirrornlp);
	return (rval);
}

/*
 * resync all mirrors (in background)
 */
int
meta_mirror_resync_all(
	mdsetname_t	*sp,
	daddr_t		size,
	md_error_t	*ep
)
{
	mm_unit_list_t	*mirrors[MD_PASS_MAX + 1];
	mm_pass_num_t	pass, max_pass;
	int		rval = 0, fval;

	/* should have a set */
	assert(sp != NULL);

	/* get mirrors */
	(void) memset(mirrors, 0, sizeof (mirrors));
	if (setup_units(sp, mirrors, ep) != 0)
		return (-1);

	/* fork a process */
	if ((fval = md_daemonize(sp, ep)) != 0) {
		/*
		 * md_daemonize will fork off a process.  The is the
		 * parent or error.
		 */
		if (fval > 0) {
			free_units(mirrors);
			return (0);
		}
		mdclrerror(ep);
	}
	/*
	 * Closing stdin/out/err here.
	 * In case this was called thru rsh, the calling process on the other
	 * side will know, it doesn't have to wait until all the resyncs have
	 * finished.
	 * Also initialise the rpc client pool so that this process will use
	 * a unique pool of clients. If we don't do this, all of the forked
	 * clients will end up using the same pool of clients which can result
	 * in hung clients.
	 */
	if (meta_is_mn_set(sp, ep)) {
		(void) close(0);
		(void) close(1);
		(void) close(2);
		mdmn_clients = NULL;
	}
	assert((fval == 0) || (fval == -1));

	/*
	 * Determine which pass level is the highest that contains mirrors to
	 * resync. We only need to wait for completion of earlier levels below
	 * this high watermark. If all mirrors are at the same pass level
	 * there is no requirement to wait for completion.
	 */

	max_pass = 1;
	for (pass = MD_PASS_MAX; pass > 1; --pass) {
		if (mirrors[pass] != NULL) {
			max_pass = pass;
			break;
		}
	}

	/*
	 * max_pass now contains the highest pass-level with resyncable mirrors
	 */

	/* do passes */
	for (pass = 1; (pass <= MD_PASS_MAX); ++pass) {
		int			dispatched = 0;
		unsigned		howlong = 1;
		mm_unit_list_t		*lp;

		/* skip empty passes */
		if (mirrors[pass] == NULL)
			continue;

		/* dispatch all resyncs in pass */
		for (lp = mirrors[pass]; (lp != NULL); lp = lp->next) {
			if (meta_is_mn_set(sp, ep)) {
				if (meta_mn_send_setsync(sp, lp->namep,
				    size, ep) != 0) {
					rval = -1;
					lp->done = 1;
				} else {
					++dispatched;
				}
			} else {
				if (meta_mirror_resync(sp, lp->namep, size, ep,
				    MD_RESYNC_START) != 0) {
					rval = -1;
					lp->done = 1;
				} else {
					++dispatched;
				}
			}
		}

		/*
		 * Wait for them to finish iff we are at a level lower than
		 * max_pass. This orders the resyncs into distinct levels.
		 * I.e. level 2 resyncs won't start until all level 1 ones
		 * have completed.
		 */
		if (pass == max_pass)
			continue;

		howlong = 1;
		while (dispatched > 0) {

			/* wait a while */
			(void) sleep(howlong);

			/* see if any finished */
			for (lp = mirrors[pass]; lp != NULL; lp = lp->next) {
				md_resync_ioctl_t	ri;

				if (lp->done)
					continue;

				(void) memset(&ri, '\0', sizeof (ri));
				ri.ri_mnum = meta_getminor(lp->namep->dev);
				MD_SETDRIVERNAME(&ri, MD_MIRROR, sp->setno);
				if (metaioctl(MD_IOCGETSYNC, &ri, &ri.mde,
				    lp->namep->cname) != 0) {
					(void) mdstealerror(ep, &ri.mde);
					rval = -1;
					lp->done = 1;
					--dispatched;
				} else if (! (ri.ri_flags & MD_RI_INPROGRESS)) {
					lp->done = 1;
					--dispatched;
				}
			}

			/* wait a little longer next time */
			if (howlong < 10)
				++howlong;
		}
	}

	/* cleanup, return success */
	free_units(mirrors);
	if (fval == 0)  /* we are the child process so exit */
		exit(0);
	return (rval);
}

/*
 * meta_mn_mirror_resync_all:
 * -------------------------
 * Resync all mirrors associated with given set (arg). Called when master
 * node is adding a node to a diskset.  Only want to initiate the resync on
 * the current node.
 */
void *
meta_mn_mirror_resync_all(void *arg)
{
	set_t		setno = *((set_t *)arg);
	mdsetname_t	*sp;
	mm_unit_list_t	*mirrors[MD_PASS_MAX + 1];
	mm_pass_num_t	pass, max_pass;
	md_error_t	mde = mdnullerror;
	int		fval;


	/* should have a set */
	assert(setno != NULL);

	if ((sp = metasetnosetname(setno, &mde)) == NULL) {
		mde_perror(&mde, "");
		return (NULL);
	}

	if (!(meta_is_mn_set(sp, &mde))) {
		mde_perror(&mde, "");
		return (NULL);
	}

	/* fork a process */
	if ((fval = md_daemonize(sp, &mde)) != 0) {
		/*
		 * md_daemonize will fork off a process.  The is the
		 * parent or error.
		 */
		if (fval > 0) {
			return (NULL);
		}
		mde_perror(&mde, "");
		return (NULL);
	}
	/*
	 * Child process should never return back to rpc.metad, but
	 * should exit.
	 * Flush all internally cached data inherited from parent process
	 * since cached data will be cleared when parent process RPC request
	 * has completed (which is possibly before this child process
	 * can complete).
	 * Child process can retrieve and cache its own copy of data from
	 * rpc.metad that won't be changed by the parent process.
	 *
	 * Reset md_in_daemon since this child will be a client of rpc.metad
	 * not part of the rpc.metad daemon itself.
	 * md_in_daemon is used by rpc.metad so that libmeta can tell if
	 * this thread is rpc.metad or any other thread.  (If this thread
	 * was rpc.metad it could use some short circuit code to get data
	 * directly from rpc.metad instead of doing an RPC call to rpc.metad).
	 */
	md_in_daemon = 0;
	metaflushsetname(sp);
	sr_cache_flush_setno(setno);
	if ((sp = metasetnosetname(setno, &mde)) == NULL) {
		mde_perror(&mde, "");
		md_exit(sp, 1);
	}

	if (meta_lock(sp, TRUE, &mde) != 0) {
		mde_perror(&mde, "");
		md_exit(sp, 1);
	}

	/*
	 * Closing stdin/out/err here.
	 */
	(void) close(0);
	(void) close(1);
	(void) close(2);
	assert(fval == 0);

	/* get mirrors */
	(void) memset(mirrors, 0, sizeof (mirrors));
	if (setup_units(sp, mirrors, &mde) != 0) {
		(void) meta_unlock(sp, &mde);
		md_exit(sp, 1);
	}

	/*
	 * Determine which pass level is the highest that contains mirrors to
	 * resync. We only need to wait for completion of earlier levels below
	 * this high watermark. If all mirrors are at the same pass level
	 * there is no requirement to wait for completion.
	 */
	max_pass = 1;
	for (pass = MD_PASS_MAX; pass > 1; --pass) {
		if (mirrors[pass] != NULL) {
			max_pass = pass;
			break;
		}
	}

	/*
	 * max_pass now contains the highest pass-level with resyncable mirrors
	 */
	/* do passes */
	for (pass = 1; (pass <= MD_PASS_MAX); ++pass) {
		int			dispatched = 0;
		unsigned		howlong = 1;
		mm_unit_list_t		*lp;

		/* skip empty passes */
		if (mirrors[pass] == NULL)
			continue;

		/* dispatch all resyncs in pass */
		for (lp = mirrors[pass]; (lp != NULL); lp = lp->next) {
			if (meta_mirror_resync(sp, lp->namep, 0, &mde,
			    MD_RESYNC_FORCE_MNSTART) != 0) {
				mdclrerror(&mde);
				lp->done = 1;
			} else {
				++dispatched;
			}
		}

		/*
		 * Wait for them to finish iff we are at a level lower than
		 * max_pass. This orders the resyncs into distinct levels.
		 * I.e. level 2 resyncs won't start until all level 1 ones
		 * have completed.
		 */
		if (pass == max_pass)
			continue;

		howlong = 1;
		while (dispatched > 0) {

			/* wait a while */
			(void) sleep(howlong);

			/* see if any finished */
			for (lp = mirrors[pass]; lp != NULL; lp = lp->next) {
				md_resync_ioctl_t	ri;

				if (lp->done)
					continue;

				(void) memset(&ri, '\0', sizeof (ri));
				ri.ri_mnum = meta_getminor(lp->namep->dev);
				MD_SETDRIVERNAME(&ri, MD_MIRROR, sp->setno);
				if (metaioctl(MD_IOCGETSYNC, &ri, &ri.mde,
				    lp->namep->cname) != 0) {
					mdclrerror(&mde);
					lp->done = 1;
					--dispatched;
				} else if (! (ri.ri_flags & MD_RI_INPROGRESS)) {
					lp->done = 1;
					--dispatched;
				}
			}

			/* wait a little longer next time */
			if (howlong < 10)
				++howlong;
		}
	}

	/* cleanup, return success */
	free_units(mirrors);
	(void) meta_unlock(sp, &mde);
	md_exit(sp, 0);
	/*NOTREACHED*/
	return (NULL);
}

/*
 * meta_mirror_resync_process:
 * --------------------------
 * Modify any resync that is in progress on this node for the given set.
 *
 * Input Parameters:
 *	sp	setname to scan for mirrors
 *	cmd	action to take:
 *		MD_RESYNC_KILL	- kill all resync threads
 *		MD_RESYNC_BLOCK	- block all resync threads
 *		MD_RESYNC_UNBLOCK - resume all resync threads
 * Output Parameters
 *	ep	error return structure
 *
 * meta_lock for this set should be held on entry.
 */
static void
meta_mirror_resync_process(mdsetname_t *sp, md_error_t *ep, md_resync_cmd_t cmd)
{
	mm_unit_list_t	*mirrors[MD_PASS_MAX + 1];
	mm_pass_num_t	pass;

	/* Grab all the mirrors from the set (if any) */
	(void) memset(mirrors, 0, sizeof (mirrors));
	if (setup_units(sp, mirrors, ep) != 0)
		return;

	/* do passes */
	for (pass = 1; (pass <= MD_PASS_MAX); ++pass) {
		mm_unit_list_t		*lp;

		/* skip empty passes */
		if (mirrors[pass] == NULL)
			continue;

		/* Process all resyncs in pass */
		for (lp = mirrors[pass]; (lp != NULL); lp = lp->next) {
			(void) meta_mirror_resync(sp, lp->namep, 0, ep,
			    cmd);
		}
	}

	/* Clear up mirror units */
	free_units(mirrors);
}

/*
 * meta_mirror_resync_process_all:
 * ------------------------------
 * Issue the given resync command to all mirrors contained in all multi-node
 * sets.
 *
 * Input Parameters:
 *	cmd	- MD_RESYNC_KILL, MD_RESYNC_BLOCK, MD_RESYNC_UNBLOCK
 */
static void
meta_mirror_resync_process_all(md_resync_cmd_t cmd)
{
	set_t		setno, max_sets;
	md_error_t	mde = mdnullerror;
	mdsetname_t	*this_sp;
	md_set_desc	*sd;

	/*
	 * Traverse all sets looking for multi-node capable ones.
	 */
	max_sets = get_max_sets(&mde);
	for (setno = 1; setno < max_sets; setno++) {
		mde = mdnullerror;
		if (this_sp = metasetnosetname(setno, &mde)) {
			if ((sd = metaget_setdesc(this_sp, &mde)) == NULL)
				continue;
			if (!MD_MNSET_DESC(sd))
				continue;

			if (meta_lock(this_sp, TRUE, &mde)) {
				continue;
			}
			meta_mirror_resync_process(this_sp, &mde, cmd);
			(void) meta_unlock(this_sp, &mde);
		}
	}
}

/*
 * meta_mirror_resync_kill_all:
 * ---------------------------
 * Abort any resync that is in progress on this node. Scan all sets for all
 * mirrors.
 * Note: this routine is provided for future use. For example to kill all
 *	 resyncs on a node this could be used as long as the
 *	 mddoors / rpc.mdcommd tuple is running on all members of the cluster.
 */
void
meta_mirror_resync_kill_all(void)
{
	meta_mirror_resync_process_all(MD_RESYNC_KILL);
}

/*
 * meta_mirror_resync_block_all:
 * ----------------------------
 * Block all resyncs that are in progress. This causes the resync state to
 * freeze on this machine, and can be resumed by calling
 * meta_mirror_resync_unblock_all.
 */
void
meta_mirror_resync_block_all(void)
{
	meta_mirror_resync_process_all(MD_RESYNC_BLOCK);
}

/*
 * meta_mirror_resync_unblock_all:
 * ------------------------------
 * Unblock all previously blocked resync threads on this node.
 */
void
meta_mirror_resync_unblock_all(void)
{
	meta_mirror_resync_process_all(MD_RESYNC_UNBLOCK);
}

/*
 * meta_mirror_resync_unblock:
 * --------------------------
 * Unblock any previously blocked resync threads for the given set.
 * meta_lock for this set should be held on entry.
 */
void
meta_mirror_resync_unblock(mdsetname_t *sp)
{
	md_error_t	mde = mdnullerror;

	meta_mirror_resync_process(sp, &mde, MD_RESYNC_UNBLOCK);
}

/*
 * meta_mirror_resync_kill:
 * -----------------------
 * Kill any resync threads running on mirrors in the given set.
 * Called when releasing a set (meta_set_prv.c`halt_set)
 */
void
meta_mirror_resync_kill(mdsetname_t *sp)
{
	md_error_t	mde = mdnullerror;

	meta_mirror_resync_process(sp, &mde, MD_RESYNC_KILL);
}