common/os/sched.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
/*	  All Rights Reserved	*/

#include <sys/param.h>
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/cpuvar.h>
#include <sys/var.h>
#include <sys/tuneable.h>
#include <sys/cmn_err.h>
#include <sys/buf.h>
#include <sys/disp.h>
#include <sys/vmsystm.h>
#include <sys/vmparam.h>
#include <sys/class.h>
#include <sys/vtrace.h>
#include <sys/modctl.h>
#include <sys/debug.h>
#include <sys/tnf_probe.h>
#include <sys/procfs.h>

#include <vm/seg.h>
#include <vm/seg_kp.h>
#include <vm/as.h>
#include <vm/rm.h>
#include <vm/seg_kmem.h>
#include <sys/callb.h>

/*
 * The swapper sleeps on runout when there is no one to swap in.
 * It sleeps on runin when it could not find space to swap someone
 * in or after swapping someone in.
 */
char	runout;
char	runin;
char	wake_sched;	/* flag tells clock to wake swapper on next tick */
char	wake_sched_sec;	/* flag tells clock to wake swapper after a second */

/*
 * The swapper swaps processes to reduce memory demand and runs
 * when avefree < desfree.  The swapper resorts to SOFTSWAP when
 * avefree < desfree which results in swapping out all processes
 * sleeping for more than maxslp seconds.  HARDSWAP occurs when the
 * system is on the verge of thrashing and this results in swapping
 * out runnable threads or threads sleeping for less than maxslp secs.
 *
 * The swapper runs through all the active processes in the system
 * and invokes the scheduling class specific swapin/swapout routine
 * for every thread in the process to obtain an effective priority
 * for the process.  A priority of -1 implies that the thread isn't
 * swappable.  This effective priority is used to find the most
 * eligible process to swapout or swapin.
 *
 * NOTE:  Threads which have been swapped are not linked on any
 *	  queue and their dispatcher lock points at the "swapped_lock".
 *
 * Processes containing threads with the TS_DONT_SWAP flag set cannot be
 * swapped out immediately by the swapper.  This is due to the fact that
 * such threads may be holding locks which may be needed by the swapper
 * to push its pages out.  The TS_SWAPENQ flag is set on such threads
 * to prevent them running in user mode.  When such threads reach a
 * safe point (i.e., are not holding any locks - CL_TRAPRET), they
 * queue themseleves onto the swap queue which is processed by the
 * swapper.  This results in reducing memory demand when the system
 * is desparate for memory as the thread can't run in user mode.
 *
 * The swap queue consists of threads, linked via t_link, which are
 * haven't been swapped, are runnable but not on the run queue.  The
 * swap queue is protected by the "swapped_lock".  The dispatcher
 * lock (t_lockp) of all threads on the swap queue points at the
 * "swapped_lock".  Thus, the entire queue and/or threads on the
 * queue can be locked by acquiring "swapped_lock".
 */
static kthread_t *tswap_queue;
extern disp_lock_t swapped_lock; /* protects swap queue and threads on it */

int	maxslp = 0;
pgcnt_t	avefree;	/* 5 sec moving average of free memory */
pgcnt_t	avefree30;	/* 30 sec moving average of free memory */

/*
 * Minimum size used to decide if sufficient memory is available
 * before a process is swapped in.  This is necessary since in most
 * cases the actual size of a process (p_swrss) being swapped in
 * is usually 2 pages (kernel stack pages).  This is due to the fact
 * almost all user pages of a process are stolen by pageout before
 * the swapper decides to swapout it out.
 */
int	min_procsize = 12;

static int	swapin(proc_t *);
static int	swapout(proc_t *, uint_t *, int);
static void	process_swap_queue();

#ifdef __sparc
extern void lwp_swapin(kthread_t *);
#endif /* __sparc */

/*
 * Counters to keep track of the number of swapins or swapouts.
 */
uint_t tot_swapped_in, tot_swapped_out;
uint_t softswap, hardswap, swapqswap;

/*
 * Macro to determine if a process is eligble to be swapped.
 */
#define	not_swappable(p)					\
	(((p)->p_flag & SSYS) || (p)->p_stat == SIDL ||		\
	    (p)->p_stat == SZOMB || (p)->p_as == NULL ||	\
	    (p)->p_as == &kas)

/*
 * Memory scheduler.
 */
void
sched()
{
	kthread_id_t	t;
	pri_t		proc_pri;
	pri_t		thread_pri;
	pri_t		swapin_pri;
	int		desperate;
	pgcnt_t		needs;
	int		divisor;
	proc_t		*prp;
	proc_t		*swapout_prp;
	proc_t		*swapin_prp;
	spgcnt_t	avail;
	int		chosen_pri;
	time_t		swapout_time;
	time_t		swapin_proc_time;
	callb_cpr_t	cprinfo;
	kmutex_t	swap_cpr_lock;

	mutex_init(&swap_cpr_lock, NULL, MUTEX_DEFAULT, NULL);
	CALLB_CPR_INIT(&cprinfo, &swap_cpr_lock, callb_generic_cpr, "sched");
	if (maxslp == 0)
		maxslp = MAXSLP;
loop:
	needs = 0;
	desperate = 0;

	swapin_pri = v.v_nglobpris;
	swapin_prp = NULL;
	chosen_pri = -1;

	process_swap_queue();

	/*
	 * Set desperate if
	 * 	1.  At least 2 runnable processes (on average).
	 *	2.  Short (5 sec) and longer (30 sec) average is less
	 *	    than minfree and desfree respectively.
	 *	3.  Pagein + pageout rate is excessive.
	 */
	if (avenrun[0] >= 2 * FSCALE &&
	    (MAX(avefree, avefree30) < desfree) &&
	    (pginrate + pgoutrate > maxpgio || avefree < minfree)) {
		TRACE_4(TR_FAC_SCHED, TR_DESPERATE,
		    "desp:avefree: %d, avefree30: %d, freemem: %d"
		    " pginrate: %d\n", avefree, avefree30, freemem, pginrate);
		desperate = 1;
		goto unload;
	}

	/*
	 * Search list of processes to swapin and swapout deadwood.
	 */
	swapin_proc_time = 0;
top:
	mutex_enter(&pidlock);
	for (prp = practive; prp != NULL; prp = prp->p_next) {
		if (not_swappable(prp))
			continue;

		/*
		 * Look at processes with at least one swapped lwp.
		 */
		if (prp->p_swapcnt) {
			time_t proc_time;

			/*
			 * Higher priority processes are good candidates
			 * to swapin.
			 */
			mutex_enter(&prp->p_lock);
			proc_pri = -1;
			t = prp->p_tlist;
			proc_time = 0;
			do {
				if (t->t_schedflag & TS_LOAD)
					continue;

				thread_lock(t);
				thread_pri = CL_SWAPIN(t, 0);
				thread_unlock(t);

				if (t->t_stime - proc_time > 0)
					proc_time = t->t_stime;
				if (thread_pri > proc_pri)
					proc_pri = thread_pri;
			} while ((t = t->t_forw) != prp->p_tlist);
			mutex_exit(&prp->p_lock);

			if (proc_pri == -1)
				continue;

			TRACE_3(TR_FAC_SCHED, TR_CHOOSE_SWAPIN,
			    "prp %p epri %d proc_time %d",
			    prp, proc_pri, proc_time);

			/*
			 * Swapin processes with a high effective priority.
			 */
			if (swapin_prp == NULL || proc_pri > chosen_pri) {
				swapin_prp = prp;
				chosen_pri = proc_pri;
				swapin_pri = proc_pri;
				swapin_proc_time = proc_time;
			}
		} else {
			/*
			 * No need to soft swap if we have sufficient
			 * memory.
			 */
			if (avefree > desfree ||
			    avefree < desfree && freemem > desfree)
				continue;

			/*
			 * Skip processes that are exiting
			 * or whose address spaces are locked.
			 */
			mutex_enter(&prp->p_lock);
			if ((prp->p_flag & SEXITING) ||
			    (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) {
				mutex_exit(&prp->p_lock);
				continue;
			}

			/*
			 * Softswapping to kick out deadwood.
			 */
			proc_pri = -1;
			t = prp->p_tlist;
			do {
				if ((t->t_schedflag & (TS_SWAPENQ |
				    TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD)
					continue;

				thread_lock(t);
				thread_pri = CL_SWAPOUT(t, SOFTSWAP);
				thread_unlock(t);
				if (thread_pri > proc_pri)
					proc_pri = thread_pri;
			} while ((t = t->t_forw) != prp->p_tlist);

			if (proc_pri != -1) {
				uint_t swrss;

				mutex_exit(&pidlock);

				TRACE_1(TR_FAC_SCHED, TR_SOFTSWAP,
				    "softswap:prp %p", prp);

				(void) swapout(prp, &swrss, SOFTSWAP);
				softswap++;
				prp->p_swrss += swrss;
				mutex_exit(&prp->p_lock);
				goto top;
			}
			mutex_exit(&prp->p_lock);
		}
	}
	if (swapin_prp != NULL)
		mutex_enter(&swapin_prp->p_lock);
	mutex_exit(&pidlock);

	if (swapin_prp == NULL) {
		TRACE_3(TR_FAC_SCHED, TR_RUNOUT,
		"schedrunout:runout nswapped: %d, avefree: %ld freemem: %ld",
		    nswapped, avefree, freemem);

		t = curthread;
		thread_lock(t);
		runout++;
		t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART);
		t->t_whystop = PR_SUSPENDED;
		t->t_whatstop = SUSPEND_NORMAL;
		(void) new_mstate(t, LMS_SLEEP);
		mutex_enter(&swap_cpr_lock);
		CALLB_CPR_SAFE_BEGIN(&cprinfo);
		mutex_exit(&swap_cpr_lock);
		thread_stop(t);		/* change state and drop lock */
		swtch();
		mutex_enter(&swap_cpr_lock);
		CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock);
		mutex_exit(&swap_cpr_lock);
		goto loop;
	}

	/*
	 * Decide how deserving this process is to be brought in.
	 * Needs is an estimate of how much core the process will
	 * need.  If the process has been out for a while, then we
	 * will bring it in with 1/2 the core needed, otherwise
	 * we are conservative.
	 */
	divisor = 1;
	swapout_time = (ddi_get_lbolt() - swapin_proc_time) / hz;
	if (swapout_time > maxslp / 2)
		divisor = 2;

	needs = MIN(swapin_prp->p_swrss, lotsfree);
	needs = MAX(needs, min_procsize);
	needs = needs / divisor;

	/*
	 * Use freemem, since we want processes to be swapped
	 * in quickly.
	 */
	avail = freemem - deficit;
	if (avail > (spgcnt_t)needs) {
		deficit += needs;

		TRACE_2(TR_FAC_SCHED, TR_SWAPIN_VALUES,
		    "swapin_values: prp %p needs %lu", swapin_prp, needs);

		if (swapin(swapin_prp)) {
			mutex_exit(&swapin_prp->p_lock);
			goto loop;
		}
		deficit -= MIN(needs, deficit);
		mutex_exit(&swapin_prp->p_lock);
	} else {
		mutex_exit(&swapin_prp->p_lock);
		/*
		 * If deficit is high, too many processes have been
		 * swapped in so wait a sec before attempting to
		 * swapin more.
		 */
		if (freemem > needs) {
			TRACE_2(TR_FAC_SCHED, TR_HIGH_DEFICIT,
			    "deficit: prp %p needs %lu", swapin_prp, needs);
			goto block;
		}
	}

	TRACE_2(TR_FAC_SCHED, TR_UNLOAD,
	    "unload: prp %p needs %lu", swapin_prp, needs);

unload:
	/*
	 * Unload all unloadable modules, free all other memory
	 * resources we can find, then look for a thread to hardswap.
	 */
	modreap();
	segkp_cache_free();

	swapout_prp = NULL;
	mutex_enter(&pidlock);
	for (prp = practive; prp != NULL; prp = prp->p_next) {

		/*
		 * No need to soft swap if we have sufficient
		 * memory.
		 */
		if (not_swappable(prp))
			continue;

		if (avefree > minfree ||
		    avefree < minfree && freemem > desfree) {
			swapout_prp = NULL;
			break;
		}

		/*
		 * Skip processes that are exiting
		 * or whose address spaces are locked.
		 */
		mutex_enter(&prp->p_lock);
		if ((prp->p_flag & SEXITING) ||
		    (prp->p_as != NULL && AS_ISPGLCK(prp->p_as))) {
			mutex_exit(&prp->p_lock);
			continue;
		}

		proc_pri = -1;
		t = prp->p_tlist;
		do {
			if ((t->t_schedflag & (TS_SWAPENQ |
			    TS_ON_SWAPQ | TS_LOAD)) != TS_LOAD)
				continue;

			thread_lock(t);
			thread_pri = CL_SWAPOUT(t, HARDSWAP);
			thread_unlock(t);
			if (thread_pri > proc_pri)
				proc_pri = thread_pri;
		} while ((t = t->t_forw) != prp->p_tlist);

		mutex_exit(&prp->p_lock);
		if (proc_pri == -1)
			continue;

		/*
		 * Swapout processes sleeping with a lower priority
		 * than the one currently being swapped in, if any.
		 */
		if (swapin_prp == NULL || swapin_pri > proc_pri) {
			TRACE_2(TR_FAC_SCHED, TR_CHOOSE_SWAPOUT,
			    "hardswap: prp %p needs %lu", prp, needs);

			if (swapout_prp == NULL || proc_pri < chosen_pri) {
				swapout_prp = prp;
				chosen_pri = proc_pri;
			}
		}
	}

	/*
	 * Acquire the "p_lock" before dropping "pidlock"
	 * to prevent the proc structure from being freed
	 * if the process exits before swapout completes.
	 */
	if (swapout_prp != NULL)
		mutex_enter(&swapout_prp->p_lock);
	mutex_exit(&pidlock);

	if ((prp = swapout_prp) != NULL) {
		uint_t swrss = 0;
		int swapped;

		swapped = swapout(prp, &swrss, HARDSWAP);
		if (swapped) {
			/*
			 * If desperate, we want to give the space obtained
			 * by swapping this process out to processes in core,
			 * so we give them a chance by increasing deficit.
			 */
			prp->p_swrss += swrss;
			if (desperate)
				deficit += MIN(prp->p_swrss, lotsfree);
			hardswap++;
		}
		mutex_exit(&swapout_prp->p_lock);

		if (swapped)
			goto loop;
	}

	/*
	 * Delay for 1 second and look again later.
	 */
	TRACE_3(TR_FAC_SCHED, TR_RUNIN,
	    "schedrunin:runin nswapped: %d, avefree: %ld freemem: %ld",
	    nswapped, avefree, freemem);

block:
	t = curthread;
	thread_lock(t);
	runin++;
	t->t_schedflag |= (TS_ALLSTART & ~TS_CSTART);
	t->t_whystop = PR_SUSPENDED;
	t->t_whatstop = SUSPEND_NORMAL;
	(void) new_mstate(t, LMS_SLEEP);
	mutex_enter(&swap_cpr_lock);
	CALLB_CPR_SAFE_BEGIN(&cprinfo);
	mutex_exit(&swap_cpr_lock);
	thread_stop(t);		/* change to stop state and drop lock */
	swtch();
	mutex_enter(&swap_cpr_lock);
	CALLB_CPR_SAFE_END(&cprinfo, &swap_cpr_lock);
	mutex_exit(&swap_cpr_lock);
	goto loop;
}

/*
 * Remove the specified thread from the swap queue.
 */
static void
swapdeq(kthread_id_t tp)
{
	kthread_id_t *tpp;

	ASSERT(THREAD_LOCK_HELD(tp));
	ASSERT(tp->t_schedflag & TS_ON_SWAPQ);

	tpp = &tswap_queue;
	for (;;) {
		ASSERT(*tpp != NULL);
		if (*tpp == tp)
			break;
		tpp = &(*tpp)->t_link;
	}
	*tpp = tp->t_link;
	tp->t_schedflag &= ~TS_ON_SWAPQ;
}

/*
 * Swap in lwps.  Returns nonzero on success (i.e., if at least one lwp is
 * swapped in) and 0 on failure.
 */
static int
swapin(proc_t *pp)
{
	kthread_id_t tp;
	int err;
	int num_swapped_in = 0;
	struct cpu *cpup = CPU;
	pri_t thread_pri;

	ASSERT(MUTEX_HELD(&pp->p_lock));
	ASSERT(pp->p_swapcnt);

top:
	tp = pp->p_tlist;
	do {
		/*
		 * Only swapin eligible lwps (specified by the scheduling
		 * class) which are unloaded and ready to run.
		 */
		thread_lock(tp);
		thread_pri = CL_SWAPIN(tp, 0);
		if (thread_pri != -1 && tp->t_state == TS_RUN &&
		    (tp->t_schedflag & TS_LOAD) == 0) {
			size_t stack_size;
			pgcnt_t stack_pages;

			ASSERT((tp->t_schedflag & TS_ON_SWAPQ) == 0);

			thread_unlock(tp);
			/*
			 * Now drop the p_lock since the stack needs
			 * to brought in.
			 */
			mutex_exit(&pp->p_lock);

			stack_size = swapsize(tp->t_swap);
			stack_pages = btopr(stack_size);
			/* Kernel probe */
			TNF_PROBE_4(swapin_lwp, "vm swap swapin", /* CSTYLED */,
			    tnf_pid,		pid,		pp->p_pid,
			    tnf_lwpid,		lwpid,		tp->t_tid,
			    tnf_kthread_id,	tid,		tp,
			    tnf_ulong,		page_count,	stack_pages);

			rw_enter(&kas.a_lock, RW_READER);
			err = segkp_fault(segkp->s_as->a_hat, segkp,
			    tp->t_swap, stack_size, F_SOFTLOCK, S_OTHER);
			rw_exit(&kas.a_lock);

			/*
			 * Re-acquire the p_lock.
			 */
			mutex_enter(&pp->p_lock);
			if (err) {
				num_swapped_in = 0;
				break;
			} else {
#ifdef __sparc
				lwp_swapin(tp);
#endif /* __sparc */
				CPU_STATS_ADDQ(cpup, vm, swapin, 1);
				CPU_STATS_ADDQ(cpup, vm, pgswapin,
				    stack_pages);

				pp->p_swapcnt--;
				pp->p_swrss -= stack_pages;

				thread_lock(tp);
				tp->t_schedflag |= TS_LOAD;
				dq_sruninc(tp);

				/* set swapin time */
				tp->t_stime = ddi_get_lbolt();
				thread_unlock(tp);

				nswapped--;
				tot_swapped_in++;
				num_swapped_in++;

				TRACE_2(TR_FAC_SCHED, TR_SWAPIN,
				    "swapin: pp %p stack_pages %lu",
				    pp, stack_pages);
				goto top;
			}
		}
		thread_unlock(tp);
	} while ((tp = tp->t_forw) != pp->p_tlist);
	return (num_swapped_in);
}

/*
 * Swap out lwps.  Returns nonzero on success (i.e., if at least one lwp is
 * swapped out) and 0 on failure.
 */
static int
swapout(proc_t *pp, uint_t *swrss, int swapflags)
{
	kthread_id_t tp;
	pgcnt_t ws_pages = 0;
	int err;
	int swapped_lwps = 0;
	struct as *as = pp->p_as;
	struct cpu *cpup = CPU;
	pri_t thread_pri;

	ASSERT(MUTEX_HELD(&pp->p_lock));

	if (pp->p_flag & SEXITING)
		return (0);

top:
	tp = pp->p_tlist;
	do {
		klwp_t *lwp = ttolwp(tp);

		/*
		 * Swapout eligible lwps (specified by the scheduling
		 * class) which don't have TS_DONT_SWAP set.  Set the
		 * "intent to swap" flag (TS_SWAPENQ) on threads
		 * which have TS_DONT_SWAP set so that they can be
		 * swapped if and when they reach a safe point.
		 */
		thread_lock(tp);
		thread_pri = CL_SWAPOUT(tp, swapflags);
		if (thread_pri != -1) {
			if (tp->t_schedflag & TS_DONT_SWAP) {
				tp->t_schedflag |= TS_SWAPENQ;
				tp->t_trapret = 1;
				aston(tp);
			} else {
				pgcnt_t stack_pages;
				size_t stack_size;

				ASSERT((tp->t_schedflag &
				    (TS_DONT_SWAP | TS_LOAD)) == TS_LOAD);

				if (lock_try(&tp->t_lock)) {
					/*
					 * Remove thread from the swap_queue.
					 */
					if (tp->t_schedflag & TS_ON_SWAPQ) {
						ASSERT(!(tp->t_schedflag &
						    TS_SWAPENQ));
						swapdeq(tp);
					} else if (tp->t_state == TS_RUN)
						dq_srundec(tp);

					tp->t_schedflag &=
					    ~(TS_LOAD | TS_SWAPENQ);
					lock_clear(&tp->t_lock);

					/*
					 * Set swapout time if the thread isn't
					 * sleeping.
					 */
					if (tp->t_state != TS_SLEEP)
						tp->t_stime = ddi_get_lbolt();
					thread_unlock(tp);

					nswapped++;
					tot_swapped_out++;

					lwp->lwp_ru.nswap++;

					/*
					 * Now drop the p_lock since the
					 * stack needs to pushed out.
					 */
					mutex_exit(&pp->p_lock);

					stack_size = swapsize(tp->t_swap);
					stack_pages = btopr(stack_size);
					ws_pages += stack_pages;
					/* Kernel probe */
					TNF_PROBE_4(swapout_lwp,
					    "vm swap swapout",
					    /* CSTYLED */,
					    tnf_pid, pid, pp->p_pid,
					    tnf_lwpid, lwpid, tp->t_tid,
					    tnf_kthread_id, tid, tp,
					    tnf_ulong, page_count,
					    stack_pages);

					rw_enter(&kas.a_lock, RW_READER);
					err = segkp_fault(segkp->s_as->a_hat,
					    segkp, tp->t_swap, stack_size,
					    F_SOFTUNLOCK, S_WRITE);
					rw_exit(&kas.a_lock);

					if (err) {
						cmn_err(CE_PANIC,
						    "swapout: segkp_fault "
						    "failed err: %d", err);
					}
					CPU_STATS_ADDQ(cpup,
					    vm, pgswapout, stack_pages);

					mutex_enter(&pp->p_lock);
					pp->p_swapcnt++;
					swapped_lwps++;
					goto top;
				}
			}
		}
		thread_unlock(tp);
	} while ((tp = tp->t_forw) != pp->p_tlist);

	/*
	 * Unload address space when all lwps are swapped out.
	 */
	if (pp->p_swapcnt == pp->p_lwpcnt) {
		size_t as_size = 0;

		/*
		 * Avoid invoking as_swapout() if the process has
		 * no MMU resources since pageout will eventually
		 * steal pages belonging to this address space.  This
		 * saves CPU cycles as the number of pages that are
		 * potentially freed or pushed out by the segment
		 * swapout operation is very small.
		 */
		if (rm_asrss(pp->p_as) != 0)
			as_size = as_swapout(as);

		CPU_STATS_ADDQ(cpup, vm, pgswapout, btop(as_size));
		CPU_STATS_ADDQ(cpup, vm, swapout, 1);
		ws_pages += btop(as_size);

		TRACE_2(TR_FAC_SCHED, TR_SWAPOUT,
		    "swapout: pp %p pages_pushed %lu", pp, ws_pages);
		/* Kernel probe */
		TNF_PROBE_2(swapout_process, "vm swap swapout", /* CSTYLED */,
		    tnf_pid,	pid,		pp->p_pid,
		    tnf_ulong,	page_count,	ws_pages);
	}
	*swrss = ws_pages;
	return (swapped_lwps);
}

void
swapout_lwp(klwp_t *lwp)
{
	kthread_id_t tp = curthread;

	ASSERT(curthread == lwptot(lwp));

	/*
	 * Don't insert the thread onto the swap queue if
	 * sufficient memory is available.
	 */
	if (avefree > desfree || avefree < desfree && freemem > desfree) {
		thread_lock(tp);
		tp->t_schedflag &= ~TS_SWAPENQ;
		thread_unlock(tp);
		return;
	}

	/*
	 * Lock the thread, then move it to the swapped queue from the
	 * onproc queue and set its state to be TS_RUN.
	 */
	thread_lock(tp);
	ASSERT(tp->t_state == TS_ONPROC);
	if (tp->t_schedflag & TS_SWAPENQ) {
		tp->t_schedflag &= ~TS_SWAPENQ;

		/*
		 * Set the state of this thread to be runnable
		 * and move it from the onproc queue to the swap queue.
		 */
		disp_swapped_enq(tp);

		/*
		 * Insert the thread onto the swap queue.
		 */
		tp->t_link = tswap_queue;
		tswap_queue = tp;
		tp->t_schedflag |= TS_ON_SWAPQ;

		thread_unlock_nopreempt(tp);

		TRACE_1(TR_FAC_SCHED, TR_SWAPOUT_LWP, "swapout_lwp:%x", lwp);

		swtch();
	} else {
		thread_unlock(tp);
	}
}

/*
 * Swap all threads on the swap queue.
 */
static void
process_swap_queue(void)
{
	kthread_id_t tp;
	uint_t ws_pages;
	proc_t *pp;
	struct cpu *cpup = CPU;
	klwp_t *lwp;
	int err;

	if (tswap_queue == NULL)
		return;

	/*
	 * Acquire the "swapped_lock" which locks the swap queue,
	 * and unload the stacks of all threads on it.
	 */
	disp_lock_enter(&swapped_lock);
	while ((tp = tswap_queue) != NULL) {
		pgcnt_t stack_pages;
		size_t stack_size;

		tswap_queue = tp->t_link;
		tp->t_link = NULL;

		/*
		 * Drop the "dispatcher lock" before acquiring "t_lock"
		 * to avoid spinning on it since the thread at the front
		 * of the swap queue could be pinned before giving up
		 * its "t_lock" in resume.
		 */
		disp_lock_exit(&swapped_lock);
		lock_set(&tp->t_lock);

		/*
		 * Now, re-acquire the "swapped_lock".  Acquiring this lock
		 * results in locking the thread since its dispatcher lock
		 * (t_lockp) is the "swapped_lock".
		 */
		disp_lock_enter(&swapped_lock);
		ASSERT(tp->t_state == TS_RUN);
		ASSERT(tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ));

		tp->t_schedflag &= ~(TS_LOAD | TS_ON_SWAPQ);
		tp->t_stime = ddi_get_lbolt();		/* swapout time */
		disp_lock_exit(&swapped_lock);
		lock_clear(&tp->t_lock);

		lwp = ttolwp(tp);
		lwp->lwp_ru.nswap++;

		pp = ttoproc(tp);
		stack_size = swapsize(tp->t_swap);
		stack_pages = btopr(stack_size);

		/* Kernel probe */
		TNF_PROBE_4(swapout_lwp, "vm swap swapout", /* CSTYLED */,
		    tnf_pid,		pid,		pp->p_pid,
		    tnf_lwpid,		lwpid,		tp->t_tid,
		    tnf_kthread_id,	tid,		tp,
		    tnf_ulong,		page_count,	stack_pages);

		rw_enter(&kas.a_lock, RW_READER);
		err = segkp_fault(segkp->s_as->a_hat, segkp, tp->t_swap,
		    stack_size, F_SOFTUNLOCK, S_WRITE);
		rw_exit(&kas.a_lock);

		if (err) {
			cmn_err(CE_PANIC,
			"process_swap_list: segkp_fault failed err: %d", err);
		}
		CPU_STATS_ADDQ(cpup, vm, pgswapout, stack_pages);

		nswapped++;
		tot_swapped_out++;
		swapqswap++;

		/*
		 * Don't need p_lock since the swapper is the only
		 * thread which increments/decrements p_swapcnt and p_swrss.
		 */
		ws_pages = stack_pages;
		pp->p_swapcnt++;

		TRACE_1(TR_FAC_SCHED, TR_SWAPQ_LWP, "swaplist: pp %p", pp);

		/*
		 * Unload address space when all lwps are swapped out.
		 */
		if (pp->p_swapcnt == pp->p_lwpcnt) {
			size_t as_size = 0;

			if (rm_asrss(pp->p_as) != 0)
				as_size = as_swapout(pp->p_as);

			CPU_STATS_ADDQ(cpup, vm, pgswapout,
			    btop(as_size));
			CPU_STATS_ADDQ(cpup, vm, swapout, 1);

			ws_pages += btop(as_size);

			TRACE_2(TR_FAC_SCHED, TR_SWAPQ_PROC,
			    "swaplist_proc: pp %p pages_pushed: %lu",
			    pp, ws_pages);
			/* Kernel probe */
			TNF_PROBE_2(swapout_process, "vm swap swapout",
			    /* CSTYLED */,
			    tnf_pid,	pid,		pp->p_pid,
			    tnf_ulong,	page_count,	ws_pages);
		}
		pp->p_swrss += ws_pages;
		disp_lock_enter(&swapped_lock);
	}
	disp_lock_exit(&swapped_lock);
}