Home | History | Annotate | Download | only in sdbc
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 /*
     22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
     23  * Use is subject to license terms.
     24  */
     25 
     26 #include <errno.h>
     27 #include <stdio.h>
     28 #include <unistd.h>
     29 #include <strings.h>
     30 #include <stdlib.h>
     31 
     32 #include <sys/types.h>
     33 #include <sys/time.h>
     34 #include <sys/nsctl/sdbc_ioctl.h>
     35 #include <sys/nsctl/rdc_ioctl.h>
     36 #include <sys/nsctl/sd_bcache.h>
     37 #include <sys/nsctl/sd_conf.h>
     38 #include <sys/nsctl/rdc_io.h>
     39 #include <sys/nsctl/rdc_bitmap.h>
     40 #include <sys/unistat/spcs_s_u.h>
     41 #include <curses.h>
     42 
     43 static rdc_status_t *rdc_status;
     44 static rdc_u_info_t *rdc_info;
     45 static int rdc_maxsets;
     46 static int rdc_enabled_sets;
     47 
     48 static unsigned prev_time, delta_time;
     49 #ifdef m88k
     50 extern unsigned *usec_ptr;
     51 #endif
     52 static int bright = 0;
     53 
     54 extern int sdbc_max_devices;
     55 
     56 extern _sd_stats_t *cs_cur;
     57 extern _sd_stats_t *cs_prev;
     58 extern _sd_stats_t *cs_persec;
     59 
     60 extern int *on_off;
     61 extern int *dual_on_off;
     62 extern int *updates_prev;
     63 extern double *rate_prev;
     64 extern int *samples;
     65 
     66 int		range_num = 0;
     67 int		screen = 0;
     68 int		dual_screen = 0;
     69 static		int rnum = 0;
     70 
     71 typedef struct {
     72 	int lb, ub;
     73 } range_t;
     74 range_t ranges[100];
     75 
     76 extern int range_first();
     77 extern int range_next(int);
     78 extern int range_last();
     79 
     80 static int dual_initted = 0;
     81 static char status[11][30];
     82 
     83 unsigned dc_delta_time, dc_prev_time;
     84 
     85 #ifdef m88k
     86 #define	USEC_INIT()	usec_ptr = (unsigned int *)timer_init()
     87 #define	USEC_READ()	(*usec_ptr)
     88 #else /* !m88k */
     89 #define	USEC_INIT()	USEC_START()
     90 #include <sys/time.h>
     91 static struct timeval Usec_time;
     92 static int Usec_started = 0;
     93 
     94 void total_display(void);
     95 void disp_stats(void);
     96 void do_calc(void);
     97 void init_dual(void);
     98 void calc_time(void);
     99 void calc_completion(int, int, int);
    100 void disp_total_stats(void);
    101 void display_cache(void);
    102 
    103 #define	DISPLEN 16
    104 
    105 static void
    106 USEC_START(void)
    107 {
    108 	if (!Usec_started) {
    109 		(void) gettimeofday(&Usec_time, NULL);
    110 		Usec_started = 1;
    111 	}
    112 }
    113 
    114 static unsigned int
    115 USEC_READ()
    116 {
    117 	struct timeval tv;
    118 	if (!Usec_started)
    119 		USEC_START();
    120 
    121 	(void) gettimeofday(&tv, NULL);
    122 	return (unsigned)((tv.tv_sec - Usec_time.tv_sec) * 1000000 +
    123 	    (tv.tv_usec - Usec_time.tv_usec));
    124 }
    125 #endif /* m88k */
    126 
    127 #define	SAMPLE_RATE 5
    128 
    129 /*
    130  * refresh curses window to file
    131  */
    132 void
    133 wrefresh_file(WINDOW *win, int fd)
    134 {
    135 	char buf[8192], c, *cp = buf, *line, *blank, *empty;
    136 	int x, y;
    137 
    138 	empty = NULL;		/* cull trailing empty lines */
    139 	for (y = 0; y < win->_maxy; y++) {
    140 		line = cp;
    141 		blank = NULL;	/* cull trailing blanks */
    142 		for (x = 0; x < win->_maxx; x++) {
    143 			c = (win->_y[y][x]) & A_CHARTEXT;
    144 			if (c != ' ')
    145 				blank = NULL;
    146 			else if (blank == NULL)
    147 				blank = cp;
    148 			*cp++ = c;
    149 		}
    150 		if (blank)
    151 			cp = blank;
    152 		if (line != cp)
    153 			empty = NULL;
    154 		else if (empty == NULL)
    155 			empty = cp + 1;
    156 		*cp++ = '\n';
    157 	}
    158 	if (empty)
    159 		cp = empty;
    160 	*cp++ = '\f'; *cp++ = '\n'; *cp = '\0';
    161 	/* cp is eliminated by short _maxy and _maxx, it won't overflow */
    162 	/* LINTED, cp - buf won't be > INT32_MAX */
    163 	(void) write(fd, buf, cp - buf);
    164 }
    165 
    166 
    167 int
    168 higher(int high)
    169 {
    170 	int i;
    171 
    172 	for (i = high + 1; i <= sdbc_max_devices; i++) {
    173 		if (cs_cur->st_shared[i].sh_alloc)
    174 			return (i);
    175 	}
    176 	return (0);
    177 }
    178 
    179 int
    180 is_dirty()
    181 {
    182 	int i, dirty = 0;
    183 	spcs_s_info_t ustats;
    184 
    185 	if (SDBC_IOCTL(SDBC_STATS, cs_cur, 0, 0, 0, 0,
    186 	    &ustats) == SPCS_S_ERROR) {
    187 		perror("Could not get stats from kernel");
    188 		if (ustats) {
    189 			spcs_s_report(ustats, stderr);
    190 			spcs_s_ufree(&ustats);
    191 		}
    192 		return (-errno);
    193 	}
    194 	if (cs_cur->st_cachesize == 0)
    195 		return (0);
    196 
    197 	for (i = 0; i < cs_cur->st_count; i++)  {
    198 		if (cs_cur->st_shared[i].sh_alloc)
    199 			dirty += cs_cur->st_shared[i].sh_numdirty;
    200 	}
    201 
    202 	return (dirty != 0);
    203 }
    204 
    205 void
    206 display_cache(void)
    207 {
    208 	static int first = 1;
    209 	spcs_s_info_t ustats;
    210 
    211 	if (SDBC_IOCTL(SDBC_STATS, cs_cur, 0, 0, 0, 0, &ustats) ==
    212 	    SPCS_S_ERROR) {
    213 		perror("sd_stats");
    214 		if (ustats) {
    215 			spcs_s_report(ustats, stderr);
    216 			spcs_s_ufree(&ustats);
    217 		}
    218 	}
    219 
    220 	do_calc();
    221 	if (first) {
    222 		prev_time = USEC_READ();
    223 		first = 0;
    224 	} else
    225 		disp_stats();
    226 }
    227 
    228 void
    229 total_display(void)
    230 {
    231 	spcs_s_info_t ustats;
    232 
    233 	if (SDBC_IOCTL(SDBC_STATS, cs_cur, 0, 0, 0, 0, &ustats) ==
    234 	    SPCS_S_ERROR) {
    235 		if (ustats) {
    236 			spcs_s_report(ustats, stderr);
    237 			spcs_s_ufree(&ustats);
    238 		}
    239 		perror("sd_stats");
    240 	}
    241 	disp_total_stats();
    242 }
    243 
    244 
    245 int
    246 range_first()
    247 {
    248 	rnum = 0;
    249 	return (ranges[rnum].lb);
    250 }
    251 
    252 int
    253 range_next(int cd)
    254 {
    255 	if (ranges[rnum].ub > cd)
    256 		return (cd + 1);
    257 	if (range_num > rnum)
    258 		rnum++;
    259 	else
    260 		return (cd + 1);
    261 	return (ranges[rnum].lb);
    262 }
    263 
    264 int
    265 range_last() {
    266 	return (ranges[range_num].ub);
    267 }
    268 
    269 
    270 void
    271 set_dual_on_off()
    272 {
    273 	int i, j, ct = 0, newct = 0;
    274 
    275 	for (i = range_first(); i < rdc_enabled_sets && i <= range_last();
    276 	    i = range_next(i)) {
    277 		if (rdc_info[i].flags & RDC_ENABLED) {
    278 			ct++;
    279 			if (ct > dual_screen * ((LINES - 9) / 2))
    280 				break;
    281 		}
    282 	}
    283 	if (((i >= rdc_enabled_sets) ||
    284 	    (i > range_last())) && (dual_screen > 0)) {
    285 		dual_screen--;
    286 		set_dual_on_off();
    287 	} else {
    288 		bzero(dual_on_off, sdbc_max_devices * sizeof (int));
    289 		for (j = i; j < rdc_enabled_sets && j <= range_last();
    290 		    j = range_next(j)) {
    291 			if (rdc_info[j].flags & RDC_ENABLED) {
    292 				newct++;
    293 				if (newct <= (LINES - 9) / 2) {
    294 					dual_on_off[j] = 1;
    295 				} else
    296 					break;
    297 			}
    298 		}
    299 	}
    300 }
    301 
    302 
    303 void
    304 set_on_off()
    305 {
    306 	int i, j, ct = 0, newct = 0;
    307 
    308 	for (i = range_first(); i <= range_last(); i = range_next(i)) {
    309 		if (cs_cur->st_shared[i].sh_alloc) {
    310 			ct++;
    311 			if (ct > screen*((LINES - 9) / 2))
    312 				break;
    313 		}
    314 	}
    315 	if ((i > range_last()) && (screen > 0)) {
    316 		screen--;
    317 		set_on_off();
    318 	} else {
    319 		bzero(on_off, sdbc_max_devices * sizeof (int));
    320 		for (j = i; j <= range_last(); j = range_next(j)) {
    321 			if (cs_cur->st_shared[j].sh_alloc) {
    322 				newct++;
    323 				if (newct <= (LINES - 9) / 2)
    324 					on_off[j] = 1;
    325 				else
    326 					break;
    327 			}
    328 		}
    329 	}
    330 }
    331 
    332 void
    333 disp_stats(void)
    334 {
    335 	double	read_s, write_s, access_s, readp, writep;
    336 	double	rmiss_s, wmiss_s;
    337 	double	elapsed = delta_time / 1000000.0;
    338 	double  kbps = elapsed * 1024.0; /* for Kbytes per second */
    339 	int	rtotal, wtotal, i, j;
    340 	double	throughput = 0.0, rthroughput = 0.0;
    341 	double	creads = 0.0, cwrites = 0.0;
    342 	char	status_bit, down = 0;
    343 	int	len;
    344 	char	fn[19];
    345 
    346 	if (delta_time != 0) {
    347 		read_s  = cs_persec->st_rdhits / elapsed;
    348 		write_s = cs_persec->st_wrhits / elapsed;
    349 		rmiss_s = cs_persec->st_rdmiss / elapsed;
    350 		wmiss_s = cs_persec->st_wrmiss / elapsed;
    351 		access_s = (cs_persec->st_wrhits + cs_persec->st_rdhits +
    352 		    cs_persec->st_rdmiss + cs_persec->st_wrmiss) / elapsed;
    353 	} else
    354 		read_s = write_s = access_s = 0.0;
    355 
    356 	rtotal = cs_persec->st_rdhits + cs_persec->st_rdmiss;
    357 	wtotal = cs_persec->st_wrhits + cs_persec->st_wrmiss;
    358 	if (rtotal != 0)
    359 		readp = cs_persec->st_rdhits / (double)rtotal;
    360 	else
    361 		readp = 0.0;
    362 
    363 	if (wtotal != 0) {
    364 		writep = cs_persec->st_wrhits / (double)wtotal;
    365 	} else
    366 		writep = 0.0;
    367 
    368 	set_on_off();
    369 	if (cs_cur->st_cachesize == 0)
    370 		(void) mvprintw(0, 20, "****** Storage Cache Disabled ******");
    371 	else
    372 		(void) mvprintw(0, 20, "******      Storage Cache     ******");
    373 	(void) mvprintw(2, 26, "disk_io       cache          write_blocks");
    374 	(void) attron(A_UNDERLINE);
    375 	(void) mvprintw(3, 1, " cd cached_partition  reads writes  reads writes"
    376 	    "  dirty todisk failed");
    377 	(void) attroff(A_UNDERLINE);
    378 	for (i = 0, j = 0; j < cs_cur->st_count; i++) {
    379 		if (i >= sdbc_max_devices)
    380 			break;
    381 		if (cs_cur->st_shared[i].sh_alloc)  {
    382 			cs_persec->st_shared[i].sh_disk_write /= kbps;
    383 			cs_persec->st_shared[i].sh_disk_read  /= kbps;
    384 			cs_persec->st_shared[i].sh_cache_write /= kbps;
    385 			cs_persec->st_shared[i].sh_cache_read /= kbps;
    386 			rthroughput += cs_persec->st_shared[i].sh_disk_read;
    387 			throughput += cs_persec->st_shared[i].sh_disk_write;
    388 			creads += cs_persec->st_shared[i].sh_cache_read;
    389 			cwrites += cs_persec->st_shared[i].sh_cache_write;
    390 			if (!down)
    391 				down = cs_cur->st_shared[i].sh_failed;
    392 			if (cs_cur->st_shared[i].sh_failed && bright) {
    393 				status_bit = '*';
    394 			} else
    395 				status_bit = ' ';
    396 			if ((len = strlen(cs_cur->st_shared[i].sh_filename))
    397 			    > 15) {
    398 				strcpy(fn, "...");
    399 				strcat(fn, cs_cur->st_shared[i].sh_filename +
    400 				    len - 12);
    401 			} else
    402 				strcpy(fn, cs_cur->st_shared[i].sh_filename);
    403 			if (on_off[i]) {
    404 				(void) mvprintw(4 + j, 1,
    405 				    "%3d %-15s%c %6d %6d %6d %6d %6d %6d %6d",
    406 				    cs_cur->st_shared[i].sh_cd,
    407 				    fn,
    408 				    status_bit,
    409 				    cs_persec->st_shared[i].sh_disk_read,
    410 				    cs_persec->st_shared[i].sh_disk_write,
    411 				    cs_persec->st_shared[i].sh_cache_read,
    412 				    cs_persec->st_shared[i].sh_cache_write,
    413 				    cs_cur->st_shared[i].sh_numdirty,
    414 				    cs_cur->st_shared[i].sh_numio,
    415 				    cs_cur->st_shared[i].sh_numfail);
    416 				j++;
    417 			}
    418 		}
    419 	}
    420 	bright = !bright;
    421 
    422 	(void) mvprintw(4 + j, 22, "------ ------ ------ ------");
    423 	(void) mvprintw(5 + j, 6, " Kbytes/s total:%6d %6d %6d %6d",
    424 	    (int)rthroughput, (int)throughput,
    425 	    (int)creads, (int)cwrites);
    426 	(void) mvprintw(7 + j, 1, "accesses/s");
    427 	(void) mvprintw(7 + j, 15, "read/s    write/s   %%readh   %%writeh");
    428 
    429 	(void) attron(A_UNDERLINE);
    430 	(void) mvprintw(8 + j, 1, "            ");
    431 	(void) mvprintw(8 + j, 13,
    432 	    "                                                ");
    433 	(void) mvprintw(8 + j, 13, "(misses/s) (misses/s)");
    434 	(void) attroff(A_UNDERLINE);
    435 
    436 	(void) mvprintw(9 + j, 0, "%10.2lf    %7.2f    %7.2f   %6.1f    %6.1f",
    437 	    access_s, read_s, write_s, readp * 100.0, writep * 100.0);
    438 	(void) mvprintw(10 + j, 0, "             (%7.2f ) (%7.2f )\n\n",
    439 	    rmiss_s, wmiss_s);
    440 
    441 	if (down)
    442 		(void) mvprintw(20 + j, 1, "* -- disk off-line");
    443 }
    444 
    445 void
    446 do_calc(void)
    447 {
    448 	int i, j;
    449 
    450 	delta_time = USEC_READ() - prev_time;
    451 
    452 	cs_persec->st_rdhits = cs_cur->st_rdhits - cs_prev->st_rdhits;
    453 	cs_persec->st_rdmiss = cs_cur->st_rdmiss - cs_prev->st_rdmiss;
    454 	cs_persec->st_wrhits = cs_cur->st_wrhits - cs_prev->st_wrhits;
    455 	cs_persec->st_wrmiss = cs_cur->st_wrmiss - cs_prev->st_wrmiss;
    456 
    457 	for (i = 0, j = 0; j < cs_cur->st_count; i++) {
    458 		if (i >= sdbc_max_devices)
    459 			break;
    460 		if (cs_cur->st_shared[i].sh_alloc) {
    461 			cs_persec->st_shared[i].sh_disk_write =
    462 			    FBA_SIZE(cs_cur->st_shared[i].sh_disk_write -
    463 			    cs_prev->st_shared[i].sh_disk_write);
    464 			cs_persec->st_shared[i].sh_disk_read =
    465 			    FBA_SIZE(cs_cur->st_shared[i].sh_disk_read -
    466 			    cs_prev->st_shared[i].sh_disk_read);
    467 			cs_persec->st_shared[i].sh_cache_read =
    468 			    FBA_SIZE(cs_cur->st_shared[i].sh_cache_read -
    469 			    cs_prev->st_shared[i].sh_cache_read);
    470 			cs_persec->st_shared[i].sh_cache_write =
    471 			    FBA_SIZE(cs_cur->st_shared[i].sh_cache_write -
    472 			    cs_prev->st_shared[i].sh_cache_write);
    473 			j++;
    474 		}
    475 	}
    476 	(void) memcpy((char *) cs_prev, (char *) cs_cur, sizeof (_sd_stats_t) +
    477 	    (sdbc_max_devices - 1) * sizeof (_sd_shared_t));
    478 	prev_time = USEC_READ();
    479 }
    480 
    481 
    482 void
    483 init_dual(void)
    484 {
    485 #define	IND_ENABLED		0
    486 #define	IND_RESYNC  		1
    487 #define	IND_RESYNC_REVERSE	2
    488 #define	IND_VOLUME_DOWN		3
    489 #define	IND_MIRROR_DOWN		4
    490 #define	IND_LOGGING		5
    491 #define	IND_RESYNC_NEEDED	6
    492 #define	IND_REV_RESYNC_NEEDED	7
    493 #define	IND_BITMAP_FAILED	8
    494 #define	IND_FULL_SYNC_NEEDED	9
    495 #define	IND_FCAL_FAILED		10
    496 	strcpy(status[IND_ENABLED], "replicating");
    497 	strcpy(status[IND_RESYNC], "sync");
    498 	strcpy(status[IND_RESYNC_REVERSE], "rev sync");
    499 	strcpy(status[IND_VOLUME_DOWN], "volume down");
    500 	strcpy(status[IND_MIRROR_DOWN], "mirror down");
    501 	strcpy(status[IND_LOGGING], "logging");
    502 	strcpy(status[IND_RESYNC_NEEDED], "need sync");
    503 	strcpy(status[IND_REV_RESYNC_NEEDED], "need rev sync");
    504 	strcpy(status[IND_BITMAP_FAILED], "bitmap failed");
    505 	strcpy(status[IND_FULL_SYNC_NEEDED], "full sync needed");
    506 	strcpy(status[IND_FCAL_FAILED], "fcal failed");
    507 	dual_initted = 1;
    508 }
    509 
    510 
    511 int
    512 rdc_get_maxsets(void)
    513 {
    514 	rdc_status_t rdc_status;
    515 	spcs_s_info_t ustatus;
    516 	int rc;
    517 
    518 	rdc_status.nset = 0;
    519 	ustatus = spcs_s_ucreate();
    520 
    521 	rc = RDC_IOCTL(RDC_STATUS, &rdc_status, 0, 0, 0, 0, ustatus);
    522 	spcs_s_ufree(&ustatus);
    523 
    524 	if (rc == SPCS_S_ERROR)
    525 		return (-1);
    526 
    527 	return (rdc_status.maxsets);
    528 }
    529 
    530 int
    531 dual_stats()
    532 {
    533 	int ind, i, k, len;
    534 	int stars, size, segs;
    535 	int rdcindex;
    536 	float pct;
    537 	char	fn[19];
    538 	char *phost;
    539 	char *shost;
    540 	char *pfile;
    541 	char *sfile;
    542 	char lhost[16];
    543 	spcs_s_info_t ustats = NULL;
    544 
    545 	(void) gethostname(lhost, 16);
    546 
    547 	if (rdc_maxsets <= 0)
    548 		rdc_maxsets = rdc_get_maxsets();
    549 
    550 	if (rdc_maxsets < 0)
    551 		goto no_stats;
    552 
    553 	if (!rdc_status) {
    554 		rdc_status = malloc(sizeof (rdc_status_t) +
    555 			(sizeof (rdc_set_t) * (rdc_maxsets - 1)));
    556 		if (!rdc_status) {
    557 no_stats:
    558 			(void) mvprintw(0, 20,
    559 				"****** Dual Copy Not Available ******");
    560 			return (-1);
    561 		}
    562 
    563 		rdc_info = rdc_status->rdc_set;
    564 	}
    565 
    566 	rdc_status->nset = rdc_maxsets;
    567 	ustats = spcs_s_ucreate();
    568 
    569 	size = RDC_IOCTL(RDC_STATUS, rdc_status, 0, 0, 0, 0, ustats);
    570 	if (size == SPCS_S_ERROR) {
    571 		if (ustats) {
    572 			spcs_s_report(ustats, stderr);
    573 			spcs_s_ufree(&ustats);
    574 		}
    575 		(void) mvprintw(0, 20, "****** Dual Copy Not Available ******");
    576 		return (-1);
    577 	}
    578 	spcs_s_ufree(&ustats);
    579 	rdc_enabled_sets = rdc_status->nset;
    580 
    581 	if (!dual_initted)
    582 		init_dual();
    583 
    584 	set_dual_on_off();
    585 
    586 	calc_time();
    587 
    588 	(void) mvprintw(0, 20, "****** Dual Copy Statistics ******");
    589 	(void) attron(A_UNDERLINE);
    590 	(void) mvprintw(2,  0, "primary");
    591 	(void) mvprintw(2, 22, "link status");
    592 	(void) mvprintw(2, 36, "secondary");
    593 	(void) mvprintw(2, 54, "dual copy status");
    594 	(void) attroff(A_UNDERLINE);
    595 
    596 	for (rdcindex = 0, k = 0; rdcindex < rdc_enabled_sets; rdcindex++)  {
    597 		if (!(rdc_info[rdcindex].flags & RDC_ENABLED) ||
    598 		    !dual_on_off[rdcindex])
    599 			continue;
    600 
    601 		if (rdc_info[rdcindex].sync_flags & RDC_VOL_FAILED)
    602 			ind = IND_VOLUME_DOWN;
    603 		else if (rdc_info[rdcindex].flags & RDC_FCAL_FAILED)
    604 			ind = IND_FCAL_FAILED;
    605 		else if (rdc_info[rdcindex].bmap_flags & RDC_BMP_FAILED)
    606 			ind = IND_BITMAP_FAILED;
    607 		else if (rdc_info[rdcindex].flags & RDC_LOGGING) {
    608 			if (rdc_info[rdcindex].sync_flags &
    609 			    RDC_SYNC_NEEDED)
    610 				ind = IND_RESYNC_NEEDED;
    611 			else if (rdc_info[rdcindex].sync_flags &
    612 			    RDC_RSYNC_NEEDED)
    613 				ind = IND_REV_RESYNC_NEEDED;
    614 			else
    615 				ind = IND_LOGGING;
    616 		} else if ((rdc_info[rdcindex].flags & RDC_SLAVE) &&
    617 		    (rdc_info[rdcindex].flags & RDC_SYNCING)) {
    618 			if (rdc_info[rdcindex].flags & RDC_PRIMARY)
    619 				ind = IND_RESYNC_REVERSE;
    620 			else
    621 				ind = IND_RESYNC;
    622 		} else if (rdc_info[rdcindex].flags & RDC_SYNCING) {
    623 			if (rdc_info[rdcindex].flags & RDC_PRIMARY)
    624 				ind = IND_RESYNC;
    625 			else
    626 				ind = IND_RESYNC_REVERSE;
    627 		} else
    628 			ind = IND_ENABLED;
    629 
    630 		phost = rdc_info[rdcindex].primary.intf;
    631 		pfile = rdc_info[rdcindex].primary.file;
    632 		shost = rdc_info[rdcindex].secondary.intf;
    633 		sfile = rdc_info[rdcindex].secondary.file;
    634 
    635 		if ((len = strlen(phost)) > 8) {
    636 			(void) mvprintw(4 + k, 0, ".%+7s:",
    637 				phost + len - 7);
    638 		} else
    639 			(void) mvprintw(4 + k, 0, "%+8s:", phost);
    640 
    641 		if ((len = strlen(pfile)) > DISPLEN) {
    642 			(void) mvprintw(4 + k, 9, "...%-13s",
    643 			    pfile + len - DISPLEN + 3);
    644 		} else
    645 			(void) mvprintw(4 + k, 9, "%-16s", pfile);
    646 
    647 		(void) attron(A_BOLD);
    648 		(void) mvprintw(4 + k, 26, "*");
    649 		(void) mvprintw(4 + k, 28, "*");
    650 
    651 		(void) mvprintw(4 + k, 56, "%-8s", status[ind]);
    652 		(void) attroff(A_BOLD);
    653 
    654 		if (ind == IND_RESYNC_REVERSE) {
    655 			if (bright && !(rdc_info[rdcindex].flags & RDC_LOGGING))
    656 				(void) mvprintw(4 + k, 27, "<");
    657 			if (rdc_info[rdcindex].flags & RDC_PRIMARY &&
    658 			    !(rdc_info[rdcindex].flags & RDC_LOGGING))
    659 				calc_completion(rdcindex,
    660 				rdc_info[rdcindex].bits_set, 4 + k);
    661 		} else if (ind == IND_RESYNC) {
    662 			if (bright && !(rdc_info[rdcindex].flags & RDC_LOGGING))
    663 				(void) mvprintw(4 + k, 27, ">");
    664 			if (rdc_info[rdcindex].flags & RDC_PRIMARY &&
    665 			    !(rdc_info[rdcindex].flags & RDC_LOGGING))
    666 				calc_completion(rdcindex,
    667 				rdc_info[rdcindex].bits_set, 4 + k);
    668 		} else if (ind == IND_LOGGING)
    669 			(void) mvprintw(4 + k, 27, ".");
    670 		else if (ind == IND_ENABLED)
    671 			(void) mvprintw(4 + k, 27, "=");
    672 
    673 		if ((len = strlen(shost)) > 8) {
    674 			(void) mvprintw(4 + k, 30, ".%+7s:",
    675 				shost + len - 7);
    676 		} else
    677 			(void) mvprintw(4 + k, 30, "%+8s:", shost);
    678 
    679 		if ((len = strlen(sfile)) > DISPLEN) {
    680 			(void) mvprintw(4 + k, 39, "...%-13s",
    681 			sfile + len - DISPLEN + 3);
    682 		} else
    683 			(void) mvprintw(4 + k, 39, "%-16s", sfile);
    684 
    685 		k++;
    686 	}
    687 
    688 	k += 5;
    689 	(void) attron(A_UNDERLINE);
    690 	for (i = 0; i < 80; i++)
    691 		(void) mvprintw(k, i, " ");
    692 	k += 2;
    693 	(void) mvprintw(k,  0, "partition");
    694 	(void) mvprintw(k, 16, "recovery needed");
    695 	(void) mvprintw(k, 48, "recovery completed");
    696 	(void) attroff(A_UNDERLINE);
    697 	k += 2;
    698 
    699 	for (rdcindex = 0; rdcindex < rdc_enabled_sets; rdcindex++)  {
    700 		if (!(rdc_info[rdcindex].flags & RDC_ENABLED) ||
    701 		    !dual_on_off[rdcindex])
    702 			continue;
    703 
    704 		if (!(rdc_info[rdcindex].flags & RDC_PRIMARY)) {
    705 			continue;
    706 		}
    707 		if (!(rdc_info[rdcindex].flags & RDC_SLAVE) &&
    708 		    !(rdc_info[rdcindex].flags & RDC_SYNCING) &&
    709 		    !(rdc_info[rdcindex].flags & RDC_LOGGING)) {
    710 			continue;
    711 		}
    712 
    713 		len = strlen(rdc_info[rdcindex].secondary.file);
    714 		if (len > 15) {
    715 			strcpy(fn, "...");
    716 			strcat(fn,
    717 			    rdc_info[rdcindex].secondary.file + len - 12);
    718 		} else
    719 			strcpy(fn, rdc_info[rdcindex].secondary.file);
    720 		(void) mvprintw(k, 0, "%-15s", fn);
    721 
    722 		segs = FBA_TO_LOG_LEN(rdc_info[rdcindex].volume_size);
    723 		pct  = segs ?
    724 		    ((float)rdc_info[rdcindex].bits_set / (float)segs) : 0.0;
    725 		stars = (int)(pct * 20.0);
    726 		while (stars > 0) {
    727 			(void) mvprintw(k, 16 + stars, "*");
    728 			stars--;
    729 		}
    730 		(void) attron(A_BOLD);
    731 		(void) mvprintw(k, 16, "[");
    732 		(void) mvprintw(k, 37, "]");
    733 		(void) attroff(A_BOLD);
    734 		(void) mvprintw(k, 39, "%6.2f%%", pct * 100.0);
    735 
    736 		if (rdc_info[rdcindex].flags & RDC_SYNCING)
    737 			pct = ((float)rdc_info[rdcindex].sync_pos /
    738 			    (float)rdc_info[rdcindex].volume_size);
    739 		else
    740 			pct = 0.0;
    741 		stars = (int)(pct * 20.0);
    742 		while (stars > 0) {
    743 			(void) mvprintw(k, 48 + stars, "*");
    744 			stars--;
    745 		}
    746 		(void) attron(A_BOLD);
    747 		(void) mvprintw(k, 48, "[");
    748 		(void) mvprintw(k, 69, "]");
    749 		(void) attroff(A_BOLD);
    750 		(void) mvprintw(k, 70, "%6.2f%%", pct * 100.0);
    751 		k++;
    752 	}
    753 	bright = !bright;
    754 	return (0);
    755 }
    756 
    757 /*
    758  * Calculate a time interval in milliseconds using the
    759  * micosecond counter
    760  */
    761 void
    762 calc_time(void)
    763 {
    764 	unsigned int cur;
    765 
    766 	cur = USEC_READ();
    767 	dc_delta_time = cur > dc_prev_time ? cur - dc_prev_time :
    768 		cur + 0xFFFFFFFF - dc_prev_time;
    769 	dc_delta_time /= 1000;
    770 	dc_prev_time = cur;
    771 }
    772 
    773 /*
    774  * Calculate estimated time of completion of resync
    775  */
    776 void
    777 calc_completion(int cd, int updates_left, int l)
    778 {
    779 	int delta_done;
    780 	double rate;
    781 	long time_left;
    782 	long hours;
    783 	long minutes;
    784 	static int initted = 0;
    785 
    786 	if (!initted) {
    787 		updates_prev[cd] = updates_left;
    788 		initted = 1;
    789 		return;
    790 	}
    791 
    792 	/*
    793 	 * Caclulate updates since last check
    794 	 */
    795 	delta_done = updates_prev[cd] - updates_left;
    796 	updates_prev[cd] = updates_left;
    797 
    798 	/*
    799 	 * If no updates, don't bother estimating completion time
    800 	 */
    801 	if (delta_done <= 0) {
    802 		samples[cd] = 0;
    803 		return;
    804 	}
    805 
    806 	rate = delta_done * 1000.0 / dc_delta_time;
    807 
    808 	/*
    809 	 * Calculate rate of updates as a weighted average
    810 	 * of previous and current rate
    811 	 */
    812 	if (rate_prev[cd] && samples[cd] > SAMPLE_RATE)
    813 		rate = (rate_prev[cd] * 4.0 + rate) / 5.0;
    814 	rate_prev[cd] = rate;
    815 	samples[cd]++;
    816 
    817 	/*
    818 	 * Get enough samples before making estimate
    819 	 */
    820 	if (samples[cd]++ < SAMPLE_RATE)
    821 		return;
    822 
    823 	time_left = (long)(updates_left/rate);   /* time left in seconds */
    824 
    825 	if (time_left < 0)
    826 		return;
    827 
    828 	hours = time_left / (60 * 60);
    829 	time_left -= hours * (60 * 60);
    830 	minutes = time_left / 60;
    831 	time_left -= minutes * 60;
    832 	(void) mvprintw(l, 67,
    833 	    "time %02d:%02d:%02d  \n", hours, minutes, time_left);
    834 }
    835 
    836 void
    837 disp_total_stats(void)
    838 {
    839 	unsigned int	read_s, write_s, access_s;
    840 	double readp, writep;
    841 	unsigned int	rmiss_s, wmiss_s;
    842 	double  kbps = 2.0;
    843 	int	rtotal, wtotal, i, j;
    844 	unsigned int throughput = 0, rthroughput = 0, creads = 0, cwrites = 0;
    845 	char	status_bit, down = 0;
    846 	int	len;
    847 	char	fn[19];
    848 
    849 	read_s  = cs_cur->st_rdhits;
    850 	write_s = cs_cur->st_wrhits;
    851 	rmiss_s = cs_cur->st_rdmiss;
    852 	wmiss_s = cs_cur->st_wrmiss;
    853 	access_s = (read_s + write_s + rmiss_s + wmiss_s);
    854 
    855 	rtotal = cs_cur->st_rdhits + cs_cur->st_rdmiss;
    856 	wtotal = cs_cur->st_wrhits + cs_cur->st_wrmiss;
    857 	if (rtotal != 0)
    858 		readp = cs_cur->st_rdhits / (double)rtotal;
    859 	else
    860 		readp = 0.0;
    861 
    862 	if (wtotal != 0)
    863 		writep = cs_cur->st_wrhits / (double)wtotal;
    864 	else
    865 		writep = 0.0;
    866 
    867 	set_on_off();
    868 	(void) mvprintw(0, 14,
    869 	    "******     Storage Cache (Cumulative)      ******");
    870 	(void) mvprintw(2, 30, "disk_io                  cache");
    871 	(void) attron(A_UNDERLINE);
    872 	(void) mvprintw(3,  1,
    873 	    " cd cached_partition      reads     writes      reads     writes");
    874 	(void) attroff(A_UNDERLINE);
    875 	for (i = 0, j = 0; j < cs_cur->st_count; i++) {
    876 		if (i >= sdbc_max_devices)
    877 			break;
    878 		if (cs_cur->st_shared[i].sh_alloc)  {
    879 			cs_cur->st_shared[i].sh_disk_write /= kbps;
    880 			cs_cur->st_shared[i].sh_disk_read /= kbps;
    881 			cs_cur->st_shared[i].sh_cache_write /= kbps;
    882 			cs_cur->st_shared[i].sh_cache_read /= kbps;
    883 			rthroughput += cs_cur->st_shared[i].sh_disk_read;
    884 			throughput += cs_cur->st_shared[i].sh_disk_write;
    885 			creads += cs_cur->st_shared[i].sh_cache_read;
    886 			cwrites += cs_cur->st_shared[i].sh_cache_write;
    887 			if (!down)
    888 				down = cs_cur->st_shared[i].sh_failed;
    889 			if (cs_cur->st_shared[i].sh_failed && bright)
    890 				status_bit = '*';
    891 			else
    892 				status_bit = ' ';
    893 			if ((len =
    894 			    strlen(cs_cur->st_shared[i].sh_filename)) > 15) {
    895 				strcpy(fn, "...");
    896 				strcat(fn, cs_cur->st_shared[i].sh_filename +
    897 				    len - 12);
    898 			} else
    899 				strcpy(fn, cs_cur->st_shared[i].sh_filename);
    900 
    901 			if (on_off[i]) {
    902 				(void) mvprintw(4 + j, 1,
    903 				    "%3d %-15s%c %10u %10u %10u %10u",
    904 				    cs_cur->st_shared[i].sh_cd,
    905 				    fn,
    906 				    status_bit,
    907 				    cs_cur->st_shared[i].sh_disk_read,
    908 				    cs_cur->st_shared[i].sh_disk_write,
    909 				    cs_cur->st_shared[i].sh_cache_read,
    910 				    cs_cur->st_shared[i].sh_cache_write);
    911 				j++;
    912 			}
    913 		}
    914 	}
    915 	bright = !bright;
    916 
    917 	(void) mvprintw(4 + j, 22,
    918 	    "---------- ---------- ---------- ----------");
    919 	(void) mvprintw(5 + j, 8, " Kbytes total:%10u %10u %10u %10u",
    920 	    (int)rthroughput, (int)throughput,
    921 	    (int)creads, (int)cwrites);
    922 	(void) mvprintw(7 + j, 1, " accesses");
    923 	(void) mvprintw(7 + j, 18, "read        write    %%readh  %%writeh");
    924 
    925 	(void) attron(A_UNDERLINE);
    926 	(void) mvprintw(8 + j, 1, "            ");
    927 	(void) mvprintw(8 + j, 13,
    928 	    "                                                ");
    929 	(void) mvprintw(8 + j, 11, "(    misses) (    misses)");
    930 	(void) attroff(A_UNDERLINE);
    931 
    932 	(void) mvprintw(9 + j, 0, "%10u  %10u   %10u    %6.1f   %6.1f",
    933 	    access_s, read_s, write_s, readp*100.0, writep*100.0);
    934 	(void) mvprintw(10 + j, 0,
    935 	    "           (%10u) (%10u)\n\n", rmiss_s, wmiss_s);
    936 
    937 	(void) attron(A_UNDERLINE);
    938 	(void) mvprintw(13 + j, 1, "cachesize  blocksize");
    939 	(void) attroff(A_UNDERLINE);
    940 	(void) mvprintw(14 + j, 1, "%8dK %10d", cs_cur->st_cachesize / 1024,
    941 	    cs_cur->st_blksize);
    942 
    943 	(void) attron(A_UNDERLINE);
    944 	(void) mvprintw(16 + j, 1, "Write blocks available:");
    945 	(void) attroff(A_UNDERLINE);
    946 	(void) mvprintw(17 + j, 1, "Net 0: %6d", cs_cur->st_wlru_inq);
    947 
    948 	(void) attron(A_UNDERLINE);
    949 	(void) mvprintw(19 + j, 1, "LRU stats:  Blocks	Requeued    Optimized");
    950 	(void) attroff(A_UNDERLINE);
    951 	(void) mvprintw(20 + j, 7, "%12d %12u %12u", cs_cur->st_lru_blocks,
    952 	    cs_cur->st_lru_req, cs_cur->st_lru_noreq);
    953 
    954 	if (down)
    955 		(void) mvprintw(25 + j, 1, "* -- disk off-line");
    956 }
    957