1 789 ahrens /* 2 789 ahrens * CDDL HEADER START 3 789 ahrens * 4 789 ahrens * The contents of this file are subject to the terms of the 5 1491 ahrens * Common Development and Distribution License (the "License"). 6 1491 ahrens * You may not use this file except in compliance with the License. 7 789 ahrens * 8 789 ahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 789 ahrens * or http://www.opensolaris.org/os/licensing. 10 789 ahrens * See the License for the specific language governing permissions 11 789 ahrens * and limitations under the License. 12 789 ahrens * 13 789 ahrens * When distributing Covered Code, include this CDDL HEADER in each 14 789 ahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 789 ahrens * If applicable, add the following below this CDDL HEADER, with the 16 789 ahrens * fields enclosed by brackets "[]" replaced with your own identifying 17 789 ahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18 789 ahrens * 19 789 ahrens * CDDL HEADER END 20 789 ahrens */ 21 789 ahrens /* 22 9643 Eric * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 789 ahrens * Use is subject to license terms. 24 789 ahrens */ 25 789 ahrens 26 789 ahrens /* 27 789 ahrens * The 512-byte leaf is broken into 32 16-byte chunks. 28 789 ahrens * chunk number n means l_chunk[n], even though the header precedes it. 29 789 ahrens * the names are stored null-terminated. 30 789 ahrens */ 31 789 ahrens 32 10922 Jeff #include <sys/zio.h> 33 9643 Eric #include <sys/spa.h> 34 9643 Eric #include <sys/dmu.h> 35 789 ahrens #include <sys/zfs_context.h> 36 9643 Eric #include <sys/fs/zfs.h> 37 789 ahrens #include <sys/zap.h> 38 789 ahrens #include <sys/zap_impl.h> 39 789 ahrens #include <sys/zap_leaf.h> 40 5331 amw 41 5331 amw static uint16_t *zap_leaf_rehash_entry(zap_leaf_t *l, uint16_t entry); 42 789 ahrens 43 789 ahrens #define CHAIN_END 0xffff /* end of the chunk chain */ 44 789 ahrens 45 1491 ahrens /* half the (current) minimum block size */ 46 789 ahrens #define MAX_ARRAY_BYTES (8<<10) 47 789 ahrens 48 789 ahrens #define LEAF_HASH(l, h) \ 49 1491 ahrens ((ZAP_LEAF_HASH_NUMENTRIES(l)-1) & \ 50 1578 ahrens ((h) >> (64 - ZAP_LEAF_HASH_SHIFT(l)-(l)->l_phys->l_hdr.lh_prefix_len))) 51 789 ahrens 52 789 ahrens #define LEAF_HASH_ENTPTR(l, h) (&(l)->l_phys->l_hash[LEAF_HASH(l, h)]) 53 789 ahrens 54 789 ahrens 55 789 ahrens static void 56 789 ahrens zap_memset(void *a, int c, size_t n) 57 789 ahrens { 58 789 ahrens char *cp = a; 59 789 ahrens char *cpend = cp + n; 60 789 ahrens 61 789 ahrens while (cp < cpend) 62 789 ahrens *cp++ = c; 63 789 ahrens } 64 789 ahrens 65 789 ahrens static void 66 789 ahrens stv(int len, void *addr, uint64_t value) 67 789 ahrens { 68 789 ahrens switch (len) { 69 789 ahrens case 1: 70 789 ahrens *(uint8_t *)addr = value; 71 789 ahrens return; 72 789 ahrens case 2: 73 789 ahrens *(uint16_t *)addr = value; 74 789 ahrens return; 75 789 ahrens case 4: 76 789 ahrens *(uint32_t *)addr = value; 77 789 ahrens return; 78 789 ahrens case 8: 79 789 ahrens *(uint64_t *)addr = value; 80 789 ahrens return; 81 789 ahrens } 82 789 ahrens ASSERT(!"bad int len"); 83 789 ahrens } 84 789 ahrens 85 789 ahrens static uint64_t 86 789 ahrens ldv(int len, const void *addr) 87 789 ahrens { 88 789 ahrens switch (len) { 89 789 ahrens case 1: 90 789 ahrens return (*(uint8_t *)addr); 91 789 ahrens case 2: 92 789 ahrens return (*(uint16_t *)addr); 93 789 ahrens case 4: 94 789 ahrens return (*(uint32_t *)addr); 95 789 ahrens case 8: 96 789 ahrens return (*(uint64_t *)addr); 97 789 ahrens } 98 789 ahrens ASSERT(!"bad int len"); 99 2856 nd150628 return (0xFEEDFACEDEADBEEFULL); 100 789 ahrens } 101 789 ahrens 102 789 ahrens void 103 1491 ahrens zap_leaf_byteswap(zap_leaf_phys_t *buf, int size) 104 789 ahrens { 105 789 ahrens int i; 106 1491 ahrens zap_leaf_t l; 107 1491 ahrens l.l_bs = highbit(size)-1; 108 1491 ahrens l.l_phys = buf; 109 789 ahrens 110 1578 ahrens buf->l_hdr.lh_block_type = BSWAP_64(buf->l_hdr.lh_block_type); 111 1578 ahrens buf->l_hdr.lh_prefix = BSWAP_64(buf->l_hdr.lh_prefix); 112 1578 ahrens buf->l_hdr.lh_magic = BSWAP_32(buf->l_hdr.lh_magic); 113 1578 ahrens buf->l_hdr.lh_nfree = BSWAP_16(buf->l_hdr.lh_nfree); 114 1578 ahrens buf->l_hdr.lh_nentries = BSWAP_16(buf->l_hdr.lh_nentries); 115 1578 ahrens buf->l_hdr.lh_prefix_len = BSWAP_16(buf->l_hdr.lh_prefix_len); 116 789 ahrens buf->l_hdr.lh_freelist = BSWAP_16(buf->l_hdr.lh_freelist); 117 789 ahrens 118 1491 ahrens for (i = 0; i < ZAP_LEAF_HASH_NUMENTRIES(&l); i++) 119 789 ahrens buf->l_hash[i] = BSWAP_16(buf->l_hash[i]); 120 789 ahrens 121 1491 ahrens for (i = 0; i < ZAP_LEAF_NUMCHUNKS(&l); i++) { 122 1491 ahrens zap_leaf_chunk_t *lc = &ZAP_LEAF_CHUNK(&l, i); 123 789 ahrens struct zap_leaf_entry *le; 124 789 ahrens 125 1491 ahrens switch (lc->l_free.lf_type) { 126 1491 ahrens case ZAP_CHUNK_ENTRY: 127 1491 ahrens le = &lc->l_entry; 128 789 ahrens 129 1578 ahrens le->le_type = BSWAP_8(le->le_type); 130 11165 Matthew le->le_value_intlen = BSWAP_8(le->le_value_intlen); 131 1578 ahrens le->le_next = BSWAP_16(le->le_next); 132 1578 ahrens le->le_name_chunk = BSWAP_16(le->le_name_chunk); 133 11165 Matthew le->le_name_numints = BSWAP_16(le->le_name_numints); 134 1578 ahrens le->le_value_chunk = BSWAP_16(le->le_value_chunk); 135 11165 Matthew le->le_value_numints = BSWAP_16(le->le_value_numints); 136 1578 ahrens le->le_cd = BSWAP_32(le->le_cd); 137 1578 ahrens le->le_hash = BSWAP_64(le->le_hash); 138 789 ahrens break; 139 1491 ahrens case ZAP_CHUNK_FREE: 140 1578 ahrens lc->l_free.lf_type = BSWAP_8(lc->l_free.lf_type); 141 1578 ahrens lc->l_free.lf_next = BSWAP_16(lc->l_free.lf_next); 142 789 ahrens break; 143 1491 ahrens case ZAP_CHUNK_ARRAY: 144 1578 ahrens lc->l_array.la_type = BSWAP_8(lc->l_array.la_type); 145 1578 ahrens lc->l_array.la_next = BSWAP_16(lc->l_array.la_next); 146 789 ahrens /* la_array doesn't need swapping */ 147 789 ahrens break; 148 789 ahrens default: 149 789 ahrens ASSERT(!"bad leaf type"); 150 789 ahrens } 151 789 ahrens } 152 789 ahrens } 153 789 ahrens 154 789 ahrens void 155 5498 timh zap_leaf_init(zap_leaf_t *l, boolean_t sort) 156 789 ahrens { 157 789 ahrens int i; 158 789 ahrens 159 1491 ahrens l->l_bs = highbit(l->l_dbuf->db_size)-1; 160 789 ahrens zap_memset(&l->l_phys->l_hdr, 0, sizeof (struct zap_leaf_header)); 161 1491 ahrens zap_memset(l->l_phys->l_hash, CHAIN_END, 2*ZAP_LEAF_HASH_NUMENTRIES(l)); 162 1491 ahrens for (i = 0; i < ZAP_LEAF_NUMCHUNKS(l); i++) { 163 1491 ahrens ZAP_LEAF_CHUNK(l, i).l_free.lf_type = ZAP_CHUNK_FREE; 164 1491 ahrens ZAP_LEAF_CHUNK(l, i).l_free.lf_next = i+1; 165 789 ahrens } 166 1491 ahrens ZAP_LEAF_CHUNK(l, ZAP_LEAF_NUMCHUNKS(l)-1).l_free.lf_next = CHAIN_END; 167 1578 ahrens l->l_phys->l_hdr.lh_block_type = ZBT_LEAF; 168 1578 ahrens l->l_phys->l_hdr.lh_magic = ZAP_LEAF_MAGIC; 169 1578 ahrens l->l_phys->l_hdr.lh_nfree = ZAP_LEAF_NUMCHUNKS(l); 170 5498 timh if (sort) 171 5331 amw l->l_phys->l_hdr.lh_flags |= ZLF_ENTRIES_CDSORTED; 172 789 ahrens } 173 789 ahrens 174 789 ahrens /* 175 789 ahrens * Routines which manipulate leaf chunks (l_chunk[]). 176 789 ahrens */ 177 789 ahrens 178 789 ahrens static uint16_t 179 789 ahrens zap_leaf_chunk_alloc(zap_leaf_t *l) 180 789 ahrens { 181 789 ahrens int chunk; 182 789 ahrens 183 1578 ahrens ASSERT(l->l_phys->l_hdr.lh_nfree > 0); 184 789 ahrens 185 789 ahrens chunk = l->l_phys->l_hdr.lh_freelist; 186 1491 ahrens ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l)); 187 1491 ahrens ASSERT3U(ZAP_LEAF_CHUNK(l, chunk).l_free.lf_type, ==, ZAP_CHUNK_FREE); 188 789 ahrens 189 1491 ahrens l->l_phys->l_hdr.lh_freelist = ZAP_LEAF_CHUNK(l, chunk).l_free.lf_next; 190 789 ahrens 191 1578 ahrens l->l_phys->l_hdr.lh_nfree--; 192 789 ahrens 193 789 ahrens return (chunk); 194 789 ahrens } 195 789 ahrens 196 789 ahrens static void 197 789 ahrens zap_leaf_chunk_free(zap_leaf_t *l, uint16_t chunk) 198 789 ahrens { 199 1491 ahrens struct zap_leaf_free *zlf = &ZAP_LEAF_CHUNK(l, chunk).l_free; 200 1578 ahrens ASSERT3U(l->l_phys->l_hdr.lh_nfree, <, ZAP_LEAF_NUMCHUNKS(l)); 201 1491 ahrens ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l)); 202 1491 ahrens ASSERT(zlf->lf_type != ZAP_CHUNK_FREE); 203 789 ahrens 204 1491 ahrens zlf->lf_type = ZAP_CHUNK_FREE; 205 789 ahrens zlf->lf_next = l->l_phys->l_hdr.lh_freelist; 206 789 ahrens bzero(zlf->lf_pad, sizeof (zlf->lf_pad)); /* help it to compress */ 207 789 ahrens l->l_phys->l_hdr.lh_freelist = chunk; 208 789 ahrens 209 1578 ahrens l->l_phys->l_hdr.lh_nfree++; 210 789 ahrens } 211 789 ahrens 212 789 ahrens /* 213 789 ahrens * Routines which manipulate leaf arrays (zap_leaf_array type chunks). 214 789 ahrens */ 215 789 ahrens 216 789 ahrens static uint16_t 217 1578 ahrens zap_leaf_array_create(zap_leaf_t *l, const char *buf, 218 11165 Matthew int integer_size, int num_integers) 219 789 ahrens { 220 789 ahrens uint16_t chunk_head; 221 789 ahrens uint16_t *chunkp = &chunk_head; 222 789 ahrens int byten = 0; 223 789 ahrens uint64_t value; 224 789 ahrens int shift = (integer_size-1)*8; 225 789 ahrens int len = num_integers; 226 789 ahrens 227 789 ahrens ASSERT3U(num_integers * integer_size, <, MAX_ARRAY_BYTES); 228 789 ahrens 229 789 ahrens while (len > 0) { 230 789 ahrens uint16_t chunk = zap_leaf_chunk_alloc(l); 231 1491 ahrens struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(l, chunk).l_array; 232 789 ahrens int i; 233 789 ahrens 234 1491 ahrens la->la_type = ZAP_CHUNK_ARRAY; 235 789 ahrens for (i = 0; i < ZAP_LEAF_ARRAY_BYTES; i++) { 236 789 ahrens if (byten == 0) 237 789 ahrens value = ldv(integer_size, buf); 238 3052 ahrens la->la_array[i] = value >> shift; 239 789 ahrens value <<= 8; 240 789 ahrens if (++byten == integer_size) { 241 789 ahrens byten = 0; 242 789 ahrens buf += integer_size; 243 789 ahrens if (--len == 0) 244 789 ahrens break; 245 789 ahrens } 246 789 ahrens } 247 789 ahrens 248 789 ahrens *chunkp = chunk; 249 789 ahrens chunkp = &la->la_next; 250 789 ahrens } 251 789 ahrens *chunkp = CHAIN_END; 252 789 ahrens 253 789 ahrens return (chunk_head); 254 789 ahrens } 255 789 ahrens 256 789 ahrens static void 257 1578 ahrens zap_leaf_array_free(zap_leaf_t *l, uint16_t *chunkp) 258 789 ahrens { 259 789 ahrens uint16_t chunk = *chunkp; 260 789 ahrens 261 789 ahrens *chunkp = CHAIN_END; 262 789 ahrens 263 789 ahrens while (chunk != CHAIN_END) { 264 1491 ahrens int nextchunk = ZAP_LEAF_CHUNK(l, chunk).l_array.la_next; 265 1491 ahrens ASSERT3U(ZAP_LEAF_CHUNK(l, chunk).l_array.la_type, ==, 266 1491 ahrens ZAP_CHUNK_ARRAY); 267 789 ahrens zap_leaf_chunk_free(l, chunk); 268 789 ahrens chunk = nextchunk; 269 789 ahrens } 270 789 ahrens } 271 789 ahrens 272 789 ahrens /* array_len and buf_len are in integers, not bytes */ 273 789 ahrens static void 274 1578 ahrens zap_leaf_array_read(zap_leaf_t *l, uint16_t chunk, 275 789 ahrens int array_int_len, int array_len, int buf_int_len, uint64_t buf_len, 276 10922 Jeff void *buf) 277 789 ahrens { 278 789 ahrens int len = MIN(array_len, buf_len); 279 789 ahrens int byten = 0; 280 789 ahrens uint64_t value = 0; 281 10922 Jeff char *p = buf; 282 789 ahrens 283 789 ahrens ASSERT3U(array_int_len, <=, buf_int_len); 284 789 ahrens 285 885 ahrens /* Fast path for one 8-byte integer */ 286 885 ahrens if (array_int_len == 8 && buf_int_len == 8 && len == 1) { 287 1491 ahrens struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(l, chunk).l_array; 288 899 bonwick uint8_t *ip = la->la_array; 289 10922 Jeff uint64_t *buf64 = buf; 290 899 bonwick 291 899 bonwick *buf64 = (uint64_t)ip[0] << 56 | (uint64_t)ip[1] << 48 | 292 899 bonwick (uint64_t)ip[2] << 40 | (uint64_t)ip[3] << 32 | 293 899 bonwick (uint64_t)ip[4] << 24 | (uint64_t)ip[5] << 16 | 294 899 bonwick (uint64_t)ip[6] << 8 | (uint64_t)ip[7]; 295 885 ahrens return; 296 885 ahrens } 297 885 ahrens 298 885 ahrens /* Fast path for an array of 1-byte integers (eg. the entry name) */ 299 885 ahrens if (array_int_len == 1 && buf_int_len == 1 && 300 885 ahrens buf_len > array_len + ZAP_LEAF_ARRAY_BYTES) { 301 885 ahrens while (chunk != CHAIN_END) { 302 885 ahrens struct zap_leaf_array *la = 303 1491 ahrens &ZAP_LEAF_CHUNK(l, chunk).l_array; 304 10922 Jeff bcopy(la->la_array, p, ZAP_LEAF_ARRAY_BYTES); 305 10922 Jeff p += ZAP_LEAF_ARRAY_BYTES; 306 885 ahrens chunk = la->la_next; 307 885 ahrens } 308 885 ahrens return; 309 885 ahrens } 310 885 ahrens 311 789 ahrens while (len > 0) { 312 1491 ahrens struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(l, chunk).l_array; 313 789 ahrens int i; 314 789 ahrens 315 1491 ahrens ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l)); 316 789 ahrens for (i = 0; i < ZAP_LEAF_ARRAY_BYTES && len > 0; i++) { 317 789 ahrens value = (value << 8) | la->la_array[i]; 318 789 ahrens byten++; 319 789 ahrens if (byten == array_int_len) { 320 10922 Jeff stv(buf_int_len, p, value); 321 789 ahrens byten = 0; 322 789 ahrens len--; 323 789 ahrens if (len == 0) 324 789 ahrens return; 325 10922 Jeff p += buf_int_len; 326 789 ahrens } 327 789 ahrens } 328 789 ahrens chunk = la->la_next; 329 789 ahrens } 330 789 ahrens } 331 789 ahrens 332 5331 amw static boolean_t 333 11165 Matthew zap_leaf_array_match(zap_leaf_t *l, zap_name_t *zn, 334 11165 Matthew int chunk, int array_numints) 335 789 ahrens { 336 789 ahrens int bseen = 0; 337 789 ahrens 338 10922 Jeff if (zap_getflags(zn->zn_zap) & ZAP_FLAG_UINT64_KEY) { 339 10922 Jeff uint64_t *thiskey; 340 10922 Jeff boolean_t match; 341 10922 Jeff 342 10922 Jeff ASSERT(zn->zn_key_intlen == sizeof (*thiskey)); 343 11165 Matthew thiskey = kmem_alloc(array_numints * sizeof (*thiskey), 344 11165 Matthew KM_SLEEP); 345 10922 Jeff 346 11165 Matthew zap_leaf_array_read(l, chunk, sizeof (*thiskey), array_numints, 347 11165 Matthew sizeof (*thiskey), array_numints, thiskey); 348 10922 Jeff match = bcmp(thiskey, zn->zn_key_orig, 349 11165 Matthew array_numints * sizeof (*thiskey)) == 0; 350 11165 Matthew kmem_free(thiskey, array_numints * sizeof (*thiskey)); 351 10922 Jeff return (match); 352 10922 Jeff } 353 10922 Jeff 354 11165 Matthew ASSERT(zn->zn_key_intlen == 1); 355 5331 amw if (zn->zn_matchtype == MT_FIRST) { 356 11165 Matthew char *thisname = kmem_alloc(array_numints, KM_SLEEP); 357 5331 amw boolean_t match; 358 5331 amw 359 11165 Matthew zap_leaf_array_read(l, chunk, sizeof (char), array_numints, 360 11165 Matthew sizeof (char), array_numints, thisname); 361 5331 amw match = zap_match(zn, thisname); 362 11165 Matthew kmem_free(thisname, array_numints); 363 5331 amw return (match); 364 5331 amw } 365 5331 amw 366 10922 Jeff /* 367 10922 Jeff * Fast path for exact matching. 368 10922 Jeff * First check that the lengths match, so that we don't read 369 10922 Jeff * past the end of the zn_key_orig array. 370 10922 Jeff */ 371 11165 Matthew if (array_numints != zn->zn_key_orig_numints) 372 10922 Jeff return (B_FALSE); 373 11165 Matthew while (bseen < array_numints) { 374 1491 ahrens struct zap_leaf_array *la = &ZAP_LEAF_CHUNK(l, chunk).l_array; 375 11165 Matthew int toread = MIN(array_numints - bseen, ZAP_LEAF_ARRAY_BYTES); 376 1491 ahrens ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l)); 377 10922 Jeff if (bcmp(la->la_array, (char *)zn->zn_key_orig + bseen, toread)) 378 789 ahrens break; 379 789 ahrens chunk = la->la_next; 380 789 ahrens bseen += toread; 381 789 ahrens } 382 11165 Matthew return (bseen == array_numints); 383 789 ahrens } 384 789 ahrens 385 789 ahrens /* 386 789 ahrens * Routines which manipulate leaf entries. 387 789 ahrens */ 388 789 ahrens 389 789 ahrens int 390 5331 amw zap_leaf_lookup(zap_leaf_t *l, zap_name_t *zn, zap_entry_handle_t *zeh) 391 789 ahrens { 392 789 ahrens uint16_t *chunkp; 393 789 ahrens struct zap_leaf_entry *le; 394 789 ahrens 395 1578 ahrens ASSERT3U(l->l_phys->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC); 396 789 ahrens 397 5331 amw again: 398 5331 amw for (chunkp = LEAF_HASH_ENTPTR(l, zn->zn_hash); 399 789 ahrens *chunkp != CHAIN_END; chunkp = &le->le_next) { 400 789 ahrens uint16_t chunk = *chunkp; 401 1491 ahrens le = ZAP_LEAF_ENTRY(l, chunk); 402 789 ahrens 403 1491 ahrens ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l)); 404 1491 ahrens ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY); 405 789 ahrens 406 5331 amw if (le->le_hash != zn->zn_hash) 407 789 ahrens continue; 408 789 ahrens 409 5331 amw /* 410 5331 amw * NB: the entry chain is always sorted by cd on 411 5331 amw * normalized zap objects, so this will find the 412 5331 amw * lowest-cd match for MT_FIRST. 413 5331 amw */ 414 5331 amw ASSERT(zn->zn_matchtype == MT_EXACT || 415 5331 amw (l->l_phys->l_hdr.lh_flags & ZLF_ENTRIES_CDSORTED)); 416 5331 amw if (zap_leaf_array_match(l, zn, le->le_name_chunk, 417 11165 Matthew le->le_name_numints)) { 418 11165 Matthew zeh->zeh_num_integers = le->le_value_numints; 419 11165 Matthew zeh->zeh_integer_size = le->le_value_intlen; 420 789 ahrens zeh->zeh_cd = le->le_cd; 421 789 ahrens zeh->zeh_hash = le->le_hash; 422 789 ahrens zeh->zeh_chunkp = chunkp; 423 1578 ahrens zeh->zeh_leaf = l; 424 789 ahrens return (0); 425 789 ahrens } 426 5331 amw } 427 5331 amw 428 5331 amw /* 429 5331 amw * NB: we could of course do this in one pass, but that would be 430 5331 amw * a pain. We'll see if MT_BEST is even used much. 431 5331 amw */ 432 5331 amw if (zn->zn_matchtype == MT_BEST) { 433 5331 amw zn->zn_matchtype = MT_FIRST; 434 5331 amw goto again; 435 789 ahrens } 436 789 ahrens 437 789 ahrens return (ENOENT); 438 789 ahrens } 439 789 ahrens 440 789 ahrens /* Return (h1,cd1 >= h2,cd2) */ 441 885 ahrens #define HCD_GTEQ(h1, cd1, h2, cd2) \ 442 885 ahrens ((h1 > h2) ? TRUE : ((h1 == h2 && cd1 >= cd2) ? TRUE : FALSE)) 443 789 ahrens 444 789 ahrens int 445 789 ahrens zap_leaf_lookup_closest(zap_leaf_t *l, 446 789 ahrens uint64_t h, uint32_t cd, zap_entry_handle_t *zeh) 447 789 ahrens { 448 789 ahrens uint16_t chunk; 449 789 ahrens uint64_t besth = -1ULL; 450 10922 Jeff uint32_t bestcd = -1U; 451 1491 ahrens uint16_t bestlh = ZAP_LEAF_HASH_NUMENTRIES(l)-1; 452 789 ahrens uint16_t lh; 453 789 ahrens struct zap_leaf_entry *le; 454 789 ahrens 455 1578 ahrens ASSERT3U(l->l_phys->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC); 456 789 ahrens 457 789 ahrens for (lh = LEAF_HASH(l, h); lh <= bestlh; lh++) { 458 789 ahrens for (chunk = l->l_phys->l_hash[lh]; 459 789 ahrens chunk != CHAIN_END; chunk = le->le_next) { 460 1491 ahrens le = ZAP_LEAF_ENTRY(l, chunk); 461 789 ahrens 462 1491 ahrens ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l)); 463 1491 ahrens ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY); 464 789 ahrens 465 885 ahrens if (HCD_GTEQ(le->le_hash, le->le_cd, h, cd) && 466 885 ahrens HCD_GTEQ(besth, bestcd, le->le_hash, le->le_cd)) { 467 789 ahrens ASSERT3U(bestlh, >=, lh); 468 789 ahrens bestlh = lh; 469 789 ahrens besth = le->le_hash; 470 789 ahrens bestcd = le->le_cd; 471 789 ahrens 472 11165 Matthew zeh->zeh_num_integers = le->le_value_numints; 473 11165 Matthew zeh->zeh_integer_size = le->le_value_intlen; 474 789 ahrens zeh->zeh_cd = le->le_cd; 475 789 ahrens zeh->zeh_hash = le->le_hash; 476 789 ahrens zeh->zeh_fakechunk = chunk; 477 789 ahrens zeh->zeh_chunkp = &zeh->zeh_fakechunk; 478 1578 ahrens zeh->zeh_leaf = l; 479 789 ahrens } 480 789 ahrens } 481 789 ahrens } 482 789 ahrens 483 10922 Jeff return (bestcd == -1U ? ENOENT : 0); 484 789 ahrens } 485 789 ahrens 486 789 ahrens int 487 789 ahrens zap_entry_read(const zap_entry_handle_t *zeh, 488 789 ahrens uint8_t integer_size, uint64_t num_integers, void *buf) 489 789 ahrens { 490 1491 ahrens struct zap_leaf_entry *le = 491 1578 ahrens ZAP_LEAF_ENTRY(zeh->zeh_leaf, *zeh->zeh_chunkp); 492 1491 ahrens ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY); 493 789 ahrens 494 11165 Matthew if (le->le_value_intlen > integer_size) 495 789 ahrens return (EINVAL); 496 789 ahrens 497 11165 Matthew zap_leaf_array_read(zeh->zeh_leaf, le->le_value_chunk, 498 11165 Matthew le->le_value_intlen, le->le_value_numints, 499 11165 Matthew integer_size, num_integers, buf); 500 789 ahrens 501 789 ahrens if (zeh->zeh_num_integers > num_integers) 502 789 ahrens return (EOVERFLOW); 503 789 ahrens return (0); 504 789 ahrens 505 789 ahrens } 506 789 ahrens 507 789 ahrens int 508 10922 Jeff zap_entry_read_name(zap_t *zap, const zap_entry_handle_t *zeh, uint16_t buflen, 509 10922 Jeff char *buf) 510 789 ahrens { 511 1491 ahrens struct zap_leaf_entry *le = 512 1578 ahrens ZAP_LEAF_ENTRY(zeh->zeh_leaf, *zeh->zeh_chunkp); 513 1491 ahrens ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY); 514 789 ahrens 515 10922 Jeff if (zap_getflags(zap) & ZAP_FLAG_UINT64_KEY) { 516 10922 Jeff zap_leaf_array_read(zeh->zeh_leaf, le->le_name_chunk, 8, 517 11165 Matthew le->le_name_numints, 8, buflen / 8, buf); 518 10922 Jeff } else { 519 10922 Jeff zap_leaf_array_read(zeh->zeh_leaf, le->le_name_chunk, 1, 520 11165 Matthew le->le_name_numints, 1, buflen, buf); 521 10922 Jeff } 522 11165 Matthew if (le->le_name_numints > buflen) 523 789 ahrens return (EOVERFLOW); 524 789 ahrens return (0); 525 789 ahrens } 526 789 ahrens 527 789 ahrens int 528 789 ahrens zap_entry_update(zap_entry_handle_t *zeh, 529 789 ahrens uint8_t integer_size, uint64_t num_integers, const void *buf) 530 789 ahrens { 531 789 ahrens int delta_chunks; 532 1578 ahrens zap_leaf_t *l = zeh->zeh_leaf; 533 1578 ahrens struct zap_leaf_entry *le = ZAP_LEAF_ENTRY(l, *zeh->zeh_chunkp); 534 789 ahrens 535 1578 ahrens delta_chunks = ZAP_LEAF_ARRAY_NCHUNKS(num_integers * integer_size) - 536 11165 Matthew ZAP_LEAF_ARRAY_NCHUNKS(le->le_value_numints * le->le_value_intlen); 537 789 ahrens 538 1578 ahrens if ((int)l->l_phys->l_hdr.lh_nfree < delta_chunks) 539 789 ahrens return (EAGAIN); 540 789 ahrens 541 789 ahrens /* 542 789 ahrens * We should search other chained leaves (via 543 789 ahrens * zap_entry_remove,create?) otherwise returning EAGAIN will 544 789 ahrens * just send us into an infinite loop if we have to chain 545 789 ahrens * another leaf block, rather than being able to split this 546 789 ahrens * block. 547 789 ahrens */ 548 789 ahrens 549 1578 ahrens zap_leaf_array_free(l, &le->le_value_chunk); 550 789 ahrens le->le_value_chunk = 551 1578 ahrens zap_leaf_array_create(l, buf, integer_size, num_integers); 552 11165 Matthew le->le_value_numints = num_integers; 553 11165 Matthew le->le_value_intlen = integer_size; 554 789 ahrens return (0); 555 789 ahrens } 556 789 ahrens 557 789 ahrens void 558 789 ahrens zap_entry_remove(zap_entry_handle_t *zeh) 559 789 ahrens { 560 789 ahrens uint16_t entry_chunk; 561 789 ahrens struct zap_leaf_entry *le; 562 1578 ahrens zap_leaf_t *l = zeh->zeh_leaf; 563 789 ahrens 564 789 ahrens ASSERT3P(zeh->zeh_chunkp, !=, &zeh->zeh_fakechunk); 565 789 ahrens 566 789 ahrens entry_chunk = *zeh->zeh_chunkp; 567 1491 ahrens le = ZAP_LEAF_ENTRY(l, entry_chunk); 568 1491 ahrens ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY); 569 789 ahrens 570 1578 ahrens zap_leaf_array_free(l, &le->le_name_chunk); 571 1578 ahrens zap_leaf_array_free(l, &le->le_value_chunk); 572 789 ahrens 573 789 ahrens *zeh->zeh_chunkp = le->le_next; 574 789 ahrens zap_leaf_chunk_free(l, entry_chunk); 575 789 ahrens 576 1578 ahrens l->l_phys->l_hdr.lh_nentries--; 577 789 ahrens } 578 789 ahrens 579 789 ahrens int 580 10922 Jeff zap_entry_create(zap_leaf_t *l, zap_name_t *zn, uint32_t cd, 581 789 ahrens uint8_t integer_size, uint64_t num_integers, const void *buf, 582 789 ahrens zap_entry_handle_t *zeh) 583 789 ahrens { 584 789 ahrens uint16_t chunk; 585 789 ahrens uint16_t *chunkp; 586 789 ahrens struct zap_leaf_entry *le; 587 10922 Jeff uint64_t valuelen; 588 789 ahrens int numchunks; 589 10922 Jeff uint64_t h = zn->zn_hash; 590 789 ahrens 591 789 ahrens valuelen = integer_size * num_integers; 592 789 ahrens 593 11165 Matthew numchunks = 1 + ZAP_LEAF_ARRAY_NCHUNKS(zn->zn_key_orig_numints * 594 11165 Matthew zn->zn_key_intlen) + ZAP_LEAF_ARRAY_NCHUNKS(valuelen); 595 1491 ahrens if (numchunks > ZAP_LEAF_NUMCHUNKS(l)) 596 789 ahrens return (E2BIG); 597 789 ahrens 598 10922 Jeff if (cd == ZAP_NEED_CD) { 599 5331 amw /* find the lowest unused cd */ 600 5331 amw if (l->l_phys->l_hdr.lh_flags & ZLF_ENTRIES_CDSORTED) { 601 5331 amw cd = 0; 602 5331 amw 603 1578 ahrens for (chunk = *LEAF_HASH_ENTPTR(l, h); 604 1578 ahrens chunk != CHAIN_END; chunk = le->le_next) { 605 1578 ahrens le = ZAP_LEAF_ENTRY(l, chunk); 606 5331 amw if (le->le_cd > cd) 607 1578 ahrens break; 608 5331 amw if (le->le_hash == h) { 609 5331 amw ASSERT3U(cd, ==, le->le_cd); 610 5331 amw cd++; 611 789 ahrens } 612 789 ahrens } 613 5331 amw } else { 614 5331 amw /* old unsorted format; do it the O(n^2) way */ 615 10922 Jeff for (cd = 0; ; cd++) { 616 5331 amw for (chunk = *LEAF_HASH_ENTPTR(l, h); 617 5331 amw chunk != CHAIN_END; chunk = le->le_next) { 618 5331 amw le = ZAP_LEAF_ENTRY(l, chunk); 619 5331 amw if (le->le_hash == h && 620 5331 amw le->le_cd == cd) { 621 5331 amw break; 622 5331 amw } 623 5331 amw } 624 5331 amw /* If this cd is not in use, we are good. */ 625 5331 amw if (chunk == CHAIN_END) 626 5331 amw break; 627 5331 amw } 628 789 ahrens } 629 5331 amw /* 630 10922 Jeff * We would run out of space in a block before we could 631 10922 Jeff * store enough entries to run out of CD values. 632 5331 amw */ 633 10922 Jeff ASSERT3U(cd, <, zap_maxcd(zn->zn_zap)); 634 789 ahrens } 635 789 ahrens 636 1578 ahrens if (l->l_phys->l_hdr.lh_nfree < numchunks) 637 789 ahrens return (EAGAIN); 638 789 ahrens 639 789 ahrens /* make the entry */ 640 789 ahrens chunk = zap_leaf_chunk_alloc(l); 641 1491 ahrens le = ZAP_LEAF_ENTRY(l, chunk); 642 1491 ahrens le->le_type = ZAP_CHUNK_ENTRY; 643 10922 Jeff le->le_name_chunk = zap_leaf_array_create(l, zn->zn_key_orig, 644 11165 Matthew zn->zn_key_intlen, zn->zn_key_orig_numints); 645 11165 Matthew le->le_name_numints = zn->zn_key_orig_numints; 646 789 ahrens le->le_value_chunk = 647 1578 ahrens zap_leaf_array_create(l, buf, integer_size, num_integers); 648 11165 Matthew le->le_value_numints = num_integers; 649 11165 Matthew le->le_value_intlen = integer_size; 650 789 ahrens le->le_hash = h; 651 789 ahrens le->le_cd = cd; 652 789 ahrens 653 789 ahrens /* link it into the hash chain */ 654 5331 amw /* XXX if we did the search above, we could just use that */ 655 5331 amw chunkp = zap_leaf_rehash_entry(l, chunk); 656 789 ahrens 657 1578 ahrens l->l_phys->l_hdr.lh_nentries++; 658 789 ahrens 659 1578 ahrens zeh->zeh_leaf = l; 660 789 ahrens zeh->zeh_num_integers = num_integers; 661 11165 Matthew zeh->zeh_integer_size = le->le_value_intlen; 662 789 ahrens zeh->zeh_cd = le->le_cd; 663 789 ahrens zeh->zeh_hash = le->le_hash; 664 789 ahrens zeh->zeh_chunkp = chunkp; 665 789 ahrens 666 789 ahrens return (0); 667 789 ahrens } 668 789 ahrens 669 789 ahrens /* 670 5331 amw * Determine if there is another entry with the same normalized form. 671 5331 amw * For performance purposes, either zn or name must be provided (the 672 5331 amw * other can be NULL). Note, there usually won't be any hash 673 5331 amw * conflicts, in which case we don't need the concatenated/normalized 674 5331 amw * form of the name. But all callers have one of these on hand anyway, 675 5331 amw * so might as well take advantage. A cleaner but slower interface 676 5331 amw * would accept neither argument, and compute the normalized name as 677 5331 amw * needed (using zap_name_alloc(zap_entry_read_name(zeh))). 678 5331 amw */ 679 5331 amw boolean_t 680 5331 amw zap_entry_normalization_conflict(zap_entry_handle_t *zeh, zap_name_t *zn, 681 5331 amw const char *name, zap_t *zap) 682 5331 amw { 683 5331 amw uint64_t chunk; 684 5331 amw struct zap_leaf_entry *le; 685 5331 amw boolean_t allocdzn = B_FALSE; 686 5331 amw 687 5331 amw if (zap->zap_normflags == 0) 688 5331 amw return (B_FALSE); 689 5331 amw 690 5331 amw for (chunk = *LEAF_HASH_ENTPTR(zeh->zeh_leaf, zeh->zeh_hash); 691 5331 amw chunk != CHAIN_END; chunk = le->le_next) { 692 5331 amw le = ZAP_LEAF_ENTRY(zeh->zeh_leaf, chunk); 693 5331 amw if (le->le_hash != zeh->zeh_hash) 694 5331 amw continue; 695 5331 amw if (le->le_cd == zeh->zeh_cd) 696 5331 amw continue; 697 5331 amw 698 5331 amw if (zn == NULL) { 699 5331 amw zn = zap_name_alloc(zap, name, MT_FIRST); 700 5331 amw allocdzn = B_TRUE; 701 5331 amw } 702 5331 amw if (zap_leaf_array_match(zeh->zeh_leaf, zn, 703 11165 Matthew le->le_name_chunk, le->le_name_numints)) { 704 5331 amw if (allocdzn) 705 5331 amw zap_name_free(zn); 706 5331 amw return (B_TRUE); 707 5331 amw } 708 5331 amw } 709 5331 amw if (allocdzn) 710 5331 amw zap_name_free(zn); 711 5331 amw return (B_FALSE); 712 5331 amw } 713 5331 amw 714 5331 amw /* 715 789 ahrens * Routines for transferring entries between leafs. 716 789 ahrens */ 717 789 ahrens 718 5331 amw static uint16_t * 719 789 ahrens zap_leaf_rehash_entry(zap_leaf_t *l, uint16_t entry) 720 789 ahrens { 721 1491 ahrens struct zap_leaf_entry *le = ZAP_LEAF_ENTRY(l, entry); 722 5331 amw struct zap_leaf_entry *le2; 723 5331 amw uint16_t *chunkp; 724 5331 amw 725 5331 amw /* 726 5331 amw * keep the entry chain sorted by cd 727 5331 amw * NB: this will not cause problems for unsorted leafs, though 728 5331 amw * it is unnecessary there. 729 5331 amw */ 730 5331 amw for (chunkp = LEAF_HASH_ENTPTR(l, le->le_hash); 731 5331 amw *chunkp != CHAIN_END; chunkp = &le2->le_next) { 732 5331 amw le2 = ZAP_LEAF_ENTRY(l, *chunkp); 733 5331 amw if (le2->le_cd > le->le_cd) 734 5331 amw break; 735 5331 amw } 736 5331 amw 737 5331 amw le->le_next = *chunkp; 738 5331 amw *chunkp = entry; 739 5331 amw return (chunkp); 740 789 ahrens } 741 789 ahrens 742 789 ahrens static uint16_t 743 789 ahrens zap_leaf_transfer_array(zap_leaf_t *l, uint16_t chunk, zap_leaf_t *nl) 744 789 ahrens { 745 789 ahrens uint16_t new_chunk; 746 789 ahrens uint16_t *nchunkp = &new_chunk; 747 789 ahrens 748 789 ahrens while (chunk != CHAIN_END) { 749 789 ahrens uint16_t nchunk = zap_leaf_chunk_alloc(nl); 750 789 ahrens struct zap_leaf_array *nla = 751 1491 ahrens &ZAP_LEAF_CHUNK(nl, nchunk).l_array; 752 789 ahrens struct zap_leaf_array *la = 753 1491 ahrens &ZAP_LEAF_CHUNK(l, chunk).l_array; 754 789 ahrens int nextchunk = la->la_next; 755 789 ahrens 756 1491 ahrens ASSERT3U(chunk, <, ZAP_LEAF_NUMCHUNKS(l)); 757 1491 ahrens ASSERT3U(nchunk, <, ZAP_LEAF_NUMCHUNKS(l)); 758 789 ahrens 759 1578 ahrens *nla = *la; /* structure assignment */ 760 789 ahrens 761 789 ahrens zap_leaf_chunk_free(l, chunk); 762 789 ahrens chunk = nextchunk; 763 789 ahrens *nchunkp = nchunk; 764 789 ahrens nchunkp = &nla->la_next; 765 789 ahrens } 766 789 ahrens *nchunkp = CHAIN_END; 767 789 ahrens return (new_chunk); 768 789 ahrens } 769 789 ahrens 770 789 ahrens static void 771 1578 ahrens zap_leaf_transfer_entry(zap_leaf_t *l, int entry, zap_leaf_t *nl) 772 789 ahrens { 773 789 ahrens struct zap_leaf_entry *le, *nle; 774 1578 ahrens uint16_t chunk; 775 789 ahrens 776 1491 ahrens le = ZAP_LEAF_ENTRY(l, entry); 777 1491 ahrens ASSERT3U(le->le_type, ==, ZAP_CHUNK_ENTRY); 778 789 ahrens 779 789 ahrens chunk = zap_leaf_chunk_alloc(nl); 780 1491 ahrens nle = ZAP_LEAF_ENTRY(nl, chunk); 781 1578 ahrens *nle = *le; /* structure assignment */ 782 789 ahrens 783 5331 amw (void) zap_leaf_rehash_entry(nl, chunk); 784 789 ahrens 785 789 ahrens nle->le_name_chunk = zap_leaf_transfer_array(l, le->le_name_chunk, nl); 786 789 ahrens nle->le_value_chunk = 787 789 ahrens zap_leaf_transfer_array(l, le->le_value_chunk, nl); 788 789 ahrens 789 789 ahrens zap_leaf_chunk_free(l, entry); 790 789 ahrens 791 1578 ahrens l->l_phys->l_hdr.lh_nentries--; 792 1578 ahrens nl->l_phys->l_hdr.lh_nentries++; 793 789 ahrens } 794 789 ahrens 795 789 ahrens /* 796 1578 ahrens * Transfer the entries whose hash prefix ends in 1 to the new leaf. 797 789 ahrens */ 798 1578 ahrens void 799 5498 timh zap_leaf_split(zap_leaf_t *l, zap_leaf_t *nl, boolean_t sort) 800 789 ahrens { 801 789 ahrens int i; 802 1578 ahrens int bit = 64 - 1 - l->l_phys->l_hdr.lh_prefix_len; 803 789 ahrens 804 1578 ahrens /* set new prefix and prefix_len */ 805 1578 ahrens l->l_phys->l_hdr.lh_prefix <<= 1; 806 1578 ahrens l->l_phys->l_hdr.lh_prefix_len++; 807 1578 ahrens nl->l_phys->l_hdr.lh_prefix = l->l_phys->l_hdr.lh_prefix | 1; 808 1578 ahrens nl->l_phys->l_hdr.lh_prefix_len = l->l_phys->l_hdr.lh_prefix_len; 809 1578 ahrens 810 789 ahrens /* break existing hash chains */ 811 1491 ahrens zap_memset(l->l_phys->l_hash, CHAIN_END, 2*ZAP_LEAF_HASH_NUMENTRIES(l)); 812 5331 amw 813 5498 timh if (sort) 814 5331 amw l->l_phys->l_hdr.lh_flags |= ZLF_ENTRIES_CDSORTED; 815 789 ahrens 816 1578 ahrens /* 817 1578 ahrens * Transfer entries whose hash bit 'bit' is set to nl; rehash 818 1578 ahrens * the remaining entries 819 1578 ahrens * 820 1578 ahrens * NB: We could find entries via the hashtable instead. That 821 1578 ahrens * would be O(hashents+numents) rather than O(numblks+numents), 822 1578 ahrens * but this accesses memory more sequentially, and when we're 823 1578 ahrens * called, the block is usually pretty full. 824 1578 ahrens */ 825 1491 ahrens for (i = 0; i < ZAP_LEAF_NUMCHUNKS(l); i++) { 826 1491 ahrens struct zap_leaf_entry *le = ZAP_LEAF_ENTRY(l, i); 827 1491 ahrens if (le->le_type != ZAP_CHUNK_ENTRY) 828 789 ahrens continue; 829 789 ahrens 830 1578 ahrens if (le->le_hash & (1ULL << bit)) 831 1578 ahrens zap_leaf_transfer_entry(l, i, nl); 832 1578 ahrens else 833 5331 amw (void) zap_leaf_rehash_entry(l, i); 834 789 ahrens } 835 789 ahrens } 836 789 ahrens 837 789 ahrens void 838 1578 ahrens zap_leaf_stats(zap_t *zap, zap_leaf_t *l, zap_stats_t *zs) 839 789 ahrens { 840 1578 ahrens int i, n; 841 789 ahrens 842 1578 ahrens n = zap->zap_f.zap_phys->zap_ptrtbl.zt_shift - 843 1578 ahrens l->l_phys->l_hdr.lh_prefix_len; 844 789 ahrens n = MIN(n, ZAP_HISTOGRAM_SIZE-1); 845 789 ahrens zs->zs_leafs_with_2n_pointers[n]++; 846 789 ahrens 847 789 ahrens 848 1578 ahrens n = l->l_phys->l_hdr.lh_nentries/5; 849 1578 ahrens n = MIN(n, ZAP_HISTOGRAM_SIZE-1); 850 1578 ahrens zs->zs_blocks_with_n5_entries[n]++; 851 789 ahrens 852 1578 ahrens n = ((1<<FZAP_BLOCK_SHIFT(zap)) - 853 1578 ahrens l->l_phys->l_hdr.lh_nfree * (ZAP_LEAF_ARRAY_BYTES+1))*10 / 854 1578 ahrens (1<<FZAP_BLOCK_SHIFT(zap)); 855 1578 ahrens n = MIN(n, ZAP_HISTOGRAM_SIZE-1); 856 1578 ahrens zs->zs_blocks_n_tenths_full[n]++; 857 789 ahrens 858 1578 ahrens for (i = 0; i < ZAP_LEAF_HASH_NUMENTRIES(l); i++) { 859 1578 ahrens int nentries = 0; 860 1578 ahrens int chunk = l->l_phys->l_hash[i]; 861 789 ahrens 862 1578 ahrens while (chunk != CHAIN_END) { 863 1578 ahrens struct zap_leaf_entry *le = 864 1578 ahrens ZAP_LEAF_ENTRY(l, chunk); 865 789 ahrens 866 11165 Matthew n = 1 + ZAP_LEAF_ARRAY_NCHUNKS(le->le_name_numints) + 867 11165 Matthew ZAP_LEAF_ARRAY_NCHUNKS(le->le_value_numints * 868 11165 Matthew le->le_value_intlen); 869 1578 ahrens n = MIN(n, ZAP_HISTOGRAM_SIZE-1); 870 1578 ahrens zs->zs_entries_using_n_chunks[n]++; 871 789 ahrens 872 1578 ahrens chunk = le->le_next; 873 1578 ahrens nentries++; 874 789 ahrens } 875 789 ahrens 876 1578 ahrens n = nentries; 877 1578 ahrens n = MIN(n, ZAP_HISTOGRAM_SIZE-1); 878 1578 ahrens zs->zs_buckets_with_n_entries[n]++; 879 1578 ahrens } 880 789 ahrens } 881