1 0 stevel /* 2 0 stevel * CDDL HEADER START 3 0 stevel * 4 0 stevel * The contents of this file are subject to the terms of the 5 1253 lq150181 * Common Development and Distribution License (the "License"). 6 1455 andrei * You may not use this file except in compliance with the License. 7 0 stevel * 8 0 stevel * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 0 stevel * or http://www.opensolaris.org/os/licensing. 10 0 stevel * See the License for the specific language governing permissions 11 0 stevel * and limitations under the License. 12 0 stevel * 13 0 stevel * When distributing Covered Code, include this CDDL HEADER in each 14 0 stevel * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 0 stevel * If applicable, add the following below this CDDL HEADER, with the 16 0 stevel * fields enclosed by brackets "[]" replaced with your own identifying 17 0 stevel * information: Portions Copyright [yyyy] [name of copyright owner] 18 0 stevel * 19 0 stevel * CDDL HEADER END 20 0 stevel */ 21 1253 lq150181 22 0 stevel /* 23 8803 Jonathan * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 0 stevel * Use is subject to license terms. 25 0 stevel */ 26 0 stevel 27 0 stevel #include <sys/machsystm.h> 28 0 stevel #include <sys/archsystm.h> 29 0 stevel #include <sys/vm.h> 30 0 stevel #include <sys/cpu.h> 31 0 stevel #include <sys/atomic.h> 32 0 stevel #include <sys/reboot.h> 33 0 stevel #include <sys/kdi.h> 34 0 stevel #include <sys/bootconf.h> 35 0 stevel #include <sys/memlist_plat.h> 36 0 stevel #include <sys/memlist_impl.h> 37 0 stevel #include <sys/prom_plat.h> 38 0 stevel #include <sys/prom_isa.h> 39 0 stevel #include <sys/autoconf.h> 40 0 stevel #include <sys/intreg.h> 41 0 stevel #include <sys/ivintr.h> 42 0 stevel #include <sys/fpu/fpusystm.h> 43 0 stevel #include <sys/iommutsb.h> 44 0 stevel #include <vm/vm_dep.h> 45 0 stevel #include <vm/seg_dev.h> 46 0 stevel #include <vm/seg_kmem.h> 47 0 stevel #include <vm/seg_kpm.h> 48 0 stevel #include <vm/seg_map.h> 49 0 stevel #include <vm/seg_kp.h> 50 0 stevel #include <sys/sysconf.h> 51 0 stevel #include <vm/hat_sfmmu.h> 52 0 stevel #include <sys/kobj.h> 53 0 stevel #include <sys/sun4asi.h> 54 0 stevel #include <sys/clconf.h> 55 0 stevel #include <sys/platform_module.h> 56 0 stevel #include <sys/panic.h> 57 0 stevel #include <sys/cpu_sgnblk_defs.h> 58 0 stevel #include <sys/clock.h> 59 0 stevel #include <sys/cmn_err.h> 60 0 stevel #include <sys/promif.h> 61 0 stevel #include <sys/prom_debug.h> 62 0 stevel #include <sys/traptrace.h> 63 0 stevel #include <sys/memnode.h> 64 0 stevel #include <sys/mem_cage.h> 65 2241 huah #include <sys/mmu.h> 66 0 stevel 67 0 stevel extern void setup_trap_table(void); 68 4050 jb145095 extern int cpu_intrq_setup(struct cpu *); 69 0 stevel extern void cpu_intrq_register(struct cpu *); 70 0 stevel extern void contig_mem_init(void); 71 4204 ha137994 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 72 0 stevel extern void mach_dump_buffer_init(void); 73 0 stevel extern void mach_descrip_init(void); 74 1991 heppo extern void mach_descrip_startup_fini(void); 75 0 stevel extern void mach_memscrub(void); 76 0 stevel extern void mach_fpras(void); 77 0 stevel extern void mach_cpu_halt_idle(void); 78 0 stevel extern void mach_hw_copy_limit(void); 79 1991 heppo extern void load_mach_drivers(void); 80 0 stevel extern void load_tod_module(void); 81 0 stevel #pragma weak load_tod_module 82 0 stevel 83 0 stevel extern int ndata_alloc_mmfsa(struct memlist *ndata); 84 0 stevel #pragma weak ndata_alloc_mmfsa 85 1991 heppo 86 1991 heppo extern void cif_init(void); 87 1991 heppo #pragma weak cif_init 88 0 stevel 89 0 stevel extern void parse_idprom(void); 90 0 stevel extern void add_vx_handler(char *, int, void (*)(cell_t *)); 91 0 stevel extern void mem_config_init(void); 92 0 stevel extern void memseg_remap_init(void); 93 0 stevel 94 2296 ae112802 extern void mach_kpm_init(void); 95 6880 dv142724 extern void pcf_init(); 96 4878 blakej extern int size_pse_array(pgcnt_t, int); 97 8906 Eric extern void pg_init(); 98 2296 ae112802 99 0 stevel /* 100 0 stevel * External Data: 101 0 stevel */ 102 0 stevel extern int vac_size; /* cache size in bytes */ 103 0 stevel extern uint_t vac_mask; /* VAC alignment consistency mask */ 104 0 stevel extern uint_t vac_colors; 105 0 stevel 106 0 stevel /* 107 0 stevel * Global Data Definitions: 108 0 stevel */ 109 0 stevel 110 0 stevel /* 111 0 stevel * XXX - Don't port this to new architectures 112 0 stevel * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 113 0 stevel * 'romp' has no use with a prom with an IEEE 1275 client interface. 114 0 stevel * The driver doesn't use the value, but it depends on the symbol. 115 0 stevel */ 116 0 stevel void *romp; /* veritas driver won't load without romp 4154976 */ 117 0 stevel /* 118 0 stevel * Declare these as initialized data so we can patch them. 119 0 stevel */ 120 0 stevel pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 121 0 stevel pgcnt_t segkpsize = 122 0 stevel btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 123 9281 Prakash uint_t segmap_percent = 6; /* Size of segmap segment */ 124 0 stevel 125 0 stevel int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 126 0 stevel int vac_copyback = 1; 127 0 stevel char *cache_mode = NULL; 128 0 stevel int use_mix = 1; 129 0 stevel int prom_debug = 0; 130 0 stevel 131 0 stevel caddr_t boot_tba; /* %tba at boot - used by kmdb */ 132 0 stevel uint_t tba_taken_over = 0; 133 0 stevel 134 0 stevel caddr_t s_text; /* start of kernel text segment */ 135 0 stevel caddr_t e_text; /* end of kernel text segment */ 136 0 stevel caddr_t s_data; /* start of kernel data segment */ 137 0 stevel caddr_t e_data; /* end of kernel data segment */ 138 0 stevel 139 0 stevel caddr_t modtext; /* beginning of module text */ 140 0 stevel size_t modtext_sz; /* size of module text */ 141 0 stevel caddr_t moddata; /* beginning of module data reserve */ 142 0 stevel caddr_t e_moddata; /* end of module data reserve */ 143 0 stevel 144 0 stevel /* 145 0 stevel * End of first block of contiguous kernel in 32-bit virtual address space 146 0 stevel */ 147 0 stevel caddr_t econtig32; /* end of first blk of contiguous kernel */ 148 0 stevel 149 0 stevel caddr_t ncbase; /* beginning of non-cached segment */ 150 0 stevel caddr_t ncend; /* end of non-cached segment */ 151 0 stevel 152 0 stevel size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 153 0 stevel caddr_t nalloc_base; /* beginning of nucleus allocation */ 154 0 stevel caddr_t nalloc_end; /* end of nucleus allocatable memory */ 155 0 stevel caddr_t valloc_base; /* beginning of kvalloc segment */ 156 0 stevel 157 0 stevel caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 158 0 stevel caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 159 5648 setje size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 160 3764 dp78419 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 161 3764 dp78419 int kmem64_szc; /* page size code */ 162 3764 dp78419 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 163 0 stevel 164 2296 ae112802 uintptr_t shm_alignment; /* VAC address consistency modulus */ 165 0 stevel struct memlist *phys_install; /* Total installed physical memory */ 166 0 stevel struct memlist *phys_avail; /* Available (unreserved) physical memory */ 167 0 stevel struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 168 5648 setje struct memlist *nopp_list; /* pages with no backing page structs */ 169 0 stevel struct memlist ndata; /* memlist of nucleus allocatable memory */ 170 0 stevel int memexp_flag; /* memory expansion card flag */ 171 0 stevel uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 172 0 stevel pgcnt_t obp_pages; /* Physical pages used by OBP */ 173 0 stevel 174 0 stevel /* 175 0 stevel * VM data structures 176 0 stevel */ 177 0 stevel long page_hashsz; /* Size of page hash table (power of two) */ 178 0 stevel struct page *pp_base; /* Base of system page struct array */ 179 0 stevel size_t pp_sz; /* Size in bytes of page struct array */ 180 0 stevel struct page **page_hash; /* Page hash table */ 181 4878 blakej pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 182 4878 blakej size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 183 4878 blakej int pse_shift; /* log2(pse_table_size) */ 184 0 stevel struct seg ktextseg; /* Segment used for kernel executable image */ 185 0 stevel struct seg kvalloc; /* Segment used for "valloc" mapping */ 186 0 stevel struct seg kpseg; /* Segment used for pageable kernel virt mem */ 187 0 stevel struct seg ktexthole; /* Segment used for nucleus text hole */ 188 0 stevel struct seg kmapseg; /* Segment used for generic kernel mappings */ 189 0 stevel struct seg kpmseg; /* Segment used for physical mapping */ 190 0 stevel struct seg kdebugseg; /* Segment used for the kernel debugger */ 191 0 stevel 192 5648 setje void *kpm_pp_base; /* Base of system kpm_page array */ 193 0 stevel size_t kpm_pp_sz; /* Size of system kpm_page array */ 194 0 stevel pgcnt_t kpm_npages; /* How many kpm pages are managed */ 195 0 stevel 196 0 stevel struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 197 0 stevel struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 198 0 stevel struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 199 0 stevel 200 3290 johansen int segzio_fromheap = 0; /* zio allocations occur from heap */ 201 3290 johansen caddr_t segzio_base; /* Base address of segzio */ 202 3290 johansen pgcnt_t segziosize = 0; /* size of zio segment in pages */ 203 10106 Jason 204 10106 Jason /* 205 10106 Jason * A static DR page_t VA map is reserved that can map the page structures 206 10106 Jason * for a domain's entire RA space. The pages that backs this space are 207 10106 Jason * dynamically allocated and need not be physically contiguous. The DR 208 10106 Jason * map size is derived from KPM size. 209 10106 Jason */ 210 10106 Jason int ppvm_enable = 0; /* Static virtual map for page structs */ 211 10106 Jason page_t *ppvm_base; /* Base of page struct map */ 212 10106 Jason pgcnt_t ppvm_size = 0; /* Size of page struct map */ 213 3290 johansen 214 0 stevel /* 215 0 stevel * debugger pages (if allocated) 216 0 stevel */ 217 0 stevel struct vnode kdebugvp; 218 3446 mrj 219 3446 mrj /* 220 3446 mrj * VA range available to the debugger 221 3446 mrj */ 222 3446 mrj const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 223 3446 mrj const size_t kdi_segdebugsize = SEGDEBUGSIZE; 224 0 stevel 225 0 stevel /* 226 0 stevel * Segment for relocated kernel structures in 64-bit large RAM kernels 227 0 stevel */ 228 0 stevel struct seg kmem64; 229 0 stevel 230 5648 setje struct memseg *memseg_free; 231 5648 setje 232 0 stevel struct vnode unused_pages_vp; 233 0 stevel 234 0 stevel /* 235 0 stevel * VM data structures allocated early during boot. 236 0 stevel */ 237 0 stevel size_t pagehash_sz; 238 0 stevel uint64_t memlist_sz; 239 0 stevel 240 0 stevel char tbr_wr_addr_inited = 0; 241 0 stevel 242 5358 jc25722 caddr_t mpo_heap32_buf = NULL; 243 5358 jc25722 size_t mpo_heap32_bufsz = 0; 244 0 stevel 245 0 stevel /* 246 0 stevel * Static Routines: 247 0 stevel */ 248 5648 setje static int ndata_alloc_memseg(struct memlist *, size_t); 249 5648 setje static void memlist_new(uint64_t, uint64_t, struct memlist **); 250 5648 setje static void memlist_add(uint64_t, uint64_t, 251 5648 setje struct memlist **, struct memlist **); 252 5648 setje static void kphysm_init(void); 253 0 stevel static void kvm_init(void); 254 5648 setje static void install_kmem64_tte(void); 255 0 stevel 256 0 stevel static void startup_init(void); 257 0 stevel static void startup_memlist(void); 258 0 stevel static void startup_modules(void); 259 0 stevel static void startup_bop_gone(void); 260 0 stevel static void startup_vm(void); 261 0 stevel static void startup_end(void); 262 0 stevel static void setup_cage_params(void); 263 1253 lq150181 static void startup_create_io_node(void); 264 0 stevel 265 0 stevel static pgcnt_t npages; 266 0 stevel static struct memlist *memlist; 267 0 stevel void *memlist_end; 268 0 stevel 269 0 stevel static pgcnt_t bop_alloc_pages; 270 0 stevel static caddr_t hblk_base; 271 0 stevel uint_t hblk_alloc_dynamic = 0; 272 0 stevel uint_t hblk1_min = H1MIN; 273 0 stevel 274 0 stevel 275 0 stevel /* 276 0 stevel * Hooks for unsupported platforms and down-rev firmware 277 0 stevel */ 278 0 stevel int iam_positron(void); 279 0 stevel #pragma weak iam_positron 280 0 stevel static void do_prom_version_check(void); 281 0 stevel 282 0 stevel /* 283 0 stevel * After receiving a thermal interrupt, this is the number of seconds 284 0 stevel * to delay before shutting off the system, assuming 285 0 stevel * shutdown fails. Use /etc/system to change the delay if this isn't 286 0 stevel * large enough. 287 0 stevel */ 288 0 stevel int thermal_powerdown_delay = 1200; 289 0 stevel 290 0 stevel /* 291 0 stevel * Used to hold off page relocations into the cage until OBP has completed 292 0 stevel * its boot-time handoff of its resources to the kernel. 293 0 stevel */ 294 0 stevel int page_relocate_ready = 0; 295 0 stevel 296 0 stevel /* 297 7218 svemuri * Indicate if kmem64 allocation was done in small chunks 298 7218 svemuri */ 299 7218 svemuri int kmem64_smchunks = 0; 300 7218 svemuri 301 7218 svemuri /* 302 0 stevel * Enable some debugging messages concerning memory usage... 303 0 stevel */ 304 0 stevel #ifdef DEBUGGING_MEM 305 0 stevel static int debugging_mem; 306 0 stevel static void 307 0 stevel printmemlist(char *title, struct memlist *list) 308 0 stevel { 309 0 stevel if (!debugging_mem) 310 0 stevel return; 311 0 stevel 312 0 stevel printf("%s\n", title); 313 0 stevel 314 0 stevel while (list) { 315 0 stevel prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 316 0 stevel (uint32_t)(list->address >> 32), (uint32_t)list->address, 317 0 stevel (uint32_t)(list->size >> 32), (uint32_t)(list->size)); 318 0 stevel list = list->next; 319 0 stevel } 320 0 stevel } 321 0 stevel 322 0 stevel void 323 0 stevel printmemseg(struct memseg *memseg) 324 0 stevel { 325 0 stevel if (!debugging_mem) 326 0 stevel return; 327 0 stevel 328 0 stevel printf("memseg\n"); 329 0 stevel 330 0 stevel while (memseg) { 331 0 stevel prom_printf("\tpage = 0x%p, epage = 0x%p, " 332 0 stevel "pfn = 0x%x, epfn = 0x%x\n", 333 0 stevel memseg->pages, memseg->epages, 334 0 stevel memseg->pages_base, memseg->pages_end); 335 0 stevel memseg = memseg->next; 336 0 stevel } 337 0 stevel } 338 0 stevel 339 0 stevel #define debug_pause(str) halt((str)) 340 0 stevel #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 341 0 stevel #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 342 0 stevel #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 343 0 stevel #define MPRINTF3(str, a, b, c) \ 344 0 stevel if (debugging_mem) prom_printf((str), (a), (b), (c)) 345 0 stevel #else /* DEBUGGING_MEM */ 346 0 stevel #define MPRINTF(str) 347 0 stevel #define MPRINTF1(str, a) 348 0 stevel #define MPRINTF2(str, a, b) 349 0 stevel #define MPRINTF3(str, a, b, c) 350 0 stevel #endif /* DEBUGGING_MEM */ 351 0 stevel 352 0 stevel 353 0 stevel /* 354 0 stevel * 355 0 stevel * Kernel's Virtual Memory Layout. 356 0 stevel * /-----------------------\ 357 0 stevel * 0xFFFFFFFF.FFFFFFFF -| |- 358 0 stevel * | OBP's virtual page | 359 0 stevel * | tables | 360 0 stevel * 0xFFFFFFFC.00000000 -|-----------------------|- 361 0 stevel * : : 362 0 stevel * : : 363 3290 johansen * -|-----------------------|- 364 3290 johansen * | segzio | (base and size vary) 365 0 stevel * 0xFFFFFE00.00000000 -|-----------------------|- 366 0 stevel * | | Ultrasparc I/II support 367 0 stevel * | segkpm segment | up to 2TB of physical 368 0 stevel * | (64-bit kernel ONLY) | memory, VAC has 2 colors 369 0 stevel * | | 370 0 stevel * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 371 0 stevel * : : 372 0 stevel * : : 373 0 stevel * 0xFFFFF810.00000000 -|-----------------------|- hole_end 374 0 stevel * | | ^ 375 0 stevel * | UltraSPARC I/II call | | 376 0 stevel * | bug requires an extra | | 377 0 stevel * | 4 GB of space between | | 378 0 stevel * | hole and used RAM | | 379 0 stevel * | | | 380 0 stevel * 0xFFFFF800.00000000 -|-----------------------|- | 381 0 stevel * | | | 382 0 stevel * | Virtual Address Hole | UltraSPARC 383 0 stevel * | on UltraSPARC I/II | I/II * ONLY * 384 0 stevel * | | | 385 0 stevel * 0x00000800.00000000 -|-----------------------|- | 386 0 stevel * | | | 387 0 stevel * | UltraSPARC I/II call | | 388 0 stevel * | bug requires an extra | | 389 0 stevel * | 4 GB of space between | | 390 0 stevel * | hole and used RAM | | 391 0 stevel * | | v 392 0 stevel * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 393 0 stevel * : : ^ 394 0 stevel * : : | 395 5648 setje * |-----------------------| | 396 5648 setje * | | | 397 5648 setje * | ecache flush area | | 398 5648 setje * | (twice largest e$) | | 399 5648 setje * | | | 400 3764 dp78419 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 401 3764 dp78419 * | overmapped area | alignend_end | 402 3764 dp78419 * | (kmem64_alignsize | | 403 3764 dp78419 * | boundary) | | 404 0 stevel * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 405 0 stevel * | | | 406 0 stevel * | 64-bit kernel ONLY | | 407 0 stevel * | | | 408 0 stevel * | kmem64 segment | | 409 0 stevel * | | | 410 0 stevel * | (Relocated extra HME | Approximately 411 0 stevel * | block allocations, | 1 TB of virtual 412 0 stevel * | memnode freelists, | address space 413 0 stevel * | HME hash buckets, | | 414 0 stevel * | mml_table, kpmp_table,| | 415 0 stevel * | page_t array and | | 416 0 stevel * | hashblock pool to | | 417 0 stevel * | avoid hard-coded | | 418 0 stevel * | 32-bit vaddr | | 419 0 stevel * | limitations) | | 420 0 stevel * | | v 421 0 stevel * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 422 0 stevel * | | 423 0 stevel * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 424 0 stevel * | | 425 0 stevel * 0x00000300.00000000 -|-----------------------|- SYSBASE 426 0 stevel * : : 427 0 stevel * : : 428 0 stevel * -|-----------------------|- 429 0 stevel * | | 430 0 stevel * | segmap segment | SEGMAPSIZE (1/8th physmem, 431 0 stevel * | | 256G MAX) 432 0 stevel * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 433 0 stevel * : : 434 0 stevel * : : 435 0 stevel * -|-----------------------|- 436 0 stevel * | | 437 0 stevel * | segkp | SEGKPSIZE (2GB) 438 0 stevel * | | 439 0 stevel * | | 440 0 stevel * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 441 0 stevel * | | 442 0 stevel * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 443 0 stevel * | | (SEGKPBASE - 0x400000) 444 0 stevel * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 445 0 stevel * | | (MEMSCRUBBASE - NCARGS) 446 0 stevel * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 447 0 stevel * | | (ARGSBASE - PPMAPSIZE) 448 0 stevel * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 449 0 stevel * | | 450 0 stevel * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 451 0 stevel * | | 452 0 stevel * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 453 0 stevel * : : 454 0 stevel * : : 455 0 stevel * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 456 0 stevel * | | 457 0 stevel * | OBP | 458 0 stevel * | | 459 0 stevel * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 460 0 stevel * | kmdb | 461 0 stevel * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 462 0 stevel * : : 463 0 stevel * : : 464 0 stevel * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 465 0 stevel * | | 466 0 stevel * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 467 0 stevel * | | ~64MB) 468 5648 setje * 0x00000000.70002000 -|-----------------------| 469 0 stevel * | panicbuf | 470 5648 setje * 0x00000000.70000000 -|-----------------------|- SYSBASE32 471 5648 setje * | boot-time | 472 5648 setje * | temporary space | 473 5648 setje * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 474 0 stevel * : : 475 0 stevel * : : 476 0 stevel * | | 477 0 stevel * |-----------------------|- econtig32 478 0 stevel * | vm structures | 479 0 stevel * 0x00000000.01C00000 |-----------------------|- nalloc_end 480 0 stevel * | TSBs | 481 0 stevel * |-----------------------|- end/nalloc_base 482 0 stevel * | kernel data & bss | 483 0 stevel * 0x00000000.01800000 -|-----------------------| 484 0 stevel * : nucleus text hole : 485 0 stevel * 0x00000000.01400000 -|-----------------------| 486 0 stevel * : : 487 0 stevel * |-----------------------| 488 0 stevel * | module text | 489 0 stevel * |-----------------------|- e_text/modtext 490 0 stevel * | kernel text | 491 0 stevel * |-----------------------| 492 0 stevel * | trap table (48k) | 493 0 stevel * 0x00000000.01000000 -|-----------------------|- KERNELBASE 494 0 stevel * | reserved for trapstat |} TSTAT_TOTAL_SIZE 495 0 stevel * |-----------------------| 496 0 stevel * | | 497 0 stevel * | invalid | 498 0 stevel * | | 499 0 stevel * 0x00000000.00000000 _|_______________________| 500 0 stevel * 501 0 stevel * 502 0 stevel * 503 0 stevel * 32-bit User Virtual Memory Layout. 504 0 stevel * /-----------------------\ 505 0 stevel * | | 506 0 stevel * | invalid | 507 0 stevel * | | 508 0 stevel * 0xFFC00000 -|-----------------------|- USERLIMIT 509 0 stevel * | user stack | 510 0 stevel * : : 511 0 stevel * : : 512 0 stevel * : : 513 0 stevel * | user data | 514 0 stevel * -|-----------------------|- 515 0 stevel * | user text | 516 0 stevel * 0x00002000 -|-----------------------|- 517 0 stevel * | invalid | 518 0 stevel * 0x00000000 _|_______________________| 519 0 stevel * 520 0 stevel * 521 0 stevel * 522 0 stevel * 64-bit User Virtual Memory Layout. 523 0 stevel * /-----------------------\ 524 0 stevel * | | 525 0 stevel * | invalid | 526 0 stevel * | | 527 0 stevel * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 528 0 stevel * | user stack | 529 0 stevel * : : 530 0 stevel * : : 531 0 stevel * : : 532 0 stevel * | user data | 533 0 stevel * -|-----------------------|- 534 0 stevel * | user text | 535 5648 setje * 0x00000000.01000000 -|-----------------------|- 536 0 stevel * | invalid | 537 0 stevel * 0x00000000.00000000 _|_______________________| 538 0 stevel */ 539 0 stevel 540 0 stevel extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 541 0 stevel extern uint64_t ecache_flush_address(void); 542 0 stevel 543 0 stevel #pragma weak load_platform_modules 544 1772 jl139090 #pragma weak plat_startup_memlist 545 0 stevel #pragma weak ecache_init_scrub_flush_area 546 0 stevel #pragma weak ecache_flush_address 547 0 stevel 548 0 stevel 549 0 stevel /* 550 0 stevel * By default the DR Cage is enabled for maximum OS 551 0 stevel * MPSS performance. Users needing to disable the cage mechanism 552 0 stevel * can set this variable to zero via /etc/system. 553 0 stevel * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 554 0 stevel * will result in loss of DR functionality. 555 0 stevel * Platforms wishing to disable kernel Cage by default 556 0 stevel * should do so in their set_platform_defaults() routine. 557 0 stevel */ 558 0 stevel int kernel_cage_enable = 1; 559 0 stevel 560 0 stevel static void 561 0 stevel setup_cage_params(void) 562 0 stevel { 563 0 stevel void (*func)(void); 564 0 stevel 565 0 stevel func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 566 0 stevel if (func != NULL) { 567 0 stevel (*func)(); 568 0 stevel return; 569 0 stevel } 570 0 stevel 571 0 stevel if (kernel_cage_enable == 0) { 572 0 stevel return; 573 0 stevel } 574 4266 dp78419 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 575 0 stevel 576 0 stevel if (kcage_on) { 577 0 stevel cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 578 0 stevel } else { 579 0 stevel cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 580 0 stevel } 581 0 stevel 582 0 stevel } 583 0 stevel 584 0 stevel /* 585 0 stevel * Machine-dependent startup code 586 0 stevel */ 587 0 stevel void 588 0 stevel startup(void) 589 0 stevel { 590 0 stevel startup_init(); 591 0 stevel if (&startup_platform) 592 0 stevel startup_platform(); 593 0 stevel startup_memlist(); 594 0 stevel startup_modules(); 595 0 stevel setup_cage_params(); 596 0 stevel startup_bop_gone(); 597 0 stevel startup_vm(); 598 0 stevel startup_end(); 599 0 stevel } 600 0 stevel 601 0 stevel struct regs sync_reg_buf; 602 0 stevel uint64_t sync_tt; 603 0 stevel 604 0 stevel void 605 0 stevel sync_handler(void) 606 0 stevel { 607 5084 johnlev struct panic_trap_info ti; 608 0 stevel int i; 609 0 stevel 610 0 stevel /* 611 0 stevel * Prevent trying to talk to the other CPUs since they are 612 0 stevel * sitting in the prom and won't reply. 613 0 stevel */ 614 0 stevel for (i = 0; i < NCPU; i++) { 615 0 stevel if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 616 0 stevel cpu[i]->cpu_flags &= ~CPU_READY; 617 0 stevel cpu[i]->cpu_flags |= CPU_QUIESCED; 618 0 stevel CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 619 0 stevel } 620 0 stevel } 621 0 stevel 622 0 stevel /* 623 0 stevel * We've managed to get here without going through the 624 0 stevel * normal panic code path. Try and save some useful 625 0 stevel * information. 626 0 stevel */ 627 0 stevel if (!panicstr && (curthread->t_panic_trap == NULL)) { 628 0 stevel ti.trap_type = sync_tt; 629 0 stevel ti.trap_regs = &sync_reg_buf; 630 0 stevel ti.trap_addr = NULL; 631 0 stevel ti.trap_mmu_fsr = 0x0; 632 0 stevel 633 0 stevel curthread->t_panic_trap = &ti; 634 0 stevel } 635 0 stevel 636 0 stevel /* 637 0 stevel * If we're re-entering the panic path, update the signature 638 0 stevel * block so that the SC knows we're in the second part of panic. 639 0 stevel */ 640 0 stevel if (panicstr) 641 0 stevel CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 642 0 stevel 643 0 stevel nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 644 0 stevel panic("sync initiated"); 645 0 stevel } 646 0 stevel 647 0 stevel 648 0 stevel static void 649 0 stevel startup_init(void) 650 0 stevel { 651 0 stevel /* 652 0 stevel * We want to save the registers while we're still in OBP 653 0 stevel * so that we know they haven't been fiddled with since. 654 0 stevel * (In principle, OBP can't change them just because it 655 0 stevel * makes a callback, but we'd rather not depend on that 656 0 stevel * behavior.) 657 0 stevel */ 658 0 stevel char sync_str[] = 659 4528 paulsan "warning @ warning off : sync " 660 4528 paulsan "%%tl-c %%tstate h# %p x! " 661 4528 paulsan "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 662 4528 paulsan "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 663 4528 paulsan "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 664 4528 paulsan "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 665 4528 paulsan "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 666 4528 paulsan "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 667 4528 paulsan "%%y h# %p l! %%tl-c %%tt h# %p x! " 668 4528 paulsan "sync ; warning !"; 669 0 stevel 670 0 stevel /* 671 0 stevel * 20 == num of %p substrings 672 0 stevel * 16 == max num of chars %p will expand to. 673 0 stevel */ 674 0 stevel char bp[sizeof (sync_str) + 16 * 20]; 675 0 stevel 676 0 stevel /* 677 0 stevel * Initialize ptl1 stack for the 1st CPU. 678 0 stevel */ 679 0 stevel ptl1_init_cpu(&cpu0); 680 0 stevel 681 0 stevel /* 682 0 stevel * Initialize the address map for cache consistent mappings 683 0 stevel * to random pages; must be done after vac_size is set. 684 0 stevel */ 685 0 stevel ppmapinit(); 686 0 stevel 687 0 stevel /* 688 0 stevel * Initialize the PROM callback handler. 689 0 stevel */ 690 0 stevel init_vx_handler(); 691 0 stevel 692 0 stevel /* 693 0 stevel * have prom call sync_callback() to handle the sync and 694 0 stevel * save some useful information which will be stored in the 695 0 stevel * core file later. 696 0 stevel */ 697 0 stevel (void) sprintf((char *)bp, sync_str, 698 4528 paulsan (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 699 4528 paulsan (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 700 4528 paulsan (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 701 4528 paulsan (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 702 4528 paulsan (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 703 4528 paulsan (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 704 4528 paulsan (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 705 4528 paulsan (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 706 4528 paulsan (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 707 4528 paulsan (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 708 0 stevel prom_interpret(bp, 0, 0, 0, 0, 0); 709 0 stevel add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 710 0 stevel } 711 0 stevel 712 5648 setje 713 5648 setje size_t 714 5648 setje calc_pp_sz(pgcnt_t npages) 715 5648 setje { 716 5648 setje 717 5648 setje return (npages * sizeof (struct page)); 718 5648 setje } 719 5648 setje 720 5648 setje size_t 721 5648 setje calc_kpmpp_sz(pgcnt_t npages) 722 5648 setje { 723 5648 setje 724 5648 setje kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 725 5648 setje kpm_pgsz = 1ull << kpm_pgshft; 726 5648 setje kpm_pgoff = kpm_pgsz - 1; 727 5648 setje kpmp2pshft = kpm_pgshft - PAGESHIFT; 728 5648 setje kpmpnpgs = 1 << kpmp2pshft; 729 5648 setje 730 5648 setje if (kpm_smallpages == 0) { 731 5648 setje /* 732 5648 setje * Avoid fragmentation problems in kphysm_init() 733 5648 setje * by allocating for all of physical memory 734 5648 setje */ 735 5648 setje kpm_npages = ptokpmpr(physinstalled); 736 5648 setje return (kpm_npages * sizeof (kpm_page_t)); 737 5648 setje } else { 738 5648 setje kpm_npages = npages; 739 5648 setje return (kpm_npages * sizeof (kpm_spage_t)); 740 5648 setje } 741 5648 setje } 742 5648 setje 743 5648 setje size_t 744 5648 setje calc_pagehash_sz(pgcnt_t npages) 745 5648 setje { 746 5648 setje 747 5648 setje /* 748 5648 setje * The page structure hash table size is a power of 2 749 5648 setje * such that the average hash chain length is PAGE_HASHAVELEN. 750 5648 setje */ 751 5648 setje page_hashsz = npages / PAGE_HASHAVELEN; 752 5648 setje page_hashsz = 1 << highbit(page_hashsz); 753 5648 setje return (page_hashsz * sizeof (struct page *)); 754 5648 setje } 755 5648 setje 756 7218 svemuri int testkmem64_smchunks = 0; 757 7218 svemuri 758 7218 svemuri int 759 5648 setje alloc_kmem64(caddr_t base, caddr_t end) 760 5648 setje { 761 5648 setje int i; 762 5648 setje caddr_t aligned_end = NULL; 763 7218 svemuri 764 7218 svemuri if (testkmem64_smchunks) 765 7218 svemuri return (1); 766 5648 setje 767 5648 setje /* 768 5648 setje * Make one large memory alloc after figuring out the 64-bit size. This 769 5648 setje * will enable use of the largest page size appropriate for the system 770 5648 setje * architecture. 771 5648 setje */ 772 5648 setje ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 773 5648 setje ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 774 5648 setje for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 775 5648 setje size_t alloc_size, alignsize; 776 5648 setje #if !defined(C_OBP) 777 5648 setje unsigned long long pa; 778 5648 setje #endif /* !C_OBP */ 779 5648 setje 780 5648 setje if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 781 5648 setje continue; 782 5648 setje alignsize = TTEBYTES(i); 783 5648 setje kmem64_szc = i; 784 5648 setje 785 5648 setje /* limit page size for small memory */ 786 5648 setje if (mmu_btop(alignsize) > (npages >> 2)) 787 5648 setje continue; 788 5648 setje 789 5648 setje aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 790 5648 setje alloc_size = aligned_end - base; 791 5648 setje #if !defined(C_OBP) 792 5648 setje if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 793 5648 setje if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 794 5648 setje kmem64_pabase = pa; 795 5648 setje kmem64_aligned_end = aligned_end; 796 5648 setje install_kmem64_tte(); 797 5648 setje break; 798 5648 setje } else { 799 5648 setje prom_free_phys(alloc_size, pa); 800 5648 setje } 801 5648 setje } 802 5648 setje #else /* !C_OBP */ 803 5648 setje if (prom_alloc(base, alloc_size, alignsize) == base) { 804 5648 setje kmem64_pabase = va_to_pa(kmem64_base); 805 5648 setje kmem64_aligned_end = aligned_end; 806 5648 setje break; 807 5648 setje } 808 5648 setje #endif /* !C_OBP */ 809 5648 setje if (i == TTE8K) { 810 7218 svemuri #ifdef sun4v 811 7218 svemuri /* return failure to try small allocations */ 812 7218 svemuri return (1); 813 7218 svemuri #else 814 5648 setje prom_panic("kmem64 allocation failure"); 815 7218 svemuri #endif 816 5648 setje } 817 5648 setje } 818 5648 setje ASSERT(aligned_end != NULL); 819 7218 svemuri return (0); 820 5648 setje } 821 5648 setje 822 5648 setje static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 823 0 stevel static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 824 0 stevel 825 5648 setje #define IVSIZE roundup(((MAXIVNUM * sizeof (intr_vec_t *)) + \ 826 5648 setje (MAX_RSVD_IV * sizeof (intr_vec_t)) + \ 827 5648 setje (MAX_RSVD_IVX * sizeof (intr_vecx_t))), PAGESIZE) 828 0 stevel 829 3764 dp78419 #if !defined(C_OBP) 830 3764 dp78419 /* 831 3764 dp78419 * Install a temporary tte handler in OBP for kmem64 area. 832 3764 dp78419 * 833 3764 dp78419 * We map kmem64 area with large pages before the trap table is taken 834 3764 dp78419 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 835 3764 dp78419 * the same area. Duplicate tlb entries with different page sizes 836 3764 dp78419 * cause unpredicatble behavior. To avoid this, we don't create 837 3764 dp78419 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 838 3764 dp78419 * OBP). Instead, we manage translations with a temporary va>tte-data 839 3764 dp78419 * handler (kmem64-tte). This handler is replaced by unix-tte when 840 3764 dp78419 * the trap table is taken over. 841 3764 dp78419 * 842 3764 dp78419 * The temporary handler knows the physical address of the kmem64 843 3764 dp78419 * area. It uses the prom's pgmap@ Forth word for other addresses. 844 3764 dp78419 * 845 3764 dp78419 * We have to use BOP_ALLOC() method for C-OBP platforms because 846 3764 dp78419 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 847 3764 dp78419 * sun4u platforms. On sun4u we flush tlb after trap table is taken 848 3764 dp78419 * over if we use large pages for kernel heap and kmem64. Since sun4u 849 3764 dp78419 * prom (unlike sun4v) calls va>tte-data first for client address 850 3764 dp78419 * translation prom's ttes for kmem64 can't get into TLB even if we 851 3764 dp78419 * later switch to prom's trap table again. C-OBP uses 4M pages for 852 3764 dp78419 * client mappings when possible so on all platforms we get the 853 3764 dp78419 * benefit from large mappings for kmem64 area immediately during 854 3764 dp78419 * boot. 855 3764 dp78419 * 856 3764 dp78419 * pseudo code: 857 3764 dp78419 * if (context != 0) { 858 3764 dp78419 * return false 859 3764 dp78419 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 860 3764 dp78419 * tte = tte_template + 861 3764 dp78419 * (((miss_va & pagemask) - kmem64_base)); 862 3764 dp78419 * return tte, true 863 3764 dp78419 * } else { 864 3764 dp78419 * return pgmap@ result 865 3764 dp78419 * } 866 3764 dp78419 */ 867 3764 dp78419 char kmem64_obp_str[] = 868 5648 setje "h# %lx constant kmem64-base " 869 5648 setje "h# %lx constant kmem64-end " 870 5648 setje "h# %lx constant kmem64-pagemask " 871 5648 setje "h# %lx constant kmem64-template " 872 3764 dp78419 873 3764 dp78419 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 874 3764 dp78419 " if ( addr ) " 875 3764 dp78419 " drop false exit then ( false ) " 876 5648 setje " dup kmem64-base kmem64-end within if ( addr ) " 877 5648 setje " kmem64-pagemask and ( addr' ) " 878 5648 setje " kmem64-base - ( addr' ) " 879 5648 setje " kmem64-template + ( tte ) " 880 3764 dp78419 " true ( tte true ) " 881 3764 dp78419 " else ( addr ) " 882 3764 dp78419 " pgmap@ ( tte ) " 883 3764 dp78419 " dup 0< if true else drop false then ( tte true | false ) " 884 3764 dp78419 " then ( tte true | false ) " 885 3764 dp78419 "; " 886 3764 dp78419 887 3764 dp78419 "' kmem64-tte is va>tte-data " 888 3764 dp78419 ; 889 3764 dp78419 890 5648 setje static void 891 3764 dp78419 install_kmem64_tte() 892 3764 dp78419 { 893 3764 dp78419 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 894 3764 dp78419 tte_t tte; 895 3764 dp78419 896 3764 dp78419 PRM_DEBUG(kmem64_pabase); 897 3764 dp78419 PRM_DEBUG(kmem64_szc); 898 3764 dp78419 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 899 10271 Jason PROC_DATA | HAT_NOSYNC, kmem64_szc); 900 3764 dp78419 PRM_DEBUG(tte.ll); 901 3764 dp78419 (void) sprintf(b, kmem64_obp_str, 902 3764 dp78419 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 903 3764 dp78419 ASSERT(strlen(b) < sizeof (b)); 904 3764 dp78419 prom_interpret(b, 0, 0, 0, 0, 0); 905 3764 dp78419 } 906 3764 dp78419 #endif /* !C_OBP */ 907 3764 dp78419 908 0 stevel /* 909 0 stevel * As OBP takes up some RAM when the system boots, pages will already be "lost" 910 0 stevel * to the system and reflected in npages by the time we see it. 911 0 stevel * 912 0 stevel * We only want to allocate kernel structures in the 64-bit virtual address 913 0 stevel * space on systems with enough RAM to make the overhead of keeping track of 914 0 stevel * an extra kernel memory segment worthwhile. 915 0 stevel * 916 0 stevel * Since OBP has already performed its memory allocations by this point, if we 917 0 stevel * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 918 0 stevel * memory in the 64-bit virtual address space; otherwise keep allocations 919 0 stevel * contiguous with we've mapped so far in the 32-bit virtual address space. 920 0 stevel */ 921 0 stevel #define MINMOVE_RAM_MB ((size_t)1900) 922 0 stevel #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 923 5872 setje #define BYTES_TO_MB(b) ((b) / 1048576ul) 924 0 stevel 925 0 stevel pgcnt_t tune_npages = (pgcnt_t) 926 0 stevel (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 927 3733 dp78419 928 3733 dp78419 #pragma weak page_set_colorequiv_arr_cpu 929 3733 dp78419 extern void page_set_colorequiv_arr_cpu(void); 930 5648 setje extern void page_set_colorequiv_arr(void); 931 5648 setje 932 5872 setje static pgcnt_t ramdisk_npages; 933 5872 setje static struct memlist *old_phys_avail; 934 5872 setje 935 5872 setje kcage_dir_t kcage_startup_dir = KCAGE_DOWN; 936 0 stevel 937 0 stevel static void 938 0 stevel startup_memlist(void) 939 0 stevel { 940 5648 setje size_t hmehash_sz, pagelist_sz, tt_sz; 941 5648 setje size_t psetable_sz; 942 0 stevel caddr_t alloc_base; 943 0 stevel caddr_t memspace; 944 0 stevel struct memlist *cur; 945 0 stevel size_t syslimit = (size_t)SYSLIMIT; 946 0 stevel size_t sysbase = (size_t)SYSBASE; 947 0 stevel 948 0 stevel /* 949 0 stevel * Initialize enough of the system to allow kmem_alloc to work by 950 0 stevel * calling boot to allocate its memory until the time that 951 0 stevel * kvm_init is completed. The page structs are allocated after 952 0 stevel * rounding up end to the nearest page boundary; the memsegs are 953 0 stevel * initialized and the space they use comes from the kernel heap. 954 0 stevel * With appropriate initialization, they can be reallocated later 955 0 stevel * to a size appropriate for the machine's configuration. 956 0 stevel * 957 0 stevel * At this point, memory is allocated for things that will never 958 0 stevel * need to be freed, this used to be "valloced". This allows a 959 0 stevel * savings as the pages don't need page structures to describe 960 0 stevel * them because them will not be managed by the vm system. 961 0 stevel */ 962 0 stevel 963 0 stevel /* 964 0 stevel * We're loaded by boot with the following configuration (as 965 0 stevel * specified in the sun4u/conf/Mapfile): 966 0 stevel * 967 0 stevel * text: 4 MB chunk aligned on a 4MB boundary 968 0 stevel * data & bss: 4 MB chunk aligned on a 4MB boundary 969 0 stevel * 970 0 stevel * These two chunks will eventually be mapped by 2 locked 4MB 971 0 stevel * ttes and will represent the nucleus of the kernel. This gives 972 0 stevel * us some free space that is already allocated, some or all of 973 0 stevel * which is made available to kernel module text. 974 0 stevel * 975 0 stevel * The free space in the data-bss chunk is used for nucleus 976 0 stevel * allocatable data structures and we reserve it using the 977 0 stevel * nalloc_base and nalloc_end variables. This space is currently 978 0 stevel * being used for hat data structures required for tlb miss 979 0 stevel * handling operations. We align nalloc_base to a l2 cache 980 0 stevel * linesize because this is the line size the hardware uses to 981 0 stevel * maintain cache coherency. 982 5648 setje * 512K is carved out for module data. 983 0 stevel */ 984 0 stevel 985 5648 setje moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 986 5648 setje e_moddata = moddata + MODDATA; 987 0 stevel nalloc_base = e_moddata; 988 0 stevel 989 0 stevel nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 990 0 stevel valloc_base = nalloc_base; 991 0 stevel 992 0 stevel /* 993 0 stevel * Calculate the start of the data segment. 994 0 stevel */ 995 5648 setje if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 996 5648 setje prom_panic("nucleus data overflow"); 997 0 stevel 998 0 stevel PRM_DEBUG(moddata); 999 0 stevel PRM_DEBUG(nalloc_base); 1000 0 stevel PRM_DEBUG(nalloc_end); 1001 0 stevel 1002 0 stevel /* 1003 0 stevel * Remember any slop after e_text so we can give it to the modules. 1004 0 stevel */ 1005 0 stevel PRM_DEBUG(e_text); 1006 0 stevel modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 1007 3791 kchow if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 1008 3764 dp78419 prom_panic("nucleus text overflow"); 1009 0 stevel modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 1010 0 stevel modtext; 1011 0 stevel PRM_DEBUG(modtext); 1012 0 stevel PRM_DEBUG(modtext_sz); 1013 0 stevel 1014 5648 setje init_boot_memlists(); 1015 0 stevel copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1016 0 stevel &boot_physavail, &boot_physavail_len, 1017 0 stevel &boot_virtavail, &boot_virtavail_len); 1018 5648 setje 1019 0 stevel /* 1020 0 stevel * Remember what the physically available highest page is 1021 0 stevel * so that dumpsys works properly, and find out how much 1022 0 stevel * memory is installed. 1023 0 stevel */ 1024 0 stevel installed_top_size_memlist_array(boot_physinstalled, 1025 0 stevel boot_physinstalled_len, &physmax, &physinstalled); 1026 0 stevel PRM_DEBUG(physinstalled); 1027 0 stevel PRM_DEBUG(physmax); 1028 0 stevel 1029 0 stevel /* Fill out memory nodes config structure */ 1030 0 stevel startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 1031 0 stevel 1032 0 stevel /* 1033 5648 setje * npages is the maximum of available physical memory possible. 1034 5648 setje * (ie. it will never be more than this) 1035 5648 setje * 1036 5648 setje * When we boot from a ramdisk, the ramdisk memory isn't free, so 1037 5648 setje * using phys_avail will underestimate what will end up being freed. 1038 5648 setje * A better initial guess is just total memory minus the kernel text 1039 0 stevel */ 1040 5648 setje npages = physinstalled - btop(MMU_PAGESIZE4M); 1041 0 stevel 1042 0 stevel /* 1043 5648 setje * First allocate things that can go in the nucleus data page 1044 5648 setje * (fault status, TSBs, dmv, CPUs) 1045 0 stevel */ 1046 0 stevel ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 1047 0 stevel 1048 0 stevel if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1049 0 stevel cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1050 0 stevel 1051 0 stevel if (ndata_alloc_tsbs(&ndata, npages) != 0) 1052 0 stevel cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1053 0 stevel 1054 0 stevel if (ndata_alloc_dmv(&ndata) != 0) 1055 0 stevel cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1056 0 stevel 1057 5648 setje if (ndata_alloc_page_mutexs(&ndata) != 0) 1058 0 stevel cmn_err(CE_PANIC, 1059 0 stevel "no more nucleus memory after page free lists alloc"); 1060 0 stevel 1061 5648 setje if (ndata_alloc_hat(&ndata, npages) != 0) 1062 0 stevel cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1063 0 stevel 1064 5648 setje if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 1065 5648 setje cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 1066 0 stevel 1067 0 stevel /* 1068 0 stevel * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1069 0 stevel * 1070 0 stevel * There are comments all over the SFMMU code warning of dire 1071 0 stevel * consequences if the TSBs are moved out of 32-bit space. This 1072 0 stevel * is largely because the asm code uses "sethi %hi(addr)"-type 1073 0 stevel * instructions which will not provide the expected result if the 1074 0 stevel * address is a 64-bit one. 1075 0 stevel * 1076 0 stevel * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1077 0 stevel */ 1078 0 stevel alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1079 5648 setje PRM_DEBUG(alloc_base); 1080 5648 setje 1081 0 stevel alloc_base = sfmmu_ktsb_alloc(alloc_base); 1082 0 stevel alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1083 0 stevel PRM_DEBUG(alloc_base); 1084 0 stevel 1085 0 stevel /* 1086 0 stevel * Allocate IOMMU TSB array. We do this here so that the physical 1087 0 stevel * memory gets deducted from the PROM's physical memory list. 1088 0 stevel */ 1089 0 stevel alloc_base = iommu_tsb_init(alloc_base); 1090 5648 setje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1091 0 stevel PRM_DEBUG(alloc_base); 1092 5648 setje 1093 5648 setje /* 1094 5648 setje * Allow for an early allocation of physically contiguous memory. 1095 5648 setje */ 1096 5648 setje alloc_base = contig_mem_prealloc(alloc_base, npages); 1097 0 stevel 1098 0 stevel /* 1099 1772 jl139090 * Platforms like Starcat and OPL need special structures assigned in 1100 1772 jl139090 * 32-bit virtual address space because their probing routines execute 1101 1772 jl139090 * FCode, and FCode can't handle 64-bit virtual addresses... 1102 0 stevel */ 1103 1772 jl139090 if (&plat_startup_memlist) { 1104 1772 jl139090 alloc_base = plat_startup_memlist(alloc_base); 1105 0 stevel alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1106 0 stevel ecache_alignsize); 1107 0 stevel PRM_DEBUG(alloc_base); 1108 0 stevel } 1109 0 stevel 1110 0 stevel /* 1111 0 stevel * Save off where the contiguous allocations to date have ended 1112 0 stevel * in econtig32. 1113 0 stevel */ 1114 0 stevel econtig32 = alloc_base; 1115 0 stevel PRM_DEBUG(econtig32); 1116 0 stevel if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1117 0 stevel cmn_err(CE_PANIC, "econtig32 too big"); 1118 0 stevel 1119 5648 setje pp_sz = calc_pp_sz(npages); 1120 5648 setje PRM_DEBUG(pp_sz); 1121 5648 setje if (kpm_enable) { 1122 5648 setje kpm_pp_sz = calc_kpmpp_sz(npages); 1123 5648 setje PRM_DEBUG(kpm_pp_sz); 1124 5648 setje } 1125 5648 setje 1126 5648 setje hmehash_sz = calc_hmehash_sz(npages); 1127 5648 setje PRM_DEBUG(hmehash_sz); 1128 5648 setje 1129 5648 setje pagehash_sz = calc_pagehash_sz(npages); 1130 5648 setje PRM_DEBUG(pagehash_sz); 1131 5648 setje 1132 5648 setje pagelist_sz = calc_free_pagelist_sz(); 1133 5648 setje PRM_DEBUG(pagelist_sz); 1134 5648 setje 1135 5648 setje #ifdef TRAPTRACE 1136 5648 setje tt_sz = calc_traptrace_sz(); 1137 5648 setje PRM_DEBUG(tt_sz); 1138 5648 setje #else 1139 5648 setje tt_sz = 0; 1140 5648 setje #endif /* TRAPTRACE */ 1141 0 stevel 1142 3764 dp78419 /* 1143 5648 setje * Place the array that protects pp->p_selock in the kmem64 wad. 1144 5648 setje */ 1145 6582 setje pse_shift = size_pse_array(npages, max_ncpus); 1146 5648 setje PRM_DEBUG(pse_shift); 1147 5648 setje pse_table_size = 1 << pse_shift; 1148 5648 setje PRM_DEBUG(pse_table_size); 1149 5648 setje psetable_sz = roundup( 1150 5648 setje pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 1151 5648 setje PRM_DEBUG(psetable_sz); 1152 5648 setje 1153 5648 setje /* 1154 5648 setje * Now allocate the whole wad 1155 5648 setje */ 1156 5648 setje kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 1157 5648 setje pagelist_sz + tt_sz + psetable_sz; 1158 5648 setje kmem64_sz = roundup(kmem64_sz, PAGESIZE); 1159 5648 setje kmem64_base = (caddr_t)syslimit; 1160 5648 setje kmem64_end = kmem64_base + kmem64_sz; 1161 7218 svemuri if (alloc_kmem64(kmem64_base, kmem64_end)) { 1162 7218 svemuri /* 1163 7218 svemuri * Attempt for kmem64 to allocate one big 1164 7218 svemuri * contiguous chunk of memory failed. 1165 7218 svemuri * We get here because we are sun4v. 1166 7218 svemuri * We will proceed by breaking up 1167 7218 svemuri * the allocation into two attempts. 1168 7218 svemuri * First, we allocate kpm_pp_sz, hmehash_sz, 1169 7218 svemuri * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as 1170 7218 svemuri * one contiguous chunk. This is a much smaller 1171 7218 svemuri * chunk and we should get it, if not we panic. 1172 7218 svemuri * Note that hmehash and tt need to be physically 1173 7218 svemuri * (in the real address sense) contiguous. 1174 7218 svemuri * Next, we use bop_alloc_chunk() to 1175 7218 svemuri * to allocate the page_t structures. 1176 7218 svemuri * This will allow the page_t to be allocated 1177 7218 svemuri * in multiple smaller chunks. 1178 7218 svemuri * In doing so, the assumption that page_t is 1179 7218 svemuri * physically contiguous no longer hold, this is ok 1180 7218 svemuri * for sun4v but not for sun4u. 1181 7218 svemuri */ 1182 7218 svemuri size_t tmp_size; 1183 7218 svemuri caddr_t tmp_base; 1184 7218 svemuri 1185 7218 svemuri pp_sz = roundup(pp_sz, PAGESIZE); 1186 7218 svemuri 1187 7218 svemuri /* 1188 7218 svemuri * Allocate kpm_pp_sz, hmehash_sz, 1189 7218 svemuri * pagehash_sz, pagelist_sz, tt_sz & psetable_sz 1190 7218 svemuri */ 1191 7218 svemuri tmp_base = kmem64_base + pp_sz; 1192 7218 svemuri tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz + 1193 7218 svemuri pagelist_sz + tt_sz + psetable_sz, PAGESIZE); 1194 7218 svemuri if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0) 1195 7218 svemuri prom_panic("kmem64 prom_alloc contig failed"); 1196 7218 svemuri PRM_DEBUG(tmp_base); 1197 7218 svemuri PRM_DEBUG(tmp_size); 1198 7218 svemuri 1199 7218 svemuri /* 1200 7218 svemuri * Allocate the page_ts 1201 7218 svemuri */ 1202 7218 svemuri if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0) 1203 7218 svemuri prom_panic("kmem64 bop_alloc_chunk page_t failed"); 1204 7218 svemuri PRM_DEBUG(kmem64_base); 1205 7218 svemuri PRM_DEBUG(pp_sz); 1206 7218 svemuri 1207 7218 svemuri kmem64_aligned_end = kmem64_base + pp_sz + tmp_size; 1208 7218 svemuri ASSERT(kmem64_aligned_end >= kmem64_end); 1209 7218 svemuri 1210 7218 svemuri kmem64_smchunks = 1; 1211 7218 svemuri } else { 1212 7218 svemuri 1213 7218 svemuri /* 1214 7218 svemuri * We need to adjust pp_sz for the normal 1215 7218 svemuri * case where kmem64 can allocate one large chunk 1216 7218 svemuri */ 1217 7218 svemuri if (kpm_smallpages == 0) { 1218 7218 svemuri npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 1219 7218 svemuri } else { 1220 7218 svemuri npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) + 1221 7218 svemuri sizeof (kpm_spage_t)); 1222 7218 svemuri } 1223 7218 svemuri pp_sz = npages * sizeof (struct page); 1224 7218 svemuri } 1225 7218 svemuri 1226 5648 setje if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 1227 5648 setje cmn_err(CE_PANIC, "not enough kmem64 space"); 1228 5648 setje PRM_DEBUG(kmem64_base); 1229 5648 setje PRM_DEBUG(kmem64_end); 1230 5648 setje PRM_DEBUG(kmem64_aligned_end); 1231 5648 setje 1232 5648 setje /* 1233 5648 setje * ... and divy it up 1234 3764 dp78419 */ 1235 3764 dp78419 alloc_base = kmem64_base; 1236 5872 setje 1237 5648 setje pp_base = (page_t *)alloc_base; 1238 5648 setje alloc_base += pp_sz; 1239 5648 setje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1240 5648 setje PRM_DEBUG(pp_base); 1241 5648 setje PRM_DEBUG(npages); 1242 0 stevel 1243 5648 setje if (kpm_enable) { 1244 5648 setje kpm_pp_base = alloc_base; 1245 5648 setje if (kpm_smallpages == 0) { 1246 5648 setje /* kpm_npages based on physinstalled, don't reset */ 1247 5648 setje kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 1248 5648 setje } else { 1249 5648 setje kpm_npages = ptokpmpr(npages); 1250 5648 setje kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 1251 5648 setje } 1252 5648 setje alloc_base += kpm_pp_sz; 1253 5648 setje alloc_base = 1254 5648 setje (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1255 5648 setje PRM_DEBUG(kpm_pp_base); 1256 0 stevel } 1257 4204 ha137994 1258 5648 setje alloc_base = alloc_hmehash(alloc_base); 1259 5648 setje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1260 5648 setje PRM_DEBUG(alloc_base); 1261 0 stevel 1262 5648 setje page_hash = (page_t **)alloc_base; 1263 5648 setje alloc_base += pagehash_sz; 1264 5648 setje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1265 5648 setje PRM_DEBUG(page_hash); 1266 0 stevel 1267 5648 setje alloc_base = alloc_page_freelists(alloc_base); 1268 5648 setje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1269 5648 setje PRM_DEBUG(alloc_base); 1270 0 stevel 1271 5648 setje #ifdef TRAPTRACE 1272 5648 setje ttrace_buf = alloc_base; 1273 5648 setje alloc_base += tt_sz; 1274 5648 setje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1275 5648 setje PRM_DEBUG(alloc_base); 1276 5648 setje #endif /* TRAPTRACE */ 1277 0 stevel 1278 5648 setje pse_mutex = (pad_mutex_t *)alloc_base; 1279 5648 setje alloc_base += psetable_sz; 1280 5648 setje alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1281 5648 setje PRM_DEBUG(alloc_base); 1282 0 stevel 1283 7218 svemuri /* 1284 7218 svemuri * Note that if we use small chunk allocations for 1285 7218 svemuri * kmem64, we need to ensure kmem64_end is the same as 1286 7218 svemuri * kmem64_aligned_end to prevent subsequent logic from 1287 7218 svemuri * trying to reuse the overmapping. 1288 7218 svemuri * Otherwise we adjust kmem64_end to what we really allocated. 1289 7218 svemuri */ 1290 7218 svemuri if (kmem64_smchunks) { 1291 7218 svemuri kmem64_end = kmem64_aligned_end; 1292 7218 svemuri } else { 1293 7218 svemuri kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 1294 7218 svemuri } 1295 5648 setje kmem64_sz = kmem64_end - kmem64_base; 1296 0 stevel 1297 0 stevel if (&ecache_init_scrub_flush_area) { 1298 5648 setje alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 1299 5648 setje ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 1300 0 stevel } 1301 0 stevel 1302 0 stevel /* 1303 0 stevel * If physmem is patched to be non-zero, use it instead of 1304 0 stevel * the monitor value unless physmem is larger than the total 1305 0 stevel * amount of memory on hand. 1306 0 stevel */ 1307 0 stevel if (physmem == 0 || physmem > npages) 1308 0 stevel physmem = npages; 1309 0 stevel 1310 0 stevel /* 1311 5648 setje * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 1312 5648 setje * is mounted as root. This memory is held down by OBP and unlike 1313 5648 setje * the stub boot_archive is never released. 1314 5648 setje * 1315 5648 setje * In order to get things sized correctly on lower memory 1316 5648 setje * machines (where the memory used by the ramdisk represents 1317 5648 setje * a significant portion of memory), physmem is adjusted. 1318 5648 setje * 1319 5648 setje * This is done by subtracting the ramdisk_size which is set 1320 5648 setje * to the size of the ramdisk (in Kb) in /etc/system at the 1321 5648 setje * time the miniroot archive is constructed. 1322 0 stevel */ 1323 5872 setje if (root_is_ramdisk == B_TRUE) { 1324 5872 setje ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE; 1325 5872 setje physmem -= ramdisk_npages; 1326 5872 setje } 1327 0 stevel 1328 5648 setje if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 1329 5648 setje cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 1330 0 stevel 1331 0 stevel /* 1332 5648 setje * Allocate space for the interrupt vector table. 1333 0 stevel */ 1334 5648 setje memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 1335 2973 govinda if (memspace != (caddr_t)intr_vec_table) 1336 3764 dp78419 prom_panic("interrupt vector table allocation failure"); 1337 0 stevel 1338 0 stevel /* 1339 0 stevel * Between now and when we finish copying in the memory lists, 1340 0 stevel * allocations happen so the space gets fragmented and the 1341 5648 setje * lists longer. Leave enough space for lists twice as 1342 5648 setje * long as we have now; then roundup to a pagesize. 1343 0 stevel */ 1344 5648 setje memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 1345 5648 setje prom_phys_avail_len() + prom_virt_avail_len()); 1346 5648 setje memlist_sz *= 2; 1347 5648 setje memlist_sz = roundup(memlist_sz, PAGESIZE); 1348 5648 setje memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 1349 0 stevel if (memspace == NULL) 1350 5648 setje cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 1351 0 stevel 1352 0 stevel memlist = (struct memlist *)memspace; 1353 0 stevel memlist_end = (char *)memspace + memlist_sz; 1354 0 stevel PRM_DEBUG(memlist); 1355 0 stevel PRM_DEBUG(memlist_end); 1356 5648 setje 1357 0 stevel PRM_DEBUG(sysbase); 1358 0 stevel PRM_DEBUG(syslimit); 1359 0 stevel kernelheap_init((void *)sysbase, (void *)syslimit, 1360 0 stevel (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1361 0 stevel 1362 0 stevel /* 1363 0 stevel * Take the most current snapshot we can by calling mem-update. 1364 0 stevel */ 1365 0 stevel copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1366 0 stevel &boot_physavail, &boot_physavail_len, 1367 0 stevel &boot_virtavail, &boot_virtavail_len); 1368 0 stevel 1369 0 stevel /* 1370 5648 setje * Remove the space used by prom_alloc from the kernel heap 1371 0 stevel * plus the area actually used by the OBP (if any) 1372 0 stevel * ignoring virtual addresses in virt_avail, above syslimit. 1373 0 stevel */ 1374 0 stevel virt_avail = memlist; 1375 0 stevel copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1376 0 stevel 1377 0 stevel for (cur = virt_avail; cur->next; cur = cur->next) { 1378 0 stevel uint64_t range_base, range_size; 1379 0 stevel 1380 0 stevel if ((range_base = cur->address + cur->size) < (uint64_t)sysbase) 1381 0 stevel continue; 1382 0 stevel if (range_base >= (uint64_t)syslimit) 1383 0 stevel break; 1384 0 stevel /* 1385 0 stevel * Limit the range to end at syslimit. 1386 0 stevel */ 1387 0 stevel range_size = MIN(cur->next->address, 1388 0 stevel (uint64_t)syslimit) - range_base; 1389 0 stevel (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1390 0 stevel 0, 0, (void *)range_base, (void *)(range_base + range_size), 1391 0 stevel VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1392 0 stevel } 1393 0 stevel 1394 0 stevel phys_avail = memlist; 1395 5648 setje copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1396 0 stevel 1397 0 stevel /* 1398 4104 blakej * Add any extra memory at the end of the ndata region if there's at 1399 4104 blakej * least a page to add. There might be a few more pages available in 1400 4104 blakej * the middle of the ndata region, but for now they are ignored. 1401 0 stevel */ 1402 4104 blakej nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1403 4104 blakej if (nalloc_base == NULL) 1404 0 stevel nalloc_base = nalloc_end; 1405 0 stevel ndata_remain_sz = nalloc_end - nalloc_base; 1406 0 stevel 1407 5648 setje /* 1408 5648 setje * Copy physinstalled list into kernel space. 1409 5648 setje */ 1410 5648 setje phys_install = memlist; 1411 5648 setje copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1412 0 stevel 1413 5648 setje /* 1414 5648 setje * Create list of physical addrs we don't need pp's for: 1415 5648 setje * kernel text 4M page 1416 5648 setje * kernel data 4M page - ndata_remain_sz 1417 5648 setje * kmem64 pages 1418 5648 setje * 1419 5648 setje * NB if adding any pages here, make sure no kpm page 1420 5648 setje * overlaps can occur (see ASSERTs in kphysm_memsegs) 1421 5648 setje */ 1422 5648 setje nopp_list = memlist; 1423 5648 setje memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 1424 5648 setje memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 1425 5648 setje &memlist, &nopp_list); 1426 7218 svemuri 1427 7218 svemuri /* Don't add to nopp_list if kmem64 was allocated in smchunks */ 1428 7218 svemuri if (!kmem64_smchunks) 1429 7218 svemuri memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 1430 0 stevel 1431 0 stevel if ((caddr_t)memlist > (memspace + memlist_sz)) 1432 3764 dp78419 prom_panic("memlist overflow"); 1433 6880 dv142724 1434 6880 dv142724 /* 1435 6880 dv142724 * Size the pcf array based on the number of cpus in the box at 1436 6880 dv142724 * boot time. 1437 6880 dv142724 */ 1438 6880 dv142724 pcf_init(); 1439 0 stevel 1440 0 stevel /* 1441 0 stevel * Initialize the page structures from the memory lists. 1442 0 stevel */ 1443 5648 setje kphysm_init(); 1444 0 stevel 1445 0 stevel availrmem_initial = availrmem = freemem; 1446 0 stevel PRM_DEBUG(availrmem); 1447 0 stevel 1448 0 stevel /* 1449 0 stevel * Some of the locks depend on page_hashsz being set! 1450 0 stevel * kmem_init() depends on this; so, keep it here. 1451 0 stevel */ 1452 0 stevel page_lock_init(); 1453 0 stevel 1454 0 stevel /* 1455 0 stevel * Initialize kernel memory allocator. 1456 0 stevel */ 1457 0 stevel kmem_init(); 1458 3717 dp78419 1459 3717 dp78419 /* 1460 3717 dp78419 * Factor in colorequiv to check additional 'equivalent' bins 1461 3717 dp78419 */ 1462 3733 dp78419 if (&page_set_colorequiv_arr_cpu != NULL) 1463 3733 dp78419 page_set_colorequiv_arr_cpu(); 1464 3733 dp78419 else 1465 3733 dp78419 page_set_colorequiv_arr(); 1466 0 stevel 1467 0 stevel /* 1468 0 stevel * Initialize bp_mapin(). 1469 0 stevel */ 1470 0 stevel bp_init(shm_alignment, HAT_STRICTORDER); 1471 0 stevel 1472 0 stevel /* 1473 5358 jc25722 * Reserve space for panicbuf, intr_vec_table, reserved interrupt 1474 5358 jc25722 * vector data structures and MPO mblock structs from the 32-bit heap. 1475 0 stevel */ 1476 0 stevel (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0, 1477 0 stevel panicbuf, panicbuf + PANICBUFSIZE, 1478 0 stevel VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1479 0 stevel 1480 0 stevel (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0, 1481 2973 govinda intr_vec_table, (caddr_t)intr_vec_table + IVSIZE, 1482 0 stevel VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1483 0 stevel 1484 5358 jc25722 if (mpo_heap32_bufsz > (size_t)0) { 1485 5358 jc25722 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1486 5358 jc25722 PAGESIZE, 0, 0, mpo_heap32_buf, 1487 5358 jc25722 mpo_heap32_buf + mpo_heap32_bufsz, 1488 5358 jc25722 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1489 5358 jc25722 } 1490 0 stevel mem_config_init(); 1491 0 stevel } 1492 0 stevel 1493 0 stevel static void 1494 0 stevel startup_modules(void) 1495 0 stevel { 1496 5648 setje int nhblk1, nhblk8; 1497 0 stevel size_t nhblksz; 1498 3764 dp78419 pgcnt_t pages_per_hblk; 1499 0 stevel size_t hme8blk_sz, hme1blk_sz; 1500 0 stevel 1501 0 stevel /* 1502 0 stevel * Let the platforms have a chance to change default 1503 0 stevel * values before reading system file. 1504 0 stevel */ 1505 0 stevel if (&set_platform_defaults) 1506 0 stevel set_platform_defaults(); 1507 0 stevel 1508 0 stevel /* 1509 0 stevel * Calculate default settings of system parameters based upon 1510 0 stevel * maxusers, yet allow to be overridden via the /etc/system file. 1511 0 stevel */ 1512 0 stevel param_calc(0); 1513 0 stevel 1514 0 stevel mod_setup(); 1515 0 stevel 1516 0 stevel /* 1517 0 stevel * If this is a positron, complain and halt. 1518 0 stevel */ 1519 0 stevel if (&iam_positron && iam_positron()) { 1520 0 stevel cmn_err(CE_WARN, "This hardware platform is not supported" 1521 0 stevel " by this release of Solaris.\n"); 1522 0 stevel #ifdef DEBUG 1523 0 stevel prom_enter_mon(); /* Type 'go' to resume */ 1524 0 stevel cmn_err(CE_WARN, "Booting an unsupported platform.\n"); 1525 0 stevel cmn_err(CE_WARN, "Booting with down-rev firmware.\n"); 1526 0 stevel 1527 0 stevel #else /* DEBUG */ 1528 0 stevel halt(0); 1529 0 stevel #endif /* DEBUG */ 1530 0 stevel } 1531 0 stevel 1532 0 stevel /* 1533 0 stevel * If we are running firmware that isn't 64-bit ready 1534 0 stevel * then complain and halt. 1535 0 stevel */ 1536 0 stevel do_prom_version_check(); 1537 0 stevel 1538 0 stevel /* 1539 0 stevel * Initialize system parameters 1540 0 stevel */ 1541 0 stevel param_init(); 1542 0 stevel 1543 0 stevel /* 1544 0 stevel * maxmem is the amount of physical memory we're playing with. 1545 0 stevel */ 1546 0 stevel maxmem = physmem; 1547 0 stevel 1548 0 stevel /* Set segkp limits. */ 1549 3446 mrj ncbase = kdi_segdebugbase; 1550 3446 mrj ncend = kdi_segdebugbase; 1551 0 stevel 1552 0 stevel /* 1553 0 stevel * Initialize the hat layer. 1554 0 stevel */ 1555 0 stevel hat_init(); 1556 0 stevel 1557 0 stevel /* 1558 0 stevel * Initialize segment management stuff. 1559 0 stevel */ 1560 0 stevel seg_init(); 1561 0 stevel 1562 0 stevel /* 1563 0 stevel * Create the va>tte handler, so the prom can understand 1564 0 stevel * kernel translations. The handler is installed later, just 1565 0 stevel * as we are about to take over the trap table from the prom. 1566 0 stevel */ 1567 0 stevel create_va_to_tte(); 1568 0 stevel 1569 0 stevel /* 1570 0 stevel * Load the forthdebugger (optional) 1571 0 stevel */ 1572 0 stevel forthdebug_init(); 1573 0 stevel 1574 0 stevel /* 1575 0 stevel * Create OBP node for console input callbacks 1576 0 stevel * if it is needed. 1577 0 stevel */ 1578 1253 lq150181 startup_create_io_node(); 1579 0 stevel 1580 0 stevel if (modloadonly("fs", "specfs") == -1) 1581 0 stevel halt("Can't load specfs"); 1582 0 stevel 1583 0 stevel if (modloadonly("fs", "devfs") == -1) 1584 0 stevel halt("Can't load devfs"); 1585 0 stevel 1586 11173 Jonathan if (modloadonly("fs", "procfs") == -1) 1587 11173 Jonathan halt("Can't load procfs"); 1588 11173 Jonathan 1589 0 stevel if (modloadonly("misc", "swapgeneric") == -1) 1590 0 stevel halt("Can't load swapgeneric"); 1591 1676 jpk 1592 1676 jpk (void) modloadonly("sys", "lbl_edition"); 1593 0 stevel 1594 0 stevel dispinit(); 1595 0 stevel 1596 0 stevel /* 1597 0 stevel * Infer meanings to the members of the idprom buffer. 1598 0 stevel */ 1599 0 stevel parse_idprom(); 1600 0 stevel 1601 0 stevel /* Read cluster configuration data. */ 1602 0 stevel clconf_init(); 1603 0 stevel 1604 0 stevel setup_ddi(); 1605 0 stevel 1606 0 stevel /* 1607 0 stevel * Lets take this opportunity to load the root device. 1608 0 stevel */ 1609 0 stevel if (loadrootmodules() != 0) 1610 0 stevel debug_enter("Can't load the root filesystem"); 1611 0 stevel 1612 0 stevel /* 1613 0 stevel * Load tod driver module for the tod part found on this system. 1614 0 stevel * Recompute the cpu frequency/delays based on tod as tod part 1615 0 stevel * tends to keep time more accurately. 1616 0 stevel */ 1617 0 stevel if (&load_tod_module) 1618 0 stevel load_tod_module(); 1619 0 stevel 1620 0 stevel /* 1621 0 stevel * Allow platforms to load modules which might 1622 0 stevel * be needed after bootops are gone. 1623 0 stevel */ 1624 0 stevel if (&load_platform_modules) 1625 0 stevel load_platform_modules(); 1626 0 stevel 1627 0 stevel setcpudelay(); 1628 0 stevel 1629 0 stevel copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1630 0 stevel &boot_physavail, &boot_physavail_len, 1631 0 stevel &boot_virtavail, &boot_virtavail_len); 1632 0 stevel 1633 3764 dp78419 /* 1634 3764 dp78419 * Calculation and allocation of hmeblks needed to remap 1635 3764 dp78419 * the memory allocated by PROM till now. 1636 3764 dp78419 * Overestimate the number of hblk1 elements by assuming 1637 3764 dp78419 * worst case of TTE64K mappings. 1638 3764 dp78419 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1639 3764 dp78419 */ 1640 3764 dp78419 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1641 3764 dp78419 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1642 3764 dp78419 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1643 3764 dp78419 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1644 3764 dp78419 1645 0 stevel bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1646 0 stevel 1647 3764 dp78419 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1648 0 stevel hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1649 0 stevel hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1650 0 stevel 1651 3764 dp78419 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1652 3764 dp78419 1653 0 stevel pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1654 3764 dp78419 nhblk8 = 0; 1655 3764 dp78419 while (bop_alloc_pages > 1) { 1656 3764 dp78419 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1657 3764 dp78419 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1658 3764 dp78419 bop_alloc_pages *= hme8blk_sz; 1659 3764 dp78419 bop_alloc_pages = btopr(bop_alloc_pages); 1660 3764 dp78419 } 1661 3764 dp78419 nhblk8 += 2; 1662 0 stevel 1663 0 stevel /* 1664 0 stevel * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1665 0 stevel * boundary, the number of hblk8's needed to map the entries in the 1666 0 stevel * boot_virtavail list needs to be adjusted to take this into 1667 0 stevel * consideration. Thus, we need to add additional hblk8's since it 1668 0 stevel * is possible that an hblk8 will not have all 8 slots used due to 1669 0 stevel * alignment constraints. Since there were boot_virtavail_len entries 1670 0 stevel * in that list, we need to add that many hblk8's to the number 1671 0 stevel * already calculated to make sure we don't underestimate. 1672 0 stevel */ 1673 0 stevel nhblk8 += boot_virtavail_len; 1674 0 stevel nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1675 0 stevel 1676 0 stevel /* Allocate in pagesize chunks */ 1677 0 stevel nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1678 0 stevel hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1679 0 stevel sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1680 0 stevel } 1681 0 stevel 1682 0 stevel static void 1683 0 stevel startup_bop_gone(void) 1684 0 stevel { 1685 0 stevel 1686 0 stevel /* 1687 1991 heppo * Destroy the MD initialized at startup 1688 1991 heppo * The startup initializes the MD framework 1689 1991 heppo * using prom and BOP alloc free it now. 1690 1991 heppo */ 1691 1991 heppo mach_descrip_startup_fini(); 1692 1991 heppo 1693 1991 heppo /* 1694 5648 setje * We're done with prom allocations. 1695 0 stevel */ 1696 5648 setje bop_fini(); 1697 0 stevel 1698 0 stevel copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1699 0 stevel &boot_physavail, &boot_physavail_len, 1700 0 stevel &boot_virtavail, &boot_virtavail_len); 1701 0 stevel 1702 0 stevel /* 1703 0 stevel * setup physically contiguous area twice as large as the ecache. 1704 0 stevel * this is used while doing displacement flush of ecaches 1705 0 stevel */ 1706 0 stevel if (&ecache_flush_address) { 1707 0 stevel ecache_flushaddr = ecache_flush_address(); 1708 0 stevel if (ecache_flushaddr == (uint64_t)-1) { 1709 0 stevel cmn_err(CE_PANIC, 1710 0 stevel "startup: no memory to set ecache_flushaddr"); 1711 0 stevel } 1712 0 stevel } 1713 0 stevel 1714 0 stevel /* 1715 0 stevel * Virtual available next. 1716 0 stevel */ 1717 0 stevel ASSERT(virt_avail != NULL); 1718 0 stevel memlist_free_list(virt_avail); 1719 0 stevel virt_avail = memlist; 1720 0 stevel copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1721 0 stevel 1722 0 stevel } 1723 0 stevel 1724 0 stevel 1725 0 stevel /* 1726 0 stevel * startup_fixup_physavail - called from mach_sfmmu.c after the final 1727 0 stevel * allocations have been performed. We can't call it in startup_bop_gone 1728 0 stevel * since later operations can cause obp to allocate more memory. 1729 0 stevel */ 1730 0 stevel void 1731 0 stevel startup_fixup_physavail(void) 1732 0 stevel { 1733 0 stevel struct memlist *cur; 1734 3764 dp78419 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1735 3764 dp78419 1736 3764 dp78419 PRM_DEBUG(kmem64_overmap_size); 1737 0 stevel 1738 0 stevel /* 1739 0 stevel * take the most current snapshot we can by calling mem-update 1740 0 stevel */ 1741 0 stevel copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1742 0 stevel &boot_physavail, &boot_physavail_len, 1743 0 stevel &boot_virtavail, &boot_virtavail_len); 1744 0 stevel 1745 0 stevel /* 1746 0 stevel * Copy phys_avail list, again. 1747 0 stevel * Both the kernel/boot and the prom have been allocating 1748 0 stevel * from the original list we copied earlier. 1749 0 stevel */ 1750 0 stevel cur = memlist; 1751 5648 setje copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1752 3764 dp78419 1753 3764 dp78419 /* 1754 3764 dp78419 * Add any unused kmem64 memory from overmapped page 1755 3764 dp78419 * (Note: va_to_pa does not work for kmem64_end) 1756 3764 dp78419 */ 1757 3764 dp78419 if (kmem64_overmap_size) { 1758 3764 dp78419 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1759 5648 setje kmem64_overmap_size, &memlist, &cur); 1760 3764 dp78419 } 1761 0 stevel 1762 0 stevel /* 1763 5648 setje * Add any extra memory after e_data we added to the phys_avail list 1764 0 stevel * back to the old list. 1765 0 stevel */ 1766 0 stevel if (ndata_remain_sz >= MMU_PAGESIZE) 1767 0 stevel memlist_add(va_to_pa(nalloc_base), 1768 0 stevel (uint64_t)ndata_remain_sz, &memlist, &cur); 1769 0 stevel 1770 0 stevel /* 1771 0 stevel * There isn't any bounds checking on the memlist area 1772 0 stevel * so ensure it hasn't overgrown. 1773 0 stevel */ 1774 0 stevel if ((caddr_t)memlist > (caddr_t)memlist_end) 1775 0 stevel cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1776 0 stevel 1777 0 stevel /* 1778 0 stevel * The kernel removes the pages that were allocated for it from 1779 0 stevel * the freelist, but we now have to find any -extra- pages that 1780 0 stevel * the prom has allocated for it's own book-keeping, and remove 1781 0 stevel * them from the freelist too. sigh. 1782 0 stevel */ 1783 5648 setje sync_memlists(phys_avail, cur); 1784 0 stevel 1785 0 stevel ASSERT(phys_avail != NULL); 1786 5872 setje 1787 5872 setje old_phys_avail = phys_avail; 1788 0 stevel phys_avail = cur; 1789 5872 setje } 1790 0 stevel 1791 5872 setje void 1792 5872 setje update_kcage_ranges(uint64_t addr, uint64_t len) 1793 5872 setje { 1794 5872 setje pfn_t base = btop(addr); 1795 5872 setje pgcnt_t num = btop(len); 1796 5872 setje int rv; 1797 5872 setje 1798 5872 setje rv = kcage_range_add(base, num, kcage_startup_dir); 1799 5872 setje 1800 5872 setje if (rv == ENOMEM) { 1801 5872 setje cmn_err(CE_WARN, "%ld megabytes not available to kernel cage", 1802 5872 setje (len == 0 ? 0 : BYTES_TO_MB(len))); 1803 5872 setje } else if (rv != 0) { 1804 5872 setje /* catch this in debug kernels */ 1805 5872 setje ASSERT(0); 1806 5872 setje 1807 5872 setje cmn_err(CE_WARN, "unexpected kcage_range_add" 1808 5872 setje " return value %d", rv); 1809 5872 setje } 1810 0 stevel } 1811 0 stevel 1812 0 stevel static void 1813 0 stevel startup_vm(void) 1814 0 stevel { 1815 0 stevel size_t i; 1816 0 stevel struct segmap_crargs a; 1817 0 stevel struct segkpm_crargs b; 1818 0 stevel 1819 0 stevel uint64_t avmem; 1820 0 stevel caddr_t va; 1821 0 stevel pgcnt_t max_phys_segkp; 1822 0 stevel int mnode; 1823 0 stevel 1824 2991 susans extern int use_brk_lpg, use_stk_lpg; 1825 423 davemq 1826 0 stevel /* 1827 0 stevel * get prom's mappings, create hments for them and switch 1828 0 stevel * to the kernel context. 1829 0 stevel */ 1830 0 stevel hat_kern_setup(); 1831 0 stevel 1832 0 stevel /* 1833 0 stevel * Take over trap table 1834 0 stevel */ 1835 0 stevel setup_trap_table(); 1836 0 stevel 1837 0 stevel /* 1838 0 stevel * Install the va>tte handler, so that the prom can handle 1839 0 stevel * misses and understand the kernel table layout in case 1840 0 stevel * we need call into the prom. 1841 0 stevel */ 1842 0 stevel install_va_to_tte(); 1843 0 stevel 1844 0 stevel /* 1845 0 stevel * Set a flag to indicate that the tba has been taken over. 1846 0 stevel */ 1847 0 stevel tba_taken_over = 1; 1848 0 stevel 1849 0 stevel /* initialize MMU primary context register */ 1850 0 stevel mmu_init_kcontext(); 1851 0 stevel 1852 0 stevel /* 1853 0 stevel * The boot cpu can now take interrupts, x-calls, x-traps 1854 0 stevel */ 1855 0 stevel CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1856 0 stevel CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1857 0 stevel 1858 0 stevel /* 1859 0 stevel * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1860 0 stevel */ 1861 0 stevel tbr_wr_addr_inited = 1; 1862 0 stevel 1863 0 stevel /* 1864 0 stevel * Initialize VM system, and map kernel address space. 1865 0 stevel */ 1866 0 stevel kvm_init(); 1867 0 stevel 1868 5872 setje ASSERT(old_phys_avail != NULL && phys_avail != NULL); 1869 5872 setje if (kernel_cage_enable) { 1870 5872 setje diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges); 1871 5872 setje } 1872 5872 setje memlist_free_list(old_phys_avail); 1873 5872 setje 1874 0 stevel /* 1875 0 stevel * If the following is true, someone has patched 1876 0 stevel * phsymem to be less than the number of pages that 1877 0 stevel * the system actually has. Remove pages until system 1878 0 stevel * memory is limited to the requested amount. Since we 1879 0 stevel * have allocated page structures for all pages, we 1880 0 stevel * correct the amount of memory we want to remove 1881 0 stevel * by the size of the memory used to hold page structures 1882 0 stevel * for the non-used pages. 1883 0 stevel */ 1884 5872 setje if (physmem + ramdisk_npages < npages) { 1885 0 stevel pgcnt_t diff, off; 1886 0 stevel struct page *pp; 1887 0 stevel struct seg kseg; 1888 0 stevel 1889 0 stevel cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1890 0 stevel 1891 0 stevel off = 0; 1892 5872 setje diff = npages - (physmem + ramdisk_npages); 1893 0 stevel diff -= mmu_btopr(diff * sizeof (struct page)); 1894 0 stevel kseg.s_as = &kas; 1895 0 stevel while (diff--) { 1896 0 stevel pp = page_create_va(&unused_pages_vp, (offset_t)off, 1897 0 stevel MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1898 0 stevel &kseg, (caddr_t)off); 1899 0 stevel if (pp == NULL) 1900 0 stevel cmn_err(CE_PANIC, "limited physmem too much!"); 1901 0 stevel page_io_unlock(pp); 1902 0 stevel page_downgrade(pp); 1903 0 stevel availrmem--; 1904 0 stevel off += MMU_PAGESIZE; 1905 0 stevel } 1906 0 stevel } 1907 0 stevel 1908 0 stevel /* 1909 0 stevel * When printing memory, show the total as physmem less 1910 0 stevel * that stolen by a debugger. 1911 0 stevel */ 1912 0 stevel cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1913 0 stevel (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1914 0 stevel (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1915 0 stevel 1916 0 stevel avmem = (uint64_t)freemem << PAGESHIFT; 1917 0 stevel cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1918 423 davemq 1919 2991 susans /* 1920 2991 susans * For small memory systems disable automatic large pages. 1921 2991 susans */ 1922 2991 susans if (physmem < privm_lpg_min_physmem) { 1923 423 davemq use_brk_lpg = 0; 1924 423 davemq use_stk_lpg = 0; 1925 423 davemq } 1926 0 stevel 1927 0 stevel /* 1928 0 stevel * Perform platform specific freelist processing 1929 0 stevel */ 1930 0 stevel if (&plat_freelist_process) { 1931 0 stevel for (mnode = 0; mnode < max_mem_nodes; mnode++) 1932 0 stevel if (mem_node_config[mnode].exists) 1933 0 stevel plat_freelist_process(mnode); 1934 0 stevel } 1935 0 stevel 1936 0 stevel /* 1937 0 stevel * Initialize the segkp segment type. We position it 1938 0 stevel * after the configured tables and buffers (whose end 1939 0 stevel * is given by econtig) and before V_WKBASE_ADDR. 1940 0 stevel * Also in this area is segkmap (size SEGMAPSIZE). 1941 0 stevel */ 1942 0 stevel 1943 0 stevel /* XXX - cache alignment? */ 1944 0 stevel va = (caddr_t)SEGKPBASE; 1945 0 stevel ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1946 0 stevel 1947 0 stevel max_phys_segkp = (physmem * 2); 1948 0 stevel 1949 0 stevel if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1950 0 stevel segkpsize = btop(SEGKPDEFSIZE); 1951 0 stevel cmn_err(CE_WARN, "Illegal value for segkpsize. " 1952 0 stevel "segkpsize has been reset to %ld pages", segkpsize); 1953 0 stevel } 1954 0 stevel 1955 0 stevel i = ptob(MIN(segkpsize, max_phys_segkp)); 1956 0 stevel 1957 0 stevel rw_enter(&kas.a_lock, RW_WRITER); 1958 0 stevel if (seg_attach(&kas, va, i, segkp) < 0) 1959 0 stevel cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1960 0 stevel if (segkp_create(segkp) != 0) 1961 0 stevel cmn_err(CE_PANIC, "startup: segkp_create failed"); 1962 0 stevel rw_exit(&kas.a_lock); 1963 0 stevel 1964 0 stevel /* 1965 0 stevel * kpm segment 1966 0 stevel */ 1967 0 stevel segmap_kpm = kpm_enable && 1968 4528 paulsan segmap_kpm && PAGESIZE == MAXBSIZE; 1969 0 stevel 1970 0 stevel if (kpm_enable) { 1971 0 stevel rw_enter(&kas.a_lock, RW_WRITER); 1972 0 stevel 1973 0 stevel /* 1974 0 stevel * The segkpm virtual range range is larger than the 1975 0 stevel * actual physical memory size and also covers gaps in 1976 0 stevel * the physical address range for the following reasons: 1977 0 stevel * . keep conversion between segkpm and physical addresses 1978 0 stevel * simple, cheap and unambiguous. 1979 0 stevel * . avoid extension/shrink of the the segkpm in case of DR. 1980 0 stevel * . avoid complexity for handling of virtual addressed 1981 0 stevel * caches, segkpm and the regular mapping scheme must be 1982 0 stevel * kept in sync wrt. the virtual color of mapped pages. 1983 0 stevel * Any accesses to virtual segkpm ranges not backed by 1984 0 stevel * physical memory will fall through the memseg pfn hash 1985 0 stevel * and will be handled in segkpm_fault. 1986 0 stevel * Additional kpm_size spaces needed for vac alias prevention. 1987 0 stevel */ 1988 0 stevel if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 1989 0 stevel segkpm) < 0) 1990 0 stevel cmn_err(CE_PANIC, "cannot attach segkpm"); 1991 0 stevel 1992 0 stevel b.prot = PROT_READ | PROT_WRITE; 1993 0 stevel b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 1994 0 stevel 1995 0 stevel if (segkpm_create(segkpm, (caddr_t)&b) != 0) 1996 0 stevel panic("segkpm_create segkpm"); 1997 0 stevel 1998 0 stevel rw_exit(&kas.a_lock); 1999 2296 ae112802 2000 2296 ae112802 mach_kpm_init(); 2001 0 stevel } 2002 0 stevel 2003 10106 Jason va = kpm_vbase + (kpm_size * vac_colors); 2004 10106 Jason 2005 3290 johansen if (!segzio_fromheap) { 2006 3290 johansen size_t size; 2007 3552 johansen size_t physmem_b = mmu_ptob(physmem); 2008 3290 johansen 2009 3290 johansen /* size is in bytes, segziosize is in pages */ 2010 3290 johansen if (segziosize == 0) { 2011 3552 johansen size = physmem_b; 2012 3290 johansen } else { 2013 3290 johansen size = mmu_ptob(segziosize); 2014 3290 johansen } 2015 3290 johansen 2016 3290 johansen if (size < SEGZIOMINSIZE) { 2017 3290 johansen size = SEGZIOMINSIZE; 2018 3552 johansen } else if (size > SEGZIOMAXSIZE) { 2019 3552 johansen size = SEGZIOMAXSIZE; 2020 3552 johansen /* 2021 3552 johansen * On 64-bit x86, we only have 2TB of KVA. This exists 2022 3552 johansen * for parity with x86. 2023 3552 johansen * 2024 3552 johansen * SEGZIOMAXSIZE is capped at 512gb so that segzio 2025 3552 johansen * doesn't consume all of KVA. However, if we have a 2026 3552 johansen * system that has more thant 512gb of physical memory, 2027 3552 johansen * we can actually consume about half of the difference 2028 3552 johansen * between 512gb and the rest of the available physical 2029 3552 johansen * memory. 2030 3552 johansen */ 2031 3552 johansen if (physmem_b > SEGZIOMAXSIZE) { 2032 3552 johansen size += (physmem_b - SEGZIOMAXSIZE) / 2; 2033 3552 johansen } 2034 3290 johansen } 2035 3290 johansen segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2036 3290 johansen /* put the base of the ZIO segment after the kpm segment */ 2037 10106 Jason segzio_base = va; 2038 10106 Jason va += mmu_ptob(segziosize); 2039 3290 johansen PRM_DEBUG(segziosize); 2040 3290 johansen PRM_DEBUG(segzio_base); 2041 3290 johansen 2042 3290 johansen /* 2043 3290 johansen * On some platforms, kvm_init is called after the kpm 2044 3290 johansen * sizes have been determined. On SPARC, kvm_init is called 2045 3290 johansen * before, so we have to attach the kzioseg after kvm is 2046 3290 johansen * initialized, otherwise we'll try to allocate from the boot 2047 3290 johansen * area since the kernel heap hasn't yet been configured. 2048 3290 johansen */ 2049 3290 johansen rw_enter(&kas.a_lock, RW_WRITER); 2050 3290 johansen 2051 3290 johansen (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2052 3290 johansen &kzioseg); 2053 3290 johansen (void) segkmem_zio_create(&kzioseg); 2054 3290 johansen 2055 3290 johansen /* create zio area covering new segment */ 2056 3290 johansen segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2057 3290 johansen 2058 3290 johansen rw_exit(&kas.a_lock); 2059 3290 johansen } 2060 3290 johansen 2061 10106 Jason if (ppvm_enable) { 2062 10106 Jason caddr_t ppvm_max; 2063 10106 Jason 2064 10106 Jason /* 2065 10106 Jason * ppvm refers to the static VA space used to map 2066 10106 Jason * the page_t's for dynamically added memory. 2067 10106 Jason * 2068 10106 Jason * ppvm_base should not cross a potential VA hole. 2069 10106 Jason * 2070 10106 Jason * ppvm_size should be large enough to map the 2071 10106 Jason * page_t's needed to manage all of KPM range. 2072 10106 Jason */ 2073 10106 Jason ppvm_size = 2074 10106 Jason roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t), 2075 10106 Jason MMU_PAGESIZE); 2076 10106 Jason ppvm_max = (caddr_t)(0ull - ppvm_size); 2077 10106 Jason ppvm_base = (page_t *)va; 2078 10106 Jason 2079 10106 Jason if ((caddr_t)ppvm_base <= hole_end) { 2080 10106 Jason cmn_err(CE_WARN, 2081 10106 Jason "Memory DR disabled: invalid DR map base: 0x%p\n", 2082 10106 Jason (void *)ppvm_base); 2083 10106 Jason ppvm_enable = 0; 2084 10106 Jason } else if ((caddr_t)ppvm_base > ppvm_max) { 2085 10106 Jason uint64_t diff = (caddr_t)ppvm_base - ppvm_max; 2086 10106 Jason 2087 10106 Jason cmn_err(CE_WARN, 2088 10106 Jason "Memory DR disabled: insufficient DR map size:" 2089 10106 Jason " 0x%lx (needed 0x%lx)\n", 2090 10106 Jason ppvm_size - diff, ppvm_size); 2091 10106 Jason ppvm_enable = 0; 2092 10106 Jason } 2093 10106 Jason PRM_DEBUG(ppvm_size); 2094 10106 Jason PRM_DEBUG(ppvm_base); 2095 10106 Jason } 2096 3290 johansen 2097 0 stevel /* 2098 0 stevel * Now create generic mapping segment. This mapping 2099 0 stevel * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2100 0 stevel * virtual address is greater than the amount of free 2101 0 stevel * memory that is available, then we trim back the 2102 0 stevel * segment size to that amount 2103 0 stevel */ 2104 0 stevel va = (caddr_t)SEGMAPBASE; 2105 0 stevel 2106 0 stevel /* 2107 0 stevel * 1201049: segkmap base address must be MAXBSIZE aligned 2108 0 stevel */ 2109 0 stevel ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2110 0 stevel 2111 0 stevel /* 2112 0 stevel * Set size of segmap to percentage of freemem at boot, 2113 0 stevel * but stay within the allowable range 2114 0 stevel * Note we take percentage before converting from pages 2115 0 stevel * to bytes to avoid an overflow on 32-bit kernels. 2116 0 stevel */ 2117 0 stevel i = mmu_ptob((freemem * segmap_percent) / 100); 2118 0 stevel 2119 0 stevel if (i < MINMAPSIZE) 2120 0 stevel i = MINMAPSIZE; 2121 0 stevel 2122 0 stevel if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2123 0 stevel i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2124 0 stevel 2125 0 stevel i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2126 0 stevel 2127 0 stevel rw_enter(&kas.a_lock, RW_WRITER); 2128 0 stevel if (seg_attach(&kas, va, i, segkmap) < 0) 2129 0 stevel cmn_err(CE_PANIC, "cannot attach segkmap"); 2130 0 stevel 2131 0 stevel a.prot = PROT_READ | PROT_WRITE; 2132 0 stevel a.shmsize = shm_alignment; 2133 0 stevel a.nfreelist = 0; /* use segmap driver defaults */ 2134 0 stevel 2135 0 stevel if (segmap_create(segkmap, (caddr_t)&a) != 0) 2136 0 stevel panic("segmap_create segkmap"); 2137 0 stevel rw_exit(&kas.a_lock); 2138 0 stevel 2139 0 stevel segdev_init(); 2140 0 stevel } 2141 0 stevel 2142 0 stevel static void 2143 0 stevel startup_end(void) 2144 0 stevel { 2145 0 stevel if ((caddr_t)memlist > (caddr_t)memlist_end) 2146 0 stevel panic("memlist overflow 2"); 2147 0 stevel memlist_free_block((caddr_t)memlist, 2148 0 stevel ((caddr_t)memlist_end - (caddr_t)memlist)); 2149 0 stevel memlist = NULL; 2150 0 stevel 2151 0 stevel /* enable page_relocation since OBP is now done */ 2152 0 stevel page_relocate_ready = 1; 2153 0 stevel 2154 0 stevel /* 2155 0 stevel * Perform tasks that get done after most of the VM 2156 0 stevel * initialization has been done but before the clock 2157 0 stevel * and other devices get started. 2158 0 stevel */ 2159 0 stevel kern_setup1(); 2160 0 stevel 2161 0 stevel /* 2162 8803 Jonathan * Perform CPC initialization for this CPU. 2163 8803 Jonathan */ 2164 8803 Jonathan kcpc_hw_init(); 2165 8803 Jonathan 2166 8803 Jonathan /* 2167 0 stevel * Intialize the VM arenas for allocating physically 2168 0 stevel * contiguus memory chunk for interrupt queues snd 2169 0 stevel * allocate/register boot cpu's queues, if any and 2170 0 stevel * allocate dump buffer for sun4v systems to store 2171 0 stevel * extra crash information during crash dump 2172 0 stevel */ 2173 0 stevel contig_mem_init(); 2174 0 stevel mach_descrip_init(); 2175 4050 jb145095 2176 4050 jb145095 if (cpu_intrq_setup(CPU)) { 2177 4050 jb145095 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2178 4050 jb145095 } 2179 0 stevel cpu_intrq_register(CPU); 2180 1077 svemuri mach_htraptrace_setup(CPU->cpu_id); 2181 1077 svemuri mach_htraptrace_configure(CPU->cpu_id); 2182 0 stevel mach_dump_buffer_init(); 2183 0 stevel 2184 0 stevel /* 2185 0 stevel * Initialize interrupt related stuff 2186 0 stevel */ 2187 1455 andrei cpu_intr_alloc(CPU, NINTR_THREADS); 2188 0 stevel 2189 0 stevel (void) splzs(); /* allow hi clock ints but not zs */ 2190 0 stevel 2191 0 stevel /* 2192 0 stevel * Initialize errors. 2193 0 stevel */ 2194 0 stevel error_init(); 2195 0 stevel 2196 0 stevel /* 2197 0 stevel * Note that we may have already used kernel bcopy before this 2198 0 stevel * point - but if you really care about this, adb the use_hw_* 2199 0 stevel * variables to 0 before rebooting. 2200 0 stevel */ 2201 0 stevel mach_hw_copy_limit(); 2202 0 stevel 2203 0 stevel /* 2204 0 stevel * Install the "real" preemption guards before DDI services 2205 0 stevel * are available. 2206 0 stevel */ 2207 0 stevel (void) prom_set_preprom(kern_preprom); 2208 0 stevel (void) prom_set_postprom(kern_postprom); 2209 0 stevel CPU->cpu_m.mutex_ready = 1; 2210 0 stevel 2211 0 stevel /* 2212 0 stevel * Initialize segnf (kernel support for non-faulting loads). 2213 0 stevel */ 2214 0 stevel segnf_init(); 2215 0 stevel 2216 0 stevel /* 2217 0 stevel * Configure the root devinfo node. 2218 0 stevel */ 2219 0 stevel configure(); /* set up devices */ 2220 0 stevel mach_cpu_halt_idle(); 2221 0 stevel } 2222 0 stevel 2223 0 stevel 2224 0 stevel void 2225 0 stevel post_startup(void) 2226 0 stevel { 2227 0 stevel #ifdef PTL1_PANIC_DEBUG 2228 0 stevel extern void init_ptl1_thread(void); 2229 0 stevel #endif /* PTL1_PANIC_DEBUG */ 2230 0 stevel extern void abort_sequence_init(void); 2231 0 stevel 2232 0 stevel /* 2233 0 stevel * Set the system wide, processor-specific flags to be passed 2234 0 stevel * to userland via the aux vector for performance hints and 2235 0 stevel * instruction set extensions. 2236 0 stevel */ 2237 0 stevel bind_hwcap(); 2238 0 stevel 2239 0 stevel /* 2240 0 stevel * Startup memory scrubber (if any) 2241 0 stevel */ 2242 0 stevel mach_memscrub(); 2243 0 stevel 2244 0 stevel /* 2245 0 stevel * Allocate soft interrupt to handle abort sequence. 2246 0 stevel */ 2247 0 stevel abort_sequence_init(); 2248 0 stevel 2249 0 stevel /* 2250 0 stevel * Configure the rest of the system. 2251 0 stevel * Perform forceloading tasks for /etc/system. 2252 0 stevel */ 2253 0 stevel (void) mod_sysctl(SYS_FORCELOAD, NULL); 2254 0 stevel /* 2255 0 stevel * ON4.0: Force /proc module in until clock interrupt handle fixed 2256 0 stevel * ON4.0: This must be fixed or restated in /etc/systems. 2257 0 stevel */ 2258 0 stevel (void) modload("fs", "procfs"); 2259 0 stevel 2260 1991 heppo /* load machine class specific drivers */ 2261 1991 heppo load_mach_drivers(); 2262 1991 heppo 2263 1991 heppo /* load platform specific drivers */ 2264 0 stevel if (&load_platform_drivers) 2265 0 stevel load_platform_drivers(); 2266 0 stevel 2267 0 stevel /* load vis simulation module, if we are running w/fpu off */ 2268 0 stevel if (!fpu_exists) { 2269 0 stevel if (modload("misc", "vis") == -1) 2270 0 stevel halt("Can't load vis"); 2271 0 stevel } 2272 0 stevel 2273 0 stevel mach_fpras(); 2274 0 stevel 2275 0 stevel maxmem = freemem; 2276 0 stevel 2277 8906 Eric pg_init(); 2278 8906 Eric 2279 0 stevel #ifdef PTL1_PANIC_DEBUG 2280 0 stevel init_ptl1_thread(); 2281 0 stevel #endif /* PTL1_PANIC_DEBUG */ 2282 0 stevel } 2283 0 stevel 2284 0 stevel #ifdef PTL1_PANIC_DEBUG 2285 0 stevel int ptl1_panic_test = 0; 2286 0 stevel int ptl1_panic_xc_one_test = 0; 2287 0 stevel int ptl1_panic_xc_all_test = 0; 2288 0 stevel int ptl1_panic_xt_one_test = 0; 2289 0 stevel int ptl1_panic_xt_all_test = 0; 2290 0 stevel kthread_id_t ptl1_thread_p = NULL; 2291 0 stevel kcondvar_t ptl1_cv; 2292 0 stevel kmutex_t ptl1_mutex; 2293 0 stevel int ptl1_recurse_count_threshold = 0x40; 2294 0 stevel int ptl1_recurse_trap_threshold = 0x3d; 2295 0 stevel extern void ptl1_recurse(int, int); 2296 0 stevel extern void ptl1_panic_xt(int, int); 2297 0 stevel 2298 0 stevel /* 2299 0 stevel * Called once per second by timeout() to wake up 2300 0 stevel * the ptl1_panic thread to see if it should cause 2301 0 stevel * a trap to the ptl1_panic() code. 2302 0 stevel */ 2303 0 stevel /* ARGSUSED */ 2304 0 stevel static void 2305 0 stevel ptl1_wakeup(void *arg) 2306 0 stevel { 2307 0 stevel mutex_enter(&ptl1_mutex); 2308 0 stevel cv_signal(&ptl1_cv); 2309 0 stevel mutex_exit(&ptl1_mutex); 2310 0 stevel } 2311 0 stevel 2312 0 stevel /* 2313 0 stevel * ptl1_panic cross call function: 2314 0 stevel * Needed because xc_one() and xc_some() can pass 2315 0 stevel * 64 bit args but ptl1_recurse() expects ints. 2316 0 stevel */ 2317 0 stevel static void 2318 0 stevel ptl1_panic_xc(void) 2319 0 stevel { 2320 0 stevel ptl1_recurse(ptl1_recurse_count_threshold, 2321 0 stevel ptl1_recurse_trap_threshold); 2322 0 stevel } 2323 0 stevel 2324 0 stevel /* 2325 0 stevel * The ptl1 thread waits for a global flag to be set 2326 0 stevel * and uses the recurse thresholds to set the stack depth 2327 0 stevel * to cause a ptl1_panic() directly via a call to ptl1_recurse 2328 0 stevel * or indirectly via the cross call and cross trap functions. 2329 0 stevel * 2330 0 stevel * This is useful testing stack overflows and normal 2331 0 stevel * ptl1_panic() states with a know stack frame. 2332 0 stevel * 2333 0 stevel * ptl1_recurse() is an asm function in ptl1_panic.s that 2334 0 stevel * sets the {In, Local, Out, and Global} registers to a 2335 0 stevel * know state on the stack and just prior to causing a 2336 0 stevel * test ptl1_panic trap. 2337 0 stevel */ 2338 0 stevel static void 2339 0 stevel ptl1_thread(void) 2340 0 stevel { 2341 0 stevel mutex_enter(&ptl1_mutex); 2342 0 stevel while (ptl1_thread_p) { 2343 0 stevel cpuset_t other_cpus; 2344 0 stevel int cpu_id; 2345 0 stevel int my_cpu_id; 2346 0 stevel int target_cpu_id; 2347 0 stevel int target_found; 2348 0 stevel 2349 0 stevel if (ptl1_panic_test) { 2350 0 stevel ptl1_recurse(ptl1_recurse_count_threshold, 2351 0 stevel ptl1_recurse_trap_threshold); 2352 0 stevel } 2353 0 stevel 2354 0 stevel /* 2355 0 stevel * Find potential targets for x-call and x-trap, 2356 0 stevel * if any exist while preempt is disabled we 2357 0 stevel * start a ptl1_panic if requested via a 2358 0 stevel * globals. 2359 0 stevel */ 2360 0 stevel kpreempt_disable(); 2361 0 stevel my_cpu_id = CPU->cpu_id; 2362 0 stevel other_cpus = cpu_ready_set; 2363 0 stevel CPUSET_DEL(other_cpus, CPU->cpu_id); 2364 0 stevel target_found = 0; 2365 0 stevel if (!CPUSET_ISNULL(other_cpus)) { 2366 0 stevel /* 2367 0 stevel * Pick the first one 2368 0 stevel */ 2369 0 stevel for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2370 0 stevel if (cpu_id == my_cpu_id) 2371 0 stevel continue; 2372 0 stevel 2373 0 stevel if (CPU_XCALL_READY(cpu_id)) { 2374 0 stevel target_cpu_id = cpu_id; 2375 0 stevel target_found = 1; 2376 0 stevel break; 2377 0 stevel } 2378 0 stevel } 2379 0 stevel ASSERT(target_found); 2380 0 stevel 2381 0 stevel if (ptl1_panic_xc_one_test) { 2382 0 stevel xc_one(target_cpu_id, 2383 0 stevel (xcfunc_t *)ptl1_panic_xc, 0, 0); 2384 0 stevel } 2385 0 stevel if (ptl1_panic_xc_all_test) { 2386 0 stevel xc_some(other_cpus, 2387 0 stevel (xcfunc_t *)ptl1_panic_xc, 0, 0); 2388 0 stevel } 2389 0 stevel if (ptl1_panic_xt_one_test) { 2390 0 stevel xt_one(target_cpu_id, 2391 0 stevel (xcfunc_t *)ptl1_panic_xt, 0, 0); 2392 0 stevel } 2393 0 stevel if (ptl1_panic_xt_all_test) { 2394 0 stevel xt_some(other_cpus, 2395 0 stevel (xcfunc_t *)ptl1_panic_xt, 0, 0); 2396 0 stevel } 2397 0 stevel } 2398 0 stevel kpreempt_enable(); 2399 0 stevel (void) timeout(ptl1_wakeup, NULL, hz); 2400 0 stevel (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2401 0 stevel } 2402 0 stevel mutex_exit(&ptl1_mutex); 2403 0 stevel } 2404 0 stevel 2405 0 stevel /* 2406 0 stevel * Called during early startup to create the ptl1_thread 2407 0 stevel */ 2408 0 stevel void 2409 0 stevel init_ptl1_thread(void) 2410 0 stevel { 2411 0 stevel ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2412 0 stevel &p0, TS_RUN, 0); 2413 0 stevel } 2414 0 stevel #endif /* PTL1_PANIC_DEBUG */ 2415 0 stevel 2416 0 stevel 2417 5648 setje static void 2418 5648 setje memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 2419 5648 setje { 2420 5648 setje struct memlist *new; 2421 5648 setje 2422 5648 setje new = *memlistp; 2423 5648 setje new->address = start; 2424 5648 setje new->size = len; 2425 5648 setje *memlistp = new + 1; 2426 5648 setje } 2427 5648 setje 2428 0 stevel /* 2429 0 stevel * Add to a memory list. 2430 0 stevel * start = start of new memory segment 2431 0 stevel * len = length of new memory segment in bytes 2432 0 stevel * memlistp = pointer to array of available memory segment structures 2433 0 stevel * curmemlistp = memory list to which to add segment. 2434 0 stevel */ 2435 0 stevel static void 2436 0 stevel memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2437 0 stevel struct memlist **curmemlistp) 2438 0 stevel { 2439 5648 setje struct memlist *new = *memlistp; 2440 0 stevel 2441 5648 setje memlist_new(start, len, memlistp); 2442 5648 setje memlist_insert(new, curmemlistp); 2443 5648 setje } 2444 0 stevel 2445 5648 setje static int 2446 5648 setje ndata_alloc_memseg(struct memlist *ndata, size_t avail) 2447 5648 setje { 2448 5648 setje int nseg; 2449 5648 setje size_t memseg_sz; 2450 5648 setje struct memseg *msp; 2451 5648 setje 2452 5648 setje /* 2453 5648 setje * The memseg list is for the chunks of physical memory that 2454 5648 setje * will be managed by the vm system. The number calculated is 2455 5648 setje * a guess as boot may fragment it more when memory allocations 2456 5648 setje * are made before kphysm_init(). 2457 5648 setje */ 2458 5648 setje memseg_sz = (avail + 10) * sizeof (struct memseg); 2459 5648 setje memseg_sz = roundup(memseg_sz, PAGESIZE); 2460 5648 setje nseg = memseg_sz / sizeof (struct memseg); 2461 5648 setje msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 2462 5648 setje if (msp == NULL) 2463 5648 setje return (1); 2464 5648 setje PRM_DEBUG(memseg_free); 2465 5648 setje 2466 5648 setje while (nseg--) { 2467 5648 setje msp->next = memseg_free; 2468 5648 setje memseg_free = msp; 2469 5648 setje msp++; 2470 5648 setje } 2471 5648 setje return (0); 2472 0 stevel } 2473 0 stevel 2474 0 stevel /* 2475 0 stevel * In the case of architectures that support dynamic addition of 2476 0 stevel * memory at run-time there are two cases where memsegs need to 2477 0 stevel * be initialized and added to the memseg list. 2478 0 stevel * 1) memsegs that are constructed at startup. 2479 0 stevel * 2) memsegs that are constructed at run-time on 2480 0 stevel * hot-plug capable architectures. 2481 0 stevel * This code was originally part of the function kphysm_init(). 2482 0 stevel */ 2483 0 stevel 2484 0 stevel static void 2485 0 stevel memseg_list_add(struct memseg *memsegp) 2486 0 stevel { 2487 0 stevel struct memseg **prev_memsegp; 2488 0 stevel pgcnt_t num; 2489 0 stevel 2490 0 stevel /* insert in memseg list, decreasing number of pages order */ 2491 0 stevel 2492 0 stevel num = MSEG_NPAGES(memsegp); 2493 0 stevel 2494 0 stevel for (prev_memsegp = &memsegs; *prev_memsegp; 2495 0 stevel prev_memsegp = &((*prev_memsegp)->next)) { 2496 0 stevel if (num > MSEG_NPAGES(*prev_memsegp)) 2497 0 stevel break; 2498 0 stevel } 2499 0 stevel 2500 0 stevel memsegp->next = *prev_memsegp; 2501 0 stevel *prev_memsegp = memsegp; 2502 0 stevel 2503 0 stevel if (kpm_enable) { 2504 0 stevel memsegp->nextpa = (memsegp->next) ? 2505 4528 paulsan va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2506 0 stevel 2507 0 stevel if (prev_memsegp != &memsegs) { 2508 0 stevel struct memseg *msp; 2509 0 stevel msp = (struct memseg *)((caddr_t)prev_memsegp - 2510 4528 paulsan offsetof(struct memseg, next)); 2511 0 stevel msp->nextpa = va_to_pa(memsegp); 2512 0 stevel } else { 2513 0 stevel memsegspa = va_to_pa(memsegs); 2514 0 stevel } 2515 0 stevel } 2516 0 stevel } 2517 0 stevel 2518 0 stevel /* 2519 0 stevel * PSM add_physmem_cb(). US-II and newer processors have some 2520 0 stevel * flavor of the prefetch capability implemented. We exploit 2521 0 stevel * this capability for optimum performance. 2522 0 stevel */ 2523 0 stevel #define PREFETCH_BYTES 64 2524 0 stevel 2525 0 stevel void 2526 0 stevel add_physmem_cb(page_t *pp, pfn_t pnum) 2527 0 stevel { 2528 0 stevel extern void prefetch_page_w(void *); 2529 0 stevel 2530 0 stevel pp->p_pagenum = pnum; 2531 0 stevel 2532 0 stevel /* 2533 0 stevel * Prefetch one more page_t into E$. To prevent future 2534 0 stevel * mishaps with the sizeof(page_t) changing on us, we 2535 0 stevel * catch this on debug kernels if we can't bring in the 2536 0 stevel * entire hpage with 2 PREFETCH_BYTES reads. See 2537 0 stevel * also, sun4u/cpu/cpu_module.c 2538 0 stevel */ 2539 0 stevel /*LINTED*/ 2540 0 stevel ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2541 0 stevel prefetch_page_w((char *)pp); 2542 0 stevel } 2543 0 stevel 2544 0 stevel /* 2545 5648 setje * Find memseg with given pfn 2546 5648 setje */ 2547 5648 setje static struct memseg * 2548 5648 setje memseg_find(pfn_t base, pfn_t *next) 2549 5648 setje { 2550 5648 setje struct memseg *seg; 2551 5648 setje 2552 5648 setje if (next != NULL) 2553 5648 setje *next = LONG_MAX; 2554 5648 setje for (seg = memsegs; seg != NULL; seg = seg->next) { 2555 5648 setje if (base >= seg->pages_base && base < seg->pages_end) 2556 5648 setje return (seg); 2557 5648 setje if (next != NULL && seg->pages_base > base && 2558 5648 setje seg->pages_base < *next) 2559 5648 setje *next = seg->pages_base; 2560 5648 setje } 2561 5648 setje return (NULL); 2562 5648 setje } 2563 5648 setje 2564 5648 setje /* 2565 5648 setje * Put page allocated by OBP on prom_ppages 2566 0 stevel */ 2567 0 stevel static void 2568 5648 setje kphysm_erase(uint64_t addr, uint64_t len) 2569 0 stevel { 2570 5648 setje struct page *pp; 2571 5648 setje struct memseg *seg; 2572 5648 setje pfn_t base = btop(addr), next; 2573 5648 setje pgcnt_t num = btop(len); 2574 5648 setje 2575 5648 setje while (num != 0) { 2576 5648 setje pgcnt_t off, left; 2577 5648 setje 2578 5648 setje seg = memseg_find(base, &next); 2579 5648 setje if (seg == NULL) { 2580 5648 setje if (next == LONG_MAX) 2581 5648 setje break; 2582 5648 setje left = MIN(next - base, num); 2583 5648 setje base += left, num -= left; 2584 5648 setje continue; 2585 5648 setje } 2586 5648 setje off = base - seg->pages_base; 2587 5648 setje pp = seg->pages + off; 2588 5648 setje left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 2589 5648 setje while (num != left) { 2590 5648 setje /* 2591 5648 setje * init it, lock it, and hashin on prom_pages vp. 2592 5648 setje * 2593 10106 Jason * Mark it as NONRELOC to let DR know the page 2594 10106 Jason * is locked long term, otherwise DR hangs when 2595 10106 Jason * trying to remove those pages. 2596 10106 Jason * 2597 5648 setje * XXX vnode offsets on the prom_ppages vnode 2598 5648 setje * are page numbers (gack) for >32 bit 2599 5648 setje * physical memory machines. 2600 5648 setje */ 2601 10106 Jason PP_SETNORELOC(pp); 2602 5648 setje add_physmem_cb(pp, base); 2603 5648 setje if (page_trylock(pp, SE_EXCL) == 0) 2604 5648 setje cmn_err(CE_PANIC, "prom page locked"); 2605 11185 Sean (void) page_hashin(pp, &promvp, 2606 5648 setje (offset_t)base, NULL); 2607 5648 setje (void) page_pp_lock(pp, 0, 1); 2608 5648 setje pp++, base++, num--; 2609 5648 setje } 2610 5648 setje } 2611 5648 setje } 2612 5648 setje 2613 5648 setje static page_t *ppnext; 2614 5648 setje static pgcnt_t ppleft; 2615 5648 setje 2616 5648 setje static void *kpm_ppnext; 2617 5648 setje static pgcnt_t kpm_ppleft; 2618 5648 setje 2619 5648 setje /* 2620 5648 setje * Create a memseg 2621 5648 setje */ 2622 5648 setje static void 2623 5648 setje kphysm_memseg(uint64_t addr, uint64_t len) 2624 5648 setje { 2625 5648 setje pfn_t base = btop(addr); 2626 5648 setje pgcnt_t num = btop(len); 2627 5648 setje struct memseg *seg; 2628 5648 setje 2629 5648 setje seg = memseg_free; 2630 5648 setje memseg_free = seg->next; 2631 5648 setje ASSERT(seg != NULL); 2632 5648 setje 2633 5648 setje seg->pages = ppnext; 2634 5648 setje seg->epages = ppnext + num; 2635 5648 setje seg->pages_base = base; 2636 5648 setje seg->pages_end = base + num; 2637 5648 setje ppnext += num; 2638 5648 setje ppleft -= num; 2639 5648 setje 2640 5648 setje if (kpm_enable) { 2641 5648 setje pgcnt_t kpnum = ptokpmpr(num); 2642 5648 setje 2643 5648 setje if (kpnum > kpm_ppleft) 2644 5648 setje panic("kphysm_memseg: kpm_pp overflow"); 2645 5648 setje seg->pagespa = va_to_pa(seg->pages); 2646 5648 setje seg->epagespa = va_to_pa(seg->epages); 2647 5648 setje seg->kpm_pbase = kpmptop(ptokpmp(base)); 2648 5648 setje seg->kpm_nkpmpgs = kpnum; 2649 5648 setje /* 2650 5648 setje * In the kpm_smallpage case, the kpm array 2651 5648 setje * is 1-1 wrt the page array 2652 5648 setje */ 2653 5648 setje if (kpm_smallpages) { 2654 5648 setje kpm_spage_t *kpm_pp = kpm_ppnext; 2655 5648 setje 2656 5648 setje kpm_ppnext = kpm_pp + kpnum; 2657 5648 setje seg->kpm_spages = kpm_pp; 2658 5648 setje seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 2659 5648 setje } else { 2660 5648 setje kpm_page_t *kpm_pp = kpm_ppnext; 2661 5648 setje 2662 5648 setje kpm_ppnext = kpm_pp + kpnum; 2663 5648 setje seg->kpm_pages = kpm_pp; 2664 5648 setje seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 2665 5648 setje /* ASSERT no kpm overlaps */ 2666 5648 setje ASSERT( 2667 5648 setje memseg_find(base - pmodkpmp(base), NULL) == NULL); 2668 5648 setje ASSERT(memseg_find( 2669 5648 setje roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 2670 5648 setje } 2671 10106 Jason kpm_ppleft -= kpnum; 2672 5648 setje } 2673 5648 setje 2674 5648 setje memseg_list_add(seg); 2675 5648 setje } 2676 5648 setje 2677 5648 setje /* 2678 5648 setje * Add range to free list 2679 5648 setje */ 2680 5648 setje void 2681 5648 setje kphysm_add(uint64_t addr, uint64_t len, int reclaim) 2682 5648 setje { 2683 5648 setje struct page *pp; 2684 5648 setje struct memseg *seg; 2685 5648 setje pfn_t base = btop(addr); 2686 5648 setje pgcnt_t num = btop(len); 2687 5648 setje 2688 5648 setje seg = memseg_find(base, NULL); 2689 5648 setje ASSERT(seg != NULL); 2690 5648 setje pp = seg->pages + (base - seg->pages_base); 2691 5648 setje 2692 5648 setje if (reclaim) { 2693 5648 setje struct page *rpp = pp; 2694 5648 setje struct page *lpp = pp + num; 2695 5648 setje 2696 5648 setje /* 2697 5648 setje * page should be locked on prom_ppages 2698 5648 setje * unhash and unlock it 2699 5648 setje */ 2700 5648 setje while (rpp < lpp) { 2701 11185 Sean ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp); 2702 10106 Jason ASSERT(PP_ISNORELOC(rpp)); 2703 10106 Jason PP_CLRNORELOC(rpp); 2704 5648 setje page_pp_unlock(rpp, 0, 1); 2705 5648 setje page_hashout(rpp, NULL); 2706 5648 setje page_unlock(rpp); 2707 5648 setje rpp++; 2708 5648 setje } 2709 5648 setje } 2710 5648 setje 2711 5648 setje /* 2712 5648 setje * add_physmem() initializes the PSM part of the page 2713 5648 setje * struct by calling the PSM back with add_physmem_cb(). 2714 5648 setje * In addition it coalesces pages into larger pages as 2715 5648 setje * it initializes them. 2716 5648 setje */ 2717 5648 setje add_physmem(pp, num, base); 2718 5648 setje } 2719 5648 setje 2720 5648 setje /* 2721 5648 setje * kphysm_init() tackles the problem of initializing physical memory. 2722 5648 setje */ 2723 5648 setje static void 2724 5648 setje kphysm_init(void) 2725 5648 setje { 2726 5648 setje struct memlist *pmem; 2727 0 stevel 2728 0 stevel ASSERT(page_hash != NULL && page_hashsz != 0); 2729 0 stevel 2730 5648 setje ppnext = pp_base; 2731 5648 setje ppleft = npages; 2732 5648 setje kpm_ppnext = kpm_pp_base; 2733 5648 setje kpm_ppleft = kpm_npages; 2734 0 stevel 2735 5648 setje /* 2736 5648 setje * installed pages not on nopp_memlist go in memseg list 2737 5648 setje */ 2738 5648 setje diff_memlists(phys_install, nopp_list, kphysm_memseg); 2739 0 stevel 2740 5648 setje /* 2741 5648 setje * Free the avail list 2742 5648 setje */ 2743 5648 setje for (pmem = phys_avail; pmem != NULL; pmem = pmem->next) 2744 5648 setje kphysm_add(pmem->address, pmem->size, 0); 2745 0 stevel 2746 5648 setje /* 2747 5648 setje * Erase pages that aren't available 2748 5648 setje */ 2749 5648 setje diff_memlists(phys_install, phys_avail, kphysm_erase); 2750 0 stevel 2751 0 stevel build_pfn_hash(); 2752 0 stevel } 2753 0 stevel 2754 0 stevel /* 2755 0 stevel * Kernel VM initialization. 2756 0 stevel * Assumptions about kernel address space ordering: 2757 0 stevel * (1) gap (user space) 2758 0 stevel * (2) kernel text 2759 0 stevel * (3) kernel data/bss 2760 0 stevel * (4) gap 2761 0 stevel * (5) kernel data structures 2762 0 stevel * (6) gap 2763 0 stevel * (7) debugger (optional) 2764 0 stevel * (8) monitor 2765 0 stevel * (9) gap (possibly null) 2766 0 stevel * (10) dvma 2767 0 stevel * (11) devices 2768 0 stevel */ 2769 0 stevel static void 2770 0 stevel kvm_init(void) 2771 0 stevel { 2772 0 stevel /* 2773 0 stevel * Put the kernel segments in kernel address space. 2774 0 stevel */ 2775 0 stevel rw_enter(&kas.a_lock, RW_WRITER); 2776 0 stevel as_avlinit(&kas); 2777 0 stevel 2778 0 stevel (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2779 0 stevel (size_t)(e_moddata - KERNELBASE), &ktextseg); 2780 0 stevel (void) segkmem_create(&ktextseg); 2781 0 stevel 2782 0 stevel (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2783 0 stevel (size_t)(MMU_PAGESIZE4M), &ktexthole); 2784 0 stevel (void) segkmem_create(&ktexthole); 2785 0 stevel 2786 0 stevel (void) seg_attach(&kas, (caddr_t)valloc_base, 2787 0 stevel (size_t)(econtig32 - valloc_base), &kvalloc); 2788 0 stevel (void) segkmem_create(&kvalloc); 2789 0 stevel 2790 0 stevel if (kmem64_base) { 2791 4528 paulsan (void) seg_attach(&kas, (caddr_t)kmem64_base, 2792 4528 paulsan (size_t)(kmem64_end - kmem64_base), &kmem64); 2793 4528 paulsan (void) segkmem_create(&kmem64); 2794 0 stevel } 2795 0 stevel 2796 0 stevel /* 2797 0 stevel * We're about to map out /boot. This is the beginning of the 2798 0 stevel * system resource management transition. We can no longer 2799 0 stevel * call into /boot for I/O or memory allocations. 2800 0 stevel */ 2801 0 stevel (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2802 0 stevel (void) segkmem_create(&kvseg); 2803 0 stevel hblk_alloc_dynamic = 1; 2804 0 stevel 2805 0 stevel /* 2806 0 stevel * we need to preallocate pages for DR operations before enabling large 2807 0 stevel * page kernel heap because of memseg_remap_init() hat_unload() hack. 2808 0 stevel */ 2809 0 stevel memseg_remap_init(); 2810 0 stevel 2811 0 stevel /* at this point we are ready to use large page heap */ 2812 0 stevel segkmem_heap_lp_init(); 2813 0 stevel 2814 0 stevel (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2815 0 stevel &kvseg32); 2816 0 stevel (void) segkmem_create(&kvseg32); 2817 0 stevel 2818 0 stevel /* 2819 0 stevel * Create a segment for the debugger. 2820 0 stevel */ 2821 3446 mrj (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2822 0 stevel (void) segkmem_create(&kdebugseg); 2823 0 stevel 2824 0 stevel rw_exit(&kas.a_lock); 2825 0 stevel } 2826 0 stevel 2827 0 stevel char obp_tte_str[] = 2828 0 stevel "h# %x constant MMU_PAGESHIFT " 2829 0 stevel "h# %x constant TTE8K " 2830 0 stevel "h# %x constant SFHME_SIZE " 2831 0 stevel "h# %x constant SFHME_TTE " 2832 0 stevel "h# %x constant HMEBLK_TAG " 2833 0 stevel "h# %x constant HMEBLK_NEXT " 2834 0 stevel "h# %x constant HMEBLK_MISC " 2835 0 stevel "h# %x constant HMEBLK_HME1 " 2836 0 stevel "h# %x constant NHMENTS " 2837 0 stevel "h# %x constant HBLK_SZMASK " 2838 0 stevel "h# %x constant HBLK_RANGE_SHIFT " 2839 0 stevel "h# %x constant HMEBP_HBLK " 2840 8187 Paul "h# %x constant HMEBLK_ENDPA " 2841 0 stevel "h# %x constant HMEBUCKET_SIZE " 2842 0 stevel "h# %x constant HTAG_SFMMUPSZ " 2843 4528 paulsan "h# %x constant HTAG_BSPAGE_SHIFT " 2844 4528 paulsan "h# %x constant HTAG_REHASH_SHIFT " 2845 4528 paulsan "h# %x constant SFMMU_INVALID_SHMERID " 2846 0 stevel "h# %x constant mmu_hashcnt " 2847 0 stevel "h# %p constant uhme_hash " 2848 0 stevel "h# %p constant khme_hash " 2849 0 stevel "h# %x constant UHMEHASH_SZ " 2850 0 stevel "h# %x constant KHMEHASH_SZ " 2851 2241 huah "h# %p constant KCONTEXT " 2852 0 stevel "h# %p constant KHATID " 2853 0 stevel "h# %x constant ASI_MEM " 2854 0 stevel 2855 0 stevel ": PHYS-X@ ( phys -- data ) " 2856 0 stevel " ASI_MEM spacex@ " 2857 0 stevel "; " 2858 0 stevel 2859 0 stevel ": PHYS-W@ ( phys -- data ) " 2860 0 stevel " ASI_MEM spacew@ " 2861 0 stevel "; " 2862 0 stevel 2863 0 stevel ": PHYS-L@ ( phys -- data ) " 2864 0 stevel " ASI_MEM spaceL@ " 2865 0 stevel "; " 2866 0 stevel 2867 0 stevel ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2868 0 stevel " 3 * MMU_PAGESHIFT + " 2869 0 stevel "; " 2870 0 stevel 2871 0 stevel ": TTE_IS_VALID ( ttep -- flag ) " 2872 0 stevel " PHYS-X@ 0< " 2873 0 stevel "; " 2874 0 stevel 2875 0 stevel ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2876 0 stevel " dup TTE8K = if " 2877 0 stevel " drop HBLK_RANGE_SHIFT " 2878 0 stevel " else " 2879 0 stevel " TTE_PAGE_SHIFT " 2880 0 stevel " then " 2881 0 stevel "; " 2882 0 stevel 2883 0 stevel ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2884 0 stevel " tuck >> swap MMU_PAGESHIFT - << " 2885 0 stevel "; " 2886 0 stevel 2887 0 stevel ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2888 0 stevel " >> over xor swap ( hash sfmmup ) " 2889 0 stevel " KHATID <> if ( hash ) " 2890 0 stevel " UHMEHASH_SZ and ( bucket ) " 2891 0 stevel " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2892 0 stevel " else ( hash ) " 2893 0 stevel " KHMEHASH_SZ and ( bucket ) " 2894 0 stevel " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2895 0 stevel " then ( hmebp ) " 2896 0 stevel "; " 2897 0 stevel 2898 0 stevel ": HME_HASH_TABLE_SEARCH " 2899 0 stevel " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2900 0 stevel " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2901 8187 Paul " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) " 2902 0 stevel " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2903 0 stevel " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2904 0 stevel " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2905 0 stevel " else " 2906 0 stevel " hmeblk_next + phys-x@ false " 2907 0 stevel " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2908 0 stevel " then " 2909 0 stevel " else " 2910 0 stevel " hmeblk_next + phys-x@ false " 2911 0 stevel " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2912 0 stevel " then " 2913 0 stevel " else " 2914 8187 Paul " drop 0 true " 2915 0 stevel " then " 2916 0 stevel " until r> drop " 2917 0 stevel "; " 2918 0 stevel 2919 0 stevel ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2920 4528 paulsan " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2921 4528 paulsan " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2922 4528 paulsan " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2923 4528 paulsan " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2924 0 stevel "; " 2925 0 stevel 2926 0 stevel ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2927 0 stevel " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2928 0 stevel " TTE8K = if ( hmeblkp addr ) " 2929 0 stevel " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2930 0 stevel " else ( hmeblkp addr ) " 2931 0 stevel " drop 0 ( hmeblkp 0 ) " 2932 0 stevel " then ( hmeblkp hme-index ) " 2933 0 stevel " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2934 0 stevel " SFHME_TTE + ( ttep ) " 2935 0 stevel "; " 2936 0 stevel 2937 0 stevel ": unix-tte ( addr cnum -- false | tte-data true ) " 2938 2241 huah " KCONTEXT = if ( addr ) " 2939 2241 huah " KHATID ( addr khatid ) " 2940 2241 huah " else ( addr ) " 2941 2241 huah " drop false exit ( false ) " 2942 2241 huah " then " 2943 2241 huah " ( addr khatid ) " 2944 0 stevel " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2945 0 stevel " 2dup swap i HME_HASH_SHIFT " 2946 0 stevel "( addr sfmmup sfmmup addr hmeshift ) " 2947 0 stevel " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2948 0 stevel " over i 4 pick " 2949 0 stevel "( addr sfmmup hmebp sfmmup rehash addr ) " 2950 0 stevel " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2951 0 stevel " HME_HASH_TABLE_SEARCH " 2952 0 stevel "( addr sfmmup { null | hmeblkp } ) " 2953 0 stevel " ?dup if ( addr sfmmup hmeblkp ) " 2954 0 stevel " nip swap HBLK_TO_TTEP ( ttep ) " 2955 0 stevel " dup TTE_IS_VALID if ( valid-ttep ) " 2956 10271 Jason " PHYS-X@ true ( tte-data true ) " 2957 0 stevel " else ( invalid-tte ) " 2958 0 stevel " drop false ( false ) " 2959 0 stevel " then ( false | tte-data true ) " 2960 0 stevel " unloop exit ( false | tte-data true ) " 2961 0 stevel " then ( addr sfmmup ) " 2962 0 stevel " loop ( addr sfmmup ) " 2963 0 stevel " 2drop false ( false ) " 2964 0 stevel "; " 2965 0 stevel ; 2966 0 stevel 2967 0 stevel void 2968 0 stevel create_va_to_tte(void) 2969 0 stevel { 2970 0 stevel char *bp; 2971 0 stevel extern int khmehash_num, uhmehash_num; 2972 0 stevel extern struct hmehash_bucket *khme_hash, *uhme_hash; 2973 0 stevel 2974 0 stevel #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2975 0 stevel 2976 0 stevel bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2977 0 stevel 2978 0 stevel /* 2979 0 stevel * Teach obp how to parse our sw ttes. 2980 0 stevel */ 2981 0 stevel (void) sprintf(bp, obp_tte_str, 2982 0 stevel MMU_PAGESHIFT, 2983 0 stevel TTE8K, 2984 0 stevel sizeof (struct sf_hment), 2985 0 stevel OFFSET(struct sf_hment, hme_tte), 2986 0 stevel OFFSET(struct hme_blk, hblk_tag), 2987 0 stevel OFFSET(struct hme_blk, hblk_nextpa), 2988 0 stevel OFFSET(struct hme_blk, hblk_misc), 2989 0 stevel OFFSET(struct hme_blk, hblk_hme), 2990 0 stevel NHMENTS, 2991 0 stevel HBLK_SZMASK, 2992 0 stevel HBLK_RANGE_SHIFT, 2993 0 stevel OFFSET(struct hmehash_bucket, hmeh_nextpa), 2994 8187 Paul HMEBLK_ENDPA, 2995 0 stevel sizeof (struct hmehash_bucket), 2996 0 stevel HTAG_SFMMUPSZ, 2997 4528 paulsan HTAG_BSPAGE_SHIFT, 2998 4528 paulsan HTAG_REHASH_SHIFT, 2999 4528 paulsan SFMMU_INVALID_SHMERID, 3000 0 stevel mmu_hashcnt, 3001 0 stevel (caddr_t)va_to_pa((caddr_t)uhme_hash), 3002 0 stevel (caddr_t)va_to_pa((caddr_t)khme_hash), 3003 0 stevel UHMEHASH_SZ, 3004 0 stevel KHMEHASH_SZ, 3005 2241 huah KCONTEXT, 3006 0 stevel KHATID, 3007 10271 Jason ASI_MEM); 3008 0 stevel prom_interpret(bp, 0, 0, 0, 0, 0); 3009 0 stevel 3010 0 stevel kobj_free(bp, MMU_PAGESIZE); 3011 0 stevel } 3012 0 stevel 3013 0 stevel void 3014 0 stevel install_va_to_tte(void) 3015 0 stevel { 3016 0 stevel /* 3017 0 stevel * advise prom that he can use unix-tte 3018 0 stevel */ 3019 0 stevel prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 3020 0 stevel } 3021 0 stevel 3022 1253 lq150181 /* 3023 3928 lq150181 * Here we add "device-type=console" for /os-io node, for currently 3024 3928 lq150181 * our kernel console output only supports displaying text and 3025 3928 lq150181 * performing cursor-positioning operations (through kernel framebuffer 3026 3928 lq150181 * driver) and it doesn't support other functionalities required for a 3027 3928 lq150181 * standard "display" device as specified in 1275 spec. The main missing 3028 3928 lq150181 * interface defined by the 1275 spec is "draw-logo". 3029 3928 lq150181 * also see the comments above prom_stdout_is_framebuffer(). 3030 1253 lq150181 */ 3031 0 stevel static char *create_node = 3032 1492 lq150181 "\" /\" find-device " 3033 0 stevel "new-device " 3034 0 stevel "\" os-io\" device-name " 3035 3928 lq150181 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 3036 0 stevel ": cb-r/w ( adr,len method$ -- #read/#written ) " 3037 0 stevel " 2>r swap 2 2r> ['] $callback catch if " 3038 0 stevel " 2drop 3drop 0 " 3039 0 stevel " then " 3040 0 stevel "; " 3041 0 stevel ": read ( adr,len -- #read ) " 3042 0 stevel " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 3043 0 stevel " ( retN ... ret1 N ) " 3044 0 stevel " ?dup if " 3045 0 stevel " swap >r 1- 0 ?do drop loop r> " 3046 0 stevel " else " 3047 0 stevel " -2 " 3048 1492 lq150181 " then " 3049 0 stevel "; " 3050 0 stevel ": write ( adr,len -- #written ) " 3051 0 stevel " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3052 0 stevel " ( retN ... ret1 N ) " 3053 0 stevel " ?dup if " 3054 0 stevel " swap >r 1- 0 ?do drop loop r> " 3055 0 stevel " else " 3056 0 stevel " 0 " 3057 0 stevel " then " 3058 0 stevel "; " 3059 0 stevel ": poll-tty ( -- ) ; " 3060 0 stevel ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3061 0 stevel ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3062 0 stevel ": cb-give/take ( $method -- ) " 3063 0 stevel " 0 -rot ['] $callback catch ?dup if " 3064 0 stevel " >r 2drop 2drop r> throw " 3065 0 stevel " else " 3066 0 stevel " 0 ?do drop loop " 3067 0 stevel " then " 3068 0 stevel "; " 3069 0 stevel ": give ( -- ) \" exit-input\" cb-give/take ; " 3070 0 stevel ": take ( -- ) \" enter-input\" cb-give/take ; " 3071 0 stevel ": open ( -- ok? ) true ; " 3072 0 stevel ": close ( -- ) ; " 3073 0 stevel "finish-device " 3074 0 stevel "device-end "; 3075 0 stevel 3076 0 stevel /* 3077 1253 lq150181 * Create the OBP input/output node (FCode serial driver). 3078 1253 lq150181 * It is needed for both USB console keyboard and for 3079 1253 lq150181 * the kernel terminal emulator. It is too early to check for a 3080 1253 lq150181 * kernel console compatible framebuffer now, so we create this 3081 1253 lq150181 * so that we're ready if we need to enable kernel terminal emulation. 3082 0 stevel * 3083 1253 lq150181 * When the USB software takes over the input device at the time 3084 1253 lq150181 * consconfig runs, OBP's stdin is redirected to this node. 3085 1253 lq150181 * Whenever the FORTH user interface is used after this switch, 3086 1253 lq150181 * the node will call back into the kernel for console input. 3087 1253 lq150181 * If a serial device such as ttya or a UART with a Type 5 keyboard 3088 1253 lq150181 * attached is used, OBP takes over the serial device when the system 3089 1253 lq150181 * goes to the debugger after the system is booted. This sharing 3090 1253 lq150181 * of the relatively simple serial device is difficult but possible. 3091 1253 lq150181 * Sharing the USB host controller is impossible due its complexity. 3092 1253 lq150181 * 3093 1253 lq150181 * Similarly to USB keyboard input redirection, after consconfig_dacf 3094 1253 lq150181 * configures a kernel console framebuffer as the standard output 3095 1253 lq150181 * device, OBP's stdout is switched to to vector through the 3096 1253 lq150181 * /os-io node into the kernel terminal emulator. 3097 0 stevel */ 3098 0 stevel static void 3099 1253 lq150181 startup_create_io_node(void) 3100 0 stevel { 3101 1253 lq150181 prom_interpret(create_node, 0, 0, 0, 0, 0); 3102 0 stevel } 3103 0 stevel 3104 0 stevel 3105 0 stevel static void 3106 0 stevel do_prom_version_check(void) 3107 0 stevel { 3108 0 stevel int i; 3109 789 ahrens pnode_t node; 3110 0 stevel char buf[64]; 3111 0 stevel static char drev[] = "Down-rev firmware detected%s\n" 3112 4528 paulsan "\tPlease upgrade to the following minimum version:\n" 3113 4528 paulsan "\t\t%s\n"; 3114 0 stevel 3115 0 stevel i = prom_version_check(buf, sizeof (buf), &node); 3116 0 stevel 3117 0 stevel if (i == PROM_VER64_OK) 3118 0 stevel return; 3119 0 stevel 3120 0 stevel if (i == PROM_VER64_UPGRADE) { 3121 0 stevel cmn_err(CE_WARN, drev, "", buf); 3122 0 stevel 3123 0 stevel #ifdef DEBUG 3124 0 stevel prom_enter_mon(); /* Type 'go' to continue */ 3125 0 stevel cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3126 0 stevel return; 3127 0 stevel #else 3128 0 stevel halt(0); 3129 0 stevel #endif 3130 0 stevel } 3131 0 stevel 3132 0 stevel /* 3133 0 stevel * The other possibility is that this is a server running 3134 0 stevel * good firmware, but down-rev firmware was detected on at 3135 0 stevel * least one other cpu board. We just complain if we see 3136 0 stevel * that. 3137 0 stevel */ 3138 0 stevel cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3139 0 stevel } 3140 0 stevel 3141 0 stevel 3142 0 stevel /* 3143 0 stevel * Must be defined in platform dependent code. 3144 0 stevel */ 3145 0 stevel extern caddr_t modtext; 3146 0 stevel extern size_t modtext_sz; 3147 0 stevel extern caddr_t moddata; 3148 0 stevel 3149 0 stevel #define HEAPTEXT_ARENA(addr) \ 3150 0 stevel ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3151 0 stevel (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3152 0 stevel (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3153 0 stevel 3154 0 stevel #define HEAPTEXT_OVERSIZED(addr) \ 3155 0 stevel ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3156 0 stevel 3157 7428 Jonathan #define HEAPTEXT_IN_NUCLEUSDATA(addr) \ 3158 7428 Jonathan (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \ 3159 7428 Jonathan ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M)) 3160 7428 Jonathan 3161 0 stevel vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3162 0 stevel vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3163 0 stevel kmutex_t texthole_lock; 3164 0 stevel 3165 0 stevel char kern_bootargs[OBP_MAXPATHLEN]; 3166 7656 Sherry char kern_bootfile[OBP_MAXPATHLEN]; 3167 0 stevel 3168 0 stevel void 3169 0 stevel kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3170 0 stevel { 3171 0 stevel uintptr_t addr, limit; 3172 0 stevel 3173 0 stevel addr = HEAPTEXT_BASE; 3174 0 stevel limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3175 0 stevel 3176 0 stevel /* 3177 0 stevel * Before we initialize the text_arena, we want to punch holes in the 3178 0 stevel * underlying heaptext_arena. This guarantees that for any text 3179 0 stevel * address we can find a text hole less than HEAPTEXT_MAPPED away. 3180 0 stevel */ 3181 0 stevel for (; addr + HEAPTEXT_UNMAPPED <= limit; 3182 0 stevel addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3183 0 stevel (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3184 0 stevel 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3185 0 stevel VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3186 0 stevel } 3187 0 stevel 3188 0 stevel /* 3189 0 stevel * Allocate one page at the oversize to break up the text region 3190 0 stevel * from the oversized region. 3191 0 stevel */ 3192 0 stevel (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3193 0 stevel (void *)limit, (void *)(limit + PAGESIZE), 3194 0 stevel VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3195 0 stevel 3196 3791 kchow *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3197 3791 kchow modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3198 0 stevel heaptext_arena, 0, VM_SLEEP); 3199 0 stevel *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3200 0 stevel segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3201 0 stevel } 3202 0 stevel 3203 0 stevel caddr_t 3204 0 stevel kobj_text_alloc(vmem_t *arena, size_t size) 3205 0 stevel { 3206 0 stevel caddr_t rval, better; 3207 0 stevel 3208 0 stevel /* 3209 0 stevel * First, try a sleeping allocation. 3210 0 stevel */ 3211 0 stevel rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3212 0 stevel 3213 0 stevel if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3214 0 stevel return (rval); 3215 0 stevel 3216 0 stevel /* 3217 0 stevel * We didn't get the area that we wanted. We're going to try to do an 3218 0 stevel * allocation with explicit constraints. 3219 0 stevel */ 3220 0 stevel better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3221 0 stevel (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3222 0 stevel VM_NOSLEEP | VM_BESTFIT); 3223 0 stevel 3224 0 stevel if (better != NULL) { 3225 0 stevel /* 3226 0 stevel * That worked. Free our first attempt and return. 3227 0 stevel */ 3228 0 stevel vmem_free(arena, rval, size); 3229 0 stevel return (better); 3230 0 stevel } 3231 0 stevel 3232 0 stevel /* 3233 0 stevel * That didn't work; we'll have to return our first attempt. 3234 0 stevel */ 3235 0 stevel return (rval); 3236 0 stevel } 3237