The Vmem system is the way to allocate a range of addresses. Of course, the Vmem system itself depends on having certain address ranges to keep its data structures; moreover, it manages them via itself: > ::vmem ! head -n 7 ADDR NAME INUSE TOTAL SUCCEED FAIL fffffffffbcca900 heap 340680704 1092913987584 17573 0 fffffffffbccb738 vmem_metadata 8761344 8781824 2058 0 fffffffffbccc570 vmem_seg 7888896 7888896 1914 0 fffffffffbccd3a8 vmem_hash 596992 602112 37 0 fffffffffbcce1e0 vmem_vmem 276640 306736 134 0 fffffffffbcd0c88 static 0 0 0 0 This should give you a pause. If "heap" is created by Vmem before the "vmem_*" arenas can be are allocated for its own internal resources, isn't this an endless recursive relationship, a vicious circle of dependencies? The answer is, of course, that some (just enough) Vmem structs are pre-allocated and help bootstrap the system. This is done by creating some global static arrays for the needed structures just large enough to bootstrap the rest, and then using structs from these arrays when creating the above arenas. In /illumos-gate/usr/src/uts/common/os/vmem.c : 224 #define VMEM_INITIAL 10 /* early vmem arenas */ 225 #define VMEM_SEG_INITIAL 200 /* early segments */ 311 static vmem_t vmem0[VMEM_INITIAL]; 312 static vmem_t *vmem_populator[VMEM_INITIAL]; 313 static uint32_t vmem_id; 314 static uint32_t vmem_populators; 315 static vmem_seg_t vmem_seg0[VMEM_SEG_INITIAL]; Then function that creates "heap", vmem_init, is called from kernelheap_init() in /illumos-gate/usr/src/uts/common/vm/seg_kmem.c : 269 heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap; // <-- think of these as constants of the kernel AS layout 270 heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE, 271 segkmem_alloc, segkmem_free); (back to vmem.c): 1741 /* 1742 * Prepare vmem for use. 1743 */ 1744 vmem_t * 1745 vmem_init(const char *heap_name, 1746 void *heap_start, size_t heap_size, size_t heap_quantum, 1747 void *(*heap_alloc)(vmem_t *, size_t, int), 1748 void (*heap_free)(vmem_t *, void *, size_t)) 1749 { 1750 uint32_t id; 1751 int nseg = VMEM_SEG_INITIAL; 1752 vmem_t *heap; 1753 1754 while (--nseg >= 0) 1755 vmem_putseg_global(&vmem_seg0[nseg]); //<--- hang it on the vmem_segfree list, one by one 1756 1757 heap = vmem_create(heap_name, // it's "heap", nice to meet ya. There's only one so created. 1758 heap_start, heap_size, heap_quantum, // see above. 1759 NULL, NULL, NULL, 0, 1760 VM_SLEEP | VMC_POPULATOR); 1761 1762 vmem_metadata_arena = vmem_create("vmem_metadata", 1763 NULL, 0, heap_quantum, 1764 vmem_alloc, vmem_free, heap, 8 * heap_quantum, 1765 VM_SLEEP | VMC_POPULATOR | VMC_NO_QCACHE); 1766 1767 vmem_seg_arena = vmem_create("vmem_seg", 1768 NULL, 0, heap_quantum, 1769 heap_alloc, heap_free, vmem_metadata_arena, 0, 1770 VM_SLEEP | VMC_POPULATOR); 1771 1772 vmem_hash_arena = vmem_create("vmem_hash", 1773 NULL, 0, 8, 1774 heap_alloc, heap_free, vmem_metadata_arena, 0, 1775 VM_SLEEP); 1776 1777 vmem_vmem_arena = vmem_create("vmem_vmem", 1778 vmem0, sizeof (vmem0), 1, // <---- note that this arena will start at vmem0 1779 heap_alloc, heap_free, vmem_metadata_arena, 0, 1780 VM_SLEEP); 1781 1782 for (id = 0; id < vmem_id; id++) // <--------- see below, gets incremented under vmem_create() 1783 (void) vmem_xalloc(vmem_vmem_arena, sizeof (vmem_t), 1784 1, 0, 0, &vmem0[id], &vmem0[id + 1], 1785 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1786 1787 return (heap); 1788 } Observe the nesting of these arenas, consistent with what ::vmem shows. What is going on in the last loop? Where is vmem_d incremented? Well, vmem_xalloc() is called to allocate a vmem_t-sized chunk "anywhere" between the minaddr &vmem0[id] and the maxaddr &vmem0[id + 1] ---i.e., where there's just enough space for one (the alignment if "1" is also trivial; phase and nocross args are both 0). The returned address (it should be &vmem0[id], of course) is discarded. The effect of these calls, then, is to mark these slots in vmem0[] as busy, i.e., already allocated segments. The function vmem_create() almost immediately tail-calls vmem_create_common(), and in that one we see another mention of vmem0[] and the atomic increment of vmem_id: 1444 static vmem_t * 1445 vmem_create_common(const char *name, void *base, size_t size, size_t quantum, 1446 void *(*afunc)(vmem_t *, size_t, int), 1447 void (*ffunc)(vmem_t *, void *, size_t), 1448 vmem_t *source, size_t qcache_max, int vmflag) 1449 { 1450 int i; 1451 size_t nqcache; 1452 vmem_t *vmp, *cur, **vmpp; 1453 vmem_seg_t *vsp; 1454 vmem_freelist_t *vfp; 1455 uint32_t id = atomic_inc_32_nv(&vmem_id); // <---- atomic increment, only here 1456 1457 if (vmem_vmem_arena != NULL) { 1458 vmp = vmem_alloc(vmem_vmem_arena, sizeof (vmem_t), 1459 vmflag & VM_KMFLAGS); 1460 } else { // <---- special case for bootstrapping 1461 ASSERT(id <= VMEM_INITIAL); 1462 vmp = &vmem0[id - 1]; 1463 } Notice that _all_ the calls to vmem_create() above will fall through to the bootstrapping case, because the vmem_vmem_arena's value gets set to non-zero only by the last call, when it succeeds; it will stay NULL throughout before then. Vmem heaps also need segments, not just vmem_t structs. The initial 200 segments come from the vmem_seg0 array and get put on the global vmem_segfree list one by one; they get consumed from that list by vmem_getseg_global() in vmem_populate(): 669 static int 670 vmem_populate(vmem_t *vmp, int vmflag) 671 { 672 char *p; 673 vmem_seg_t *vsp; 674 ssize_t nseg; 675 size_t size; 676 kmutex_t *lp; 677 int i; 678 679 while (vmp->vm_nsegfree < VMEM_MINFREE && 680 (vsp = vmem_getseg_global()) != NULL) 681 vmem_putseg(vmp, vsp); 682 683 if (vmp->vm_nsegfree >= VMEM_MINFREE) 684 return (1); 685 686 /* 687 * If we're already populating, tap the reserve. 688 */ 689 if (vmem_is_populator()) { 690 ASSERT(vmp->vm_cflags & VMC_POPULATOR); 691 return (1); 692 } 693 694 mutex_exit(&vmp->vm_lock); Notice that vmem_populate() releases a mutex, but doesn't take it! It is always meant to be called with that mutex taken; indeed, it's point is to grab enough segment structs to last the caller. See comment section 2.5, "Vmem Population", at line 192 of vmem.c. Naturally, vmem0 and vmem_seg0 are static, i.e., file-scoped.