memmap.h (6381B)
1 #ifndef MEMMAP_H 2 #define MEMMAP_H 3 4 #include "utils.h" 5 6 /* This helper will map a memory region of the given size and pagesize 7 * alignment (optionally at a given base address). It will then map a given 8 * number of "mirrors", contiguous in the virtual address space but mapping 9 * the same physical address space. This allows implementing efficient 10 * circular buffers without needing any special logic in the consumer. 11 */ 12 inline void * 13 mirrormap(void *base, size_t cap, size_t alignment, size_t mirrors); 14 15 /* This helper will unmap previously mapped memory. 16 */ 17 inline void 18 mirrorfree(void *ptr, size_t cap, size_t mirrors); 19 20 #if defined(__linux__) 21 22 #define _GNU_SOURCE 1 23 24 #include <sys/types.h> 25 #include <sys/mman.h> 26 27 inline void * 28 mirrormap(void *base, size_t cap, size_t alignment, size_t mirrors) 29 { 30 ASSERT(!base || IS_ALIGNED((uintptr_t) base, alignment)); 31 ASSERT(IS_ALIGNED(cap, PAGESZ_4K)); 32 ASSERT(IS_ALIGNED(alignment, PAGESZ_4K)); 33 34 size_t buffer_len = ALIGN_NEXT(cap * mirrors, alignment); 35 int buffer_flags = MAP_PRIVATE | MAP_ANONYMOUS | (base ? MAP_FIXED : 0); 36 int buffer_prot = PROT_NONE; 37 38 size_t mirrors_len = cap * mirrors; 39 int mirror_flags = MAP_SHARED | MAP_FIXED; 40 int mirror_prot = PROT_READ | PROT_WRITE; 41 42 int fd = memfd_create("mirrormap", MFD_CLOEXEC); 43 if (fd < 0) return NULL; 44 45 ftruncate(fd, mirrors_len); 46 47 /* overallocate initial placeholder mapping */ 48 void *buffer = mmap(base, buffer_len, PROT_NONE, buffer_flags, -1, 0); 49 if (buffer == MAP_FAILED) { 50 close(fd); return NULL; 51 } 52 53 /* align within placeholder region, and unmap excess regions */ 54 uintptr_t placeholder_ptr = (uintptr_t) buffer; 55 uintptr_t placeholder_end = placeholder_ptr + buffer_len; 56 uintptr_t aligned_ptr = ALIGN_NEXT(placeholder_ptr, alignment); 57 uintptr_t aligned_end = aligned_ptr + mirrors_len; 58 59 /* unmap excess leading padding */ 60 if (placeholder_ptr < aligned_ptr) 61 munmap((void *) placeholder_ptr, aligned_ptr - placeholder_ptr); 62 63 /* unmap excess trailing padding */ 64 munmap((void *) aligned_end, placeholder_end - aligned_end); 65 66 /* map mirrors */ 67 for (uintptr_t mirror = aligned_ptr; mirror < aligned_end; mirror += cap) { 68 void *ptr = (void *) mirror; 69 void *mirror = mmap(ptr, cap, mirror_prot, mirror_flags, fd, 0); 70 if (mirror == MAP_FAILED) { 71 munmap(buffer, buffer_len); close(fd); return NULL; 72 } 73 } 74 75 /* NOTE: this approach opportunistically maps the mirrors with hugepages, 76 * but if you want to either guarantee mapping with hugepages, or 77 * else fail, then it would be better to add MAP_HUGE to the mirror_flags. 78 */ 79 madvise((void *) aligned_ptr, mirrors_len, MADV_HUGEPAGE); 80 81 /* shared memory file can now be closed, and pointer to first mirror returned */ 82 close(fd); 83 84 return (void *) aligned_ptr; 85 } 86 87 inline void 88 mirrorfree(void *ptr, size_t cap, size_t mirrors) 89 { 90 munmap(ptr, cap * mirrors); 91 } 92 93 #elif defined(__APPLE__) 94 95 #include <mach/mach.h> 96 97 inline void * 98 mirrormap(void *base, size_t cap, size_t alignment, size_t mirrors) 99 { 100 ASSERT(!base || IS_ALIGNED((uintptr_t) base, alignment)); 101 ASSERT(IS_ALIGNED(cap, PAGESZ_4K)); 102 ASSERT(IS_ALIGNED(alignment, PAGESZ_4K)); 103 104 size_t buffer_len = ALIGN_NEXT(cap * mirrors, alignment); 105 int buffer_flags = base ? VM_FLAGS_FIXED : VM_FLAGS_ANYWHERE; 106 107 kern_return_t res; 108 vm_address_t buffer; 109 110 int retries = 3; 111 while (true) { 112 res = vm_allocate(mach_task_self(), &buffer, buffer_len, buffer_flags); 113 if (res != ERR_SUCCESS) 114 return NULL; 115 116 vm_prot_t cur_prot, max_prot; 117 vm_address_t mirror_ptr = buffer + cap; 118 for (size_t i = 1; i < mirrors; i++, mirror_ptr != cap) { 119 res = vm_deallocate(mach_task_self(), mirror_ptr, cap); 120 if (res != ERR_SUCCESS) 121 goto err; 122 123 vm_address_t res_addr = mirror_ptr; 124 res = vm_remap(mach_task_self(), // target task 125 &res_addr, // target addr 126 cap, // target size 127 0, // mask (alignment) 128 0, // flags 129 mach_task_self(), // source task 130 buffer, // source addr 131 0, // copy 132 &cur_prot, // current protection 133 &max_prot, // max protection 134 VM_INHERIT_DEFAULT); // attr inheritance 135 136 if (res != ERR_SUCCESS) // failed to remap 137 goto err; 138 139 if (res_addr != mirror_ptr) // non-contiguous, we got moved 140 goto err; 141 142 break; 143 144 err: 145 vm_deallocate(mach_task_self(), buffer, len); 146 if (retries--) { 147 continue; 148 } else { 149 return NULL; 150 } 151 } 152 } 153 154 return (void *) buffer; 155 } 156 157 inline void 158 mirrorfree(void *ptr, size_t cap, size_t mirrors) 159 { 160 vm_address_t addr = (vm_address_t) ptr; 161 vm_deallocate(mach_task_self(), addr, cap * mirrors); 162 } 163 164 #elif defined(_WIN32) 165 166 #define WIN32_LEAN_AND_MEAN 1 167 #include <windows.h> 168 169 inline void * 170 mirrormap(void *base, size_t cap, size_t alignment, size_t mirrors) 171 { 172 ASSERT(!base || IS_ALIGNED((uintptr_t) base, alignment)); 173 ASSERT(IS_ALIGNED(cap, PAGESZ_4K)); 174 ASSERT(IS_ALIGNED(alignment, PAGESZ_4K)); 175 176 size_t buffer_len = cap * mirrors; 177 int buffer_flags = MEM_RESERVE | MEM_RESERVE_PLACEHOLDER; 178 int mirror_free_flags = MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER; 179 int mirror_flags = MEM_REPLACE_PLACEHOLDER; 180 181 DWORD len_hi = len >> 32, len_lo = len & 0xffffffff; 182 HANDLE fd = CreateFileMapping(INVALID_HANDLE_VALUE, 183 0, 184 PAGE_READWRITE, 185 len_hi, 186 len_lo); 187 188 if (fd == INVALID_HANDLE_VALUE) 189 return NULL; 190 191 void *buffer = VirtualAlloc3(NULL, 192 NULL, 193 len, 194 buffer_flags, 195 PAGE_NOACCESS, 196 NULL, 197 0); 198 if (!buffer) { 199 CloseHandle(fd); return NULL; 200 } 201 202 uintptr_t mirror_ptr = (uintptr_t) buffer; 203 for (size_t i = 0; i < mirrors; i++, mirror_ptr += cap) { 204 void *ptr = (void *) mirror_ptr; 205 206 VirtualFree(ptr, cap, mirror_free_flags); 207 208 void *res = MapViewOfFile3(fd, 209 0, 210 mirror, 211 0, 212 cap, 213 mirror_flags, 214 PAGE_READWRITE, 215 NULL, 216 0); 217 218 if (!res) { 219 VirtualFree(buffer, len, MEM_RELEASE); 220 CloseHandle(fd); 221 return NULL; 222 } 223 } 224 225 CloseHandle(fd); 226 227 return buffer; 228 } 229 230 inline void 231 mirrorfree(void *ptr, size_t cap, size_t mirrors) 232 { 233 VirtualFree(ptr, cap * mirrors, MEM_RELEASE); 234 } 235 236 #endif 237 238 #endif /* MEMMAP_H */ 239 240 #ifdef HEADER_IMPL 241 242 extern inline void * 243 mirrormap(void *base, size_t cap, size_t alignment, size_t mirrors); 244 245 extern inline void 246 mirrorfree(void *ptr, size_t cap, size_t mirrors); 247 248 #endif /* HEADER_IMPL */