| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (C) 2013 Google Inc. All rights reserved. | 2 * Copyright (C) 2013 Google Inc. All rights reserved. |
| 3 * | 3 * |
| 4 * Redistribution and use in source and binary forms, with or without | 4 * Redistribution and use in source and binary forms, with or without |
| 5 * modification, are permitted provided that the following conditions are | 5 * modification, are permitted provided that the following conditions are |
| 6 * met: | 6 * met: |
| 7 * | 7 * |
| 8 * * Redistributions of source code must retain the above copyright | 8 * * Redistributions of source code must retain the above copyright |
| 9 * notice, this list of conditions and the following disclaimer. | 9 * notice, this list of conditions and the following disclaimer. |
| 10 * * Redistributions in binary form must reproduce the above | 10 * * Redistributions in binary form must reproduce the above |
| (...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 53 | 53 |
| 54 #elif OS(WIN) | 54 #elif OS(WIN) |
| 55 | 55 |
| 56 #include <windows.h> | 56 #include <windows.h> |
| 57 | 57 |
| 58 // VirtualAlloc will fail if allocation at the hint address is blocked. | 58 // VirtualAlloc will fail if allocation at the hint address is blocked. |
| 59 static const bool kHintIsAdvisory = false; | 59 static const bool kHintIsAdvisory = false; |
| 60 | 60 |
| 61 #else | 61 #else |
| 62 #error Unknown OS | 62 #error Unknown OS |
| 63 #endif // OS(POSIX) | 63 #endif // OS(POSIX) |
| 64 | 64 |
| 65 namespace WTF { | 65 namespace WTF { |
| 66 | 66 |
| 67 // This internal function wraps the OS-specific page allocation call. The | 67 // This internal function wraps the OS-specific page allocation call. The |
| 68 // behavior of the hint address is determined by the kHintIsAdvisory constant. | 68 // behavior of the hint address is determined by the kHintIsAdvisory constant. |
| 69 // If true, a non-zero hint is advisory and the returned address may differ from | 69 // If true, a non-zero hint is advisory and the returned address may differ from |
| 70 // the hint. If false, the hint is mandatory and a successful allocation will | 70 // the hint. If false, the hint is mandatory and a successful allocation will |
| 71 // not differ from the hint. | 71 // not differ from the hint. |
| 72 static void* systemAllocPages(void* hint, size_t len, PageAccessibilityConfigura
tion pageAccessibility) | 72 static void* systemAllocPages(void* hint, size_t len, PageAccessibilityConfigura
tion pageAccessibility) { |
| 73 { | 73 ASSERT(!(len & kPageAllocationGranularityOffsetMask)); |
| 74 ASSERT(!(len & kPageAllocationGranularityOffsetMask)); | 74 ASSERT(!(reinterpret_cast<uintptr_t>(hint) & kPageAllocationGranularityOffsetM
ask)); |
| 75 ASSERT(!(reinterpret_cast<uintptr_t>(hint) & kPageAllocationGranularityOffse
tMask)); | 75 void* ret; |
| 76 void* ret; | |
| 77 #if OS(WIN) | 76 #if OS(WIN) |
| 78 DWORD accessFlag = pageAccessibility == PageAccessible ? PAGE_READWRITE : PA
GE_NOACCESS; | 77 DWORD accessFlag = pageAccessibility == PageAccessible ? PAGE_READWRITE : PAGE
_NOACCESS; |
| 79 ret = VirtualAlloc(hint, len, MEM_RESERVE | MEM_COMMIT, accessFlag); | 78 ret = VirtualAlloc(hint, len, MEM_RESERVE | MEM_COMMIT, accessFlag); |
| 80 #else | 79 #else |
| 81 int accessFlag = pageAccessibility == PageAccessible ? (PROT_READ | PROT_WRI
TE) : PROT_NONE; | 80 int accessFlag = pageAccessibility == PageAccessible ? (PROT_READ | PROT_WRITE
) : PROT_NONE; |
| 82 ret = mmap(hint, len, accessFlag, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); | 81 ret = mmap(hint, len, accessFlag, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); |
| 83 if (ret == MAP_FAILED) | 82 if (ret == MAP_FAILED) |
| 84 ret = 0; | 83 ret = 0; |
| 85 #endif | 84 #endif |
| 86 return ret; | 85 return ret; |
| 87 } | 86 } |
| 88 | 87 |
| 89 // Trims base to given length and alignment. Windows returns null on failure and
frees base. | 88 // Trims base to given length and alignment. Windows returns null on failure and
frees base. |
| 90 static void* trimMapping(void *base, size_t baseLen, size_t trimLen, uintptr_t a
lign, PageAccessibilityConfiguration pageAccessibility) | 89 static void* trimMapping(void* base, size_t baseLen, size_t trimLen, uintptr_t a
lign, PageAccessibilityConfiguration pageAccessibility) { |
| 91 { | 90 size_t preSlack = reinterpret_cast<uintptr_t>(base) & (align - 1); |
| 92 size_t preSlack = reinterpret_cast<uintptr_t>(base) & (align - 1); | 91 if (preSlack) |
| 93 if (preSlack) | 92 preSlack = align - preSlack; |
| 94 preSlack = align - preSlack; | 93 size_t postSlack = baseLen - preSlack - trimLen; |
| 95 size_t postSlack = baseLen - preSlack - trimLen; | 94 ASSERT(baseLen >= trimLen || preSlack || postSlack); |
| 96 ASSERT(baseLen >= trimLen || preSlack || postSlack); | 95 ASSERT(preSlack < baseLen); |
| 97 ASSERT(preSlack < baseLen); | 96 ASSERT(postSlack < baseLen); |
| 98 ASSERT(postSlack < baseLen); | 97 void* ret = base; |
| 99 void* ret = base; | |
| 100 | 98 |
| 101 #if OS(POSIX) // On POSIX we can resize the allocation run. | 99 #if OS(POSIX) // On POSIX we can resize the allocation run. |
| 102 (void) pageAccessibility; | 100 (void)pageAccessibility; |
| 103 if (preSlack) { | 101 if (preSlack) { |
| 104 int res = munmap(base, preSlack); | 102 int res = munmap(base, preSlack); |
| 105 RELEASE_ASSERT(!res); | 103 RELEASE_ASSERT(!res); |
| 106 ret = reinterpret_cast<char*>(base) + preSlack; | 104 ret = reinterpret_cast<char*>(base) + preSlack; |
| 107 } | 105 } |
| 108 if (postSlack) { | 106 if (postSlack) { |
| 109 int res = munmap(reinterpret_cast<char*>(ret) + trimLen, postSlack); | 107 int res = munmap(reinterpret_cast<char*>(ret) + trimLen, postSlack); |
| 110 RELEASE_ASSERT(!res); | 108 RELEASE_ASSERT(!res); |
| 111 } | 109 } |
| 112 #else // On Windows we can't resize the allocation run. | 110 #else // On Windows we can't resize the allocation run. |
| 113 if (preSlack || postSlack) { | 111 if (preSlack || postSlack) { |
| 114 ret = reinterpret_cast<char*>(base) + preSlack; | 112 ret = reinterpret_cast<char*>(base) + preSlack; |
| 115 freePages(base, baseLen); | 113 freePages(base, baseLen); |
| 116 ret = systemAllocPages(ret, trimLen, pageAccessibility); | 114 ret = systemAllocPages(ret, trimLen, pageAccessibility); |
| 117 } | 115 } |
| 118 #endif | 116 #endif |
| 119 | 117 |
| 120 return ret; | 118 return ret; |
| 121 } | 119 } |
| 122 | 120 |
| 123 void* allocPages(void* addr, size_t len, size_t align, PageAccessibilityConfigur
ation pageAccessibility) | 121 void* allocPages(void* addr, size_t len, size_t align, PageAccessibilityConfigur
ation pageAccessibility) { |
| 124 { | 122 ASSERT(len >= kPageAllocationGranularity); |
| 125 ASSERT(len >= kPageAllocationGranularity); | 123 ASSERT(!(len & kPageAllocationGranularityOffsetMask)); |
| 126 ASSERT(!(len & kPageAllocationGranularityOffsetMask)); | 124 ASSERT(align >= kPageAllocationGranularity); |
| 127 ASSERT(align >= kPageAllocationGranularity); | 125 ASSERT(!(align & kPageAllocationGranularityOffsetMask)); |
| 128 ASSERT(!(align & kPageAllocationGranularityOffsetMask)); | 126 ASSERT(!(reinterpret_cast<uintptr_t>(addr) & kPageAllocationGranularityOffsetM
ask)); |
| 129 ASSERT(!(reinterpret_cast<uintptr_t>(addr) & kPageAllocationGranularityOffse
tMask)); | 127 uintptr_t alignOffsetMask = align - 1; |
| 130 uintptr_t alignOffsetMask = align - 1; | 128 uintptr_t alignBaseMask = ~alignOffsetMask; |
| 131 uintptr_t alignBaseMask = ~alignOffsetMask; | 129 ASSERT(!(reinterpret_cast<uintptr_t>(addr) & alignOffsetMask)); |
| 132 ASSERT(!(reinterpret_cast<uintptr_t>(addr) & alignOffsetMask)); | |
| 133 | 130 |
| 134 // If the client passed null as the address, choose a good one. | 131 // If the client passed null as the address, choose a good one. |
| 135 if (!addr) { | 132 if (!addr) { |
| 136 addr = getRandomPageBase(); | 133 addr = getRandomPageBase(); |
| 137 addr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(addr) & align
BaseMask); | 134 addr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(addr) & alignBase
Mask); |
| 138 } | 135 } |
| 139 | 136 |
| 140 // First try to force an exact-size, aligned allocation from our random base
. | 137 // First try to force an exact-size, aligned allocation from our random base. |
| 141 for (int count = 0; count < 3; ++count) { | 138 for (int count = 0; count < 3; ++count) { |
| 142 void* ret = systemAllocPages(addr, len, pageAccessibility); | 139 void* ret = systemAllocPages(addr, len, pageAccessibility); |
| 143 if (kHintIsAdvisory || ret) { | 140 if (kHintIsAdvisory || ret) { |
| 144 // If the alignment is to our liking, we're done. | 141 // If the alignment is to our liking, we're done. |
| 145 if (!(reinterpret_cast<uintptr_t>(ret)& alignOffsetMask)) | 142 if (!(reinterpret_cast<uintptr_t>(ret) & alignOffsetMask)) |
| 146 return ret; | 143 return ret; |
| 147 freePages(ret, len); | 144 freePages(ret, len); |
| 148 #if CPU(32BIT) | 145 #if CPU(32BIT) |
| 149 addr = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(ret)+ali
gn) & alignBaseMask); | 146 addr = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(ret) + align)
& alignBaseMask); |
| 150 #endif | 147 #endif |
| 151 } else if (!addr) { // We know we're OOM when an unhinted allocation fai
ls. | 148 } else if (!addr) { // We know we're OOM when an unhinted allocation fails. |
| 152 return nullptr; | 149 return nullptr; |
| 153 | 150 |
| 154 } else { | 151 } else { |
| 155 #if CPU(32BIT) | 152 #if CPU(32BIT) |
| 156 addr = reinterpret_cast<char*>(addr) + align; | 153 addr = reinterpret_cast<char*>(addr) + align; |
| 157 #endif | |
| 158 } | |
| 159 | |
| 160 #if !CPU(32BIT) // Keep trying random addresses on systems that have a large add
ress space. | |
| 161 addr = getRandomPageBase(); | |
| 162 addr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(addr) & align
BaseMask); | |
| 163 #endif | 154 #endif |
| 164 } | 155 } |
| 165 | 156 |
| 166 // Map a larger allocation so we can force alignment, but continue randomizi
ng only on 64-bit POSIX. | 157 #if !CPU(32BIT) // Keep trying random addresses on systems that have a large ad
dress space. |
| 167 size_t tryLen = len + (align - kPageAllocationGranularity); | 158 addr = getRandomPageBase(); |
| 168 RELEASE_ASSERT(tryLen >= len); | 159 addr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(addr) & alignBase
Mask); |
| 169 void* ret; | 160 #endif |
| 161 } |
| 170 | 162 |
| 171 do { | 163 // Map a larger allocation so we can force alignment, but continue randomizing
only on 64-bit POSIX. |
| 172 // Don't continue to burn cycles on mandatory hints (Windows). | 164 size_t tryLen = len + (align - kPageAllocationGranularity); |
| 173 addr = kHintIsAdvisory ? getRandomPageBase() : nullptr; | 165 RELEASE_ASSERT(tryLen >= len); |
| 174 ret = systemAllocPages(addr, tryLen, pageAccessibility); | 166 void* ret; |
| 167 |
| 168 do { |
| 169 // Don't continue to burn cycles on mandatory hints (Windows). |
| 170 addr = kHintIsAdvisory ? getRandomPageBase() : nullptr; |
| 171 ret = systemAllocPages(addr, tryLen, pageAccessibility); |
| 175 // The retries are for Windows, where a race can steal our mapping on resize
. | 172 // The retries are for Windows, where a race can steal our mapping on resize
. |
| 176 } while (ret && !(ret = trimMapping(ret, tryLen, len, align, pageAccessibili
ty))); | 173 } while (ret && !(ret = trimMapping(ret, tryLen, len, align, pageAccessibility
))); |
| 177 | 174 |
| 178 return ret; | 175 return ret; |
| 179 } | 176 } |
| 180 | 177 |
| 181 void freePages(void* addr, size_t len) | 178 void freePages(void* addr, size_t len) { |
| 182 { | 179 ASSERT(!(reinterpret_cast<uintptr_t>(addr) & kPageAllocationGranularityOffsetM
ask)); |
| 183 ASSERT(!(reinterpret_cast<uintptr_t>(addr) & kPageAllocationGranularityOffse
tMask)); | 180 ASSERT(!(len & kPageAllocationGranularityOffsetMask)); |
| 184 ASSERT(!(len & kPageAllocationGranularityOffsetMask)); | |
| 185 #if OS(POSIX) | 181 #if OS(POSIX) |
| 186 int ret = munmap(addr, len); | 182 int ret = munmap(addr, len); |
| 187 RELEASE_ASSERT(!ret); | 183 RELEASE_ASSERT(!ret); |
| 188 #else | 184 #else |
| 189 BOOL ret = VirtualFree(addr, 0, MEM_RELEASE); | 185 BOOL ret = VirtualFree(addr, 0, MEM_RELEASE); |
| 190 RELEASE_ASSERT(ret); | 186 RELEASE_ASSERT(ret); |
| 191 #endif | 187 #endif |
| 192 } | 188 } |
| 193 | 189 |
| 194 void setSystemPagesInaccessible(void* addr, size_t len) | 190 void setSystemPagesInaccessible(void* addr, size_t len) { |
| 195 { | 191 ASSERT(!(len & kSystemPageOffsetMask)); |
| 196 ASSERT(!(len & kSystemPageOffsetMask)); | |
| 197 #if OS(POSIX) | 192 #if OS(POSIX) |
| 198 int ret = mprotect(addr, len, PROT_NONE); | 193 int ret = mprotect(addr, len, PROT_NONE); |
| 199 RELEASE_ASSERT(!ret); | 194 RELEASE_ASSERT(!ret); |
| 200 #else | 195 #else |
| 201 BOOL ret = VirtualFree(addr, len, MEM_DECOMMIT); | 196 BOOL ret = VirtualFree(addr, len, MEM_DECOMMIT); |
| 202 RELEASE_ASSERT(ret); | 197 RELEASE_ASSERT(ret); |
| 203 #endif | 198 #endif |
| 204 } | 199 } |
| 205 | 200 |
| 206 bool setSystemPagesAccessible(void* addr, size_t len) | 201 bool setSystemPagesAccessible(void* addr, size_t len) { |
| 207 { | 202 ASSERT(!(len & kSystemPageOffsetMask)); |
| 208 ASSERT(!(len & kSystemPageOffsetMask)); | |
| 209 #if OS(POSIX) | 203 #if OS(POSIX) |
| 210 return !mprotect(addr, len, PROT_READ | PROT_WRITE); | 204 return !mprotect(addr, len, PROT_READ | PROT_WRITE); |
| 211 #else | 205 #else |
| 212 return !!VirtualAlloc(addr, len, MEM_COMMIT, PAGE_READWRITE); | 206 return !!VirtualAlloc(addr, len, MEM_COMMIT, PAGE_READWRITE); |
| 213 #endif | 207 #endif |
| 214 } | 208 } |
| 215 | 209 |
| 216 void decommitSystemPages(void* addr, size_t len) | 210 void decommitSystemPages(void* addr, size_t len) { |
| 217 { | 211 ASSERT(!(len & kSystemPageOffsetMask)); |
| 218 ASSERT(!(len & kSystemPageOffsetMask)); | |
| 219 #if OS(POSIX) | 212 #if OS(POSIX) |
| 220 int ret = madvise(addr, len, MADV_FREE); | 213 int ret = madvise(addr, len, MADV_FREE); |
| 221 RELEASE_ASSERT(!ret); | 214 RELEASE_ASSERT(!ret); |
| 222 #else | 215 #else |
| 223 setSystemPagesInaccessible(addr, len); | 216 setSystemPagesInaccessible(addr, len); |
| 224 #endif | 217 #endif |
| 225 } | 218 } |
| 226 | 219 |
| 227 void recommitSystemPages(void* addr, size_t len) | 220 void recommitSystemPages(void* addr, size_t len) { |
| 228 { | 221 ASSERT(!(len & kSystemPageOffsetMask)); |
| 229 ASSERT(!(len & kSystemPageOffsetMask)); | |
| 230 #if OS(POSIX) | 222 #if OS(POSIX) |
| 231 (void) addr; | 223 (void)addr; |
| 232 #else | 224 #else |
| 233 RELEASE_ASSERT(setSystemPagesAccessible(addr, len)); | 225 RELEASE_ASSERT(setSystemPagesAccessible(addr, len)); |
| 234 #endif | 226 #endif |
| 235 } | 227 } |
| 236 | 228 |
| 237 void discardSystemPages(void* addr, size_t len) | 229 void discardSystemPages(void* addr, size_t len) { |
| 238 { | 230 ASSERT(!(len & kSystemPageOffsetMask)); |
| 239 ASSERT(!(len & kSystemPageOffsetMask)); | |
| 240 #if OS(POSIX) | 231 #if OS(POSIX) |
| 241 // On POSIX, the implementation detail is that discard and decommit are the | 232 // On POSIX, the implementation detail is that discard and decommit are the |
| 242 // same, and lead to pages that are returned to the system immediately and | 233 // same, and lead to pages that are returned to the system immediately and |
| 243 // get replaced with zeroed pages when touched. So we just call | 234 // get replaced with zeroed pages when touched. So we just call |
| 244 // decommitSystemPages() here to avoid code duplication. | 235 // decommitSystemPages() here to avoid code duplication. |
| 245 decommitSystemPages(addr, len); | 236 decommitSystemPages(addr, len); |
| 246 #else | 237 #else |
| 247 // On Windows discarded pages are not returned to the system immediately and | 238 // On Windows discarded pages are not returned to the system immediately and |
| 248 // not guaranteed to be zeroed when returned to the application. | 239 // not guaranteed to be zeroed when returned to the application. |
| 249 using DiscardVirtualMemoryFunction = DWORD(WINAPI*)(PVOID virtualAddress, SI
ZE_T size); | 240 using DiscardVirtualMemoryFunction = DWORD(WINAPI*)(PVOID virtualAddress, SIZE
_T size); |
| 250 static DiscardVirtualMemoryFunction discardVirtualMemory = reinterpret_cast<
DiscardVirtualMemoryFunction>(-1); | 241 static DiscardVirtualMemoryFunction discardVirtualMemory = reinterpret_cast<Di
scardVirtualMemoryFunction>(-1); |
| 251 if (discardVirtualMemory == reinterpret_cast<DiscardVirtualMemoryFunction>(-
1)) | 242 if (discardVirtualMemory == reinterpret_cast<DiscardVirtualMemoryFunction>(-1)
) |
| 252 discardVirtualMemory = reinterpret_cast<DiscardVirtualMemoryFunction>(Ge
tProcAddress(GetModuleHandle(L"Kernel32.dll"), "DiscardVirtualMemory")); | 243 discardVirtualMemory = reinterpret_cast<DiscardVirtualMemoryFunction>(GetPro
cAddress(GetModuleHandle(L"Kernel32.dll"), "DiscardVirtualMemory")); |
| 253 // Use DiscardVirtualMemory when available because it releases faster than M
EM_RESET. | 244 // Use DiscardVirtualMemory when available because it releases faster than MEM
_RESET. |
| 254 DWORD ret = 1; | 245 DWORD ret = 1; |
| 255 if (discardVirtualMemory) | 246 if (discardVirtualMemory) |
| 256 ret = discardVirtualMemory(addr, len); | 247 ret = discardVirtualMemory(addr, len); |
| 257 // DiscardVirtualMemory is buggy in Win10 SP0, so fall back to MEM_RESET on
failure. | 248 // DiscardVirtualMemory is buggy in Win10 SP0, so fall back to MEM_RESET on fa
ilure. |
| 258 if (ret) { | 249 if (ret) { |
| 259 void* ret = VirtualAlloc(addr, len, MEM_RESET, PAGE_READWRITE); | 250 void* ret = VirtualAlloc(addr, len, MEM_RESET, PAGE_READWRITE); |
| 260 RELEASE_ASSERT(ret); | 251 RELEASE_ASSERT(ret); |
| 261 } | 252 } |
| 262 #endif | 253 #endif |
| 263 } | 254 } |
| 264 | 255 |
| 265 } // namespace WTF | 256 } // namespace WTF |
| 266 | |
| OLD | NEW |