OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright (C) 2013 Google Inc. All rights reserved. | |
3 * | |
4 * Redistribution and use in source and binary forms, with or without | |
5 * modification, are permitted provided that the following conditions are | |
6 * met: | |
7 * | |
8 * * Redistributions of source code must retain the above copyright | |
9 * notice, this list of conditions and the following disclaimer. | |
10 * * Redistributions in binary form must reproduce the above | |
11 * copyright notice, this list of conditions and the following disclaimer | |
12 * in the documentation and/or other materials provided with the | |
13 * distribution. | |
14 * * Neither the name of Google Inc. nor the names of its | |
15 * contributors may be used to endorse or promote products derived from | |
16 * this software without specific prior written permission. | |
17 * | |
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
29 */ | |
30 | |
31 #include "wtf/allocator/PageAllocator.h" | |
32 | |
33 #include "wtf/Assertions.h" | |
34 #include "wtf/Atomics.h" | |
35 #include "wtf/allocator/AddressSpaceRandomization.h" | |
36 | |
37 #include <limits.h> | |
38 | |
39 #if OS(POSIX) | |
40 | |
41 #include <errno.h> | |
42 #include <sys/mman.h> | |
43 | |
44 #ifndef MADV_FREE | |
45 #define MADV_FREE MADV_DONTNEED | |
46 #endif | |
47 | |
48 #ifndef MAP_ANONYMOUS | |
49 #define MAP_ANONYMOUS MAP_ANON | |
50 #endif | |
51 | |
52 // On POSIX memmap uses a nearby address if the hint address is blocked. | |
53 static const bool kHintIsAdvisory = true; | |
54 static uint32_t s_allocPageErrorCode = 0; | |
55 | |
56 #elif OS(WIN) | |
57 | |
58 #include <windows.h> | |
59 | |
60 // VirtualAlloc will fail if allocation at the hint address is blocked. | |
61 static const bool kHintIsAdvisory = false; | |
62 static uint32_t s_allocPageErrorCode = ERROR_SUCCESS; | |
63 | |
64 #else | |
65 #error Unknown OS | |
66 #endif // OS(POSIX) | |
67 | |
68 namespace WTF { | |
69 | |
70 // This internal function wraps the OS-specific page allocation call. The | |
71 // behavior of the hint address is determined by the kHintIsAdvisory constant. | |
72 // If true, a non-zero hint is advisory and the returned address may differ from | |
73 // the hint. If false, the hint is mandatory and a successful allocation will | |
74 // not differ from the hint. | |
75 static void* systemAllocPages( | |
76 void* hint, | |
77 size_t len, | |
78 PageAccessibilityConfiguration pageAccessibility) { | |
79 ASSERT(!(len & kPageAllocationGranularityOffsetMask)); | |
80 ASSERT(!(reinterpret_cast<uintptr_t>(hint) & | |
81 kPageAllocationGranularityOffsetMask)); | |
82 void* ret; | |
83 #if OS(WIN) | |
84 DWORD accessFlag = | |
85 pageAccessibility == PageAccessible ? PAGE_READWRITE : PAGE_NOACCESS; | |
86 ret = VirtualAlloc(hint, len, MEM_RESERVE | MEM_COMMIT, accessFlag); | |
87 if (!ret) | |
88 releaseStore(&s_allocPageErrorCode, GetLastError()); | |
89 #else | |
90 int accessFlag = pageAccessibility == PageAccessible | |
91 ? (PROT_READ | PROT_WRITE) | |
92 : PROT_NONE; | |
93 ret = mmap(hint, len, accessFlag, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); | |
94 if (ret == MAP_FAILED) { | |
95 releaseStore(&s_allocPageErrorCode, errno); | |
96 ret = 0; | |
97 } | |
98 #endif | |
99 return ret; | |
100 } | |
101 | |
102 // Trims base to given length and alignment. Windows returns null on failure and | |
103 // frees base. | |
104 static void* trimMapping(void* base, | |
105 size_t baseLen, | |
106 size_t trimLen, | |
107 uintptr_t align, | |
108 PageAccessibilityConfiguration pageAccessibility) { | |
109 size_t preSlack = reinterpret_cast<uintptr_t>(base) & (align - 1); | |
110 if (preSlack) | |
111 preSlack = align - preSlack; | |
112 size_t postSlack = baseLen - preSlack - trimLen; | |
113 ASSERT(baseLen >= trimLen || preSlack || postSlack); | |
114 ASSERT(preSlack < baseLen); | |
115 ASSERT(postSlack < baseLen); | |
116 void* ret = base; | |
117 | |
118 #if OS(POSIX) // On POSIX we can resize the allocation run. | |
119 (void)pageAccessibility; | |
120 if (preSlack) { | |
121 int res = munmap(base, preSlack); | |
122 RELEASE_ASSERT(!res); | |
123 ret = reinterpret_cast<char*>(base) + preSlack; | |
124 } | |
125 if (postSlack) { | |
126 int res = munmap(reinterpret_cast<char*>(ret) + trimLen, postSlack); | |
127 RELEASE_ASSERT(!res); | |
128 } | |
129 #else // On Windows we can't resize the allocation run. | |
130 if (preSlack || postSlack) { | |
131 ret = reinterpret_cast<char*>(base) + preSlack; | |
132 freePages(base, baseLen); | |
133 ret = systemAllocPages(ret, trimLen, pageAccessibility); | |
134 } | |
135 #endif | |
136 | |
137 return ret; | |
138 } | |
139 | |
140 void* allocPages(void* addr, | |
141 size_t len, | |
142 size_t align, | |
143 PageAccessibilityConfiguration pageAccessibility) { | |
144 ASSERT(len >= kPageAllocationGranularity); | |
145 ASSERT(!(len & kPageAllocationGranularityOffsetMask)); | |
146 ASSERT(align >= kPageAllocationGranularity); | |
147 ASSERT(!(align & kPageAllocationGranularityOffsetMask)); | |
148 ASSERT(!(reinterpret_cast<uintptr_t>(addr) & | |
149 kPageAllocationGranularityOffsetMask)); | |
150 uintptr_t alignOffsetMask = align - 1; | |
151 uintptr_t alignBaseMask = ~alignOffsetMask; | |
152 ASSERT(!(reinterpret_cast<uintptr_t>(addr) & alignOffsetMask)); | |
153 | |
154 // If the client passed null as the address, choose a good one. | |
155 if (!addr) { | |
156 addr = getRandomPageBase(); | |
157 addr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(addr) & | |
158 alignBaseMask); | |
159 } | |
160 | |
161 // First try to force an exact-size, aligned allocation from our random base. | |
162 for (int count = 0; count < 3; ++count) { | |
163 void* ret = systemAllocPages(addr, len, pageAccessibility); | |
164 if (kHintIsAdvisory || ret) { | |
165 // If the alignment is to our liking, we're done. | |
166 if (!(reinterpret_cast<uintptr_t>(ret) & alignOffsetMask)) | |
167 return ret; | |
168 freePages(ret, len); | |
169 #if CPU(32BIT) | |
170 addr = reinterpret_cast<void*>( | |
171 (reinterpret_cast<uintptr_t>(ret) + align) & alignBaseMask); | |
172 #endif | |
173 } else if (!addr) { // We know we're OOM when an unhinted allocation fails. | |
174 return nullptr; | |
175 | |
176 } else { | |
177 #if CPU(32BIT) | |
178 addr = reinterpret_cast<char*>(addr) + align; | |
179 #endif | |
180 } | |
181 | |
182 #if !CPU(32BIT) | |
183 // Keep trying random addresses on systems that have a large address space. | |
184 addr = getRandomPageBase(); | |
185 addr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(addr) & | |
186 alignBaseMask); | |
187 #endif | |
188 } | |
189 | |
190 // Map a larger allocation so we can force alignment, but continue randomizing | |
191 // only on 64-bit POSIX. | |
192 size_t tryLen = len + (align - kPageAllocationGranularity); | |
193 RELEASE_ASSERT(tryLen >= len); | |
194 void* ret; | |
195 | |
196 do { | |
197 // Don't continue to burn cycles on mandatory hints (Windows). | |
198 addr = kHintIsAdvisory ? getRandomPageBase() : nullptr; | |
199 ret = systemAllocPages(addr, tryLen, pageAccessibility); | |
200 // The retries are for Windows, where a race can steal our mapping on | |
201 // resize. | |
202 } while (ret && | |
203 !(ret = trimMapping(ret, tryLen, len, align, pageAccessibility))); | |
204 | |
205 return ret; | |
206 } | |
207 | |
208 void freePages(void* addr, size_t len) { | |
209 ASSERT(!(reinterpret_cast<uintptr_t>(addr) & | |
210 kPageAllocationGranularityOffsetMask)); | |
211 ASSERT(!(len & kPageAllocationGranularityOffsetMask)); | |
212 #if OS(POSIX) | |
213 int ret = munmap(addr, len); | |
214 RELEASE_ASSERT(!ret); | |
215 #else | |
216 BOOL ret = VirtualFree(addr, 0, MEM_RELEASE); | |
217 RELEASE_ASSERT(ret); | |
218 #endif | |
219 } | |
220 | |
221 void setSystemPagesInaccessible(void* addr, size_t len) { | |
222 ASSERT(!(len & kSystemPageOffsetMask)); | |
223 #if OS(POSIX) | |
224 int ret = mprotect(addr, len, PROT_NONE); | |
225 RELEASE_ASSERT(!ret); | |
226 #else | |
227 BOOL ret = VirtualFree(addr, len, MEM_DECOMMIT); | |
228 RELEASE_ASSERT(ret); | |
229 #endif | |
230 } | |
231 | |
232 bool setSystemPagesAccessible(void* addr, size_t len) { | |
233 ASSERT(!(len & kSystemPageOffsetMask)); | |
234 #if OS(POSIX) | |
235 return !mprotect(addr, len, PROT_READ | PROT_WRITE); | |
236 #else | |
237 return !!VirtualAlloc(addr, len, MEM_COMMIT, PAGE_READWRITE); | |
238 #endif | |
239 } | |
240 | |
241 void decommitSystemPages(void* addr, size_t len) { | |
242 ASSERT(!(len & kSystemPageOffsetMask)); | |
243 #if OS(POSIX) | |
244 int ret = madvise(addr, len, MADV_FREE); | |
245 RELEASE_ASSERT(!ret); | |
246 #else | |
247 setSystemPagesInaccessible(addr, len); | |
248 #endif | |
249 } | |
250 | |
251 void recommitSystemPages(void* addr, size_t len) { | |
252 ASSERT(!(len & kSystemPageOffsetMask)); | |
253 #if OS(POSIX) | |
254 (void)addr; | |
255 #else | |
256 RELEASE_ASSERT(setSystemPagesAccessible(addr, len)); | |
257 #endif | |
258 } | |
259 | |
260 void discardSystemPages(void* addr, size_t len) { | |
261 ASSERT(!(len & kSystemPageOffsetMask)); | |
262 #if OS(POSIX) | |
263 // On POSIX, the implementation detail is that discard and decommit are the | |
264 // same, and lead to pages that are returned to the system immediately and | |
265 // get replaced with zeroed pages when touched. So we just call | |
266 // decommitSystemPages() here to avoid code duplication. | |
267 decommitSystemPages(addr, len); | |
268 #else | |
269 // On Windows discarded pages are not returned to the system immediately and | |
270 // not guaranteed to be zeroed when returned to the application. | |
271 using DiscardVirtualMemoryFunction = | |
272 DWORD(WINAPI*)(PVOID virtualAddress, SIZE_T size); | |
273 static DiscardVirtualMemoryFunction discardVirtualMemory = | |
274 reinterpret_cast<DiscardVirtualMemoryFunction>(-1); | |
275 if (discardVirtualMemory == | |
276 reinterpret_cast<DiscardVirtualMemoryFunction>(-1)) | |
277 discardVirtualMemory = | |
278 reinterpret_cast<DiscardVirtualMemoryFunction>(GetProcAddress( | |
279 GetModuleHandle(L"Kernel32.dll"), "DiscardVirtualMemory")); | |
280 // Use DiscardVirtualMemory when available because it releases faster than | |
281 // MEM_RESET. | |
282 DWORD ret = 1; | |
283 if (discardVirtualMemory) | |
284 ret = discardVirtualMemory(addr, len); | |
285 // DiscardVirtualMemory is buggy in Win10 SP0, so fall back to MEM_RESET on | |
286 // failure. | |
287 if (ret) { | |
288 void* ret = VirtualAlloc(addr, len, MEM_RESET, PAGE_READWRITE); | |
289 RELEASE_ASSERT(ret); | |
290 } | |
291 #endif | |
292 } | |
293 | |
294 uint32_t getAllocPageErrorCode() { | |
295 return acquireLoad(&s_allocPageErrorCode); | |
296 } | |
297 | |
298 } // namespace WTF | |
OLD | NEW |