OLD | NEW |
1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 // Author: Ken Chen <kenchen@google.com> | 4 // Author: Ken Chen <kenchen@google.com> |
5 // | 5 // |
6 // hugepage text library to remap process executable segment with hugepages. | 6 // hugepage text library to remap process executable segment with hugepages. |
7 | 7 |
8 #include "chromeos/hugepage_text/hugepage_text.h" | 8 #include "chromeos/hugepage_text/hugepage_text.h" |
9 | 9 |
10 #include <link.h> | 10 #include <link.h> |
(...skipping 18 matching lines...) Expand all Loading... |
29 | 29 |
30 const int kHpageShift = 21; | 30 const int kHpageShift = 21; |
31 const int kHpageSize = (1 << kHpageShift); | 31 const int kHpageSize = (1 << kHpageShift); |
32 const int kHpageMask = (~(kHpageSize - 1)); | 32 const int kHpageMask = (~(kHpageSize - 1)); |
33 | 33 |
34 const int kProtection = (PROT_READ | PROT_WRITE); | 34 const int kProtection = (PROT_READ | PROT_WRITE); |
35 const int kMmapFlags = (MAP_ANONYMOUS | MAP_SHARED); | 35 const int kMmapFlags = (MAP_ANONYMOUS | MAP_SHARED); |
36 const int kMmapHtlbFlags = (kMmapFlags | MAP_HUGETLB); | 36 const int kMmapHtlbFlags = (kMmapFlags | MAP_HUGETLB); |
37 const int kMremapFlags = (MREMAP_MAYMOVE | MREMAP_FIXED); | 37 const int kMremapFlags = (MREMAP_MAYMOVE | MREMAP_FIXED); |
38 | 38 |
39 // The number of hugepages we want to use to map chrome text section | |
40 // to hugepages. With the help of AutoFDO, the hot functions are grouped | |
41 // in to a small area of the binary. | |
42 const int kNumHugePages = 15; | |
43 | |
44 // mremap syscall is always supported for small page segment on all kernels. | 39 // mremap syscall is always supported for small page segment on all kernels. |
45 // However, it is not the case for hugepage. | 40 // However, it is not the case for hugepage. |
46 // If not used carefully, mremap() a hugepage segment directly onto small page | 41 // If not used carefully, mremap() a hugepage segment directly onto small page |
47 // text segment will cause irreversible damage to the existing text mapping | 42 // text segment will cause irreversible damage to the existing text mapping |
48 // and cause process to segfault. This function will dynamically at run time | 43 // and cause process to segfault. This function will dynamically at run time |
49 // determine whether a process can safely execute mremap on a hugepage segment | 44 // determine whether a process can safely execute mremap on a hugepage segment |
50 // without taking the process down. | 45 // without taking the process down. |
51 // | 46 // |
52 // Inputs: none | 47 // Inputs: none |
53 // Return: true if mremap on hugepage segment is supported on the host OS. | 48 // Return: true if mremap on hugepage segment is supported on the host OS. |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
122 // Remaps text segment at address "vaddr" to hugepage backed mapping via mremap | 117 // Remaps text segment at address "vaddr" to hugepage backed mapping via mremap |
123 // syscall. The virtual address does not change. When this call returns, the | 118 // syscall. The virtual address does not change. When this call returns, the |
124 // backing physical memory will be changed from small page to hugetlb page. | 119 // backing physical memory will be changed from small page to hugetlb page. |
125 // | 120 // |
126 // Inputs: vaddr, the starting virtual address to remap to hugepage | 121 // Inputs: vaddr, the starting virtual address to remap to hugepage |
127 // hsize, size of the memory segment to remap in bytes | 122 // hsize, size of the memory segment to remap in bytes |
128 // Return: none | 123 // Return: none |
129 // Effect: physical backing page changed from small page to hugepage. If there | 124 // Effect: physical backing page changed from small page to hugepage. If there |
130 // are error condition, the remapping operation is aborted. | 125 // are error condition, the remapping operation is aborted. |
131 static void MremapHugetlbText(void* vaddr, const size_t hsize) { | 126 static void MremapHugetlbText(void* vaddr, const size_t hsize) { |
| 127 DCHECK_EQ(0ul, reinterpret_cast<uintptr_t>(vaddr) & ~kHpageMask); |
132 void* haddr = MAP_FAILED; | 128 void* haddr = MAP_FAILED; |
133 | 129 if (HugetlbMremapSupported()) { |
134 if ((reinterpret_cast<intptr_t>(vaddr) & ~kHpageMask) == 0 && | |
135 HugetlbMremapSupported()) { | |
136 // Try anon hugepage from static hugepage pool only if the source address | 130 // Try anon hugepage from static hugepage pool only if the source address |
137 // is hugepage aligned, otherwise, mremap below has non-recoverable error. | 131 // is hugepage aligned, otherwise, mremap below has non-recoverable error. |
138 haddr = mmap(NULL, hsize, kProtection, kMmapHtlbFlags, 0, 0); | 132 haddr = mmap(NULL, hsize, kProtection, kMmapHtlbFlags, 0, 0); |
139 } | 133 } |
140 | 134 |
141 if (haddr == MAP_FAILED) { | 135 if (haddr == MAP_FAILED) { |
142 PLOG(INFO) << "static hugepage not available, trying transparent hugepage"; | 136 PLOG(INFO) << "static hugepage not available, trying transparent hugepage"; |
143 haddr = GetTransparentHugepageMapping(hsize); | 137 haddr = GetTransparentHugepageMapping(hsize); |
144 if (haddr == NULL) | 138 if (haddr == NULL) |
145 return; | 139 return; |
(...skipping 19 matching lines...) Expand all Loading... |
165 } | 159 } |
166 | 160 |
167 // Top level text remapping function. | 161 // Top level text remapping function. |
168 // | 162 // |
169 // Inputs: vaddr, the starting virtual address to remap to hugepage | 163 // Inputs: vaddr, the starting virtual address to remap to hugepage |
170 // hsize, size of the memory segment to remap in bytes | 164 // hsize, size of the memory segment to remap in bytes |
171 // Return: none | 165 // Return: none |
172 // Effect: physical backing page changed from small page to hugepage. If there | 166 // Effect: physical backing page changed from small page to hugepage. If there |
173 // are error condition, the remaping operation is aborted. | 167 // are error condition, the remaping operation is aborted. |
174 static void RemapHugetlbText(void* vaddr, const size_t segsize) { | 168 static void RemapHugetlbText(void* vaddr, const size_t segsize) { |
175 int hsize = segsize; | 169 // remove unaligned head regions |
176 if (segsize > kHpageSize * kNumHugePages) | 170 size_t head_gap = (kHpageSize - reinterpret_cast<size_t>(vaddr) % kHpageSize) |
177 hsize = kHpageSize * kNumHugePages; | 171 % kHpageSize; |
| 172 size_t addr = reinterpret_cast<size_t>(vaddr) + head_gap; |
| 173 |
| 174 int hsize = segsize - head_gap; |
178 hsize = hsize & kHpageMask; | 175 hsize = hsize & kHpageMask; |
179 | 176 |
180 if (hsize == 0) | 177 if (hsize == 0) |
181 return; | 178 return; |
182 | 179 |
183 MremapHugetlbText(vaddr, hsize); | 180 MremapHugetlbText(reinterpret_cast<void *>(addr), hsize); |
184 } | 181 } |
185 | 182 |
186 // For a given ELF program header descriptor, iterates over all segments within | 183 // For a given ELF program header descriptor, iterates over all segments within |
187 // it and find the first segment that has PT_LOAD and is executable, call | 184 // it and find the first segment that has PT_LOAD and is executable, call |
188 // RemapHugetlbText(). | 185 // RemapHugetlbText(). |
189 // | 186 // |
190 // Inputs: info: pointer to a struct dl_phdr_info that describes the DSO. | 187 // Inputs: info: pointer to a struct dl_phdr_info that describes the DSO. |
191 // size: size of the above structure (not used in this function). | 188 // size: size of the above structure (not used in this function). |
192 // data: user param (not used in this function). | 189 // data: user param (not used in this function). |
193 // Return: always return true. The value is propagated by dl_iterate_phdr(). | 190 // Return: always return true. The value is propagated by dl_iterate_phdr(). |
(...skipping 18 matching lines...) Expand all Loading... |
212 | 209 |
213 // Main library function. This function will iterate all ELF segments and | 210 // Main library function. This function will iterate all ELF segments and |
214 // attempt to remap text segment from small page to hugepage. | 211 // attempt to remap text segment from small page to hugepage. |
215 // If remapping is successful. All error conditions are soft fail such that | 212 // If remapping is successful. All error conditions are soft fail such that |
216 // effect will be rolled back and remap operation will be aborted. | 213 // effect will be rolled back and remap operation will be aborted. |
217 void ReloadElfTextInHugePages(void) { | 214 void ReloadElfTextInHugePages(void) { |
218 dl_iterate_phdr(FilterElfHeader, 0); | 215 dl_iterate_phdr(FilterElfHeader, 0); |
219 } | 216 } |
220 | 217 |
221 } // namespace chromeos | 218 } // namespace chromeos |
OLD | NEW |