OLD | NEW |
1 // Copyright (c) 2007, Google Inc. | 1 // Copyright (c) 2007, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 20 matching lines...) Expand all Loading... |
31 // Author: Arun Sharma | 31 // Author: Arun Sharma |
32 // | 32 // |
33 // A tcmalloc system allocator that uses a memory based filesystem such as | 33 // A tcmalloc system allocator that uses a memory based filesystem such as |
34 // tmpfs or hugetlbfs | 34 // tmpfs or hugetlbfs |
35 // | 35 // |
36 // Since these only exist on linux, we only register this allocator there. | 36 // Since these only exist on linux, we only register this allocator there. |
37 | 37 |
38 #ifdef __linux | 38 #ifdef __linux |
39 | 39 |
40 #include <config.h> | 40 #include <config.h> |
41 #include <errno.h> | 41 #include <errno.h> // for errno, EINVAL |
42 #include <fcntl.h> | 42 #include <inttypes.h> // for PRId64 |
43 #include <unistd.h> | 43 #include <limits.h> // for PATH_MAX |
44 #include <inttypes.h> | 44 #include <stddef.h> // for size_t, NULL |
45 #include <sys/mman.h> | 45 #ifdef HAVE_STDINT_H |
46 #include <sys/param.h> | 46 #include <stdint.h> // for int64_t, uintptr_t |
47 #include <sys/types.h> | 47 #endif |
48 #include <sys/vfs.h> // for statfs | 48 #include <stdio.h> // for snprintf |
| 49 #include <stdlib.h> // for mkstemp |
| 50 #include <string.h> // for strerror |
| 51 #include <sys/mman.h> // for mmap, MAP_FAILED, etc |
| 52 #include <sys/statfs.h> // for fstatfs, statfs |
| 53 #include <unistd.h> // for ftruncate, off_t, unlink |
| 54 #include <new> // for operator new |
49 #include <string> | 55 #include <string> |
50 | 56 |
| 57 #include <google/malloc_extension.h> |
51 #include "base/basictypes.h" | 58 #include "base/basictypes.h" |
52 #include "base/googleinit.h" | 59 #include "base/googleinit.h" |
53 #include "base/sysinfo.h" | 60 #include "base/sysinfo.h" |
54 #include "system-alloc.h" | 61 #include "system-alloc.h" |
55 #include "internal_logging.h" | 62 #include "internal_logging.h" |
56 | 63 |
57 using std::string; | 64 using std::string; |
58 | 65 |
59 DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), | 66 DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), |
60 "Path where hugetlbfs or tmpfs is mounted. The caller is " | 67 "Path where hugetlbfs or tmpfs is mounted. The caller is " |
61 "responsible for ensuring that the path is unique and does " | 68 "responsible for ensuring that the path is unique and does " |
62 "not conflict with another process"); | 69 "not conflict with another process"); |
63 DEFINE_int64(memfs_malloc_limit_mb, | 70 DEFINE_int64(memfs_malloc_limit_mb, |
64 EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0), | 71 EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0), |
65 "Limit total allocation size to the " | 72 "Limit total allocation size to the " |
66 "specified number of MiB. 0 == no limit."); | 73 "specified number of MiB. 0 == no limit."); |
67 DEFINE_bool(memfs_malloc_abort_on_fail, | 74 DEFINE_bool(memfs_malloc_abort_on_fail, |
68 EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false), | 75 EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false), |
69 "abort() whenever memfs_malloc fails to satisfy an allocation " | 76 "abort() whenever memfs_malloc fails to satisfy an allocation " |
70 "for any reason."); | 77 "for any reason."); |
71 DEFINE_bool(memfs_malloc_ignore_mmap_fail, | 78 DEFINE_bool(memfs_malloc_ignore_mmap_fail, |
72 EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false), | 79 EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false), |
73 "Ignore failures from mmap"); | 80 "Ignore failures from mmap"); |
| 81 DEFINE_bool(memfs_malloc_map_private, |
| 82 EnvToBool("TCMALLOC_MEMFS_MAP_PRIVATE", false), |
| 83 "Use MAP_PRIVATE with mmap"); |
74 | 84 |
75 // Hugetlbfs based allocator for tcmalloc | 85 // Hugetlbfs based allocator for tcmalloc |
76 class HugetlbSysAllocator: public SysAllocator { | 86 class HugetlbSysAllocator: public SysAllocator { |
77 public: | 87 public: |
78 HugetlbSysAllocator(int fd, int page_size) | 88 explicit HugetlbSysAllocator(SysAllocator* fallback) |
79 : big_page_size_(page_size), | 89 : failed_(true), // Unusable until FlagsInitialized() is called |
80 hugetlb_fd_(fd), | 90 big_page_size_(0), |
81 hugetlb_base_(0) { | 91 hugetlb_fd_(-1), |
| 92 hugetlb_base_(0), |
| 93 fallback_(fallback) { |
82 } | 94 } |
83 | 95 |
84 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 96 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
85 | 97 |
86 void DumpStats(TCMalloc_Printer* printer); | 98 void FlagsInitialized(); |
87 | 99 |
| 100 bool failed_; // Whether failed to allocate memory. |
88 private: | 101 private: |
| 102 void* AllocInternal(size_t size, size_t *actual_size, size_t alignment); |
| 103 |
89 int64 big_page_size_; | 104 int64 big_page_size_; |
90 int hugetlb_fd_; // file descriptor for hugetlb | 105 int hugetlb_fd_; // file descriptor for hugetlb |
91 off_t hugetlb_base_; | 106 off_t hugetlb_base_; |
| 107 |
| 108 SysAllocator* fallback_; // Default system allocator to fall back to. |
92 }; | 109 }; |
93 | 110 static char hugetlb_space[sizeof(HugetlbSysAllocator)]; |
94 void HugetlbSysAllocator::DumpStats(TCMalloc_Printer* printer) { | |
95 printer->printf("HugetlbSysAllocator: failed_=%d allocated=%"PRId64"\n", | |
96 failed_, static_cast<int64_t>(hugetlb_base_)); | |
97 } | |
98 | 111 |
99 // No locking needed here since we assume that tcmalloc calls | 112 // No locking needed here since we assume that tcmalloc calls |
100 // us with an internal lock held (see tcmalloc/system-alloc.cc). | 113 // us with an internal lock held (see tcmalloc/system-alloc.cc). |
101 void* HugetlbSysAllocator::Alloc(size_t size, size_t *actual_size, | 114 void* HugetlbSysAllocator::Alloc(size_t size, size_t *actual_size, |
102 size_t alignment) { | 115 size_t alignment) { |
103 | 116 if (failed_) { |
104 // don't go any further if we haven't opened the backing file | 117 return fallback_->Alloc(size, actual_size, alignment); |
105 if (hugetlb_fd_ == -1) { | |
106 return NULL; | |
107 } | 118 } |
108 | 119 |
109 // We don't respond to allocation requests smaller than big_page_size_ unless | 120 // We don't respond to allocation requests smaller than big_page_size_ unless |
110 // the caller is willing to take more than they asked for. | 121 // the caller is ok to take more than they asked for. Used by MetaDataAlloc. |
111 if (actual_size == NULL && size < big_page_size_) { | 122 if (actual_size == NULL && size < big_page_size_) { |
112 return NULL; | 123 return fallback_->Alloc(size, actual_size, alignment); |
113 } | 124 } |
114 | 125 |
115 // Enforce huge page alignment. Be careful to deal with overflow. | 126 // Enforce huge page alignment. Be careful to deal with overflow. |
116 if (alignment < big_page_size_) alignment = big_page_size_; | 127 size_t new_alignment = alignment; |
117 size_t aligned_size = ((size + alignment - 1) / alignment) * alignment; | 128 if (new_alignment < big_page_size_) new_alignment = big_page_size_; |
| 129 size_t aligned_size = ((size + new_alignment - 1) / |
| 130 new_alignment) * new_alignment; |
118 if (aligned_size < size) { | 131 if (aligned_size < size) { |
119 return NULL; | 132 return fallback_->Alloc(size, actual_size, alignment); |
120 } | 133 } |
121 size = aligned_size; | |
122 | 134 |
| 135 void* result = AllocInternal(aligned_size, actual_size, new_alignment); |
| 136 if (result != NULL) { |
| 137 return result; |
| 138 } |
| 139 TCMalloc_MESSAGE(__FILE__, __LINE__, |
| 140 "HugetlbSysAllocator: failed_=%d allocated=%"PRId64"\n", |
| 141 failed_, static_cast<int64_t>(hugetlb_base_)); |
| 142 if (FLAGS_memfs_malloc_abort_on_fail) { |
| 143 CRASH("memfs_malloc_abort_on_fail is set\n"); |
| 144 } |
| 145 return fallback_->Alloc(size, actual_size, alignment); |
| 146 } |
| 147 |
| 148 void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, |
| 149 size_t alignment) { |
123 // Ask for extra memory if alignment > pagesize | 150 // Ask for extra memory if alignment > pagesize |
124 size_t extra = 0; | 151 size_t extra = 0; |
125 if (alignment > big_page_size_) { | 152 if (alignment > big_page_size_) { |
126 extra = alignment - big_page_size_; | 153 extra = alignment - big_page_size_; |
127 } | 154 } |
128 | 155 |
129 // Test if this allocation would put us over the limit. | 156 // Test if this allocation would put us over the limit. |
130 off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024; | 157 off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024; |
131 if (limit > 0 && hugetlb_base_ + size + extra > limit) { | 158 if (limit > 0 && hugetlb_base_ + size + extra > limit) { |
132 // Disable the allocator when there's less than one page left. | 159 // Disable the allocator when there's less than one page left. |
133 if (limit - hugetlb_base_ < big_page_size_) { | 160 if (limit - hugetlb_base_ < big_page_size_) { |
134 TCMalloc_MESSAGE(__FILE__, __LINE__, "reached memfs_malloc_limit_mb\n"); | 161 TCMalloc_MESSAGE(__FILE__, __LINE__, "reached memfs_malloc_limit_mb\n"); |
135 failed_ = true; | 162 failed_ = true; |
136 } | 163 } |
137 else { | 164 else { |
138 TCMalloc_MESSAGE(__FILE__, __LINE__, "alloc size=%"PRIuS | 165 TCMalloc_MESSAGE(__FILE__, __LINE__, "alloc size=%"PRIuS |
139 " too large while %"PRId64" bytes remain\n", | 166 " too large while %"PRId64" bytes remain\n", |
140 size, static_cast<int64_t>(limit - hugetlb_base_)); | 167 size, static_cast<int64_t>(limit - hugetlb_base_)); |
141 } | 168 } |
142 if (FLAGS_memfs_malloc_abort_on_fail) { | |
143 CRASH("memfs_malloc_abort_on_fail is set\n"); | |
144 } | |
145 return NULL; | 169 return NULL; |
146 } | 170 } |
147 | 171 |
148 // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly | 172 // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly |
149 // hugetlbfs returns EINVAL for ftruncate. | 173 // hugetlbfs returns EINVAL for ftruncate. |
150 int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); | 174 int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); |
151 if (ret != 0 && errno != EINVAL) { | 175 if (ret != 0 && errno != EINVAL) { |
152 TCMalloc_MESSAGE(__FILE__, __LINE__, "ftruncate failed: %s\n", | 176 TCMalloc_MESSAGE(__FILE__, __LINE__, "ftruncate failed: %s\n", |
153 strerror(errno)); | 177 strerror(errno)); |
154 failed_ = true; | 178 failed_ = true; |
155 if (FLAGS_memfs_malloc_abort_on_fail) { | |
156 CRASH("memfs_malloc_abort_on_fail is set\n"); | |
157 } | |
158 return NULL; | 179 return NULL; |
159 } | 180 } |
160 | 181 |
161 // Note: size + extra does not overflow since: | 182 // Note: size + extra does not overflow since: |
162 // size + alignment < (1<<NBITS). | 183 // size + alignment < (1<<NBITS). |
163 // and extra <= alignment | 184 // and extra <= alignment |
164 // therefore size + extra < (1<<NBITS) | 185 // therefore size + extra < (1<<NBITS) |
165 void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ, | 186 void *result; |
166 MAP_SHARED, hugetlb_fd_, hugetlb_base_); | 187 result = mmap(0, size + extra, PROT_WRITE|PROT_READ, |
| 188 FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED, |
| 189 hugetlb_fd_, hugetlb_base_); |
167 if (result == reinterpret_cast<void*>(MAP_FAILED)) { | 190 if (result == reinterpret_cast<void*>(MAP_FAILED)) { |
168 if (!FLAGS_memfs_malloc_ignore_mmap_fail) { | 191 if (!FLAGS_memfs_malloc_ignore_mmap_fail) { |
169 TCMalloc_MESSAGE(__FILE__, __LINE__, "mmap of size %"PRIuS" failed: %s\n", | 192 TCMalloc_MESSAGE(__FILE__, __LINE__, "mmap of size %"PRIuS" failed: %s\n", |
170 size + extra, strerror(errno)); | 193 size + extra, strerror(errno)); |
171 failed_ = true; | 194 failed_ = true; |
172 if (FLAGS_memfs_malloc_abort_on_fail) { | |
173 CRASH("memfs_malloc_abort_on_fail is set\n"); | |
174 } | |
175 } | 195 } |
176 return NULL; | 196 return NULL; |
177 } | 197 } |
178 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); | 198 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
179 | 199 |
180 // Adjust the return memory so it is aligned | 200 // Adjust the return memory so it is aligned |
181 size_t adjust = 0; | 201 size_t adjust = 0; |
182 if ((ptr & (alignment - 1)) != 0) { | 202 if ((ptr & (alignment - 1)) != 0) { |
183 adjust = alignment - (ptr & (alignment - 1)); | 203 adjust = alignment - (ptr & (alignment - 1)); |
184 } | 204 } |
185 ptr += adjust; | 205 ptr += adjust; |
186 hugetlb_base_ += (size + extra); | 206 hugetlb_base_ += (size + extra); |
187 | 207 |
188 if (actual_size) { | 208 if (actual_size) { |
189 *actual_size = size + extra - adjust; | 209 *actual_size = size + extra - adjust; |
190 } | 210 } |
191 | 211 |
192 return reinterpret_cast<void*>(ptr); | 212 return reinterpret_cast<void*>(ptr); |
193 } | 213 } |
194 | 214 |
195 static void InitSystemAllocator() { | 215 void HugetlbSysAllocator::FlagsInitialized() { |
196 if (FLAGS_memfs_malloc_path.length()) { | 216 if (FLAGS_memfs_malloc_path.length()) { |
197 char path[PATH_MAX]; | 217 char path[PATH_MAX]; |
198 int rc = snprintf(path, sizeof(path), "%s.XXXXXX", | 218 int rc = snprintf(path, sizeof(path), "%s.XXXXXX", |
199 FLAGS_memfs_malloc_path.c_str()); | 219 FLAGS_memfs_malloc_path.c_str()); |
200 if (rc < 0 || rc >= sizeof(path)) { | 220 if (rc < 0 || rc >= sizeof(path)) { |
201 CRASH("XX fatal: memfs_malloc_path too long\n"); | 221 CRASH("XX fatal: memfs_malloc_path too long\n"); |
202 } | 222 } |
203 | 223 |
204 int hugetlb_fd = mkstemp(path); | 224 int hugetlb_fd = mkstemp(path); |
205 if (hugetlb_fd == -1) { | 225 if (hugetlb_fd == -1) { |
(...skipping 10 matching lines...) Expand all Loading... |
216 } | 236 } |
217 | 237 |
218 // Use fstatfs to figure out the default page size for memfs | 238 // Use fstatfs to figure out the default page size for memfs |
219 struct statfs sfs; | 239 struct statfs sfs; |
220 if (fstatfs(hugetlb_fd, &sfs) == -1) { | 240 if (fstatfs(hugetlb_fd, &sfs) == -1) { |
221 CRASH("fatal: error fstatfs of memfs_malloc_path: %s\n", | 241 CRASH("fatal: error fstatfs of memfs_malloc_path: %s\n", |
222 strerror(errno)); | 242 strerror(errno)); |
223 } | 243 } |
224 int64 page_size = sfs.f_bsize; | 244 int64 page_size = sfs.f_bsize; |
225 | 245 |
226 SysAllocator *alloc = new HugetlbSysAllocator(hugetlb_fd, page_size); | 246 hugetlb_fd_ = hugetlb_fd; |
227 // Register ourselves with tcmalloc | 247 big_page_size_ = page_size; |
228 RegisterSystemAllocator(alloc, 0); | 248 failed_ = false; |
229 } | 249 } |
230 } | 250 } |
231 | 251 |
| 252 static void InitSystemAllocator() { |
| 253 SysAllocator *alloc = MallocExtension::instance()->GetSystemAllocator(); |
| 254 HugetlbSysAllocator *hugetlb = new (hugetlb_space) HugetlbSysAllocator(alloc); |
| 255 MallocExtension::instance()->SetSystemAllocator(hugetlb); |
| 256 } |
| 257 |
232 REGISTER_MODULE_INITIALIZER(memfs_malloc, { InitSystemAllocator(); }); | 258 REGISTER_MODULE_INITIALIZER(memfs_malloc, { InitSystemAllocator(); }); |
233 | 259 |
234 #endif /* ifdef __linux */ | 260 #endif /* ifdef __linux */ |
OLD | NEW |