| OLD | NEW |
| 1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. |
| 2 // All rights reserved. | 2 // All rights reserved. |
| 3 // | 3 // |
| 4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
| 5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
| 6 // met: | 6 // met: |
| 7 // | 7 // |
| 8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
| 9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
| 10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 49 | 49 |
| 50 #include <stddef.h> // for NULL, size_t | 50 #include <stddef.h> // for NULL, size_t |
| 51 #include <string.h> // for memset | 51 #include <string.h> // for memset |
| 52 #if defined HAVE_STDINT_H | 52 #if defined HAVE_STDINT_H |
| 53 #include <stdint.h> | 53 #include <stdint.h> |
| 54 #elif defined HAVE_INTTYPES_H | 54 #elif defined HAVE_INTTYPES_H |
| 55 #include <inttypes.h> | 55 #include <inttypes.h> |
| 56 #else | 56 #else |
| 57 #include <sys/types.h> | 57 #include <sys/types.h> |
| 58 #endif | 58 #endif |
| 59 #ifdef WIN32 | |
| 60 // TODO(jar): This is not needed when TCMalloc_PageMap1_LazyCommit has an API | |
| 61 // supporting commit and reservation of memory. | |
| 62 #include "common.h" | |
| 63 #endif | |
| 64 | |
| 65 #include "internal_logging.h" // for ASSERT | 59 #include "internal_logging.h" // for ASSERT |
| 66 | 60 |
| 67 // Single-level array | 61 // Single-level array |
| 68 template <int BITS> | 62 template <int BITS> |
| 69 class TCMalloc_PageMap1 { | 63 class TCMalloc_PageMap1 { |
| 70 private: | 64 private: |
| 71 static const int LENGTH = 1 << BITS; | 65 static const int LENGTH = 1 << BITS; |
| 72 | 66 |
| 73 void** array_; | 67 void** array_; |
| 74 | 68 |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 113 // a page number >= k. Returns NULL if no such number is found. | 107 // a page number >= k. Returns NULL if no such number is found. |
| 114 void* Next(Number k) const { | 108 void* Next(Number k) const { |
| 115 while (k < (1 << BITS)) { | 109 while (k < (1 << BITS)) { |
| 116 if (array_[k] != NULL) return array_[k]; | 110 if (array_[k] != NULL) return array_[k]; |
| 117 k++; | 111 k++; |
| 118 } | 112 } |
| 119 return NULL; | 113 return NULL; |
| 120 } | 114 } |
| 121 }; | 115 }; |
| 122 | 116 |
| 123 #ifdef WIN32 | |
| 124 // Lazy commit, single-level array. | |
| 125 // Very similar to PageMap1, except the page map is only committed as needed. | |
| 126 // Since we don't return memory to the OS, the committed portion of the map will | |
| 127 // only grow, and we'll only be called to Ensure when we really grow the heap. | |
| 128 // We maintain a bit map to help us deduce if we've already committed a range | |
| 129 // in our map. | |
| 130 template <int BITS> | |
| 131 class TCMalloc_PageMap1_LazyCommit { | |
| 132 private: | |
| 133 // Dimension of our page map array_. | |
| 134 static const int LENGTH = 1 << BITS; | |
| 135 | |
| 136 // The page map array that sits in reserved virtual space. Pages of this | |
| 137 // array are committed as they are needed. For each page of virtual memory, | |
| 138 // we potentially have a pointer to a span instance. | |
| 139 void** array_; | |
| 140 | |
| 141 // A bit vector that allows us to deduce what pages in array_ are committed. | |
| 142 // Note that 2^3 = 8 bits per char, and hence the use of the magical "3" in | |
| 143 // the array range gives us the effective "divide by 8". | |
| 144 char committed_[sizeof(void*) << (BITS - kPageShift - 3)]; | |
| 145 | |
| 146 // Given an |index| into |array_|, find the page number in |array_| that holds | |
| 147 // that element. | |
| 148 size_t ContainingPage(size_t index) const { | |
| 149 return (index * sizeof(*array_)) >> kPageShift; | |
| 150 } | |
| 151 | |
| 152 // Find out if the given page_num index in array_ is in committed memory. | |
| 153 bool IsCommitted(size_t page_num) const { | |
| 154 return committed_[page_num >> 3] & (1 << (page_num & 0x7)); | |
| 155 } | |
| 156 | |
| 157 // Remember that the given page_num index in array_ is in committed memory. | |
| 158 void SetCommitted(size_t page_num) { | |
| 159 committed_[page_num >> 3] |= (1 << (page_num & 0x7)); | |
| 160 } | |
| 161 | |
| 162 public: | |
| 163 typedef uintptr_t Number; | |
| 164 | |
| 165 explicit TCMalloc_PageMap1_LazyCommit(void* (*allocator)(size_t)) { | |
| 166 // TODO(jar): We need a reservation function, but current API to this class | |
| 167 // only provides an allocator. | |
| 168 // Get decommitted memory. We will commit as necessary. | |
| 169 array_ = reinterpret_cast<void**>(VirtualAlloc( | |
| 170 NULL, sizeof(*array_) << BITS, MEM_RESERVE, PAGE_READWRITE)); | |
| 171 | |
| 172 // Make sure we divided LENGTH evenly. | |
| 173 ASSERT(sizeof(committed_) * 8 == (LENGTH * sizeof(*array_)) >> kPageShift); | |
| 174 // Indicate that none of the pages of array_ have been committed yet. | |
| 175 memset(committed_, 0, sizeof(committed_)); | |
| 176 } | |
| 177 | |
| 178 // Ensure that the map contains initialized and committed entries in array_ to | |
| 179 // describe pages "x .. x+n-1". | |
| 180 // Returns true if successful, false if we could not ensure this. | |
| 181 // If we have to commit more memory in array_ (which also clears said memory), | |
| 182 // then we'll set some of the bits in committed_ to remember this fact. | |
| 183 // Only the bits of committed_ near end-points for calls to Ensure() are ever | |
| 184 // set, as the calls to Ensure() will never have overlapping ranges other than | |
| 185 // at their end-points. | |
| 186 // | |
| 187 // Example: Suppose the OS allocates memory in pages including 40...50, and | |
| 188 // later the OS allocates memory in pages 51...83. When the first allocation | |
| 189 // of 40...50 is observed, then Ensure of (39,51) will be called. The range | |
| 190 // shown in the arguments is extended so that tcmalloc can look to see if | |
| 191 // adjacent pages are part of a span that can be coaleced. Later, when pages | |
| 192 // 51...83 are allocated, Ensure() will be called with arguments (50,84), | |
| 193 // broadened again for the same reason. | |
| 194 // | |
| 195 // After the above, we would NEVER get a call such as Ensure(45,60), as that | |
| 196 // overlaps with the interior of prior ensured regions. We ONLY get an Ensure | |
| 197 // call when the OS has allocated memory, and since we NEVER give memory back | |
| 198 // to the OS, the OS can't possible allocate the same region to us twice, and | |
| 199 // can't induce an Ensure() on an interior of previous Ensure call. | |
| 200 // | |
| 201 // Also note that OS allocations are NOT guaranteed to be consecutive (there | |
| 202 // may be "holes" where code etc. uses the virtual addresses), or to appear in | |
| 203 // any order, such as lowest to highest, or vice versa (as other independent | |
| 204 // allocation systems in the process may be performing VirtualAllocations and | |
| 205 // VirtualFrees asynchronously.) | |
| 206 bool Ensure(Number x, size_t n) { | |
| 207 if (n > LENGTH - x) | |
| 208 return false; // We won't Ensure mapping for last pages in memory. | |
| 209 ASSERT(n > 0); | |
| 210 | |
| 211 // For a given page number in memory, calculate what page in array_ needs to | |
| 212 // be memory resident. Note that we really only need a few bytes in array_ | |
| 213 // for each page of virtual space we have to map, but we can only commit | |
| 214 // whole pages of array_. For instance, a 4K page of array_ has about 1k | |
| 215 // entries, and hence can map about 1K pages, or a total of about 4MB | |
| 216 // typically. As a result, it is possible that the first entry in array_, | |
| 217 // and the n'th entry in array_, will sit in the same page of array_. | |
| 218 size_t first_page = ContainingPage(x); | |
| 219 size_t last_page = ContainingPage(x + n - 1); | |
| 220 | |
| 221 // Check at each boundary, to see if we need to commit at that end. Some | |
| 222 // other neighbor may have already forced us to commit at either or both | |
| 223 // boundaries. | |
| 224 if (IsCommitted(first_page)) { | |
| 225 if (first_page == last_page) return true; | |
| 226 ++first_page; | |
| 227 if (IsCommitted(first_page)) { | |
| 228 if (first_page == last_page) return true; | |
| 229 ++first_page; | |
| 230 } | |
| 231 } | |
| 232 | |
| 233 if (IsCommitted(last_page)) { | |
| 234 if (first_page == last_page) return true; | |
| 235 --last_page; | |
| 236 if (IsCommitted(last_page)) { | |
| 237 if (first_page == last_page) return true; | |
| 238 --last_page; | |
| 239 } | |
| 240 } | |
| 241 | |
| 242 ASSERT(!IsCommitted(last_page)); | |
| 243 ASSERT(!IsCommitted(first_page)); | |
| 244 | |
| 245 void* start = reinterpret_cast<char*>(array_) + (first_page << kPageShift); | |
| 246 size_t length = (last_page - first_page + 1) << kPageShift; | |
| 247 | |
| 248 #ifndef NDEBUG | |
| 249 // Validate we are committing new sections, and hence we're not clearing any | |
| 250 // existing data. | |
| 251 MEMORY_BASIC_INFORMATION info = {0}; | |
| 252 size_t result = VirtualQuery(start, &info, sizeof(info)); | |
| 253 ASSERT(result); | |
| 254 ASSERT(0 == (info.State & MEM_COMMIT)); // It starts with uncommitted. | |
| 255 ASSERT(info.RegionSize >= length); // Entire length is uncommitted. | |
| 256 #endif | |
| 257 | |
| 258 // TODO(jar): We need a commit that automatically tallies metadata_bytes. | |
| 259 TCMalloc_SystemCommit(start, length); | |
| 260 tcmalloc::increment_metadata_system_bytes(length); | |
| 261 | |
| 262 #ifndef NDEBUG | |
| 263 result = VirtualQuery(start, &info, sizeof(info)); | |
| 264 ASSERT(result); | |
| 265 ASSERT(0 != (info.State & MEM_COMMIT)); // Now it is committed. | |
| 266 ASSERT(info.RegionSize >= length); // Entire length is committed. | |
| 267 #endif | |
| 268 | |
| 269 // As noted in the large comment/example describing this method, we will | |
| 270 // never be called with a range of pages very much inside this |first_page| | |
| 271 // to |last_page| range. | |
| 272 // As a result, we only need to set bits for each end of that range, and one | |
| 273 // page inside each end. | |
| 274 SetCommitted(first_page); | |
| 275 if (first_page < last_page) { | |
| 276 SetCommitted(last_page); | |
| 277 SetCommitted(first_page + 1); // These may be duplicates now. | |
| 278 SetCommitted(last_page - 1); | |
| 279 } | |
| 280 | |
| 281 return true; | |
| 282 } | |
| 283 | |
| 284 // This is a premature call to get all the meta-memory allocated, so as to | |
| 285 // avoid virtual space fragmentation. Since we pre-reserved all memory, we | |
| 286 // don't need to do anything here (we won't fragment virtual space). | |
| 287 void PreallocateMoreMemory() {} | |
| 288 | |
| 289 // Return the current value for KEY. Returns NULL if not yet set, | |
| 290 // or if k is out of range. | |
| 291 void* get(Number k) const { | |
| 292 if ((k >> BITS) > 0) { | |
| 293 return NULL; | |
| 294 } | |
| 295 return array_[k]; | |
| 296 } | |
| 297 | |
| 298 // REQUIRES "k" is in range "[0,2^BITS-1]". | |
| 299 // REQUIRES "k" has been ensured before. | |
| 300 // | |
| 301 // Sets the value for KEY. | |
| 302 void set(Number k, void* v) { | |
| 303 array_[k] = v; | |
| 304 } | |
| 305 // Return the first non-NULL pointer found in this map for | |
| 306 // a page number >= k. Returns NULL if no such number is found. | |
| 307 void* Next(Number k) const { | |
| 308 while (k < (1 << BITS)) { | |
| 309 if (array_[k] != NULL) return array_[k]; | |
| 310 k++; | |
| 311 } | |
| 312 return NULL; | |
| 313 } | |
| 314 }; | |
| 315 #endif // WIN32 | |
| 316 | |
| 317 | |
| 318 // Two-level radix tree | 117 // Two-level radix tree |
| 319 template <int BITS> | 118 template <int BITS> |
| 320 class TCMalloc_PageMap2 { | 119 class TCMalloc_PageMap2 { |
| 321 private: | 120 private: |
| 322 // Put 32 entries in the root and (2^BITS)/32 entries in each leaf. | 121 // Put 32 entries in the root and (2^BITS)/32 entries in each leaf. |
| 323 static const int ROOT_BITS = 5; | 122 static const int ROOT_BITS = 5; |
| 324 static const int ROOT_LENGTH = 1 << ROOT_BITS; | 123 static const int ROOT_LENGTH = 1 << ROOT_BITS; |
| 325 | 124 |
| 326 static const int LEAF_BITS = BITS - ROOT_BITS; | 125 static const int LEAF_BITS = BITS - ROOT_BITS; |
| 327 static const int LEAF_LENGTH = 1 << LEAF_BITS; | 126 static const int LEAF_LENGTH = 1 << LEAF_BITS; |
| (...skipping 187 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 515 } | 314 } |
| 516 // Advance to next interior entry | 315 // Advance to next interior entry |
| 517 k = ((k >> LEAF_BITS) + 1) << LEAF_BITS; | 316 k = ((k >> LEAF_BITS) + 1) << LEAF_BITS; |
| 518 } | 317 } |
| 519 } | 318 } |
| 520 return NULL; | 319 return NULL; |
| 521 } | 320 } |
| 522 }; | 321 }; |
| 523 | 322 |
| 524 #endif // TCMALLOC_PAGEMAP_H_ | 323 #endif // TCMALLOC_PAGEMAP_H_ |
| OLD | NEW |