OLD | NEW |
---|---|
1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "base/profiler/stack_sampling_profiler.h" | 5 #include "base/profiler/stack_sampling_profiler.h" |
6 | 6 |
7 #include <algorithm> | 7 #include <algorithm> |
8 #include <map> | |
9 #include <queue> | |
8 #include <utility> | 10 #include <utility> |
9 | 11 |
12 #include "base/atomic_sequence_num.h" | |
13 #include "base/atomicops.h" | |
10 #include "base/bind.h" | 14 #include "base/bind.h" |
11 #include "base/bind_helpers.h" | 15 #include "base/bind_helpers.h" |
12 #include "base/callback.h" | 16 #include "base/callback.h" |
13 #include "base/lazy_instance.h" | 17 #include "base/lazy_instance.h" |
14 #include "base/location.h" | 18 #include "base/location.h" |
15 #include "base/macros.h" | 19 #include "base/macros.h" |
20 #include "base/memory/ptr_util.h" | |
21 #include "base/memory/singleton.h" | |
16 #include "base/profiler/native_stack_sampler.h" | 22 #include "base/profiler/native_stack_sampler.h" |
17 #include "base/synchronization/lock.h" | 23 #include "base/synchronization/lock.h" |
24 #include "base/threading/thread.h" | |
18 #include "base/threading/thread_task_runner_handle.h" | 25 #include "base/threading/thread_task_runner_handle.h" |
19 #include "base/timer/elapsed_timer.h" | 26 #include "base/timer/elapsed_timer.h" |
20 | 27 |
21 namespace base { | 28 namespace base { |
22 | 29 |
23 namespace { | 30 namespace { |
24 | 31 |
25 // Used to ensure only one profiler is running at a time. | |
26 LazyInstance<Lock>::Leaky concurrent_profiling_lock = LAZY_INSTANCE_INITIALIZER; | |
27 | |
28 // AsyncRunner ---------------------------------------------------------------- | 32 // AsyncRunner ---------------------------------------------------------------- |
29 | 33 |
30 // Helper class to allow a profiler to be run completely asynchronously from the | 34 // Helper class to allow a profiler to be run completely asynchronously from the |
31 // initiator, without being concerned with the profiler's lifetime. | 35 // initiator, without being concerned with the profiler's lifetime. |
32 class AsyncRunner { | 36 class AsyncRunner { |
33 public: | 37 public: |
34 // Sets up a profiler and arranges for it to be deleted on its completed | 38 // Sets up a profiler and arranges for it to be deleted on its completed |
35 // callback. | 39 // callback. |
36 static void Run(PlatformThreadId thread_id, | 40 static void Run(PlatformThreadId thread_id, |
37 const StackSamplingProfiler::SamplingParams& params, | 41 const StackSamplingProfiler::SamplingParams& params, |
(...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
153 StackSamplingProfiler::CallStackProfile | 157 StackSamplingProfiler::CallStackProfile |
154 StackSamplingProfiler::CallStackProfile::CopyForTesting() const { | 158 StackSamplingProfiler::CallStackProfile::CopyForTesting() const { |
155 return CallStackProfile(*this); | 159 return CallStackProfile(*this); |
156 } | 160 } |
157 | 161 |
158 StackSamplingProfiler::CallStackProfile::CallStackProfile( | 162 StackSamplingProfiler::CallStackProfile::CallStackProfile( |
159 const CallStackProfile& other) = default; | 163 const CallStackProfile& other) = default; |
160 | 164 |
161 // StackSamplingProfiler::SamplingThread -------------------------------------- | 165 // StackSamplingProfiler::SamplingThread -------------------------------------- |
162 | 166 |
163 StackSamplingProfiler::SamplingThread::SamplingThread( | 167 class StackSamplingProfiler::SamplingThread : public Thread { |
164 std::unique_ptr<NativeStackSampler> native_sampler, | 168 public: |
165 const SamplingParams& params, | 169 struct CollectionContext { |
166 const CompletedCallback& completed_callback) | 170 CollectionContext(PlatformThreadId target, |
167 : native_sampler_(std::move(native_sampler)), | 171 const SamplingParams& params, |
168 params_(params), | 172 const CompletedCallback& callback, |
169 stop_event_(WaitableEvent::ResetPolicy::AUTOMATIC, | 173 std::unique_ptr<NativeStackSampler> sampler) |
170 WaitableEvent::InitialState::NOT_SIGNALED), | 174 : collection_id(next_collection_id_.GetNext()), |
171 completed_callback_(completed_callback) {} | 175 target(target), |
172 | 176 params(params), |
173 StackSamplingProfiler::SamplingThread::~SamplingThread() {} | 177 callback(callback), |
174 | 178 native_sampler(std::move(sampler)) {} |
175 void StackSamplingProfiler::SamplingThread::ThreadMain() { | 179 ~CollectionContext() {} |
176 PlatformThread::SetName("Chrome_SamplingProfilerThread"); | 180 |
177 | 181 // Updates the |next_sample_time| time based on configured parameters. |
178 // For now, just ignore any requests to profile while another profiler is | 182 // This will keep a consistent average interval between samples but will |
179 // working. | 183 // result in constant series of acquisitions, thus nearly locking out the |
180 if (!concurrent_profiling_lock.Get().Try()) | 184 // target thread, if the interval is smaller than the time it takes to |
185 // actually acquire the sample. Anything sampling that quickly is going | |
186 // to be a problem anyway so don't worry about it. | |
187 bool UpdateNextSampleTime() { | |
188 if (++sample < params.samples_per_burst) { | |
189 next_sample_time += params.sampling_interval; | |
190 return true; | |
191 } | |
192 | |
193 if (++burst < params.bursts) { | |
194 sample = 0; | |
195 next_sample_time += params.burst_interval; | |
196 return true; | |
197 } | |
198 | |
199 return false; | |
200 } | |
201 | |
202 // An identifier for this collection, used to uniquely identify it to | |
203 // outside interests. | |
204 const int collection_id; | |
205 | |
206 Time next_sample_time; | |
207 | |
208 PlatformThreadId target; | |
209 SamplingParams params; | |
210 CompletedCallback callback; | |
211 | |
212 std::unique_ptr<NativeStackSampler> native_sampler; | |
213 | |
214 // Counters that indicate the current position along the acquisition. | |
215 int burst = 0; | |
216 int sample = 0; | |
217 | |
218 // The time that a profile was started, for calculating the total duration. | |
219 Time profile_start_time; | |
220 | |
221 // The collected stack samples. The active profile is always at the back(). | |
222 CallStackProfiles profiles; | |
223 | |
224 private: | |
225 static StaticAtomicSequenceNumber next_collection_id_; | |
226 }; | |
227 | |
228 // Gets the single instance of this class. | |
229 static SamplingThread* GetInstance(); | |
230 | |
231 // Starts the thread. | |
232 void Start(); | |
233 | |
234 // Adds a new CollectionContext to the thread. This can be called externally | |
235 // from any thread. This returns an ID that can later be used to stop | |
236 // the sampling. | |
237 int Add(std::unique_ptr<CollectionContext> collection); | |
238 | |
239 // Stops an active collection based on its ID, forcing it to run its callback | |
240 // if any data has been collected. This can be called externally from any | |
241 // thread. | |
242 void Stop(int id); | |
243 | |
244 private: | |
245 SamplingThread(); | |
246 ~SamplingThread() override; | |
247 friend struct DefaultSingletonTraits<SamplingThread>; | |
248 | |
249 // Finishes a collection and reports collected data via callback. | |
250 void FinishCollection(CollectionContext* collection); | |
251 | |
252 // Records a single sample of a collection. | |
253 void RecordSample(CollectionContext* collection); | |
254 | |
255 // These methods are tasks that get posted to the internal message queue. | |
256 void StartCollectionTask(std::unique_ptr<CollectionContext> collection_ptr); | |
257 void StopCollectionTask(int id); | |
258 void PerformCollectionTask(int id); | |
259 | |
260 // Thread: | |
261 void CleanUp() override; | |
262 | |
263 // The task-runner for the sampling thread. This is saved so that it can | |
264 // be freely used by any calling thread. | |
265 scoped_refptr<SingleThreadTaskRunner> task_runner_; | |
266 Lock task_runner_lock_; | |
267 | |
268 // A map of IDs to collection contexts. Because this class is a singleton | |
269 // that is never destroyed, context objects will never be destructed except | |
270 // by explicit action. Thus, it's acceptable to pass unretained pointers | |
271 // to these objects when posting tasks. | |
272 std::map<int, std::unique_ptr<CollectionContext>> active_collections_; | |
273 | |
274 DISALLOW_COPY_AND_ASSIGN(SamplingThread); | |
275 }; | |
276 | |
277 StaticAtomicSequenceNumber StackSamplingProfiler::SamplingThread:: | |
278 CollectionContext::next_collection_id_; | |
279 | |
280 StackSamplingProfiler::SamplingThread::SamplingThread() | |
281 : Thread("Chrome_SamplingProfilerThread") {} | |
282 | |
283 StackSamplingProfiler::SamplingThread::~SamplingThread() { | |
284 Thread::Stop(); | |
285 } | |
286 | |
287 StackSamplingProfiler::SamplingThread* | |
288 StackSamplingProfiler::SamplingThread::GetInstance() { | |
289 return Singleton<SamplingThread, LeakySingletonTraits<SamplingThread>>::get(); | |
290 } | |
291 | |
292 void StackSamplingProfiler::SamplingThread::Start() { | |
293 Thread::Options options; | |
294 // Use a higher priority for a more accurate sampling interval. | |
295 options.priority = ThreadPriority::DISPLAY; | |
296 Thread::StartWithOptions(options); | |
297 } | |
298 | |
299 int StackSamplingProfiler::SamplingThread::Add( | |
300 std::unique_ptr<CollectionContext> collection) { | |
301 int id = collection->collection_id; | |
302 | |
303 AutoLock lock(task_runner_lock_); | |
304 if (!task_runner_) { | |
305 // The thread is not running. Start it and get associated runner. | |
306 Start(); | |
307 task_runner_ = task_runner(); | |
308 } | |
309 | |
310 task_runner_->PostTask( | |
311 FROM_HERE, Bind(&SamplingThread::StartCollectionTask, Unretained(this), | |
312 Passed(&collection))); | |
313 return id; | |
314 } | |
315 | |
316 void StackSamplingProfiler::SamplingThread::Stop(int id) { | |
317 AutoLock lock(task_runner_lock_); | |
318 if (!task_runner_) | |
319 return; // Everything has already stopped. | |
320 | |
321 // This can fail if the thread were to exit between acquisition of the task | |
322 // runner above and the call below. In that case, however, everything has | |
323 // stopped so there's no need to try to stop it. | |
324 task_runner_->PostTask(FROM_HERE, Bind(&SamplingThread::StopCollectionTask, | |
325 Unretained(this), id)); | |
326 } | |
327 | |
328 void StackSamplingProfiler::SamplingThread::FinishCollection( | |
329 CollectionContext* collection) { | |
330 // If there is no duration for the final profile (because it was stopped), | |
331 // calculated it now. | |
332 if (!collection->profiles.empty() && | |
333 collection->profiles.back().profile_duration == TimeDelta()) { | |
334 collection->profiles.back().profile_duration = | |
335 Time::Now() - collection->profile_start_time; | |
336 } | |
337 | |
338 // Run the associated callback, passing the collected profiles. It's okay to | |
339 // move them because this collection is about to be deleted. | |
340 collection->callback.Run(std::move(collection->profiles)); | |
341 | |
342 // Remove this collection from the map of known ones. This must be done | |
343 // last as the |collection| parameter is invalid after this point. | |
344 size_t count = active_collections_.erase(collection->collection_id); | |
345 DCHECK_EQ(1U, count); | |
346 } | |
347 | |
348 void StackSamplingProfiler::SamplingThread::RecordSample( | |
349 CollectionContext* collection) { | |
350 DCHECK(collection->native_sampler); | |
351 | |
352 // If this is the first sample of a burst, a new Profile needs to be created | |
353 // and filled. | |
354 if (collection->sample == 0) { | |
355 collection->profiles.push_back(CallStackProfile()); | |
356 CallStackProfile& profile = collection->profiles.back(); | |
357 profile.sampling_period = collection->params.sampling_interval; | |
358 collection->profile_start_time = Time::Now(); | |
359 collection->native_sampler->ProfileRecordingStarting(&profile.modules); | |
360 } | |
361 | |
362 // The currently active profile being acptured. | |
363 CallStackProfile& profile = collection->profiles.back(); | |
364 | |
365 // Record a single sample. | |
366 profile.samples.push_back(Sample()); | |
367 collection->native_sampler->RecordStackSample(&profile.samples.back()); | |
368 | |
369 // If this is the last sample of a burst, record the total time. | |
370 if (collection->sample == collection->params.samples_per_burst - 1) { | |
371 profile.profile_duration = Time::Now() - collection->profile_start_time; | |
372 collection->native_sampler->ProfileRecordingStopped(); | |
373 } | |
374 } | |
375 | |
376 void StackSamplingProfiler::SamplingThread::StartCollectionTask( | |
377 std::unique_ptr<CollectionContext> collection_ptr) { | |
378 // Ownership of the collection is going to be given to a map but a pointer | |
379 // to it will be needed later. | |
380 CollectionContext* collection = collection_ptr.get(); | |
381 active_collections_.insert( | |
382 std::make_pair(collection->collection_id, std::move(collection_ptr))); | |
383 | |
384 task_runner()->PostDelayedTask( | |
385 FROM_HERE, Bind(&SamplingThread::PerformCollectionTask, Unretained(this), | |
386 collection->collection_id), | |
387 collection->params.initial_delay); | |
388 } | |
389 | |
390 void StackSamplingProfiler::SamplingThread::StopCollectionTask(int id) { | |
391 auto found = active_collections_.find(id); | |
392 if (found == active_collections_.end()) | |
181 return; | 393 return; |
182 | 394 |
183 CallStackProfiles profiles; | 395 FinishCollection(found->second.get()); |
184 CollectProfiles(&profiles); | 396 } |
185 concurrent_profiling_lock.Get().Release(); | 397 |
186 completed_callback_.Run(std::move(profiles)); | 398 void StackSamplingProfiler::SamplingThread::PerformCollectionTask(int id) { |
187 } | 399 auto found = active_collections_.find(id); |
188 | 400 if (found == active_collections_.end()) |
Mike Wittman
2016/12/22 17:38:22
It would be good to retain the comment here indica
bcwhite
2017/01/05 16:35:58
Done.
| |
189 // Depending on how long the sampling takes and the length of the sampling | |
190 // interval, a burst of samples could take arbitrarily longer than | |
191 // samples_per_burst * sampling_interval. In this case, we (somewhat | |
192 // arbitrarily) honor the number of samples requested rather than strictly | |
193 // adhering to the sampling intervals. Once we have established users for the | |
194 // StackSamplingProfiler and the collected data to judge, we may go the other | |
195 // way or make this behavior configurable. | |
196 void StackSamplingProfiler::SamplingThread::CollectProfile( | |
197 CallStackProfile* profile, | |
198 TimeDelta* elapsed_time, | |
199 bool* was_stopped) { | |
200 ElapsedTimer profile_timer; | |
201 native_sampler_->ProfileRecordingStarting(&profile->modules); | |
202 profile->sampling_period = params_.sampling_interval; | |
203 *was_stopped = false; | |
204 TimeDelta previous_elapsed_sample_time; | |
205 for (int i = 0; i < params_.samples_per_burst; ++i) { | |
206 if (i != 0) { | |
207 // Always wait, even if for 0 seconds, so we can observe a signal on | |
208 // stop_event_. | |
209 if (stop_event_.TimedWait( | |
210 std::max(params_.sampling_interval - previous_elapsed_sample_time, | |
211 TimeDelta()))) { | |
212 *was_stopped = true; | |
213 break; | |
214 } | |
215 } | |
216 ElapsedTimer sample_timer; | |
217 profile->samples.push_back(Sample()); | |
218 native_sampler_->RecordStackSample(&profile->samples.back()); | |
219 previous_elapsed_sample_time = sample_timer.Elapsed(); | |
220 } | |
221 | |
222 *elapsed_time = profile_timer.Elapsed(); | |
223 profile->profile_duration = *elapsed_time; | |
224 native_sampler_->ProfileRecordingStopped(); | |
225 } | |
226 | |
227 // In an analogous manner to CollectProfile() and samples exceeding the expected | |
228 // total sampling time, bursts may also exceed the burst_interval. We adopt the | |
229 // same wait-and-see approach here. | |
230 void StackSamplingProfiler::SamplingThread::CollectProfiles( | |
231 CallStackProfiles* profiles) { | |
232 if (stop_event_.TimedWait(params_.initial_delay)) | |
233 return; | 401 return; |
234 | 402 |
235 TimeDelta previous_elapsed_profile_time; | 403 CollectionContext* collection = found->second.get(); |
236 for (int i = 0; i < params_.bursts; ++i) { | 404 |
237 if (i != 0) { | 405 // Handle first-run with no "next time". |
238 // Always wait, even if for 0 seconds, so we can observe a signal on | 406 if (collection->next_sample_time == Time()) |
239 // stop_event_. | 407 collection->next_sample_time = Time::Now(); |
240 if (stop_event_.TimedWait( | 408 |
241 std::max(params_.burst_interval - previous_elapsed_profile_time, | 409 // Do the collection of a single sample. |
242 TimeDelta()))) | 410 RecordSample(collection); |
243 return; | 411 |
244 } | 412 // Update the time of the next sample recording. |
245 | 413 if (collection->UpdateNextSampleTime()) { |
246 CallStackProfile profile; | 414 // Place the updated entry back on the queue. |
247 bool was_stopped = false; | 415 bool success = task_runner()->PostDelayedTask( |
248 CollectProfile(&profile, &previous_elapsed_profile_time, &was_stopped); | 416 FROM_HERE, |
249 if (!profile.samples.empty()) | 417 Bind(&SamplingThread::PerformCollectionTask, Unretained(this), id), |
250 profiles->push_back(std::move(profile)); | 418 std::max(collection->next_sample_time - Time::Now(), TimeDelta())); |
251 | 419 DCHECK(success); |
252 if (was_stopped) | 420 } else { |
253 return; | 421 // All capturing has completed so finish the collection. Let object expire. |
254 } | 422 // The |collection| variable will be invalid after this call. |
255 } | 423 FinishCollection(collection); |
256 | 424 } |
257 void StackSamplingProfiler::SamplingThread::Stop() { | 425 } |
258 stop_event_.Signal(); | 426 |
427 void StackSamplingProfiler::SamplingThread::CleanUp() { | |
428 // There should be no collections remaining when the thread stops. | |
429 DCHECK(active_collections_.empty()); | |
430 | |
431 // Let the parent clean up. | |
432 Thread::CleanUp(); | |
259 } | 433 } |
260 | 434 |
261 // StackSamplingProfiler ------------------------------------------------------ | 435 // StackSamplingProfiler ------------------------------------------------------ |
262 | 436 |
263 subtle::Atomic32 StackSamplingProfiler::process_phases_ = 0; | 437 subtle::Atomic32 StackSamplingProfiler::process_phases_ = 0; |
264 | 438 |
265 StackSamplingProfiler::SamplingParams::SamplingParams() | 439 StackSamplingProfiler::SamplingParams::SamplingParams() |
266 : initial_delay(TimeDelta::FromMilliseconds(0)), | 440 : initial_delay(TimeDelta::FromMilliseconds(0)), |
267 bursts(1), | 441 bursts(1), |
268 burst_interval(TimeDelta::FromMilliseconds(10000)), | 442 burst_interval(TimeDelta::FromMilliseconds(10000)), |
(...skipping 11 matching lines...) Expand all Loading... | |
280 PlatformThreadId thread_id, | 454 PlatformThreadId thread_id, |
281 const SamplingParams& params, | 455 const SamplingParams& params, |
282 const CompletedCallback& callback, | 456 const CompletedCallback& callback, |
283 NativeStackSamplerTestDelegate* test_delegate) | 457 NativeStackSamplerTestDelegate* test_delegate) |
284 : thread_id_(thread_id), params_(params), completed_callback_(callback), | 458 : thread_id_(thread_id), params_(params), completed_callback_(callback), |
285 test_delegate_(test_delegate) { | 459 test_delegate_(test_delegate) { |
286 } | 460 } |
287 | 461 |
288 StackSamplingProfiler::~StackSamplingProfiler() { | 462 StackSamplingProfiler::~StackSamplingProfiler() { |
289 Stop(); | 463 Stop(); |
290 if (!sampling_thread_handle_.is_null()) | |
291 PlatformThread::Join(sampling_thread_handle_); | |
292 } | 464 } |
293 | 465 |
294 // static | 466 // static |
295 void StackSamplingProfiler::StartAndRunAsync( | 467 void StackSamplingProfiler::StartAndRunAsync( |
296 PlatformThreadId thread_id, | 468 PlatformThreadId thread_id, |
297 const SamplingParams& params, | 469 const SamplingParams& params, |
298 const CompletedCallback& callback) { | 470 const CompletedCallback& callback) { |
299 CHECK(ThreadTaskRunnerHandle::Get()); | 471 CHECK(ThreadTaskRunnerHandle::Get()); |
300 AsyncRunner::Run(thread_id, params, callback); | 472 AsyncRunner::Run(thread_id, params, callback); |
301 } | 473 } |
302 | 474 |
303 void StackSamplingProfiler::Start() { | 475 void StackSamplingProfiler::Start() { |
304 if (completed_callback_.is_null()) | 476 if (completed_callback_.is_null()) |
305 return; | 477 return; |
306 | 478 |
307 std::unique_ptr<NativeStackSampler> native_sampler = | 479 std::unique_ptr<NativeStackSampler> native_sampler = |
308 NativeStackSampler::Create(thread_id_, &RecordAnnotations, | 480 NativeStackSampler::Create(thread_id_, &RecordAnnotations, |
309 test_delegate_); | 481 test_delegate_); |
310 if (!native_sampler) | 482 if (!native_sampler) |
311 return; | 483 return; |
312 | 484 |
313 sampling_thread_.reset(new SamplingThread(std::move(native_sampler), params_, | 485 collection_id_ = SamplingThread::GetInstance()->Add( |
314 completed_callback_)); | 486 MakeUnique<SamplingThread::CollectionContext>( |
315 if (!PlatformThread::Create(0, sampling_thread_.get(), | 487 thread_id_, params_, completed_callback_, std::move(native_sampler))); |
316 &sampling_thread_handle_)) | |
317 sampling_thread_.reset(); | |
318 } | 488 } |
319 | 489 |
320 void StackSamplingProfiler::Stop() { | 490 void StackSamplingProfiler::Stop() { |
321 if (sampling_thread_) | 491 SamplingThread::GetInstance()->Stop(collection_id_); |
322 sampling_thread_->Stop(); | |
323 } | 492 } |
324 | 493 |
325 // static | 494 // static |
326 void StackSamplingProfiler::SetProcessPhase(int phase) { | 495 void StackSamplingProfiler::SetProcessPhase(int phase) { |
327 DCHECK_LE(0, phase); | 496 DCHECK_LE(0, phase); |
328 DCHECK_GT(static_cast<int>(sizeof(process_phases_) * 8), phase); | 497 DCHECK_GT(static_cast<int>(sizeof(process_phases_) * 8), phase); |
329 DCHECK_EQ(0, subtle::NoBarrier_Load(&process_phases_) & (1 << phase)); | 498 DCHECK_EQ(0, subtle::NoBarrier_Load(&process_phases_) & (1 << phase)); |
330 ChangeAtomicFlags(&process_phases_, 1 << phase, 0); | 499 ChangeAtomicFlags(&process_phases_, 1 << phase, 0); |
331 } | 500 } |
332 | 501 |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
378 } | 547 } |
379 | 548 |
380 bool operator<(const StackSamplingProfiler::Frame &a, | 549 bool operator<(const StackSamplingProfiler::Frame &a, |
381 const StackSamplingProfiler::Frame &b) { | 550 const StackSamplingProfiler::Frame &b) { |
382 return (a.module_index < b.module_index) || | 551 return (a.module_index < b.module_index) || |
383 (a.module_index == b.module_index && | 552 (a.module_index == b.module_index && |
384 a.instruction_pointer < b.instruction_pointer); | 553 a.instruction_pointer < b.instruction_pointer); |
385 } | 554 } |
386 | 555 |
387 } // namespace base | 556 } // namespace base |
OLD | NEW |