Support incremental-max and sample in Profiler data

base/tracked_objects.h

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef BASE_TRACKED_OBJECTS_H_
 #define BASE_TRACKED_OBJECTS_H_
 #pragma once
 
 #include <map>
 #include <stack>
 #include <string>
 #include <vector>
 
 #include "base/base_export.h"
+#include "base/gtest_prod_util.h"

[ramant (doing other things), 2011/12/03 17:36:26]

hi Jim,
  In the tip this file is deleted. New code is in child_thread.cc in content
area.

[ramant (doing other things), 2011/12/03 22:48:39]

This comment was in the wrong file. I meant to add it renderer_tracking.cc

[jar (doing other things), 2011/12/04 00:46:22]

No problem... I understood.

On 2011/12/03 22:48:39, ramant wrote:

 #include "base/lazy_instance.h"
 #include "base/location.h"
 #include "base/profiler/tracked_time.h"
 #include "base/time.h"
 #include "base/synchronization/lock.h"
 #include "base/threading/thread_local_storage.h"
 #include "base/tracking_info.h"
 #include "base/values.h"
 
 // TrackedObjects provides a database of stats about objects (generally Tasks)
@@ skipping 165 matching lines @@
 namespace tracked_objects {
 
 //------------------------------------------------------------------------------
 // For a specific thread, and a specific birth place, the collection of all
 // death info (with tallies for each death thread, to prevent access conflicts).
 class ThreadData;
 class BASE_EXPORT BirthOnThread {
  public:
   BirthOnThread(const Location& location, const ThreadData& current);
 
-  const Location location() const { return location_; }
-  const ThreadData* birth_thread() const { return birth_thread_; }
+  const Location location() const;
+  const ThreadData* birth_thread() const;
 
  private:
   // File/lineno of birth.  This defines the essence of the task, as the context
   // of the birth (construction) often tell what the item is for.  This field
   // is const, and hence safe to access from any thread.
   const Location location_;
 
   // The thread that records births into this object.  Only this thread is
   // allowed to update birth_count_ (which changes over time).
   const ThreadData* const birth_thread_;
 
   DISALLOW_COPY_AND_ASSIGN(BirthOnThread);
 };
 
 //------------------------------------------------------------------------------
 // A class for accumulating counts of births (without bothering with a map<>).
 
 class BASE_EXPORT Births: public BirthOnThread {
  public:
   Births(const Location& location, const ThreadData& current);
 
-  int birth_count() const { return birth_count_; }
+  int birth_count() const;
 
   // When we have a birth we update the count for this BirhPLace.
-  void RecordBirth() { ++birth_count_; }
+  void RecordBirth();
 
   // When a birthplace is changed (updated), we need to decrement the counter
   // for the old instance.
-  void ForgetBirth() { --birth_count_; }  // We corrected a birth place.
+  void ForgetBirth();
 
   // Hack to quickly reset all counts to zero.
-  void Clear() { birth_count_ = 0; }
+  void Clear();
 
  private:
   // The number of births on this thread for our location_.
   int birth_count_;
 
   DISALLOW_COPY_AND_ASSIGN(Births);
 };
 
 //------------------------------------------------------------------------------
 // Basic info summarizing multiple destructions of a tracked object with a
 // single birthplace (fixed Location).  Used both on specific threads, and also
 // in snapshots when integrating assembled data.
 
 class BASE_EXPORT DeathData {
  public:
   // Default initializer.
-  DeathData() : count_(0) {}
+  DeathData();
 
   // When deaths have not yet taken place, and we gather data from all the
   // threads, we create DeathData stats that tally the number of births without
-  // a corrosponding death.
-  explicit DeathData(int count)
-      : count_(count) {}
+  // a corresponding death.
+  explicit DeathData(int count);
 
   // Update stats for a task destruction (death) that had a Run() time of
   // |duration|, and has had a queueing delay of |queue_duration|.
-  void RecordDeath(DurationInt queue_duration,
-                   DurationInt run_duration);
+  void RecordDeath(const DurationInt queue_duration,
+                   const DurationInt run_duration,
+                   int random_number);
 
-  // Metrics accessors.
-  int count() const { return count_; }
-  DurationInt run_duration() const { return run_time_.duration(); }
-  DurationInt AverageMsRunDuration() const;
-  DurationInt run_duration_max() const { return run_time_.max(); }
-  DurationInt queue_duration() const { return queue_time_.duration(); }
-  DurationInt AverageMsQueueDuration() const;
-  DurationInt queue_duration_max() const { return queue_time_.max(); }
-
-  // Accumulate metrics from other into this.  This method is never used on
-  // realtime statistics, and only used in snapshots and aggregatinos.
-  void AddDeathData(const DeathData& other);
+  // Metrics accessors, used only in tests.
+  int count() const;
+  DurationInt run_duration_sum() const;
+  DurationInt run_duration_max() const;
+  DurationInt run_duration_sample() const;
+  DurationInt queue_duration_sum() const;
+  DurationInt queue_duration_max() const;
+  DurationInt queue_duration_sample() const;
 
   // Construct a DictionaryValue instance containing all our stats. The caller
   // assumes ownership of the returned instance.
   base::DictionaryValue* ToValue() const;
 
+  // Reset the max values to zero.
+  void ResetMax();
+
   // Reset all tallies to zero. This is used as a hack on realtime data.
   void Clear();
 
  private:
-  // DeathData::Data is a helper class, useful when different metrics need to be
-  // aggregated, such as queueing times, or run times.
-  class Data {
-   public:
-    Data() : duration_(0), max_(0) {}
-    ~Data() {}
-
-    DurationInt duration() const { return duration_; }
-    DurationInt max() const { return max_; }
-
-    // Agggegate data into our state.
-    void AddData(const Data& other);
-    void AddDuration(DurationInt duration);
-
-    // Central helper function for calculating averages (correctly, in only one
-    // place).
-    DurationInt AverageMsDuration(int count) const;
-
-    // Resets all members to zero.
-    void Clear();
-
-   private:
-    DurationInt duration_;  // Sum of all durations seen.
-    DurationInt max_;       // Largest singular duration seen.
-  };
-
-
-  int count_;         // Number of deaths seen.
-  Data run_time_;    // Data about run time durations.
-  Data queue_time_;  // Data about queueing times durations.
+  // Number of runs seen.
+  int count_;
+  // Data about run time durations.
+  DurationInt run_duration_sum_;
+  DurationInt run_duration_max_;
+  DurationInt run_duration_sample_;
+  // Data about queueing times durations.
+  DurationInt queue_duration_sum_;
+  DurationInt queue_duration_max_;
+  DurationInt queue_duration_sample_;
 };
 
 //------------------------------------------------------------------------------
 // A temporary collection of data that can be sorted and summarized.  It is
 // gathered (carefully) from many threads.  Instances are held in arrays and
 // processed, filtered, and rendered.
 // The source of this data was collected on many threads, and is asynchronously
 // changing.  The data in this instance is not asynchronously changing.
 
 class BASE_EXPORT Snapshot {
  public:
   // When snapshotting a full life cycle set (birth-to-death), use this:
   Snapshot(const BirthOnThread& birth_on_thread, const ThreadData& death_thread,
            const DeathData& death_data);
 
   // When snapshotting a birth, with no death yet, use this:
   Snapshot(const BirthOnThread& birth_on_thread, int count);
 
-  const ThreadData* birth_thread() const { return birth_->birth_thread(); }
-  const Location location() const { return birth_->location(); }
-  const BirthOnThread& birth() const { return *birth_; }
-  const ThreadData* death_thread() const {return death_thread_; }
-  const DeathData& death_data() const { return death_data_; }
+  // Accessor, that provides default value when there is no death thread.
   const std::string DeathThreadName() const;
 
-  int count() const { return death_data_.count(); }
-  DurationInt run_duration() const { return death_data_.run_duration(); }
-  DurationInt AverageMsRunDuration() const {
-    return death_data_.AverageMsRunDuration();
-  }
-  DurationInt run_duration_max() const {
-    return death_data_.run_duration_max();
-  }
-  DurationInt queue_duration() const { return death_data_.queue_duration(); }
-  DurationInt AverageMsQueueDuration() const {
-    return death_data_.AverageMsQueueDuration();
-  }
-  DurationInt queue_duration_max() const {
-    return death_data_.queue_duration_max();
-  }
-
   // Construct a DictionaryValue instance containing all our data recursively.
   // The caller assumes ownership of the memory in the returned instance.
   base::DictionaryValue* ToValue() const;
 
  private:
   const BirthOnThread* birth_;  // Includes Location and birth_thread.
   const ThreadData* death_thread_;
   DeathData death_data_;
 };
 
 //------------------------------------------------------------------------------
-// DataCollector is a container class for Snapshot and BirthOnThread count
-// items.
-
-class BASE_EXPORT DataCollector {
- public:
-  typedef std::vector<Snapshot> Collection;
-
-  // Construct with a list of how many threads should contribute.  This helps us
-  // determine (in the async case) when we are done with all contributions.
-  DataCollector();
-  ~DataCollector();
-
-  // Adds all stats from the indicated thread into our arrays.  This function
-  // uses locks at the lowest level (when accessing the underlying maps which
-  // could change when not locked), and can be called from any threads.
-  void Append(const ThreadData& thread_data);
-
-  // After the accumulation phase, the following accessor is used to process the
-  // data (i.e., sort it, filter it, etc.).
-  Collection* collection();
-
-  // Adds entries for all the remaining living objects (objects that have
-  // tallied a birth, but have not yet tallied a matching death, and hence must
-  // be either running, queued up, or being held in limbo for future posting).
-  // This should be called after all known ThreadData instances have been
-  // processed using Append().
-  void AddListOfLivingObjects();
-
-  // Generates a ListValue representation of the vector of snapshots. The caller
-  // assumes ownership of the memory in the returned instance.
-  base::ListValue* ToValue() const;
-
- private:
-  typedef std::map<const BirthOnThread*, int> BirthCount;
-
-  // The array that we collect data into.
-  Collection collection_;
-
-  // The total number of births recorded at each location for which we have not
-  // seen a death count.  This map changes as we do Append() calls, and is later
-  // used by AddListOfLivingObjects() to gather up unaccounted for births.
-  BirthCount global_birth_count_;
-
-  DISALLOW_COPY_AND_ASSIGN(DataCollector);
-};
-
-//------------------------------------------------------------------------------
 // For each thread, we have a ThreadData that stores all tracking info generated
 // on this thread.  This prevents the need for locking as data accumulates.
 // We use ThreadLocalStorage to quickly identfy the current ThreadData context.
 // We also have a linked list of ThreadData instances, and that list is used to
 // harvest data from all existing instances.
 
 class BASE_EXPORT ThreadData {
  public:
   // Current allowable states of the tracking system.  The states can vary
   // between ACTIVE and DEACTIVATED, but can never go back to UNINITIALIZED.
@@ skipping 13 matching lines @@
   static void InitializeThreadContext(const std::string& suggested_name);
 
   // Using Thread Local Store, find the current instance for collecting data.
   // If an instance does not exist, construct one (and remember it for use on
   // this thread.
   // This may return NULL if the system is disabled for any reason.
   static ThreadData* Get();
 
   // Constructs a DictionaryValue instance containing all recursive results in
   // our process.  The caller assumes ownership of the memory in the returned
-  // instance.
-  static base::DictionaryValue* ToValue();
+  // instance.  During the scavenging, if |reset_value| is true, then the

[ramant (doing other things), 2011/12/03 22:48:39]

reset_value -> reset_max

[jar (doing other things), 2011/12/04 00:46:22]

Done.

+  // DeathData instances max-values are reset to zero during this scan.
+  static base::DictionaryValue* ToValue(bool reset_max);
 
   // Finds (or creates) a place to count births from the given location in this
   // thread, and increment that tally.
   // TallyABirthIfActive will returns NULL if the birth cannot be tallied.
   static Births* TallyABirthIfActive(const Location& location);
 
   // Records the end of a timed run of an object.  The |completed_task| contains
   // a pointer to a Births, the time_posted, and a delayed_start_time if any.
   // The |start_of_run| indicates when we started to perform the run of the
   // task.  The delayed_start_time is non-null for tasks that were posted as
@@ skipping 19 matching lines @@
       const TrackedTime& start_of_run,
       const TrackedTime& end_of_run);
 
   // Record the end of execution in region, generally corresponding to a scope
   // being exited.
   static void TallyRunInAScopedRegionIfTracking(
       const Births* birth,
       const TrackedTime& start_of_run,
       const TrackedTime& end_of_run);
 
-  const std::string thread_name() const { return thread_name_; }
+  const std::string thread_name() const;
 
-  // ---------------------
-  // TODO(jar):
-  // The following functions should all be private, and are only public because
-  // the collection is done externally.  We need to relocate that code from the
-  // collection class into this class, and then all these methods can be made
-  // private.
-  // (Thread safe) Get start of list of all ThreadData instances.
-  static ThreadData* first();
-  // Iterate through the null terminated list of ThreadData instances.
-  ThreadData* next() const { return next_; }
-  // Using our lock, make a copy of the specified maps.  These calls may arrive
-  // from non-local threads, and are used to quickly scan data from all threads
-  // in order to build JSON for about:profiler.
-  void SnapshotBirthMap(BirthMap *output) const;
-  void SnapshotDeathMap(DeathMap *output) const;
-  // -------- end of should be private methods.
+  // Snapshot (under a lock) copies of the maps in each ThreadData instance. For
+  // each set of maps (BirthMap and DeathMap) call the Append() method of the
+  // |target| DataCollector.  If |reset_max| is true, then the max values in
+  // each DeathData instance should be reset during the scan.
+  static void SendAllMaps(bool reset_max, class DataCollector* target);
 
   // Hack: asynchronously clear all birth counts and death tallies data values
   // in all ThreadData instances.  The numerical (zeroing) part is done without
   // use of a locks or atomics exchanges, and may (for int64 values) produce
   // bogus counts VERY rarely.
   static void ResetAllThreadData();
 
   // Initializes all statics if needed (this initialization call should be made
   // while we are single threaded). Returns false if unable to initialize.
   static bool Initialize();
@@ skipping 18 matching lines @@
   static TrackedTime Now();
 
   // This function can be called at process termination to validate that thread
   // cleanup routines have been called for at least some number of named
   // threads.
   static void EnsureCleanupWasCalled(int major_threads_shutdown_count);
 
  private:
   // Allow only tests to call ShutdownSingleThreadedCleanup.  We NEVER call it
   // in production code.
+  // TODO(jar): Make this a friend in DEBUG only, so that the optimizer has a
+  // better change of optimizing (inlining? etc.) private methods (knowing that
+  // there will be no need for an external entry point).
   friend class TrackedObjectsTest;
+  FRIEND_TEST_ALL_PREFIXES(TrackedObjectsTest, MinimalStartupShutdown);
+  FRIEND_TEST_ALL_PREFIXES(TrackedObjectsTest, TinyStartupShutdown);
 
   // Worker thread construction creates a name since there is none.
   explicit ThreadData(int thread_number);
 
   // Message loop based construction should provide a name.
   explicit ThreadData(const std::string& suggested_name);
 
   ~ThreadData();
 
   // Push this instance to the head of all_thread_data_list_head_, linking it to
   // the previous head.  This is performed after each construction, and leaves
   // the instance permanently on that list.
   void PushToHeadOfList();
 
+  // (Thread safe) Get start of list of all ThreadData instances using the lock.
+  static ThreadData* first();
+
+  // Iterate through the null terminated list of ThreadData instances.
+  ThreadData* next() const;
+
+
   // In this thread's data, record a new birth.
   Births* TallyABirth(const Location& location);
 
   // Find a place to record a death on this thread.
   void TallyADeath(const Births& birth,
                    DurationInt queue_duration,
                    DurationInt duration);
 
+  // Using our lock, make a copy of the specified maps.  This call may be made
+  // on  non-local threads, which necessitate the use of the lock to prevent
+  // the map(s) from being reallocaed while they are copied. If |reset_max| is
+  // true, then, just after we copy the DeathMap, we will set the max values to
+  // zero in the active DeathMap (not the snapshot).
+  void SnapshotMaps(bool reset_max,
+                    BirthMap* birth_map,
+                    DeathMap* death_map);
+
   // Using our lock to protect the iteration, Clear all birth and death data.
   void Reset();
 
   // This method is called by the TLS system when a thread terminates.
   // The argument may be NULL if this thread has never tracked a birth or death.
   static void OnThreadTermination(void* thread_data);
 
   // This method should be called when a worker thread terminates, so that we
   // can save all the thread data into a cache of reusable ThreadData instances.
   void OnThreadTerminationCleanup();
@@ skipping 86 matching lines @@
   DeathMap death_map_;
 
   // Lock to protect *some* access to BirthMap and DeathMap.  The maps are
   // regularly read and written on this thread, but may only be read from other
   // threads.  To support this, we acquire this lock if we are writing from this
   // thread, or reading from another thread.  For reading from this thread we
   // don't need a lock, as there is no potential for a conflict since the
   // writing is only done from this thread.
   mutable base::Lock map_lock_;
 
+  // A random number that we used to select decide which sample to keep as a
+  // representative sample in each DeathData instance.  We start it off "very
+  // random" (which is expensive), and then stir in some data gently as we see
+  // various events.
+  int32 random_number_;
+
   DISALLOW_COPY_AND_ASSIGN(ThreadData);
 };
 
 //------------------------------------------------------------------------------
+// DataCollector is a container class for Snapshot and BirthOnThread count
+// items.
+
+class BASE_EXPORT DataCollector {
+ public:
+  typedef std::vector<Snapshot> Collection;
+
+  // Construct with a list of how many threads should contribute.  This helps us
+  // determine (in the async case) when we are done with all contributions.
+  DataCollector();
+  ~DataCollector();
+
+  // Adds all stats from the indicated thread into our arrays.  Accepts copies
+  // of the birth_map and death_map, so that the data will not change during the
+  // iterations and processing.
+  void Append(const ThreadData &thread_data,
+              const ThreadData::BirthMap &birth_map,
+              const ThreadData::DeathMap &death_map);
+
+  // After the accumulation phase, the following accessor is used to process the
+  // data (i.e., sort it, filter it, etc.).
+  Collection* collection();
+
+  // Adds entries for all the remaining living objects (objects that have
+  // tallied a birth, but have not yet tallied a matching death, and hence must
+  // be either running, queued up, or being held in limbo for future posting).
+  // This should be called after all known ThreadData instances have been
+  // processed using Append().
+  void AddListOfLivingObjects();
+
+  // Generates a ListValue representation of the vector of snapshots. The caller
+  // assumes ownership of the memory in the returned instance.
+  base::ListValue* ToValue() const;
+
+ private:
+  typedef std::map<const BirthOnThread*, int> BirthCount;
+
+  // The array that we collect data into.
+  Collection collection_;
+
+  // The total number of births recorded at each location for which we have not
+  // seen a death count.  This map changes as we do Append() calls, and is later
+  // used by AddListOfLivingObjects() to gather up unaccounted for births.
+  BirthCount global_birth_count_;
+
+  DISALLOW_COPY_AND_ASSIGN(DataCollector);
+};
+
+//------------------------------------------------------------------------------
 // Provide simple way to to start global tracking, and to tear down tracking
 // when done.  The design has evolved to *not* do any teardown (and just leak
 // all allocated data structures).  As a result, we don't have any code in this
 // destructor, and perhaps this whole class should go away.
 
 class BASE_EXPORT AutoTracking {
  public:
   AutoTracking() {
     ThreadData::Initialize();
   }
 
   ~AutoTracking() {
     // TODO(jar): Consider emitting a CSV dump of the data at this point.  This
     // should be called after the message loops have all terminated (or at least
     // the main message loop is gone), so there is little chance for additional
     // tasks to be Run.
   }
 
  private:
 
   DISALLOW_COPY_AND_ASSIGN(AutoTracking);
 };
 
 }  // namespace tracked_objects
 
 #endif  // BASE_TRACKED_OBJECTS_H_