Compressed activity log database storage

chrome/browser/extensions/activity_log/fullstream_ui_policy.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/extensions/activity_log/fullstream_ui_policy.h"
 
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/files/file_path.h"
 #include "base/json/json_reader.h"
@@ skipping 15 matching lines @@
 using base::Callback;
 using base::FilePath;
 using base::Time;
 using base::Unretained;
 using content::BrowserThread;
 
 namespace constants = activity_log_constants;
 
 namespace {
 
-// Key strings for passing parameters to the ProcessAction member function.
-const char kKeyReason[] =         "fsuip.reason";
-const char kKeyDomainAction[] =   "fsuip.domact";
-const char kKeyURLTitle[] =       "fsuip.urltitle";
-const char kKeyDetailsString[] =  "fsuip.details";
-
-// Obsolete database tables: these should be dropped from the database if
-// found.
-const char* kObsoleteTables[] = {"activitylog_apis", "activitylog_blocked",
-                                 "activitylog_urls"};
-
 std::string Serialize(const base::Value* value) {
   std::string value_as_text;
   if (!value) {
     value_as_text = "null";
   } else {
     JSONStringValueSerializer serializer(&value_as_text);
     serializer.SerializeAndOmitBinaryValues(*value);
   }
   return value_as_text;
 }
 
 }  // namespace
 
 namespace extensions {
 
 const char* FullStreamUIPolicy::kTableName = "activitylog_full";
 const char* FullStreamUIPolicy::kTableContentFields[] = {
   "extension_id", "time", "action_type", "api_name", "args", "page_url",
   "page_title", "arg_url", "other"
 };
 const char* FullStreamUIPolicy::kTableFieldTypes[] = {
   "LONGVARCHAR NOT NULL", "INTEGER", "INTEGER", "LONGVARCHAR", "LONGVARCHAR",
   "LONGVARCHAR", "LONGVARCHAR", "LONGVARCHAR", "LONGVARCHAR"
 };
-const int FullStreamUIPolicy::kTableFieldCount = arraysize(kTableContentFields);
+const int FullStreamUIPolicy::kTableFieldCount =
+    arraysize(FullStreamUIPolicy::kTableContentFields);
 
 FullStreamUIPolicy::FullStreamUIPolicy(Profile* profile)
     : ActivityLogDatabasePolicy(
           profile,
           FilePath(chrome::kExtensionActivityLogFilename)) {}
 
 FullStreamUIPolicy::~FullStreamUIPolicy() {}
 
 bool FullStreamUIPolicy::InitDatabase(sql::Connection* db) {
-  // Drop old database tables.
-  for (size_t i = 0; i < arraysize(kObsoleteTables); i++) {
-    const char* table_name = kObsoleteTables[i];
-    if (db->DoesTableExist(table_name)) {
-      std::string drop_statement =
-          base::StringPrintf("DROP TABLE %s", table_name);
-      if (!db->Execute(drop_statement.c_str())) {
-        return false;
-      }
-    }
-  }
+  if (!Util::DropObsoleteTables(db))
+    return false;
 
   // Create the unified activity log entry table.
   return ActivityDatabase::InitializeTable(db,
                                            kTableName,
                                            kTableContentFields,
                                            kTableFieldTypes,
                                            arraysize(kTableContentFields));
 }
 
 bool FullStreamUIPolicy::FlushDatabase(sql::Connection* db) {
   if (queued_actions_.empty())
     return true;
 
   sql::Transaction transaction(db);
   if (!transaction.Begin())
     return false;
 
   std::string sql_str =
       "INSERT INTO " + std::string(FullStreamUIPolicy::kTableName) +
       " (extension_id, time, action_type, api_name, args, "
       "page_url, page_title, arg_url, other) VALUES (?,?,?,?,?,?,?,?,?)";
   sql::Statement statement(db->GetCachedStatement(
       sql::StatementID(SQL_FROM_HERE), sql_str.c_str()));
 
-  url_canon::Replacements<char> url_sanitizer;
-  if (!CommandLine::ForCurrentProcess()->HasSwitch(
-          switches::kEnableExtensionActivityLogTesting)) {
-    url_sanitizer.ClearQuery();
-    url_sanitizer.ClearRef();
-  }
-
   Action::ActionVector::size_type i;
   for (i = 0; i != queued_actions_.size(); ++i) {
     const Action& action = *queued_actions_[i];
     statement.Reset(true);
     statement.BindString(0, action.extension_id());
     statement.BindInt64(1, action.time().ToInternalValue());
     statement.BindInt(2, static_cast<int>(action.action_type()));
     statement.BindString(3, action.api_name());
     if (action.args()) {
       statement.BindString(4, Serialize(action.args()));
     }
     if (action.page_url().is_valid()) {
       if (action.page_incognito()) {
         statement.BindString(5, constants::kIncognitoUrl);
       } else {
-        statement.BindString(
-            5, action.page_url().ReplaceComponents(url_sanitizer).spec());
+        statement.BindString(5, action.page_url().spec());
       }
     }
     if (!action.page_title().empty() && !action.page_incognito()) {
       statement.BindString(6, action.page_title());
     }
     if (action.arg_url().is_valid()) {
       if (action.arg_incognito()) {
         statement.BindString(7, constants::kIncognitoUrl);

[felt, 2013/08/07 01:09:42]

Probably should have caught this before. Why are we doing the incognito URL for
full stream? We explicitly don't care about privacy in the full stream version.
Same question above.

[mvrable, 2013/08/07 17:01:19]

Looks like I forgot to update this code.  The intended new behavior is (unless
you think I should change it): visiting http://google.com/ in an incognito tab
should log the string "<incognito>http://google.com/" to the database.  That way
we have the full URL, but we also know that the access was to an incognito tab.

From the counting policy, we'll log simply "<incognito>" without including the
URL.

[felt, 2013/08/08 02:08:50]

Hmm. It seems like doing "<incognito>url" is overloading the field and it would
be nicer to have that info in the extras, but OTOH then we'd have "<incognito>"
in two places. So although it sits a little uncomfortably with me I do think
what you have here is the better option, but maybe we could revisit this in the
future (particularly when we start doing more complicated lookups with URL
regexes).

       } else {
-        statement.BindString(
-            7, action.arg_url().ReplaceComponents(url_sanitizer).spec());
+        statement.BindString(7, action.arg_url().spec());
       }
     }
     if (action.other()) {
       statement.BindString(8, Serialize(action.other()));
     }
 
     if (!statement.Run()) {
       LOG(ERROR) << "Activity log database I/O failed: " << sql_str;
       return false;
     }
   }
 
   if (!transaction.Commit())
     return false;
 
   queued_actions_.clear();
   return true;
 }
 
 scoped_ptr<Action::ActionVector> FullStreamUIPolicy::DoReadData(
     const std::string& extension_id,
     const int days_ago) {
+  // Ensure data is flushed to the database first so that we query over all
+  // data.
+  activity_database()->AdviseFlush(ActivityDatabase::kFlushImmediately);
+
   DCHECK_GE(days_ago, 0);
   scoped_ptr<Action::ActionVector> actions(new Action::ActionVector());
 
   sql::Connection* db = GetDatabaseConnection();
   if (!db) {
     return actions.Pass();
   }
 
-  // Compute the time bounds for that day.
-  base::Time morning_midnight = Now().LocalMidnight();
-  int64 early_bound = 0;
-  int64 late_bound = 0;
-  if (days_ago == 0) {
-      early_bound = morning_midnight.ToInternalValue();
-      late_bound = base::Time::Max().ToInternalValue();
-  } else {
-      base::Time early_time = morning_midnight -
-          base::TimeDelta::FromDays(days_ago);
-      base::Time late_time = morning_midnight -
-          base::TimeDelta::FromDays(days_ago-1);
-      early_bound = early_time.ToInternalValue();
-      late_bound = late_time.ToInternalValue();
-  }
+  int64 early_bound;
+  int64 late_bound;
+  Util::ComputeDatabaseTimeBounds(Now(), days_ago, &early_bound, &late_bound);
   std::string query_str = base::StringPrintf(
       "SELECT time, action_type, api_name, args, page_url, page_title, "
       "arg_url, other "
       "FROM %s WHERE extension_id=? AND time>? AND time<=? "
       "ORDER BY time DESC",
       kTableName);
   sql::Statement query(db->GetCachedStatement(SQL_FROM_HERE,
                                               query_str.c_str()));
   query.BindString(0, extension_id);
   query.BindInt64(1, early_bound);
@@ skipping 64 matching lines @@
   BrowserThread::PostTaskAndReplyWithResult(
       BrowserThread::DB,
       FROM_HERE,
       base::Bind(&FullStreamUIPolicy::DoReadData,
                  base::Unretained(this),
                  extension_id,
                  day),
       callback);
 }
 
-std::string FullStreamUIPolicy::GetKey(ActivityLogPolicy::KeyType key_ty) const

[felt, 2013/08/07 01:09:42]

where did this go?

[mvrable, 2013/08/07 17:01:19]

This code isn't being used any longer.  The constants have been moved to
activity_action_constants.cc, and there is no longer a need for a function to
look them up since they don't depend on which policy is installed.

[felt, 2013/08/08 02:08:50]

Ahhh OK, right, I couldn't find them in this CL because that had been done
earlier.

-{
-  switch (key_ty) {
-    case PARAM_KEY_REASON:
-      return std::string(kKeyReason);
-    case PARAM_KEY_DOM_ACTION:
-      return std::string(kKeyDomainAction);
-    case PARAM_KEY_URL_TITLE:
-      return std::string(kKeyURLTitle);
-    case PARAM_KEY_DETAILS_STRING:
-      return std::string(kKeyDetailsString);
-    default:
-      return std::string();
-  }
-}
-
 scoped_refptr<Action> FullStreamUIPolicy::ProcessArguments(
     scoped_refptr<Action> action) const {
   return action;
 }
 
 void FullStreamUIPolicy::ProcessAction(scoped_refptr<Action> action) {
   // TODO(mvrable): Right now this argument stripping updates the Action object
   // in place, which isn't good if there are other users of the object.  When
   // database writing is moved to policy class, the modifications should be
   // made locally.
   action = ProcessArguments(action);
   ScheduleAndForget(this, &FullStreamUIPolicy::QueueAction, action);
 }
 
 void FullStreamUIPolicy::QueueAction(scoped_refptr<Action> action) {
   if (activity_database()->is_db_valid()) {
     queued_actions_.push_back(action);
-    activity_database()->NotifyAction();
+    activity_database()->AdviseFlush(queued_actions_.size());
   }
 }
 
 }  // namespace extensions