Adding support for storing webrtc logs locally with an extension supplied unique id, for later uplo…

chrome/browser/media/webrtc_log_uploader.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/media/webrtc_log_uploader.h"
 
 #include "base/files/file_enumerator.h"
 #include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/logging.h"
+#include "base/pickle.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_split.h"
 #include "base/strings/stringprintf.h"
 #include "base/time/time.h"
 #include "chrome/browser/browser_process.h"
 #include "chrome/browser/media/webrtc_log_list.h"
 #include "chrome/browser/media/webrtc_log_util.h"
 #include "chrome/common/chrome_version_info.h"
 #include "chrome/common/partial_circular_buffer.h"
 #include "content/public/browser/browser_thread.h"
@@ skipping 21 matching lines @@
   post_data->append(kMultipartBoundary);
   post_data->append("\r\nContent-Disposition: form-data; name=\"");
   post_data->append(content_name);
   post_data->append("\"; filename=\"");
   post_data->append(content_name + ".gz");
   post_data->append("\"\r\nContent-Type: application/gzip\r\n\r\n");
 }
 
 // Adds |compressed_log| to |post_data|.
 void AddLogData(std::string* post_data,
-                const std::vector<uint8>& compressed_log) {
+                const std::string& compressed_log) {
   AddMultipartFileContentHeader(post_data, "webrtc_log");
-  post_data->append(reinterpret_cast<const char*>(&compressed_log[0]),
-                    compressed_log.size());
+  post_data->append(compressed_log);
   post_data->append("\r\n");
 }
 
 // Adds the RTP dump data to |post_data|.
 void AddRtpDumpData(std::string* post_data,
                     const std::string& name,
                     const std::string& dump_data) {
   AddMultipartFileContentHeader(post_data, name);
   post_data->append(dump_data.data(), dump_data.size());
   post_data->append("\r\n");
@@ skipping 63 matching lines @@
   }
   return false;
 }
 
 void WebRtcLogUploader::LoggingStoppedDontUpload() {
   content::BrowserThread::PostTask(content::BrowserThread::UI, FROM_HERE,
       base::Bind(&WebRtcLogUploader::DecreaseLogCount, base::Unretained(this)));
 }
 
 void WebRtcLogUploader::LoggingStoppedDoUpload(
-    scoped_ptr<unsigned char[]> log_buffer,
-    uint32 length,
-    const std::map<std::string, std::string>& meta_data,
+    scoped_ptr<WebRtcLogBuffer> log_buffer,
+    scoped_ptr<MetaDataMap> meta_data,
     const WebRtcLogUploadDoneData& upload_done_data) {
   DCHECK(file_thread_checker_.CalledOnValidThread());
   DCHECK(log_buffer.get());
+  DCHECK(meta_data.get());
   DCHECK(!upload_done_data.log_path.empty());
 
-  std::vector<uint8> compressed_log;
-  CompressLog(
-      &compressed_log, reinterpret_cast<uint8*>(&log_buffer[0]), length);
+  std::string compressed_log;
+  CompressLog(&compressed_log, log_buffer.get());
 
   std::string local_log_id;
 
   if (base::PathExists(upload_done_data.log_path)) {
     WebRtcLogUtil::DeleteOldWebRtcLogFiles(upload_done_data.log_path);
 
     local_log_id = base::DoubleToString(base::Time::Now().ToDoubleT());
     base::FilePath log_file_path =
         upload_done_data.log_path.AppendASCII(local_log_id)
             .AddExtension(FILE_PATH_LITERAL(".gz"));
     WriteCompressedLogToFile(compressed_log, log_file_path);
 
     base::FilePath log_list_path =
         WebRtcLogList::GetWebRtcLogListFileForDirectory(
             upload_done_data.log_path);
     AddLocallyStoredLogInfoToUploadListFile(log_list_path, local_log_id);
   }
 
   WebRtcLogUploadDoneData upload_done_data_with_log_id = upload_done_data;
   upload_done_data_with_log_id.local_log_id = local_log_id;
+  UploadCompressedLog(compressed_log, meta_data.Pass(),
+      upload_done_data_with_log_id);
+}
+
+void WebRtcLogUploader::UploadCompressedLog(
+    const std::string& compressed_log,
+    scoped_ptr<MetaDataMap> meta_data,
+    const WebRtcLogUploadDoneData& upload_done_data) {
+  DCHECK(file_thread_checker_.CalledOnValidThread());
+  DCHECK(!compressed_log.empty());
+  DCHECK(meta_data.get());
 
   scoped_ptr<std::string> post_data(new std::string());
   SetupMultipart(post_data.get(),
                  compressed_log,
                  upload_done_data.incoming_rtp_dump,
                  upload_done_data.outgoing_rtp_dump,
-                 meta_data);
+                 *meta_data.get());
 
   // If a test has set the test string pointer, write to it and skip uploading.
   // Still fire the upload callback so that we can run an extension API test
   // using the test framework for that without hanging.
   // TODO(grunell): Remove this when the api test for this feature is fully
   // implemented according to the test plan. http://crbug.com/257329.
   if (post_data_) {
     *post_data_ = *post_data;
-    NotifyUploadDone(kHttpResponseOk, "", upload_done_data_with_log_id);
+    NotifyUploadDone(kHttpResponseOk, "", upload_done_data);
     return;
   }
 
   content::BrowserThread::PostTask(
       content::BrowserThread::UI,
       FROM_HERE,
       base::Bind(&WebRtcLogUploader::CreateAndStartURLFetcher,
                  base::Unretained(this),
-                 upload_done_data_with_log_id,
+                 upload_done_data,
                  Passed(&post_data)));
 
   content::BrowserThread::PostTask(content::BrowserThread::UI, FROM_HERE,
       base::Bind(&WebRtcLogUploader::DecreaseLogCount, base::Unretained(this)));
 }
 
+void WebRtcLogUploader::UploadStoredLog(
+    const WebRtcLogUploadDoneData& upload_data) {
+  DCHECK(file_thread_checker_.CalledOnValidThread());
+  DCHECK(!upload_data.local_log_id.empty());
+  DCHECK(!upload_data.log_path.empty());
+
+  base::FilePath native_log_path =
+      upload_data.log_path.AppendASCII(upload_data.local_log_id)
+      .AddExtension(FILE_PATH_LITERAL(".gz"));
+
+  std::string compressed_log;
+  if (!base::ReadFileToString(native_log_path, &compressed_log)) {
+    DPLOG(WARNING) << "Could not read WebRTC log file.";
+    content::BrowserThread::PostTask(
+        content::BrowserThread::UI, FROM_HERE,
+        base::Bind(upload_data.callback, false, "", "Log doesn't exist."));
+    return;
+  }
+
+  WebRtcLogUploadDoneData upload_data_with_rtp = upload_data;
+
+  // Optimistically set the rtp paths to what they should be if they exist.
+  upload_data_with_rtp.incoming_rtp_dump =
+      upload_data.log_path.AppendASCII(upload_data.local_log_id)
+      .AddExtension(FILE_PATH_LITERAL(".rtp_in"));
+
+  upload_data_with_rtp.outgoing_rtp_dump =
+      upload_data.log_path.AppendASCII(upload_data.local_log_id)
+      .AddExtension(FILE_PATH_LITERAL(".rtp_out"));
+
+  scoped_ptr<MetaDataMap> meta_data(new MetaDataMap());
+  {
+    std::string meta_data_contents;
+    base::FilePath meta_path =
+        upload_data.log_path.AppendASCII(upload_data.local_log_id)
+        .AddExtension(FILE_PATH_LITERAL(".meta"));
+    if (base::ReadFileToString(meta_path, &meta_data_contents) &&
+        !meta_data_contents.empty()) {
+      Pickle pickle(&meta_data_contents[0], meta_data_contents.size());
+      PickleIterator it(pickle);
+      std::string key, value;
+      while (it.ReadString(&key) && it.ReadString(&value))
+        (*meta_data.get())[key] = value;
+    }
+  }
+
+  UploadCompressedLog(compressed_log, meta_data.Pass(), upload_data_with_rtp);
+}
+
+void WebRtcLogUploader::LoggingStoppedDoStore(
+    const WebRtcLogPaths& log_paths,
+    const std::string& log_id,
+    scoped_ptr<WebRtcLogBuffer> log_buffer,
+    scoped_ptr<MetaDataMap> meta_data,
+    const WebRtcLoggingHandlerHost::GenericDoneCallback& done_callback) {
+  DCHECK(file_thread_checker_.CalledOnValidThread());
+  DCHECK(!log_id.empty());
+  DCHECK(log_buffer.get());
+  DCHECK(!log_paths.log_path.empty());
+
+  WebRtcLogUtil::DeleteOldWebRtcLogFiles(log_paths.log_path);
+
+  base::FilePath log_list_path =
+      WebRtcLogList::GetWebRtcLogListFileForDirectory(log_paths.log_path);
+
+  // Store the native log with a ".gz" extension.
+  std::string compressed_log;
+  CompressLog(&compressed_log, log_buffer.get());
+  base::FilePath native_log_path = log_paths.log_path.AppendASCII(log_id)
+      .AddExtension(FILE_PATH_LITERAL(".gz"));
+  WriteCompressedLogToFile(compressed_log, native_log_path);
+  AddLocallyStoredLogInfoToUploadListFile(log_list_path, log_id);
+
+  // Move the rtp dump files to the log directory with a name of
+  // <log id>.rtp_[in|out].
+  if (!log_paths.incoming_rtp_dump.empty()) {
+    base::FilePath rtp_path = log_paths.log_path.AppendASCII(log_id)
+        .AddExtension(FILE_PATH_LITERAL(".rtp_in"));
+    base::Move(log_paths.incoming_rtp_dump, rtp_path);
+  }
+
+  if (!log_paths.outgoing_rtp_dump.empty()) {
+    base::FilePath rtp_path = log_paths.log_path.AppendASCII(log_id)
+        .AddExtension(FILE_PATH_LITERAL(".rtp_out"));
+    base::Move(log_paths.outgoing_rtp_dump, rtp_path);
+  }
+
+  if (meta_data.get() && !meta_data->empty()) {
+    Pickle pickle;
+    for (const auto& it : *meta_data.get()) {
+      pickle.WriteString(it.first);
+      pickle.WriteString(it.second);
+    }
+    base::FilePath meta_path = log_paths.log_path.AppendASCII(log_id)
+        .AddExtension(FILE_PATH_LITERAL(".meta"));
+    base::WriteFile(meta_path, static_cast<const char*>(pickle.data()),
+        pickle.size());
+  }
+
+  content::BrowserThread::PostTask(
+      content::BrowserThread::UI, FROM_HERE,
+      base::Bind(done_callback, true, ""));
+}
+
 void WebRtcLogUploader::StartShutdown() {
   DCHECK(create_thread_checker_.CalledOnValidThread());
   DCHECK(!shutting_down_);
 
   // Delete all URLFetchers first and clear the upload done map.
   for (UploadDoneDataMap::iterator it = upload_done_data_.begin();
        it != upload_done_data_.end();
        ++it) {
     delete it->first;
   }
   upload_done_data_.clear();
   shutting_down_ = true;
 }
 
 void WebRtcLogUploader::SetupMultipart(
     std::string* post_data,
-    const std::vector<uint8>& compressed_log,
+    const std::string& compressed_log,
     const base::FilePath& incoming_rtp_dump,
     const base::FilePath& outgoing_rtp_dump,
     const std::map<std::string, std::string>& meta_data) {
 #if defined(OS_WIN)
   const char product[] = "Chrome";
 #elif defined(OS_MACOSX)
   const char product[] = "Chrome_Mac";
 #elif defined(OS_LINUX)
 #if !defined(ADDRESS_SANITIZER)
   const char product[] = "Chrome_Linux";
@@ skipping 34 matching lines @@
     if (!rtp_dumps[i].empty() && base::PathExists(rtp_dumps[i])) {
       std::string dump_data;
       if (base::ReadFileToString(rtp_dumps[i], &dump_data))
         AddRtpDumpData(post_data, kRtpDumpNames[i], dump_data);
     }
   }
 
   net::AddMultipartFinalDelimiterForUpload(kMultipartBoundary, post_data);
 }
 
-void WebRtcLogUploader::CompressLog(std::vector<uint8>* compressed_log,
-                                    uint8* input,
-                                    uint32 input_size) {
-  PartialCircularBuffer read_pcb(input, input_size);
-
+void WebRtcLogUploader::CompressLog(std::string* compressed_log,
+                                    WebRtcLogBuffer* buffer) {
   z_stream stream = {0};
   int result = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
                             // windowBits = 15 is default, 16 is added to
                             // produce a gzip header + trailer.
                             15 + 16,
                             8,  // memLevel = 8 is default.
                             Z_DEFAULT_STRATEGY);
   DCHECK_EQ(Z_OK, result);
 
   uint8 intermediate_buffer[kIntermediateCompressionBufferBytes] = {0};
   ResizeForNextOutput(compressed_log, &stream);
   uint32 read = 0;
 
+  PartialCircularBuffer read_buffer = buffer->Read();

[perkj_chrome, 2015/02/19 11:55:53]

nit ()

[tommi (sloooow) - chröme, 2015/02/19 13:31:40]

Done.

   do {
     if (stream.avail_in == 0) {
-      read = read_pcb.Read(&intermediate_buffer[0],
-                           kIntermediateCompressionBufferBytes);
+      read = read_buffer.Read(&intermediate_buffer[0],
+                              kIntermediateCompressionBufferBytes);

[perkj_chrome, 2015/02/19 11:55:53]

sizeof(intermediate_buffer)

[tommi (sloooow) - chröme, 2015/02/19 13:31:40]

Done.

       stream.next_in = &intermediate_buffer[0];
       stream.avail_in = read;
       if (read != kIntermediateCompressionBufferBytes)
         break;
     }
     result = deflate(&stream, Z_SYNC_FLUSH);
     DCHECK_EQ(Z_OK, result);
     if (stream.avail_out == 0)
       ResizeForNextOutput(compressed_log, &stream);
   } while (true);
 
   // Ensure we have enough room in the output buffer. Easier to always just do a
   // resize than looping around and resize if needed.
   if (stream.avail_out < kIntermediateCompressionBufferBytes)
     ResizeForNextOutput(compressed_log, &stream);
 
   result = deflate(&stream, Z_FINISH);
   DCHECK_EQ(Z_STREAM_END, result);
   result = deflateEnd(&stream);
   DCHECK_EQ(Z_OK, result);
 
   compressed_log->resize(compressed_log->size() - stream.avail_out);
 }
 
-void WebRtcLogUploader::ResizeForNextOutput(std::vector<uint8>* compressed_log,
+void WebRtcLogUploader::ResizeForNextOutput(std::string* compressed_log,
                                             z_stream* stream) {
   size_t old_size = compressed_log->size() - stream->avail_out;
   compressed_log->resize(old_size + kIntermediateCompressionBufferBytes);
-  stream->next_out = &(*compressed_log)[old_size];
+  stream->next_out = reinterpret_cast<unsigned char*>(
+      &(*compressed_log)[old_size]);
   stream->avail_out = kIntermediateCompressionBufferBytes;
 }
 
 void WebRtcLogUploader::CreateAndStartURLFetcher(
     const WebRtcLogUploadDoneData& upload_done_data,
     scoped_ptr<std::string> post_data) {
   DCHECK(create_thread_checker_.CalledOnValidThread());
 
   if (shutting_down_)
     return;
 
   std::string content_type = kUploadContentType;
   content_type.append("; boundary=");
   content_type.append(kMultipartBoundary);
 
   net::URLFetcher* url_fetcher =
       net::URLFetcher::Create(GURL(kUploadURL), net::URLFetcher::POST, this);
   url_fetcher->SetRequestContext(g_browser_process->system_request_context());
   url_fetcher->SetUploadData(content_type, *post_data);
   url_fetcher->Start();
   upload_done_data_[url_fetcher] = upload_done_data;
 }
 
 void WebRtcLogUploader::DecreaseLogCount() {
   DCHECK(create_thread_checker_.CalledOnValidThread());
   --log_count_;
 }
 
 void WebRtcLogUploader::WriteCompressedLogToFile(
-    const std::vector<uint8>& compressed_log,
+    const std::string& compressed_log,
     const base::FilePath& log_file_path) {
   DCHECK(file_thread_checker_.CalledOnValidThread());
   DCHECK(!compressed_log.empty());
-  base::WriteFile(log_file_path,
-                  reinterpret_cast<const char*>(&compressed_log[0]),
-                  compressed_log.size());
+  base::WriteFile(log_file_path, &compressed_log[0], compressed_log.size());
 }
 
 void WebRtcLogUploader::AddLocallyStoredLogInfoToUploadListFile(
     const base::FilePath& upload_list_path,
     const std::string& local_log_id) {
   DCHECK(file_thread_checker_.CalledOnValidThread());
   DCHECK(!upload_list_path.empty());
   DCHECK(!local_log_id.empty());
 
   std::string contents;
@@ skipping 84 matching lines @@
     if (!success) {
       error_message = "Uploading failed, response code: " +
                       base::IntToString(response_code);
     }
     content::BrowserThread::PostTask(
         content::BrowserThread::UI, FROM_HERE,
         base::Bind(upload_done_data.callback, success, report_id,
                    error_message));
   }
 }