AU: Really delta compress the kernel if an old kernel is provided.

delta_diff_generator.cc

 // Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "update_engine/delta_diff_generator.h"
 
 #include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>
 #include <sys/stat.h>
@@ skipping 36 matching lines @@
 namespace chromeos_update_engine {
 
 typedef DeltaDiffGenerator::Block Block;
 
 namespace {
 const size_t kBlockSize = 4096;  // bytes
 const size_t kRootFSPartitionSize = 1 * 1024 * 1024 * 1024;  // 1 GiB
 const uint64_t kVersionNumber = 1;
 const uint64_t kFullUpdateChunkSize = 128 * 1024;  // bytes
 
+static const char* kInstallOperationTypes[] = {
+  "REPLACE",
+  "REPLACE_BZ",
+  "MOVE",
+  "BSDIFF"
+};
+
 // Stores all Extents for a file into 'out'. Returns true on success.
 bool GatherExtents(const string& path,
                    google::protobuf::RepeatedPtrField<Extent>* out) {
   vector<Extent> extents;
   TEST_AND_RETURN_FALSE(extent_mapper::ExtentsForFileFibmap(path, &extents));
   DeltaDiffGenerator::StoreExtents(extents, out);
   return true;
 }
 
 // Runs the bsdiff tool on two files and returns the resulting delta in
@@ skipping 89 matching lines @@
                    const string& new_root,
                    const string& path,  // within new_root
                    int data_fd,
                    off_t* data_file_size) {
   vector<char> data;
   DeltaArchiveManifest_InstallOperation operation;
 
   TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_root + path,
                                                            new_root + path,
                                                            &data,
-                                                           &operation));
+                                                           &operation,
+                                                           true));
 
   // Write the data
   if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) {
     operation.set_data_offset(*data_file_size);
     operation.set_data_length(data.size());
   }
 
   TEST_AND_RETURN_FALSE(utils::WriteAll(data_fd, &data[0], data.size()));
   *data_file_size += data.size();
 
@@ skipping 211 matching lines @@
     *op = *it;
   }
 }
 
 void CheckGraph(const Graph& graph) {
   for (Graph::const_iterator it = graph.begin(); it != graph.end(); ++it) {
     CHECK(it->op.has_type());
   }
 }
 
-// Delta compresses a kernel partition new_kernel_part with knowledge of
-// the old kernel partition old_kernel_part.
+// Delta compresses a kernel partition |new_kernel_part| with knowledge of the
+// old kernel partition |old_kernel_part|. If |old_kernel_part| is an empty
+// string, generates a full update of the partition.
 bool DeltaCompressKernelPartition(
     const string& old_kernel_part,
     const string& new_kernel_part,
     vector<DeltaArchiveManifest_InstallOperation>* ops,
     int blobs_fd,
     off_t* blobs_length) {
-  // For now, just bsdiff the kernel partition as a whole.
-  // TODO(adlr): Use knowledge of how the kernel partition is laid out
-  // to more efficiently compress it.
-
   LOG(INFO) << "Delta compressing kernel partition...";
+  LOG_IF(INFO, old_kernel_part.empty()) << "Generating full kernel update...";
 
   // Add a new install operation
   ops->resize(1);
   DeltaArchiveManifest_InstallOperation* op = &(*ops)[0];
-  op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ);
-  op->set_data_offset(*blobs_length);
 
-  // Do the actual compression
   vector<char> data;
-  TEST_AND_RETURN_FALSE(utils::ReadFile(new_kernel_part, &data));
-  TEST_AND_RETURN_FALSE(!data.empty());
+  TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_kernel_part,
+                                                           new_kernel_part,
+                                                           &data,
+                                                           op,
+                                                           false));
 
-  vector<char> data_bz;
-  TEST_AND_RETURN_FALSE(BzipCompress(data, &data_bz));
-  CHECK(!data_bz.empty());
+  // Write the data
+  if (op->type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) {
+    op->set_data_offset(*blobs_length);
+    op->set_data_length(data.size());
+  }
 
-  TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, &data_bz[0], data_bz.size()));
-  *blobs_length += data_bz.size();
+  TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, &data[0], data.size()));
+  *blobs_length += data.size();
 
-  off_t new_part_size = utils::FileSize(new_kernel_part);
-  TEST_AND_RETURN_FALSE(new_part_size >= 0);
-
-  op->set_data_length(data_bz.size());
-
-  op->set_dst_length(new_part_size);
-
-  // There's a single dest extent
-  Extent* dst_extent = op->add_dst_extents();
-  dst_extent->set_start_block(0);
-  dst_extent->set_num_blocks((new_part_size + kBlockSize - 1) / kBlockSize);
-
-  LOG(INFO) << "Done compressing kernel partition.";
+  LOG(INFO) << "Done delta compressing kernel partition: "
+            << kInstallOperationTypes[op->type()];
   return true;
 }
 
 struct DeltaObject {
   DeltaObject(const string& in_name, const int in_type, const off_t in_size)
       : name(in_name),
         type(in_type),
         size(in_size) {}
   bool operator <(const DeltaObject& object) const {
     return size < object.size;
   }
   string name;
   int type;
   off_t size;
 };
 
-static const char* kInstallOperationTypes[] = {
-  "REPLACE",
-  "REPLACE_BZ",
-  "MOVE",
-  "BSDIFF"
-};
-
 void ReportPayloadUsage(const Graph& graph,
                         const DeltaArchiveManifest& manifest,
                         const int64_t manifest_metadata_size) {
   vector<DeltaObject> objects;
   off_t total_size = 0;
 
   // Graph nodes with information about file names.
   for (Vertex::Index node = 0; node < graph.size(); node++) {
     const Vertex& vertex = graph[node];
     if (!vertex.valid) {
@@ skipping 34 matching lines @@
   }
   fprintf(stderr, kFormatString, 100.0, total_size, "", "<total>");
 }
 
 }  // namespace {}
 
 bool DeltaDiffGenerator::ReadFileToDiff(
     const string& old_filename,
     const string& new_filename,
     vector<char>* out_data,
-    DeltaArchiveManifest_InstallOperation* out_op) {
+    DeltaArchiveManifest_InstallOperation* out_op,
+    bool gather_extents) {
   // Read new data in
   vector<char> new_data;
   TEST_AND_RETURN_FALSE(utils::ReadFile(new_filename, &new_data));
 
   TEST_AND_RETURN_FALSE(!new_data.empty());
 
   vector<char> new_data_bz;
   TEST_AND_RETURN_FALSE(BzipCompress(new_data, &new_data_bz));
   CHECK(!new_data_bz.empty());
 
   vector<char> data;  // Data blob that will be written to delta file.
 
   DeltaArchiveManifest_InstallOperation operation;
   size_t current_best_size = 0;
   if (new_data.size() <= new_data_bz.size()) {
     operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE);
     current_best_size = new_data.size();
     data = new_data;
   } else {
     operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ);
     current_best_size = new_data_bz.size();
     data = new_data_bz;
   }
 
   // Do we have an original file to consider?
   struct stat old_stbuf;
-  if (0 != stat(old_filename.c_str(), &old_stbuf)) {
+  bool no_original = old_filename.empty();
+  if (!no_original && 0 != stat(old_filename.c_str(), &old_stbuf)) {
     // If stat-ing the old file fails, it should be because it doesn't exist.
     TEST_AND_RETURN_FALSE(errno == ENOTDIR || errno == ENOENT);
-  } else {
+    no_original = true;
+  }
+  if (!no_original) {
     // Read old data
     vector<char> old_data;
     TEST_AND_RETURN_FALSE(utils::ReadFile(old_filename, &old_data));
     if (old_data == new_data) {
       // No change in data.
       operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE);
       current_best_size = 0;
       data.clear();
     } else {
       // Try bsdiff of old to new data
       vector<char> bsdiff_delta;
       TEST_AND_RETURN_FALSE(
           BsdiffFiles(old_filename, new_filename, &bsdiff_delta));
       CHECK_GT(bsdiff_delta.size(), 0);
       if (bsdiff_delta.size() < current_best_size) {
         operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF);
         current_best_size = bsdiff_delta.size();
 
         data = bsdiff_delta;
       }
     }
   }
 
   // Set parameters of the operations
   CHECK_EQ(data.size(), current_best_size);
 
   if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE ||
       operation.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF) {
-    TEST_AND_RETURN_FALSE(
-        GatherExtents(old_filename, operation.mutable_src_extents()));
+    if (gather_extents) {
+      TEST_AND_RETURN_FALSE(
+          GatherExtents(old_filename, operation.mutable_src_extents()));
+    } else {
+      Extent* src_extent = operation.add_src_extents();
+      src_extent->set_start_block(0);
+      src_extent->set_num_blocks(
+          (old_stbuf.st_size + kBlockSize - 1) / kBlockSize);
+    }
     operation.set_src_length(old_stbuf.st_size);
   }
 
-  TEST_AND_RETURN_FALSE(
-      GatherExtents(new_filename, operation.mutable_dst_extents()));
+  if (gather_extents) {
+    TEST_AND_RETURN_FALSE(
+        GatherExtents(new_filename, operation.mutable_dst_extents()));
+  } else {
+    Extent* dst_extent = operation.add_dst_extents();
+    dst_extent->set_start_block(0);
+    dst_extent->set_num_blocks((new_data.size() + kBlockSize - 1) / kBlockSize);
+  }
   operation.set_dst_length(new_data.size());
 
   out_data->swap(data);
   *out_op = operation;
 
   return true;
 }
 
 bool InitializePartitionInfo(bool is_kernel,
                              const string& partition,
@@ skipping 670 matching lines @@
   TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(new_image,
                                                  &new_image_block_count,
                                                  &new_image_block_size));
   if (!old_image.empty()) {
     TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(old_image,
                                                    &old_image_block_count,
                                                    &old_image_block_size));
     TEST_AND_RETURN_FALSE(old_image_block_size == new_image_block_size);
     LOG_IF(WARNING, old_image_block_count != new_image_block_count)
         << "Old and new images have different block counts.";
-    // Sanity check kernel partition arg
-    TEST_AND_RETURN_FALSE(utils::FileSize(old_kernel_part) >= 0);
   }
   // Sanity check kernel partition arg
   TEST_AND_RETURN_FALSE(utils::FileSize(new_kernel_part) >= 0);
 
   vector<Block> blocks(max(old_image_block_count, new_image_block_count));
   LOG(INFO) << "Invalid block index: " << Vertex::kInvalidIndex;
   LOG(INFO) << "Block count: " << blocks.size();
   for (vector<Block>::size_type i = 0; i < blocks.size(); i++) {
     CHECK(blocks[i].reader == Vertex::kInvalidIndex);
     CHECK(blocks[i].writer == Vertex::kInvalidIndex);
@@ skipping 221 matching lines @@
 
   LOG(INFO) << "All done. Successfully created delta file.";
   return true;
 }
 
 const char* const kBsdiffPath = "/usr/bin/bsdiff";
 const char* const kBspatchPath = "/usr/bin/bspatch";
 const char* const kDeltaMagic = "CrAU";
 
 };  // namespace chromeos_update_engine