Add support for bsdiff of file system metadata blocks

delta_diff_generator.cc

 // Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "update_engine/delta_diff_generator.h"
 
 #include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>
 #include <sys/stat.h>
@@ skipping 12 matching lines @@
 
 #include "update_engine/bzip.h"
 #include "update_engine/cycle_breaker.h"
 #include "update_engine/extent_mapper.h"
 #include "update_engine/extent_ranges.h"
 #include "update_engine/file_writer.h"
 #include "update_engine/filesystem_iterator.h"
 #include "update_engine/full_update_generator.h"
 #include "update_engine/graph_types.h"
 #include "update_engine/graph_utils.h"
+#include "update_engine/metadata.h"
 #include "update_engine/omaha_hash_calculator.h"
 #include "update_engine/payload_signer.h"
 #include "update_engine/subprocess.h"
 #include "update_engine/topological_sort.h"
 #include "update_engine/update_metadata.pb.h"
 #include "update_engine/utils.h"
 
 using std::make_pair;
 using std::map;
 using std::max;
@@ skipping 27 matching lines @@
 
 // Stores all Extents for a file into 'out'. Returns true on success.
 bool GatherExtents(const string& path,
                    google::protobuf::RepeatedPtrField<Extent>* out) {
   vector<Extent> extents;
   TEST_AND_RETURN_FALSE(extent_mapper::ExtentsForFileFibmap(path, &extents));
   DeltaDiffGenerator::StoreExtents(extents, out);
   return true;
 }
 
-// Runs the bsdiff tool on two files and returns the resulting delta in
-// 'out'. Returns true on success.
-bool BsdiffFiles(const string& old_file,
-                 const string& new_file,
-                 vector<char>* out) {
-  const string kPatchFile = "/tmp/delta.patchXXXXXX";
-  string patch_file_path;
-
-  TEST_AND_RETURN_FALSE(
-      utils::MakeTempFile(kPatchFile, &patch_file_path, NULL));
-
-  vector<string> cmd;
-  cmd.push_back(kBsdiffPath);
-  cmd.push_back(old_file);
-  cmd.push_back(new_file);
-  cmd.push_back(patch_file_path);
-
-  int rc = 1;
-  vector<char> patch_file;
-  TEST_AND_RETURN_FALSE(Subprocess::SynchronousExec(cmd, &rc));
-  TEST_AND_RETURN_FALSE(rc == 0);
-  TEST_AND_RETURN_FALSE(utils::ReadFile(patch_file_path, out));
-  unlink(patch_file_path.c_str());
-  return true;
-}
-
-// The blocks vector contains a reader and writer for each block on the
-// filesystem that's being in-place updated. We populate the reader/writer
-// fields of blocks by calling this function.
-// For each block in 'operation' that is read or written, find that block
-// in 'blocks' and set the reader/writer field to the vertex passed.
-// 'graph' is not strictly necessary, but useful for printing out
-// error messages.
-bool AddInstallOpToBlocksVector(
-    const DeltaArchiveManifest_InstallOperation& operation,
-    vector<Block>* blocks,
-    const Graph& graph,
-    Vertex::Index vertex) {
-  // See if this is already present.
-  TEST_AND_RETURN_FALSE(operation.dst_extents_size() > 0);
-
-  enum BlockField { READER = 0, WRITER, BLOCK_FIELD_COUNT };
-  for (int field = READER; field < BLOCK_FIELD_COUNT; field++) {
-    const int extents_size =
-        (field == READER) ? operation.src_extents_size() :
-        operation.dst_extents_size();
-    const char* past_participle = (field == READER) ? "read" : "written";
-    const google::protobuf::RepeatedPtrField<Extent>& extents =
-        (field == READER) ? operation.src_extents() : operation.dst_extents();
-    Vertex::Index Block::*access_type =
-        (field == READER) ? &Block::reader : &Block::writer;
-
-    for (int i = 0; i < extents_size; i++) {
-      const Extent& extent = extents.Get(i);
-      if (extent.start_block() == kSparseHole) {
-        // Hole in sparse file. skip
-        continue;
-      }
-      for (uint64_t block = extent.start_block();
-           block < (extent.start_block() + extent.num_blocks()); block++) {
-        if ((*blocks)[block].*access_type != Vertex::kInvalidIndex) {
-          LOG(FATAL) << "Block " << block << " is already "
-                     << past_participle << " by "
-                     << (*blocks)[block].*access_type << "("
-                     << graph[(*blocks)[block].*access_type].file_name
-                     << ") and also " << vertex << "("
-                     << graph[vertex].file_name << ")";
-        }
-        (*blocks)[block].*access_type = vertex;
-      }
-    }
-  }
-  return true;
-}
-
 // For a given regular file which must exist at new_root + path, and
 // may exist at old_root + path, creates a new InstallOperation and
 // adds it to the graph. Also, populates the |blocks| array as
 // necessary, if |blocks| is non-NULL.  Also, writes the data
 // necessary to send the file down to the client into data_fd, which
 // has length *data_file_size. *data_file_size is updated
 // appropriately. If |existing_vertex| is no kInvalidIndex, use that
 // rather than allocating a new vertex. Returns true on success.
 bool DeltaReadFile(Graph* graph,
                    Vertex::Index existing_vertex,
@@ skipping 25 matching lines @@
   Vertex::Index vertex = existing_vertex;
   if (vertex == Vertex::kInvalidIndex) {
     graph->resize(graph->size() + 1);
     vertex = graph->size() - 1;
   }
   (*graph)[vertex].op = operation;
   CHECK((*graph)[vertex].op.has_type());
   (*graph)[vertex].file_name = path;
 
   if (blocks)
-    TEST_AND_RETURN_FALSE(AddInstallOpToBlocksVector((*graph)[vertex].op,
-                                                     blocks,
-                                                     *graph,
-                                                     vertex));
+    TEST_AND_RETURN_FALSE(DeltaDiffGenerator::AddInstallOpToBlocksVector(
+        (*graph)[vertex].op,
+        *graph,
+        vertex,
+        blocks));
   return true;
 }
 
 // For each regular file within new_root, creates a node in the graph,
 // determines the best way to compress it (REPLACE, REPLACE_BZ, COPY, BSDIFF),
 // and writes any necessary data to the end of data_fd.
 bool DeltaReadFiles(Graph* graph,
                     vector<Block>* blocks,
                     const string& old_root,
                     const string& new_root,
@@ skipping 1169 matching lines @@
 
       TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph,
                                            &blocks,
                                            old_root,
                                            new_root,
                                            fd,
                                            &data_file_size));
       LOG(INFO) << "done reading normal files";
       CheckGraph(graph);
 
+      LOG(INFO) << "Starting metadata processing";
+      TEST_AND_RETURN_FALSE(Metadata::DeltaReadMetadata(&graph,
+                                                        &blocks,
+                                                        old_image,
+                                                        new_image,
+                                                        fd,
+                                                        &data_file_size));
+      LOG(INFO) << "Done metadata processing";
+      CheckGraph(graph);
+
       graph.resize(graph.size() + 1);
       TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks,
                                                 fd,
                                                 &data_file_size,
                                                 new_image,
                                                 &graph.back()));
 
       // Final scratch block (if there's space)
       if (blocks.size() < (kRootFSPartitionSize / kBlockSize)) {
         scratch_vertex = graph.size();
@@ skipping 175 matching lines @@
   }
 
   int64_t manifest_metadata_size =
       strlen(kDeltaMagic) + 2 * sizeof(uint64_t) + serialized_manifest.size();
   ReportPayloadUsage(manifest, manifest_metadata_size, op_name_map);
 
   LOG(INFO) << "All done. Successfully created delta file.";
   return true;
 }
 
+// Runs the bsdiff tool on two files and returns the resulting delta in
+// 'out'. Returns true on success.
+bool DeltaDiffGenerator::BsdiffFiles(const string& old_file,
+                                     const string& new_file,
+                                     vector<char>* out) {
+  const string kPatchFile = "/tmp/delta.patchXXXXXX";
+  string patch_file_path;
+
+  TEST_AND_RETURN_FALSE(
+      utils::MakeTempFile(kPatchFile, &patch_file_path, NULL));
+
+  vector<string> cmd;
+  cmd.push_back(kBsdiffPath);
+  cmd.push_back(old_file);
+  cmd.push_back(new_file);
+  cmd.push_back(patch_file_path);
+
+  int rc = 1;
+  vector<char> patch_file;
+  TEST_AND_RETURN_FALSE(Subprocess::SynchronousExec(cmd, &rc));
+  TEST_AND_RETURN_FALSE(rc == 0);
+  TEST_AND_RETURN_FALSE(utils::ReadFile(patch_file_path, out));
+  unlink(patch_file_path.c_str());
+  return true;
+}
+
+// The |blocks| vector contains a reader and writer for each block on the
+// filesystem that's being in-place updated. We populate the reader/writer
+// fields of |blocks| by calling this function.
+// For each block in |operation| that is read or written, find that block
+// in |blocks| and set the reader/writer field to the vertex passed.
+// |graph| is not strictly necessary, but useful for printing out
+// error messages.
+bool DeltaDiffGenerator::AddInstallOpToBlocksVector(
+    const DeltaArchiveManifest_InstallOperation& operation,
+    const Graph& graph,
+    Vertex::Index vertex,
+    vector<Block>* blocks) {
+  // See if this is already present.
+  TEST_AND_RETURN_FALSE(operation.dst_extents_size() > 0);
+
+  enum BlockField { READER = 0, WRITER, BLOCK_FIELD_COUNT };
+  for (int field = READER; field < BLOCK_FIELD_COUNT; field++) {
+    const int extents_size =
+        (field == READER) ? operation.src_extents_size() :
+        operation.dst_extents_size();
+    const char* past_participle = (field == READER) ? "read" : "written";
+    const google::protobuf::RepeatedPtrField<Extent>& extents =
+        (field == READER) ? operation.src_extents() : operation.dst_extents();
+    Vertex::Index Block::*access_type =
+        (field == READER) ? &Block::reader : &Block::writer;
+
+    for (int i = 0; i < extents_size; i++) {
+      const Extent& extent = extents.Get(i);
+      if (extent.start_block() == kSparseHole) {
+        // Hole in sparse file. skip
+        continue;
+      }
+      for (uint64_t block = extent.start_block();
+           block < (extent.start_block() + extent.num_blocks()); block++) {
+        if ((*blocks)[block].*access_type != Vertex::kInvalidIndex) {
+          LOG(FATAL) << "Block " << block << " is already "
+                     << past_participle << " by "
+                     << (*blocks)[block].*access_type << "("
+                     << graph[(*blocks)[block].*access_type].file_name
+                     << ") and also " << vertex << "("
+                     << graph[vertex].file_name << ")";
+        }
+        (*blocks)[block].*access_type = vertex;
+      }
+    }
+  }
+  return true;
+}
+
 const char* const kBsdiffPath = "bsdiff";
 const char* const kBspatchPath = "bspatch";
 const char* const kDeltaMagic = "CrAU";
 
 };  // namespace chromeos_update_engine