OLD | NEW |
1 // Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 1 // Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "update_engine/delta_diff_generator.h" | 5 #include "update_engine/delta_diff_generator.h" |
6 | 6 |
7 #include <errno.h> | 7 #include <errno.h> |
8 #include <fcntl.h> | 8 #include <fcntl.h> |
9 #include <inttypes.h> | 9 #include <inttypes.h> |
10 #include <sys/stat.h> | 10 #include <sys/stat.h> |
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
104 // fields of blocks by calling this function. | 104 // fields of blocks by calling this function. |
105 // For each block in 'operation' that is read or written, find that block | 105 // For each block in 'operation' that is read or written, find that block |
106 // in 'blocks' and set the reader/writer field to the vertex passed. | 106 // in 'blocks' and set the reader/writer field to the vertex passed. |
107 // 'graph' is not strictly necessary, but useful for printing out | 107 // 'graph' is not strictly necessary, but useful for printing out |
108 // error messages. | 108 // error messages. |
109 bool AddInstallOpToBlocksVector( | 109 bool AddInstallOpToBlocksVector( |
110 const DeltaArchiveManifest_InstallOperation& operation, | 110 const DeltaArchiveManifest_InstallOperation& operation, |
111 vector<Block>* blocks, | 111 vector<Block>* blocks, |
112 const Graph& graph, | 112 const Graph& graph, |
113 Vertex::Index vertex) { | 113 Vertex::Index vertex) { |
114 LOG(INFO) << "AddInstallOpToBlocksVector(" << vertex << "), " | |
115 << graph[vertex].file_name; | |
116 // See if this is already present. | 114 // See if this is already present. |
117 TEST_AND_RETURN_FALSE(operation.dst_extents_size() > 0); | 115 TEST_AND_RETURN_FALSE(operation.dst_extents_size() > 0); |
118 | 116 |
119 enum BlockField { READER = 0, WRITER, BLOCK_FIELD_COUNT }; | 117 enum BlockField { READER = 0, WRITER, BLOCK_FIELD_COUNT }; |
120 for (int field = READER; field < BLOCK_FIELD_COUNT; field++) { | 118 for (int field = READER; field < BLOCK_FIELD_COUNT; field++) { |
121 const int extents_size = | 119 const int extents_size = |
122 (field == READER) ? operation.src_extents_size() : | 120 (field == READER) ? operation.src_extents_size() : |
123 operation.dst_extents_size(); | 121 operation.dst_extents_size(); |
124 const char* past_participle = (field == READER) ? "read" : "written"; | 122 const char* past_participle = (field == READER) ? "read" : "written"; |
125 const google::protobuf::RepeatedPtrField<Extent>& extents = | 123 const google::protobuf::RepeatedPtrField<Extent>& extents = |
126 (field == READER) ? operation.src_extents() : operation.dst_extents(); | 124 (field == READER) ? operation.src_extents() : operation.dst_extents(); |
127 Vertex::Index Block::*access_type = | 125 Vertex::Index Block::*access_type = |
128 (field == READER) ? &Block::reader : &Block::writer; | 126 (field == READER) ? &Block::reader : &Block::writer; |
129 | 127 |
130 for (int i = 0; i < extents_size; i++) { | 128 for (int i = 0; i < extents_size; i++) { |
131 const Extent& extent = extents.Get(i); | 129 const Extent& extent = extents.Get(i); |
132 if (extent.start_block() == kSparseHole) { | 130 if (extent.start_block() == kSparseHole) { |
133 // Hole in sparse file. skip | 131 // Hole in sparse file. skip |
134 continue; | 132 continue; |
135 } | 133 } |
136 for (uint64_t block = extent.start_block(); | 134 for (uint64_t block = extent.start_block(); |
137 block < (extent.start_block() + extent.num_blocks()); block++) { | 135 block < (extent.start_block() + extent.num_blocks()); block++) { |
138 LOG(INFO) << "ext: " << i << " block: " << block; | |
139 if ((*blocks)[block].*access_type != Vertex::kInvalidIndex) { | 136 if ((*blocks)[block].*access_type != Vertex::kInvalidIndex) { |
140 LOG(FATAL) << "Block " << block << " is already " | 137 LOG(FATAL) << "Block " << block << " is already " |
141 << past_participle << " by " | 138 << past_participle << " by " |
142 << (*blocks)[block].*access_type << "(" | 139 << (*blocks)[block].*access_type << "(" |
143 << graph[(*blocks)[block].*access_type].file_name | 140 << graph[(*blocks)[block].*access_type].file_name |
144 << ") and also " << vertex << "(" | 141 << ") and also " << vertex << "(" |
145 << graph[vertex].file_name << ")"; | 142 << graph[vertex].file_name << ")"; |
146 } | 143 } |
147 (*blocks)[block].*access_type = vertex; | 144 (*blocks)[block].*access_type = vertex; |
148 } | 145 } |
(...skipping 167 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
316 vector<Block>::size_type blocks_copied_count = 0; | 313 vector<Block>::size_type blocks_copied_count = 0; |
317 | 314 |
318 // For each extent in extents, write the data into BZ2_bzWrite which | 315 // For each extent in extents, write the data into BZ2_bzWrite which |
319 // sends it to an output file. | 316 // sends it to an output file. |
320 // We use the temporary buffer 'buf' to hold the data, which may be | 317 // We use the temporary buffer 'buf' to hold the data, which may be |
321 // smaller than the extent, so in that case we have to loop to get | 318 // smaller than the extent, so in that case we have to loop to get |
322 // the extent's data (that's the inner while loop). | 319 // the extent's data (that's the inner while loop). |
323 for (vector<Extent>::const_iterator it = extents.begin(); | 320 for (vector<Extent>::const_iterator it = extents.begin(); |
324 it != extents.end(); ++it) { | 321 it != extents.end(); ++it) { |
325 vector<Block>::size_type blocks_read = 0; | 322 vector<Block>::size_type blocks_read = 0; |
| 323 float printed_progress = -1; |
326 while (blocks_read < it->num_blocks()) { | 324 while (blocks_read < it->num_blocks()) { |
327 const int copy_block_cnt = | 325 const int copy_block_cnt = |
328 min(buf.size() / kBlockSize, | 326 min(buf.size() / kBlockSize, |
329 static_cast<vector<char>::size_type>( | 327 static_cast<vector<char>::size_type>( |
330 it->num_blocks() - blocks_read)); | 328 it->num_blocks() - blocks_read)); |
331 ssize_t rc = pread(image_fd, | 329 ssize_t rc = pread(image_fd, |
332 &buf[0], | 330 &buf[0], |
333 copy_block_cnt * kBlockSize, | 331 copy_block_cnt * kBlockSize, |
334 (it->start_block() + blocks_read) * kBlockSize); | 332 (it->start_block() + blocks_read) * kBlockSize); |
335 TEST_AND_RETURN_FALSE_ERRNO(rc >= 0); | 333 TEST_AND_RETURN_FALSE_ERRNO(rc >= 0); |
336 TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == | 334 TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == |
337 copy_block_cnt * kBlockSize); | 335 copy_block_cnt * kBlockSize); |
338 BZ2_bzWrite(&err, bz_file, &buf[0], copy_block_cnt * kBlockSize); | 336 BZ2_bzWrite(&err, bz_file, &buf[0], copy_block_cnt * kBlockSize); |
339 TEST_AND_RETURN_FALSE(err == BZ_OK); | 337 TEST_AND_RETURN_FALSE(err == BZ_OK); |
340 blocks_read += copy_block_cnt; | 338 blocks_read += copy_block_cnt; |
341 blocks_copied_count += copy_block_cnt; | 339 blocks_copied_count += copy_block_cnt; |
342 LOG(INFO) << "progress: " << ((float)blocks_copied_count)/block_count; | 340 float current_progress = |
| 341 static_cast<float>(blocks_copied_count) / block_count; |
| 342 if (printed_progress + 0.1 < current_progress || |
| 343 blocks_copied_count == block_count) { |
| 344 LOG(INFO) << "progress: " << current_progress; |
| 345 printed_progress = current_progress; |
| 346 } |
343 } | 347 } |
344 } | 348 } |
345 BZ2_bzWriteClose(&err, bz_file, 0, NULL, NULL); | 349 BZ2_bzWriteClose(&err, bz_file, 0, NULL, NULL); |
346 TEST_AND_RETURN_FALSE(err == BZ_OK); | 350 TEST_AND_RETURN_FALSE(err == BZ_OK); |
347 bz_file = NULL; | 351 bz_file = NULL; |
348 TEST_AND_RETURN_FALSE_ERRNO(0 == fclose(file)); | 352 TEST_AND_RETURN_FALSE_ERRNO(0 == fclose(file)); |
349 file = NULL; | 353 file = NULL; |
350 | 354 |
351 vector<char> compressed_data; | 355 vector<char> compressed_data; |
352 LOG(INFO) << "Reading compressed data off disk"; | 356 LOG(INFO) << "Reading compressed data off disk"; |
(...skipping 387 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
740 cuts.reserve(edges.size()); | 744 cuts.reserve(edges.size()); |
741 | 745 |
742 uint64_t scratch_blocks_used = 0; | 746 uint64_t scratch_blocks_used = 0; |
743 for (set<Edge>::const_iterator it = edges.begin(); | 747 for (set<Edge>::const_iterator it = edges.begin(); |
744 it != edges.end(); ++it) { | 748 it != edges.end(); ++it) { |
745 cuts.resize(cuts.size() + 1); | 749 cuts.resize(cuts.size() + 1); |
746 vector<Extent> old_extents = | 750 vector<Extent> old_extents = |
747 (*graph)[it->first].out_edges[it->second].extents; | 751 (*graph)[it->first].out_edges[it->second].extents; |
748 // Choose some scratch space | 752 // Choose some scratch space |
749 scratch_blocks_used += graph_utils::EdgeWeight(*graph, *it); | 753 scratch_blocks_used += graph_utils::EdgeWeight(*graph, *it); |
750 LOG(INFO) << "using " << graph_utils::EdgeWeight(*graph, *it) | |
751 << " scratch blocks (" | |
752 << scratch_blocks_used << ")"; | |
753 cuts.back().tmp_extents = | 754 cuts.back().tmp_extents = |
754 scratch_allocator.Allocate(graph_utils::EdgeWeight(*graph, *it)); | 755 scratch_allocator.Allocate(graph_utils::EdgeWeight(*graph, *it)); |
755 // create vertex to copy original->scratch | 756 // create vertex to copy original->scratch |
756 cuts.back().new_vertex = graph->size(); | 757 cuts.back().new_vertex = graph->size(); |
757 graph->resize(graph->size() + 1); | 758 graph->resize(graph->size() + 1); |
758 cuts.back().old_src = it->first; | 759 cuts.back().old_src = it->first; |
759 cuts.back().old_dst = it->second; | 760 cuts.back().old_dst = it->second; |
760 | 761 |
761 EdgeProperties& cut_edge_properties = | 762 EdgeProperties& cut_edge_properties = |
762 (*graph)[it->first].out_edges.find(it->second)->second; | 763 (*graph)[it->first].out_edges.find(it->second)->second; |
(...skipping 223 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
986 for (vector<CutEdgeVertexes>::const_iterator it = cuts.begin(), | 987 for (vector<CutEdgeVertexes>::const_iterator it = cuts.begin(), |
987 e = cuts.end(); it != e; ++it) { | 988 e = cuts.end(); it != e; ++it) { |
988 uint64_t cut_blocks_needed = 0; | 989 uint64_t cut_blocks_needed = 0; |
989 for (vector<Extent>::const_iterator jt = it->tmp_extents.begin(), | 990 for (vector<Extent>::const_iterator jt = it->tmp_extents.begin(), |
990 je = it->tmp_extents.end(); jt != je; ++jt) { | 991 je = it->tmp_extents.end(); jt != je; ++jt) { |
991 cut_blocks_needed += jt->num_blocks(); | 992 cut_blocks_needed += jt->num_blocks(); |
992 } | 993 } |
993 blocks_needed += cut_blocks_needed; | 994 blocks_needed += cut_blocks_needed; |
994 cuts_blocks_needed[&*it] = cut_blocks_needed; | 995 cuts_blocks_needed[&*it] = cut_blocks_needed; |
995 } | 996 } |
996 LOG(INFO) << "Need to find " << blocks_needed << " blocks for " | 997 |
997 << cuts.size() << " cuts"; | |
998 | |
999 // Find enough blocks | 998 // Find enough blocks |
1000 ExtentRanges scratch_ranges; | 999 ExtentRanges scratch_ranges; |
1001 // Each block that's supplying temp blocks and the corresponding blocks: | 1000 // Each block that's supplying temp blocks and the corresponding blocks: |
1002 typedef vector<pair<Vertex::Index, ExtentRanges> > SupplierVector; | 1001 typedef vector<pair<Vertex::Index, ExtentRanges> > SupplierVector; |
1003 SupplierVector block_suppliers; | 1002 SupplierVector block_suppliers; |
1004 uint64_t scratch_blocks_found = 0; | 1003 uint64_t scratch_blocks_found = 0; |
1005 LOG(INFO) << "scan from " << (*reverse_op_indexes)[old_dst] + 1 | |
1006 << " to " << op_indexes->size(); | |
1007 for (vector<Vertex::Index>::size_type i = (*reverse_op_indexes)[old_dst] + 1, | 1004 for (vector<Vertex::Index>::size_type i = (*reverse_op_indexes)[old_dst] + 1, |
1008 e = op_indexes->size(); i < e; ++i) { | 1005 e = op_indexes->size(); i < e; ++i) { |
1009 Vertex::Index test_node = (*op_indexes)[i]; | 1006 Vertex::Index test_node = (*op_indexes)[i]; |
1010 if (!(*graph)[test_node].valid) | 1007 if (!(*graph)[test_node].valid) |
1011 continue; | 1008 continue; |
1012 // See if this node has sufficient blocks | 1009 // See if this node has sufficient blocks |
1013 ExtentRanges ranges; | 1010 ExtentRanges ranges; |
1014 ranges.AddRepeatedExtents((*graph)[test_node].op.dst_extents()); | 1011 ranges.AddRepeatedExtents((*graph)[test_node].op.dst_extents()); |
1015 ranges.SubtractExtent(ExtentForRange( | 1012 ranges.SubtractExtent(ExtentForRange( |
1016 kTempBlockStart, kSparseHole - kTempBlockStart)); | 1013 kTempBlockStart, kSparseHole - kTempBlockStart)); |
(...skipping 12 matching lines...) Expand all Loading... |
1029 if (ranges.blocks() + scratch_blocks_found > blocks_needed) { | 1026 if (ranges.blocks() + scratch_blocks_found > blocks_needed) { |
1030 // trim down ranges | 1027 // trim down ranges |
1031 vector<Extent> new_ranges = ranges.GetExtentsForBlockCount( | 1028 vector<Extent> new_ranges = ranges.GetExtentsForBlockCount( |
1032 blocks_needed - scratch_blocks_found); | 1029 blocks_needed - scratch_blocks_found); |
1033 ranges = ExtentRanges(); | 1030 ranges = ExtentRanges(); |
1034 ranges.AddExtents(new_ranges); | 1031 ranges.AddExtents(new_ranges); |
1035 } | 1032 } |
1036 scratch_ranges.AddRanges(ranges); | 1033 scratch_ranges.AddRanges(ranges); |
1037 block_suppliers.push_back(make_pair(test_node, ranges)); | 1034 block_suppliers.push_back(make_pair(test_node, ranges)); |
1038 scratch_blocks_found += ranges.blocks(); | 1035 scratch_blocks_found += ranges.blocks(); |
1039 LOG(INFO) << "Adding " << ranges.blocks() << " blocks. Now have " | |
1040 << scratch_ranges.blocks(); | |
1041 if (scratch_ranges.blocks() >= blocks_needed) | 1036 if (scratch_ranges.blocks() >= blocks_needed) |
1042 break; | 1037 break; |
1043 } | 1038 } |
1044 if (scratch_ranges.blocks() < blocks_needed) { | 1039 if (scratch_ranges.blocks() < blocks_needed) { |
1045 LOG(INFO) << "Unable to find sufficient scratch"; | 1040 LOG(INFO) << "Unable to find sufficient scratch"; |
1046 TEST_AND_RETURN_FALSE(ConvertCutsToFull(graph, | 1041 TEST_AND_RETURN_FALSE(ConvertCutsToFull(graph, |
1047 new_root, | 1042 new_root, |
1048 data_fd, | 1043 data_fd, |
1049 data_file_size, | 1044 data_file_size, |
1050 op_indexes, | 1045 op_indexes, |
(...skipping 26 matching lines...) Expand all Loading... |
1077 real_extents); | 1072 real_extents); |
1078 | 1073 |
1079 // Fix the new node w/ the real blocks. Since the new node is just a | 1074 // Fix the new node w/ the real blocks. Since the new node is just a |
1080 // copy operation, we can replace all the dest extents w/ the real | 1075 // copy operation, we can replace all the dest extents w/ the real |
1081 // blocks. | 1076 // blocks. |
1082 DeltaArchiveManifest_InstallOperation *op = | 1077 DeltaArchiveManifest_InstallOperation *op = |
1083 &(*graph)[it->new_vertex].op; | 1078 &(*graph)[it->new_vertex].op; |
1084 op->clear_dst_extents(); | 1079 op->clear_dst_extents(); |
1085 DeltaDiffGenerator::StoreExtents(real_extents, op->mutable_dst_extents()); | 1080 DeltaDiffGenerator::StoreExtents(real_extents, op->mutable_dst_extents()); |
1086 } | 1081 } |
1087 LOG(INFO) << "Done subbing in for cut set"; | |
1088 return true; | 1082 return true; |
1089 } | 1083 } |
1090 | 1084 |
1091 } // namespace {} | 1085 } // namespace {} |
1092 | 1086 |
1093 // Returns true if |op| is a no-op operation that doesn't do any useful work | 1087 // Returns true if |op| is a no-op operation that doesn't do any useful work |
1094 // (e.g., a move operation that copies blocks onto themselves). | 1088 // (e.g., a move operation that copies blocks onto themselves). |
1095 bool DeltaDiffGenerator::IsNoopOperation( | 1089 bool DeltaDiffGenerator::IsNoopOperation( |
1096 const DeltaArchiveManifest_InstallOperation& op) { | 1090 const DeltaArchiveManifest_InstallOperation& op) { |
1097 return (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE && | 1091 return (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE && |
(...skipping 532 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1630 | 1624 |
1631 LOG(INFO) << "All done. Successfully created delta file."; | 1625 LOG(INFO) << "All done. Successfully created delta file."; |
1632 return true; | 1626 return true; |
1633 } | 1627 } |
1634 | 1628 |
1635 const char* const kBsdiffPath = "bsdiff"; | 1629 const char* const kBsdiffPath = "bsdiff"; |
1636 const char* const kBspatchPath = "bspatch"; | 1630 const char* const kBspatchPath = "bspatch"; |
1637 const char* const kDeltaMagic = "CrAU"; | 1631 const char* const kDeltaMagic = "CrAU"; |
1638 | 1632 |
1639 }; // namespace chromeos_update_engine | 1633 }; // namespace chromeos_update_engine |
OLD | NEW |