OLD | NEW |
1 // Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 1 // Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "update_engine/delta_diff_generator.h" | 5 #include "update_engine/delta_diff_generator.h" |
6 | 6 |
7 #include <errno.h> | 7 #include <errno.h> |
8 #include <fcntl.h> | 8 #include <fcntl.h> |
9 #include <inttypes.h> | 9 #include <inttypes.h> |
10 #include <sys/stat.h> | 10 #include <sys/stat.h> |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
47 namespace chromeos_update_engine { | 47 namespace chromeos_update_engine { |
48 | 48 |
49 typedef DeltaDiffGenerator::Block Block; | 49 typedef DeltaDiffGenerator::Block Block; |
50 | 50 |
51 namespace { | 51 namespace { |
52 const size_t kBlockSize = 4096; // bytes | 52 const size_t kBlockSize = 4096; // bytes |
53 const size_t kRootFSPartitionSize = 1 * 1024 * 1024 * 1024; // 1 GiB | 53 const size_t kRootFSPartitionSize = 1 * 1024 * 1024 * 1024; // 1 GiB |
54 const uint64_t kVersionNumber = 1; | 54 const uint64_t kVersionNumber = 1; |
55 const uint64_t kFullUpdateChunkSize = 128 * 1024; // bytes | 55 const uint64_t kFullUpdateChunkSize = 128 * 1024; // bytes |
56 | 56 |
| 57 static const char* kInstallOperationTypes[] = { |
| 58 "REPLACE", |
| 59 "REPLACE_BZ", |
| 60 "MOVE", |
| 61 "BSDIFF" |
| 62 }; |
| 63 |
57 // Stores all Extents for a file into 'out'. Returns true on success. | 64 // Stores all Extents for a file into 'out'. Returns true on success. |
58 bool GatherExtents(const string& path, | 65 bool GatherExtents(const string& path, |
59 google::protobuf::RepeatedPtrField<Extent>* out) { | 66 google::protobuf::RepeatedPtrField<Extent>* out) { |
60 vector<Extent> extents; | 67 vector<Extent> extents; |
61 TEST_AND_RETURN_FALSE(extent_mapper::ExtentsForFileFibmap(path, &extents)); | 68 TEST_AND_RETURN_FALSE(extent_mapper::ExtentsForFileFibmap(path, &extents)); |
62 DeltaDiffGenerator::StoreExtents(extents, out); | 69 DeltaDiffGenerator::StoreExtents(extents, out); |
63 return true; | 70 return true; |
64 } | 71 } |
65 | 72 |
66 // Runs the bsdiff tool on two files and returns the resulting delta in | 73 // Runs the bsdiff tool on two files and returns the resulting delta in |
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
156 const string& new_root, | 163 const string& new_root, |
157 const string& path, // within new_root | 164 const string& path, // within new_root |
158 int data_fd, | 165 int data_fd, |
159 off_t* data_file_size) { | 166 off_t* data_file_size) { |
160 vector<char> data; | 167 vector<char> data; |
161 DeltaArchiveManifest_InstallOperation operation; | 168 DeltaArchiveManifest_InstallOperation operation; |
162 | 169 |
163 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_root + path, | 170 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_root + path, |
164 new_root + path, | 171 new_root + path, |
165 &data, | 172 &data, |
166 &operation)); | 173 &operation, |
| 174 true)); |
167 | 175 |
168 // Write the data | 176 // Write the data |
169 if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) { | 177 if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) { |
170 operation.set_data_offset(*data_file_size); | 178 operation.set_data_offset(*data_file_size); |
171 operation.set_data_length(data.size()); | 179 operation.set_data_length(data.size()); |
172 } | 180 } |
173 | 181 |
174 TEST_AND_RETURN_FALSE(utils::WriteAll(data_fd, &data[0], data.size())); | 182 TEST_AND_RETURN_FALSE(utils::WriteAll(data_fd, &data[0], data.size())); |
175 *data_file_size += data.size(); | 183 *data_file_size += data.size(); |
176 | 184 |
(...skipping 211 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
388 *op = *it; | 396 *op = *it; |
389 } | 397 } |
390 } | 398 } |
391 | 399 |
392 void CheckGraph(const Graph& graph) { | 400 void CheckGraph(const Graph& graph) { |
393 for (Graph::const_iterator it = graph.begin(); it != graph.end(); ++it) { | 401 for (Graph::const_iterator it = graph.begin(); it != graph.end(); ++it) { |
394 CHECK(it->op.has_type()); | 402 CHECK(it->op.has_type()); |
395 } | 403 } |
396 } | 404 } |
397 | 405 |
398 // Delta compresses a kernel partition new_kernel_part with knowledge of | 406 // Delta compresses a kernel partition |new_kernel_part| with knowledge of the |
399 // the old kernel partition old_kernel_part. | 407 // old kernel partition |old_kernel_part|. If |old_kernel_part| is an empty |
| 408 // string, generates a full update of the partition. |
400 bool DeltaCompressKernelPartition( | 409 bool DeltaCompressKernelPartition( |
401 const string& old_kernel_part, | 410 const string& old_kernel_part, |
402 const string& new_kernel_part, | 411 const string& new_kernel_part, |
403 vector<DeltaArchiveManifest_InstallOperation>* ops, | 412 vector<DeltaArchiveManifest_InstallOperation>* ops, |
404 int blobs_fd, | 413 int blobs_fd, |
405 off_t* blobs_length) { | 414 off_t* blobs_length) { |
406 // For now, just bsdiff the kernel partition as a whole. | |
407 // TODO(adlr): Use knowledge of how the kernel partition is laid out | |
408 // to more efficiently compress it. | |
409 | |
410 LOG(INFO) << "Delta compressing kernel partition..."; | 415 LOG(INFO) << "Delta compressing kernel partition..."; |
| 416 LOG_IF(INFO, old_kernel_part.empty()) << "Generating full kernel update..."; |
411 | 417 |
412 // Add a new install operation | 418 // Add a new install operation |
413 ops->resize(1); | 419 ops->resize(1); |
414 DeltaArchiveManifest_InstallOperation* op = &(*ops)[0]; | 420 DeltaArchiveManifest_InstallOperation* op = &(*ops)[0]; |
415 op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ); | |
416 op->set_data_offset(*blobs_length); | |
417 | 421 |
418 // Do the actual compression | |
419 vector<char> data; | 422 vector<char> data; |
420 TEST_AND_RETURN_FALSE(utils::ReadFile(new_kernel_part, &data)); | 423 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_kernel_part, |
421 TEST_AND_RETURN_FALSE(!data.empty()); | 424 new_kernel_part, |
| 425 &data, |
| 426 op, |
| 427 false)); |
422 | 428 |
423 vector<char> data_bz; | 429 // Write the data |
424 TEST_AND_RETURN_FALSE(BzipCompress(data, &data_bz)); | 430 if (op->type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) { |
425 CHECK(!data_bz.empty()); | 431 op->set_data_offset(*blobs_length); |
| 432 op->set_data_length(data.size()); |
| 433 } |
426 | 434 |
427 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, &data_bz[0], data_bz.size())); | 435 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, &data[0], data.size())); |
428 *blobs_length += data_bz.size(); | 436 *blobs_length += data.size(); |
429 | 437 |
430 off_t new_part_size = utils::FileSize(new_kernel_part); | 438 LOG(INFO) << "Done delta compressing kernel partition: " |
431 TEST_AND_RETURN_FALSE(new_part_size >= 0); | 439 << kInstallOperationTypes[op->type()]; |
432 | |
433 op->set_data_length(data_bz.size()); | |
434 | |
435 op->set_dst_length(new_part_size); | |
436 | |
437 // There's a single dest extent | |
438 Extent* dst_extent = op->add_dst_extents(); | |
439 dst_extent->set_start_block(0); | |
440 dst_extent->set_num_blocks((new_part_size + kBlockSize - 1) / kBlockSize); | |
441 | |
442 LOG(INFO) << "Done compressing kernel partition."; | |
443 return true; | 440 return true; |
444 } | 441 } |
445 | 442 |
446 struct DeltaObject { | 443 struct DeltaObject { |
447 DeltaObject(const string& in_name, const int in_type, const off_t in_size) | 444 DeltaObject(const string& in_name, const int in_type, const off_t in_size) |
448 : name(in_name), | 445 : name(in_name), |
449 type(in_type), | 446 type(in_type), |
450 size(in_size) {} | 447 size(in_size) {} |
451 bool operator <(const DeltaObject& object) const { | 448 bool operator <(const DeltaObject& object) const { |
452 return size < object.size; | 449 return size < object.size; |
453 } | 450 } |
454 string name; | 451 string name; |
455 int type; | 452 int type; |
456 off_t size; | 453 off_t size; |
457 }; | 454 }; |
458 | 455 |
459 static const char* kInstallOperationTypes[] = { | |
460 "REPLACE", | |
461 "REPLACE_BZ", | |
462 "MOVE", | |
463 "BSDIFF" | |
464 }; | |
465 | |
466 void ReportPayloadUsage(const Graph& graph, | 456 void ReportPayloadUsage(const Graph& graph, |
467 const DeltaArchiveManifest& manifest, | 457 const DeltaArchiveManifest& manifest, |
468 const int64_t manifest_metadata_size) { | 458 const int64_t manifest_metadata_size) { |
469 vector<DeltaObject> objects; | 459 vector<DeltaObject> objects; |
470 off_t total_size = 0; | 460 off_t total_size = 0; |
471 | 461 |
472 // Graph nodes with information about file names. | 462 // Graph nodes with information about file names. |
473 for (Vertex::Index node = 0; node < graph.size(); node++) { | 463 for (Vertex::Index node = 0; node < graph.size(); node++) { |
474 const Vertex& vertex = graph[node]; | 464 const Vertex& vertex = graph[node]; |
475 if (!vertex.valid) { | 465 if (!vertex.valid) { |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
510 } | 500 } |
511 fprintf(stderr, kFormatString, 100.0, total_size, "", "<total>"); | 501 fprintf(stderr, kFormatString, 100.0, total_size, "", "<total>"); |
512 } | 502 } |
513 | 503 |
514 } // namespace {} | 504 } // namespace {} |
515 | 505 |
516 bool DeltaDiffGenerator::ReadFileToDiff( | 506 bool DeltaDiffGenerator::ReadFileToDiff( |
517 const string& old_filename, | 507 const string& old_filename, |
518 const string& new_filename, | 508 const string& new_filename, |
519 vector<char>* out_data, | 509 vector<char>* out_data, |
520 DeltaArchiveManifest_InstallOperation* out_op) { | 510 DeltaArchiveManifest_InstallOperation* out_op, |
| 511 bool gather_extents) { |
521 // Read new data in | 512 // Read new data in |
522 vector<char> new_data; | 513 vector<char> new_data; |
523 TEST_AND_RETURN_FALSE(utils::ReadFile(new_filename, &new_data)); | 514 TEST_AND_RETURN_FALSE(utils::ReadFile(new_filename, &new_data)); |
524 | 515 |
525 TEST_AND_RETURN_FALSE(!new_data.empty()); | 516 TEST_AND_RETURN_FALSE(!new_data.empty()); |
526 | 517 |
527 vector<char> new_data_bz; | 518 vector<char> new_data_bz; |
528 TEST_AND_RETURN_FALSE(BzipCompress(new_data, &new_data_bz)); | 519 TEST_AND_RETURN_FALSE(BzipCompress(new_data, &new_data_bz)); |
529 CHECK(!new_data_bz.empty()); | 520 CHECK(!new_data_bz.empty()); |
530 | 521 |
531 vector<char> data; // Data blob that will be written to delta file. | 522 vector<char> data; // Data blob that will be written to delta file. |
532 | 523 |
533 DeltaArchiveManifest_InstallOperation operation; | 524 DeltaArchiveManifest_InstallOperation operation; |
534 size_t current_best_size = 0; | 525 size_t current_best_size = 0; |
535 if (new_data.size() <= new_data_bz.size()) { | 526 if (new_data.size() <= new_data_bz.size()) { |
536 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE); | 527 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE); |
537 current_best_size = new_data.size(); | 528 current_best_size = new_data.size(); |
538 data = new_data; | 529 data = new_data; |
539 } else { | 530 } else { |
540 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ); | 531 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ); |
541 current_best_size = new_data_bz.size(); | 532 current_best_size = new_data_bz.size(); |
542 data = new_data_bz; | 533 data = new_data_bz; |
543 } | 534 } |
544 | 535 |
545 // Do we have an original file to consider? | 536 // Do we have an original file to consider? |
546 struct stat old_stbuf; | 537 struct stat old_stbuf; |
547 if (0 != stat(old_filename.c_str(), &old_stbuf)) { | 538 bool no_original = old_filename.empty(); |
| 539 if (!no_original && 0 != stat(old_filename.c_str(), &old_stbuf)) { |
548 // If stat-ing the old file fails, it should be because it doesn't exist. | 540 // If stat-ing the old file fails, it should be because it doesn't exist. |
549 TEST_AND_RETURN_FALSE(errno == ENOTDIR || errno == ENOENT); | 541 TEST_AND_RETURN_FALSE(errno == ENOTDIR || errno == ENOENT); |
550 } else { | 542 no_original = true; |
| 543 } |
| 544 if (!no_original) { |
551 // Read old data | 545 // Read old data |
552 vector<char> old_data; | 546 vector<char> old_data; |
553 TEST_AND_RETURN_FALSE(utils::ReadFile(old_filename, &old_data)); | 547 TEST_AND_RETURN_FALSE(utils::ReadFile(old_filename, &old_data)); |
554 if (old_data == new_data) { | 548 if (old_data == new_data) { |
555 // No change in data. | 549 // No change in data. |
556 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE); | 550 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE); |
557 current_best_size = 0; | 551 current_best_size = 0; |
558 data.clear(); | 552 data.clear(); |
559 } else { | 553 } else { |
560 // Try bsdiff of old to new data | 554 // Try bsdiff of old to new data |
561 vector<char> bsdiff_delta; | 555 vector<char> bsdiff_delta; |
562 TEST_AND_RETURN_FALSE( | 556 TEST_AND_RETURN_FALSE( |
563 BsdiffFiles(old_filename, new_filename, &bsdiff_delta)); | 557 BsdiffFiles(old_filename, new_filename, &bsdiff_delta)); |
564 CHECK_GT(bsdiff_delta.size(), 0); | 558 CHECK_GT(bsdiff_delta.size(), 0); |
565 if (bsdiff_delta.size() < current_best_size) { | 559 if (bsdiff_delta.size() < current_best_size) { |
566 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF); | 560 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF); |
567 current_best_size = bsdiff_delta.size(); | 561 current_best_size = bsdiff_delta.size(); |
568 | 562 |
569 data = bsdiff_delta; | 563 data = bsdiff_delta; |
570 } | 564 } |
571 } | 565 } |
572 } | 566 } |
573 | 567 |
574 // Set parameters of the operations | 568 // Set parameters of the operations |
575 CHECK_EQ(data.size(), current_best_size); | 569 CHECK_EQ(data.size(), current_best_size); |
576 | 570 |
577 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE || | 571 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE || |
578 operation.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF) { | 572 operation.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF) { |
579 TEST_AND_RETURN_FALSE( | 573 if (gather_extents) { |
580 GatherExtents(old_filename, operation.mutable_src_extents())); | 574 TEST_AND_RETURN_FALSE( |
| 575 GatherExtents(old_filename, operation.mutable_src_extents())); |
| 576 } else { |
| 577 Extent* src_extent = operation.add_src_extents(); |
| 578 src_extent->set_start_block(0); |
| 579 src_extent->set_num_blocks( |
| 580 (old_stbuf.st_size + kBlockSize - 1) / kBlockSize); |
| 581 } |
581 operation.set_src_length(old_stbuf.st_size); | 582 operation.set_src_length(old_stbuf.st_size); |
582 } | 583 } |
583 | 584 |
584 TEST_AND_RETURN_FALSE( | 585 if (gather_extents) { |
585 GatherExtents(new_filename, operation.mutable_dst_extents())); | 586 TEST_AND_RETURN_FALSE( |
| 587 GatherExtents(new_filename, operation.mutable_dst_extents())); |
| 588 } else { |
| 589 Extent* dst_extent = operation.add_dst_extents(); |
| 590 dst_extent->set_start_block(0); |
| 591 dst_extent->set_num_blocks((new_data.size() + kBlockSize - 1) / kBlockSize); |
| 592 } |
586 operation.set_dst_length(new_data.size()); | 593 operation.set_dst_length(new_data.size()); |
587 | 594 |
588 out_data->swap(data); | 595 out_data->swap(data); |
589 *out_op = operation; | 596 *out_op = operation; |
590 | 597 |
591 return true; | 598 return true; |
592 } | 599 } |
593 | 600 |
594 bool InitializePartitionInfo(bool is_kernel, | 601 bool InitializePartitionInfo(bool is_kernel, |
595 const string& partition, | 602 const string& partition, |
(...skipping 670 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1266 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(new_image, | 1273 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(new_image, |
1267 &new_image_block_count, | 1274 &new_image_block_count, |
1268 &new_image_block_size)); | 1275 &new_image_block_size)); |
1269 if (!old_image.empty()) { | 1276 if (!old_image.empty()) { |
1270 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(old_image, | 1277 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(old_image, |
1271 &old_image_block_count, | 1278 &old_image_block_count, |
1272 &old_image_block_size)); | 1279 &old_image_block_size)); |
1273 TEST_AND_RETURN_FALSE(old_image_block_size == new_image_block_size); | 1280 TEST_AND_RETURN_FALSE(old_image_block_size == new_image_block_size); |
1274 LOG_IF(WARNING, old_image_block_count != new_image_block_count) | 1281 LOG_IF(WARNING, old_image_block_count != new_image_block_count) |
1275 << "Old and new images have different block counts."; | 1282 << "Old and new images have different block counts."; |
1276 // Sanity check kernel partition arg | |
1277 TEST_AND_RETURN_FALSE(utils::FileSize(old_kernel_part) >= 0); | |
1278 } | 1283 } |
1279 // Sanity check kernel partition arg | 1284 // Sanity check kernel partition arg |
1280 TEST_AND_RETURN_FALSE(utils::FileSize(new_kernel_part) >= 0); | 1285 TEST_AND_RETURN_FALSE(utils::FileSize(new_kernel_part) >= 0); |
1281 | 1286 |
1282 vector<Block> blocks(max(old_image_block_count, new_image_block_count)); | 1287 vector<Block> blocks(max(old_image_block_count, new_image_block_count)); |
1283 LOG(INFO) << "Invalid block index: " << Vertex::kInvalidIndex; | 1288 LOG(INFO) << "Invalid block index: " << Vertex::kInvalidIndex; |
1284 LOG(INFO) << "Block count: " << blocks.size(); | 1289 LOG(INFO) << "Block count: " << blocks.size(); |
1285 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { | 1290 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { |
1286 CHECK(blocks[i].reader == Vertex::kInvalidIndex); | 1291 CHECK(blocks[i].reader == Vertex::kInvalidIndex); |
1287 CHECK(blocks[i].writer == Vertex::kInvalidIndex); | 1292 CHECK(blocks[i].writer == Vertex::kInvalidIndex); |
(...skipping 221 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1509 | 1514 |
1510 LOG(INFO) << "All done. Successfully created delta file."; | 1515 LOG(INFO) << "All done. Successfully created delta file."; |
1511 return true; | 1516 return true; |
1512 } | 1517 } |
1513 | 1518 |
1514 const char* const kBsdiffPath = "/usr/bin/bsdiff"; | 1519 const char* const kBsdiffPath = "/usr/bin/bsdiff"; |
1515 const char* const kBspatchPath = "/usr/bin/bspatch"; | 1520 const char* const kBspatchPath = "/usr/bin/bspatch"; |
1516 const char* const kDeltaMagic = "CrAU"; | 1521 const char* const kDeltaMagic = "CrAU"; |
1517 | 1522 |
1518 }; // namespace chromeos_update_engine | 1523 }; // namespace chromeos_update_engine |
OLD | NEW |