OLD | NEW |
1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "update_engine/delta_diff_generator.h" | 5 #include "update_engine/delta_diff_generator.h" |
6 #include <sys/stat.h> | 6 #include <sys/stat.h> |
7 #include <sys/types.h> | 7 #include <sys/types.h> |
8 #include <errno.h> | 8 #include <errno.h> |
9 #include <fcntl.h> | 9 #include <fcntl.h> |
10 #include <algorithm> | 10 #include <algorithm> |
(...skipping 20 matching lines...) Expand all Loading... |
31 using std::set; | 31 using std::set; |
32 using std::string; | 32 using std::string; |
33 using std::vector; | 33 using std::vector; |
34 | 34 |
35 namespace chromeos_update_engine { | 35 namespace chromeos_update_engine { |
36 | 36 |
37 typedef DeltaDiffGenerator::Block Block; | 37 typedef DeltaDiffGenerator::Block Block; |
38 | 38 |
39 namespace { | 39 namespace { |
40 const size_t kBlockSize = 4096; | 40 const size_t kBlockSize = 4096; |
41 const char* const kBsdiffPath = "/usr/bin/bsdiff"; | 41 const uint64_t kVersionNumber = 1; |
42 const uint64 kVersionNumber = 1; | |
43 const char* const kDeltaMagic = "CrAU"; | |
44 | 42 |
45 // Stores all Extents for a file into 'out'. Returns true on success. | 43 // Stores all Extents for a file into 'out'. Returns true on success. |
46 bool GatherExtents(const string& path, | 44 bool GatherExtents(const string& path, |
47 google::protobuf::RepeatedPtrField<Extent>* out) { | 45 google::protobuf::RepeatedPtrField<Extent>* out) { |
48 vector<Extent> extents; | 46 vector<Extent> extents; |
49 TEST_AND_RETURN_FALSE(extent_mapper::ExtentsForFileFibmap(path, &extents)); | 47 TEST_AND_RETURN_FALSE(extent_mapper::ExtentsForFileFibmap(path, &extents)); |
50 DeltaDiffGenerator::StoreExtents(extents, out); | 48 DeltaDiffGenerator::StoreExtents(extents, out); |
51 return true; | 49 return true; |
52 } | 50 } |
53 | 51 |
(...skipping 317 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
371 out_op->set_dst_length(kBlockSize * block_count); | 369 out_op->set_dst_length(kBlockSize * block_count); |
372 DeltaDiffGenerator::StoreExtents(extents, out_op->mutable_dst_extents()); | 370 DeltaDiffGenerator::StoreExtents(extents, out_op->mutable_dst_extents()); |
373 | 371 |
374 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, | 372 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, |
375 &compressed_data[0], | 373 &compressed_data[0], |
376 compressed_data.size())); | 374 compressed_data.size())); |
377 LOG(INFO) << "done with extra blocks"; | 375 LOG(INFO) << "done with extra blocks"; |
378 return true; | 376 return true; |
379 } | 377 } |
380 | 378 |
381 // Writes the uint64 passed in in host-endian to the file as big-endian. | 379 // Writes the uint64_t passed in in host-endian to the file as big-endian. |
382 // Returns true on success. | 380 // Returns true on success. |
383 bool WriteUint64AsBigEndian(FileWriter* writer, const uint64 value) { | 381 bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) { |
384 uint64 value_be = htobe64(value); | 382 uint64_t value_be = htobe64(value); |
385 TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be)) == | 383 TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be)) == |
386 sizeof(value_be)); | 384 sizeof(value_be)); |
387 return true; | 385 return true; |
388 } | 386 } |
389 | 387 |
390 // Adds each operation from the graph to the manifest in the order | 388 // Adds each operation from the graph to the manifest in the order |
391 // specified by 'order'. | 389 // specified by 'order'. |
392 void InstallOperationsToManifest( | 390 void InstallOperationsToManifest( |
393 const Graph& graph, | 391 const Graph& graph, |
394 const vector<Vertex::Index>& order, | 392 const vector<Vertex::Index>& order, |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
485 *out_op = operation; | 483 *out_op = operation; |
486 | 484 |
487 return true; | 485 return true; |
488 } | 486 } |
489 | 487 |
490 void DeltaDiffGenerator::SubstituteBlocks( | 488 void DeltaDiffGenerator::SubstituteBlocks( |
491 DeltaArchiveManifest_InstallOperation* op, | 489 DeltaArchiveManifest_InstallOperation* op, |
492 const vector<Extent>& remove_extents, | 490 const vector<Extent>& remove_extents, |
493 const vector<Extent>& replace_extents) { | 491 const vector<Extent>& replace_extents) { |
494 // First, expand out the blocks that op reads from | 492 // First, expand out the blocks that op reads from |
495 vector<uint64> read_blocks; | 493 vector<uint64_t> read_blocks; |
496 for (int i = 0; i < op->src_extents_size(); i++) { | 494 for (int i = 0; i < op->src_extents_size(); i++) { |
497 const Extent& extent = op->src_extents(i); | 495 const Extent& extent = op->src_extents(i); |
498 if (extent.start_block() == kSparseHole) { | 496 if (extent.start_block() == kSparseHole) { |
499 read_blocks.resize(read_blocks.size() + extent.num_blocks(), kSparseHole); | 497 read_blocks.resize(read_blocks.size() + extent.num_blocks(), kSparseHole); |
500 } else { | 498 } else { |
501 for (uint64 block = extent.start_block(); | 499 for (uint64_t block = extent.start_block(); |
502 block < (extent.start_block() + extent.num_blocks()); block++) { | 500 block < (extent.start_block() + extent.num_blocks()); block++) { |
503 read_blocks.push_back(block); | 501 read_blocks.push_back(block); |
504 } | 502 } |
505 } | 503 } |
506 } | 504 } |
507 { | 505 { |
508 // Expand remove_extents and replace_extents | 506 // Expand remove_extents and replace_extents |
509 vector<uint64> remove_extents_expanded; | 507 vector<uint64_t> remove_extents_expanded; |
510 for (vector<Extent>::const_iterator it = remove_extents.begin(); | 508 for (vector<Extent>::const_iterator it = remove_extents.begin(); |
511 it != remove_extents.end(); ++it) { | 509 it != remove_extents.end(); ++it) { |
512 const Extent& extent = *it; | 510 const Extent& extent = *it; |
513 for (uint64 block = extent.start_block(); | 511 for (uint64_t block = extent.start_block(); |
514 block < (extent.start_block() + extent.num_blocks()); block++) { | 512 block < (extent.start_block() + extent.num_blocks()); block++) { |
515 remove_extents_expanded.push_back(block); | 513 remove_extents_expanded.push_back(block); |
516 } | 514 } |
517 } | 515 } |
518 vector<uint64> replace_extents_expanded; | 516 vector<uint64_t> replace_extents_expanded; |
519 for (vector<Extent>::const_iterator it = replace_extents.begin(); | 517 for (vector<Extent>::const_iterator it = replace_extents.begin(); |
520 it != replace_extents.end(); ++it) { | 518 it != replace_extents.end(); ++it) { |
521 const Extent& extent = *it; | 519 const Extent& extent = *it; |
522 for (uint64 block = extent.start_block(); | 520 for (uint64_t block = extent.start_block(); |
523 block < (extent.start_block() + extent.num_blocks()); block++) { | 521 block < (extent.start_block() + extent.num_blocks()); block++) { |
524 replace_extents_expanded.push_back(block); | 522 replace_extents_expanded.push_back(block); |
525 } | 523 } |
526 } | 524 } |
527 CHECK_EQ(remove_extents_expanded.size(), replace_extents_expanded.size()); | 525 CHECK_EQ(remove_extents_expanded.size(), replace_extents_expanded.size()); |
528 for (vector<uint64>::size_type i = 0; | 526 for (vector<uint64_t>::size_type i = 0; |
529 i < replace_extents_expanded.size(); i++) { | 527 i < replace_extents_expanded.size(); i++) { |
530 vector<uint64>::size_type index = 0; | 528 vector<uint64_t>::size_type index = 0; |
531 CHECK(utils::VectorIndexOf(read_blocks, | 529 CHECK(utils::VectorIndexOf(read_blocks, |
532 remove_extents_expanded[i], | 530 remove_extents_expanded[i], |
533 &index)); | 531 &index)); |
534 CHECK(read_blocks[index] == remove_extents_expanded[i]); | 532 CHECK(read_blocks[index] == remove_extents_expanded[i]); |
535 read_blocks[index] = replace_extents_expanded[i]; | 533 read_blocks[index] = replace_extents_expanded[i]; |
536 } | 534 } |
537 } | 535 } |
538 // Convert read_blocks back to extents | 536 // Convert read_blocks back to extents |
539 op->clear_src_extents(); | 537 op->clear_src_extents(); |
540 vector<Extent> new_extents; | 538 vector<Extent> new_extents; |
541 for (vector<uint64>::const_iterator it = read_blocks.begin(); | 539 for (vector<uint64_t>::const_iterator it = read_blocks.begin(); |
542 it != read_blocks.end(); ++it) { | 540 it != read_blocks.end(); ++it) { |
543 graph_utils::AppendBlockToExtents(&new_extents, *it); | 541 graph_utils::AppendBlockToExtents(&new_extents, *it); |
544 } | 542 } |
545 DeltaDiffGenerator::StoreExtents(new_extents, op->mutable_src_extents()); | 543 DeltaDiffGenerator::StoreExtents(new_extents, op->mutable_src_extents()); |
546 } | 544 } |
547 | 545 |
548 bool DeltaDiffGenerator::CutEdges(Graph* graph, | 546 bool DeltaDiffGenerator::CutEdges(Graph* graph, |
549 const vector<Block>& blocks, | 547 const vector<Block>& blocks, |
550 const set<Edge>& edges) { | 548 const set<Edge>& edges) { |
551 // First, find enough scratch space for the edges we'll be cutting. | 549 // First, find enough scratch space for the edges we'll be cutting. |
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
646 int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0); | 644 int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0); |
647 TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0); | 645 TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0); |
648 ScopedFdCloser in_fd_closer(&in_fd); | 646 ScopedFdCloser in_fd_closer(&in_fd); |
649 | 647 |
650 DirectFileWriter writer; | 648 DirectFileWriter writer; |
651 TEST_AND_RETURN_FALSE( | 649 TEST_AND_RETURN_FALSE( |
652 writer.Open(new_data_blobs_path.c_str(), | 650 writer.Open(new_data_blobs_path.c_str(), |
653 O_WRONLY | O_TRUNC | O_CREAT, | 651 O_WRONLY | O_TRUNC | O_CREAT, |
654 0644) == 0); | 652 0644) == 0); |
655 ScopedFileWriterCloser writer_closer(&writer); | 653 ScopedFileWriterCloser writer_closer(&writer); |
656 uint64 out_file_size = 0; | 654 uint64_t out_file_size = 0; |
657 | 655 |
658 for (int i = 0; i < manifest->install_operations_size(); i++) { | 656 for (int i = 0; i < manifest->install_operations_size(); i++) { |
659 DeltaArchiveManifest_InstallOperation* op = | 657 DeltaArchiveManifest_InstallOperation* op = |
660 manifest->mutable_install_operations(i); | 658 manifest->mutable_install_operations(i); |
661 if (!op->has_data_offset()) | 659 if (!op->has_data_offset()) |
662 continue; | 660 continue; |
663 CHECK(op->has_data_length()); | 661 CHECK(op->has_data_length()); |
664 vector<char> buf(op->data_length()); | 662 vector<char> buf(op->data_length()); |
665 ssize_t rc = pread(in_fd, &buf[0], buf.size(), op->data_offset()); | 663 ssize_t rc = pread(in_fd, &buf[0], buf.size(), op->data_offset()); |
666 TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size())); | 664 TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size())); |
(...skipping 17 matching lines...) Expand all Loading... |
684 TEST_AND_RETURN_FALSE_ERRNO(stat(new_image.c_str(), &new_image_stbuf) == 0); | 682 TEST_AND_RETURN_FALSE_ERRNO(stat(new_image.c_str(), &new_image_stbuf) == 0); |
685 LOG_IF(WARNING, new_image_stbuf.st_size != old_image_stbuf.st_size) | 683 LOG_IF(WARNING, new_image_stbuf.st_size != old_image_stbuf.st_size) |
686 << "Old and new images are different sizes."; | 684 << "Old and new images are different sizes."; |
687 LOG_IF(FATAL, new_image_stbuf.st_size % kBlockSize) | 685 LOG_IF(FATAL, new_image_stbuf.st_size % kBlockSize) |
688 << "New image not a multiple of block size " << kBlockSize; | 686 << "New image not a multiple of block size " << kBlockSize; |
689 LOG_IF(FATAL, old_image_stbuf.st_size % kBlockSize) | 687 LOG_IF(FATAL, old_image_stbuf.st_size % kBlockSize) |
690 << "Old image not a multiple of block size " << kBlockSize; | 688 << "Old image not a multiple of block size " << kBlockSize; |
691 | 689 |
692 vector<Block> blocks(min(old_image_stbuf.st_size / kBlockSize, | 690 vector<Block> blocks(min(old_image_stbuf.st_size / kBlockSize, |
693 new_image_stbuf.st_size / kBlockSize)); | 691 new_image_stbuf.st_size / kBlockSize)); |
694 LOG(INFO) << "w:" << blocks[4097].writer; | |
695 LOG(INFO) << "invalid: " << Vertex::kInvalidIndex; | 692 LOG(INFO) << "invalid: " << Vertex::kInvalidIndex; |
696 LOG(INFO) << "len: " << blocks.size(); | 693 LOG(INFO) << "len: " << blocks.size(); |
697 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { | 694 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { |
698 CHECK(blocks[i].reader == Vertex::kInvalidIndex); | 695 CHECK(blocks[i].reader == Vertex::kInvalidIndex); |
699 CHECK(blocks[i].writer == Vertex::kInvalidIndex); | 696 CHECK(blocks[i].writer == Vertex::kInvalidIndex); |
700 } | 697 } |
701 Graph graph; | 698 Graph graph; |
702 CheckGraph(graph); | 699 CheckGraph(graph); |
703 | 700 |
704 const string kTempFileTemplate("/tmp/CrAU_temp_data.XXXXXX"); | 701 const string kTempFileTemplate("/tmp/CrAU_temp_data.XXXXXX"); |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
774 TEST_AND_RETURN_FALSE(utils::MakeTempFile( | 771 TEST_AND_RETURN_FALSE(utils::MakeTempFile( |
775 "/tmp/CrAU_temp_data.ordered.XXXXXX", | 772 "/tmp/CrAU_temp_data.ordered.XXXXXX", |
776 &ordered_blobs_path, | 773 &ordered_blobs_path, |
777 false)); | 774 false)); |
778 TEST_AND_RETURN_FALSE(ReorderDataBlobs(&manifest, | 775 TEST_AND_RETURN_FALSE(ReorderDataBlobs(&manifest, |
779 temp_file_path, | 776 temp_file_path, |
780 ordered_blobs_path)); | 777 ordered_blobs_path)); |
781 | 778 |
782 // Check that install op blobs are in order and that all blocks are written. | 779 // Check that install op blobs are in order and that all blocks are written. |
783 { | 780 { |
784 vector<uint32> written_count(blocks.size(), 0); | 781 vector<uint32_t> written_count(blocks.size(), 0); |
785 uint64 next_blob_offset = 0; | 782 uint64_t next_blob_offset = 0; |
786 for (int i = 0; i < manifest.install_operations_size(); i++) { | 783 for (int i = 0; i < manifest.install_operations_size(); i++) { |
787 const DeltaArchiveManifest_InstallOperation& op = | 784 const DeltaArchiveManifest_InstallOperation& op = |
788 manifest.install_operations(i); | 785 manifest.install_operations(i); |
789 for (int j = 0; j < op.dst_extents_size(); j++) { | 786 for (int j = 0; j < op.dst_extents_size(); j++) { |
790 const Extent& extent = op.dst_extents(j); | 787 const Extent& extent = op.dst_extents(j); |
791 for (uint64 block = extent.start_block(); | 788 for (uint64_t block = extent.start_block(); |
792 block < (extent.start_block() + extent.num_blocks()); block++) { | 789 block < (extent.start_block() + extent.num_blocks()); block++) { |
793 written_count[block]++; | 790 written_count[block]++; |
794 } | 791 } |
795 } | 792 } |
796 if (op.has_data_offset()) { | 793 if (op.has_data_offset()) { |
797 if (op.data_offset() != next_blob_offset) { | 794 if (op.data_offset() != next_blob_offset) { |
798 LOG(FATAL) << "bad blob offset! " << op.data_offset() << " != " | 795 LOG(FATAL) << "bad blob offset! " << op.data_offset() << " != " |
799 << next_blob_offset; | 796 << next_blob_offset; |
800 } | 797 } |
801 next_blob_offset += op.data_length(); | 798 next_blob_offset += op.data_length(); |
802 } | 799 } |
803 } | 800 } |
804 // check all blocks written to | 801 // check all blocks written to |
805 for (vector<uint32>::size_type i = 0; i < written_count.size(); i++) { | 802 for (vector<uint32_t>::size_type i = 0; i < written_count.size(); i++) { |
806 if (written_count[i] == 0) { | 803 if (written_count[i] == 0) { |
807 LOG(FATAL) << "block " << i << " not written!"; | 804 LOG(FATAL) << "block " << i << " not written!"; |
808 } | 805 } |
809 } | 806 } |
810 } | 807 } |
811 | 808 |
812 // Serialize protobuf | 809 // Serialize protobuf |
813 string serialized_manifest; | 810 string serialized_manifest; |
814 | 811 |
815 CheckGraph(graph); | 812 CheckGraph(graph); |
(...skipping 20 matching lines...) Expand all Loading... |
836 | 833 |
837 // Write protobuf | 834 // Write protobuf |
838 LOG(INFO) << "Writing final delta file protobuf... " | 835 LOG(INFO) << "Writing final delta file protobuf... " |
839 << serialized_manifest.size(); | 836 << serialized_manifest.size(); |
840 TEST_AND_RETURN_FALSE(writer.Write(serialized_manifest.data(), | 837 TEST_AND_RETURN_FALSE(writer.Write(serialized_manifest.data(), |
841 serialized_manifest.size()) == | 838 serialized_manifest.size()) == |
842 static_cast<ssize_t>(serialized_manifest.size())); | 839 static_cast<ssize_t>(serialized_manifest.size())); |
843 | 840 |
844 // Append the data blobs | 841 // Append the data blobs |
845 LOG(INFO) << "Writing final delta file data blobs..."; | 842 LOG(INFO) << "Writing final delta file data blobs..."; |
846 int blobs_fd = open(temp_file_path.c_str(), O_RDONLY, 0); | 843 int blobs_fd = open(ordered_blobs_path.c_str(), O_RDONLY, 0); |
847 ScopedFdCloser blobs_fd_closer(&blobs_fd); | 844 ScopedFdCloser blobs_fd_closer(&blobs_fd); |
848 TEST_AND_RETURN_FALSE(blobs_fd >= 0); | 845 TEST_AND_RETURN_FALSE(blobs_fd >= 0); |
849 for (;;) { | 846 for (;;) { |
850 char buf[kBlockSize]; | 847 char buf[kBlockSize]; |
851 ssize_t rc = read(blobs_fd, buf, sizeof(buf)); | 848 ssize_t rc = read(blobs_fd, buf, sizeof(buf)); |
852 if (0 == rc) { | 849 if (0 == rc) { |
853 // EOF | 850 // EOF |
854 break; | 851 break; |
855 } | 852 } |
856 TEST_AND_RETURN_FALSE_ERRNO(rc > 0); | 853 TEST_AND_RETURN_FALSE_ERRNO(rc > 0); |
857 TEST_AND_RETURN_FALSE(writer.Write(buf, rc) == rc); | 854 TEST_AND_RETURN_FALSE(writer.Write(buf, rc) == rc); |
858 } | 855 } |
859 | 856 |
860 LOG(INFO) << "All done. Successfully created delta file."; | 857 LOG(INFO) << "All done. Successfully created delta file."; |
861 return true; | 858 return true; |
862 } | 859 } |
863 | 860 |
864 }; // namespace chromeos_update_engine | 861 }; // namespace chromeos_update_engine |
OLD | NEW |