OLD | NEW |
1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "update_engine/delta_diff_generator.h" | 5 #include "update_engine/delta_diff_generator.h" |
6 #include <sys/stat.h> | 6 #include <sys/stat.h> |
7 #include <sys/types.h> | 7 #include <sys/types.h> |
8 #include <errno.h> | 8 #include <errno.h> |
9 #include <fcntl.h> | 9 #include <fcntl.h> |
10 #include <algorithm> | 10 #include <algorithm> |
(...skipping 372 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
383 TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be)) == | 383 TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be)) == |
384 sizeof(value_be)); | 384 sizeof(value_be)); |
385 return true; | 385 return true; |
386 } | 386 } |
387 | 387 |
388 // Adds each operation from the graph to the manifest in the order | 388 // Adds each operation from the graph to the manifest in the order |
389 // specified by 'order'. | 389 // specified by 'order'. |
390 void InstallOperationsToManifest( | 390 void InstallOperationsToManifest( |
391 const Graph& graph, | 391 const Graph& graph, |
392 const vector<Vertex::Index>& order, | 392 const vector<Vertex::Index>& order, |
| 393 const vector<DeltaArchiveManifest_InstallOperation>& kernel_ops, |
393 DeltaArchiveManifest* out_manifest) { | 394 DeltaArchiveManifest* out_manifest) { |
394 for (vector<Vertex::Index>::const_iterator it = order.begin(); | 395 for (vector<Vertex::Index>::const_iterator it = order.begin(); |
395 it != order.end(); ++it) { | 396 it != order.end(); ++it) { |
396 DeltaArchiveManifest_InstallOperation* op = | 397 DeltaArchiveManifest_InstallOperation* op = |
397 out_manifest->add_install_operations(); | 398 out_manifest->add_install_operations(); |
398 *op = graph[*it].op; | 399 *op = graph[*it].op; |
399 } | 400 } |
| 401 for (vector<DeltaArchiveManifest_InstallOperation>::const_iterator it = |
| 402 kernel_ops.begin(); it != kernel_ops.end(); ++it) { |
| 403 DeltaArchiveManifest_InstallOperation* op = |
| 404 out_manifest->add_kernel_install_operations(); |
| 405 *op = *it; |
| 406 } |
400 } | 407 } |
401 | 408 |
402 void CheckGraph(const Graph& graph) { | 409 void CheckGraph(const Graph& graph) { |
403 for (Graph::const_iterator it = graph.begin(); it != graph.end(); ++it) { | 410 for (Graph::const_iterator it = graph.begin(); it != graph.end(); ++it) { |
404 CHECK(it->op.has_type()); | 411 CHECK(it->op.has_type()); |
405 } | 412 } |
406 } | 413 } |
407 | 414 |
| 415 // Delta compresses a kernel partition new_kernel_part with knowledge of |
| 416 // the old kernel partition old_kernel_part. |
| 417 bool DeltaCompressKernelPartition( |
| 418 const string& old_kernel_part, |
| 419 const string& new_kernel_part, |
| 420 vector<DeltaArchiveManifest_InstallOperation>* ops, |
| 421 int blobs_fd, |
| 422 off_t* blobs_length) { |
| 423 // For now, just bsdiff the kernel partition as a whole. |
| 424 // TODO(adlr): Use knowledge of how the kernel partition is laid out |
| 425 // to more efficiently compress it. |
| 426 |
| 427 LOG(INFO) << "Delta compressing kernel partition..."; |
| 428 |
| 429 // Add a new install operation |
| 430 ops->resize(1); |
| 431 DeltaArchiveManifest_InstallOperation* op = &(*ops)[0]; |
| 432 op->set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF); |
| 433 op->set_data_offset(*blobs_length); |
| 434 |
| 435 // Do the actual compression |
| 436 vector<char> data; |
| 437 TEST_AND_RETURN_FALSE(BsdiffFiles(old_kernel_part, new_kernel_part, &data)); |
| 438 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, &data[0], data.size())); |
| 439 *blobs_length += data.size(); |
| 440 |
| 441 off_t old_part_size = utils::FileSize(old_kernel_part); |
| 442 TEST_AND_RETURN_FALSE(old_part_size >= 0); |
| 443 off_t new_part_size = utils::FileSize(new_kernel_part); |
| 444 TEST_AND_RETURN_FALSE(new_part_size >= 0); |
| 445 |
| 446 op->set_data_length(data.size()); |
| 447 |
| 448 op->set_src_length(old_part_size); |
| 449 op->set_dst_length(new_part_size); |
| 450 |
| 451 // Theres a single src/dest extent for each |
| 452 Extent* src_extent = op->add_src_extents(); |
| 453 src_extent->set_start_block(0); |
| 454 src_extent->set_num_blocks((old_part_size + kBlockSize - 1) / kBlockSize); |
| 455 |
| 456 Extent* dst_extent = op->add_dst_extents(); |
| 457 dst_extent->set_start_block(0); |
| 458 dst_extent->set_num_blocks((new_part_size + kBlockSize - 1) / kBlockSize); |
| 459 |
| 460 LOG(INFO) << "Done delta compressing kernel partition."; |
| 461 return true; |
| 462 } |
| 463 |
408 } // namespace {} | 464 } // namespace {} |
409 | 465 |
410 bool DeltaDiffGenerator::ReadFileToDiff( | 466 bool DeltaDiffGenerator::ReadFileToDiff( |
411 const string& old_filename, | 467 const string& old_filename, |
412 const string& new_filename, | 468 const string& new_filename, |
413 vector<char>* out_data, | 469 vector<char>* out_data, |
414 DeltaArchiveManifest_InstallOperation* out_op) { | 470 DeltaArchiveManifest_InstallOperation* out_op) { |
415 // Read new data in | 471 // Read new data in |
416 vector<char> new_data; | 472 vector<char> new_data; |
417 TEST_AND_RETURN_FALSE(utils::ReadFile(new_filename, &new_data)); | 473 TEST_AND_RETURN_FALSE(utils::ReadFile(new_filename, &new_data)); |
(...skipping 228 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
646 ScopedFdCloser in_fd_closer(&in_fd); | 702 ScopedFdCloser in_fd_closer(&in_fd); |
647 | 703 |
648 DirectFileWriter writer; | 704 DirectFileWriter writer; |
649 TEST_AND_RETURN_FALSE( | 705 TEST_AND_RETURN_FALSE( |
650 writer.Open(new_data_blobs_path.c_str(), | 706 writer.Open(new_data_blobs_path.c_str(), |
651 O_WRONLY | O_TRUNC | O_CREAT, | 707 O_WRONLY | O_TRUNC | O_CREAT, |
652 0644) == 0); | 708 0644) == 0); |
653 ScopedFileWriterCloser writer_closer(&writer); | 709 ScopedFileWriterCloser writer_closer(&writer); |
654 uint64_t out_file_size = 0; | 710 uint64_t out_file_size = 0; |
655 | 711 |
656 for (int i = 0; i < manifest->install_operations_size(); i++) { | 712 for (int i = 0; i < (manifest->install_operations_size() + |
657 DeltaArchiveManifest_InstallOperation* op = | 713 manifest->kernel_install_operations_size()); i++) { |
658 manifest->mutable_install_operations(i); | 714 DeltaArchiveManifest_InstallOperation* op = NULL; |
| 715 if (i < manifest->install_operations_size()) { |
| 716 op = manifest->mutable_install_operations(i); |
| 717 } else { |
| 718 op = manifest->mutable_kernel_install_operations( |
| 719 i - manifest->install_operations_size()); |
| 720 } |
659 if (!op->has_data_offset()) | 721 if (!op->has_data_offset()) |
660 continue; | 722 continue; |
661 CHECK(op->has_data_length()); | 723 CHECK(op->has_data_length()); |
662 vector<char> buf(op->data_length()); | 724 vector<char> buf(op->data_length()); |
663 ssize_t rc = pread(in_fd, &buf[0], buf.size(), op->data_offset()); | 725 ssize_t rc = pread(in_fd, &buf[0], buf.size(), op->data_offset()); |
664 TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size())); | 726 TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size())); |
665 | 727 |
666 op->set_data_offset(out_file_size); | 728 op->set_data_offset(out_file_size); |
667 TEST_AND_RETURN_FALSE(writer.Write(&buf[0], buf.size()) == | 729 TEST_AND_RETURN_FALSE(writer.Write(&buf[0], buf.size()) == |
668 static_cast<ssize_t>(buf.size())); | 730 static_cast<ssize_t>(buf.size())); |
669 out_file_size += buf.size(); | 731 out_file_size += buf.size(); |
670 } | 732 } |
671 return true; | 733 return true; |
672 } | 734 } |
673 | 735 |
674 bool DeltaDiffGenerator::GenerateDeltaUpdateFile(const string& old_root, | 736 bool DeltaDiffGenerator::GenerateDeltaUpdateFile( |
675 const string& old_image, | 737 const string& old_root, |
676 const string& new_root, | 738 const string& old_image, |
677 const string& new_image, | 739 const string& new_root, |
678 const string& output_path) { | 740 const string& new_image, |
| 741 const std::string& old_kernel_part, |
| 742 const std::string& new_kernel_part, |
| 743 const string& output_path) { |
679 struct stat old_image_stbuf; | 744 struct stat old_image_stbuf; |
680 TEST_AND_RETURN_FALSE_ERRNO(stat(old_image.c_str(), &old_image_stbuf) == 0); | 745 TEST_AND_RETURN_FALSE_ERRNO(stat(old_image.c_str(), &old_image_stbuf) == 0); |
681 struct stat new_image_stbuf; | 746 struct stat new_image_stbuf; |
682 TEST_AND_RETURN_FALSE_ERRNO(stat(new_image.c_str(), &new_image_stbuf) == 0); | 747 TEST_AND_RETURN_FALSE_ERRNO(stat(new_image.c_str(), &new_image_stbuf) == 0); |
683 LOG_IF(WARNING, new_image_stbuf.st_size != old_image_stbuf.st_size) | 748 LOG_IF(WARNING, new_image_stbuf.st_size != old_image_stbuf.st_size) |
684 << "Old and new images are different sizes."; | 749 << "Old and new images are different sizes."; |
685 LOG_IF(FATAL, new_image_stbuf.st_size % kBlockSize) | 750 LOG_IF(FATAL, new_image_stbuf.st_size % kBlockSize) |
686 << "New image not a multiple of block size " << kBlockSize; | 751 << "New image not a multiple of block size " << kBlockSize; |
687 LOG_IF(FATAL, old_image_stbuf.st_size % kBlockSize) | 752 LOG_IF(FATAL, old_image_stbuf.st_size % kBlockSize) |
688 << "Old image not a multiple of block size " << kBlockSize; | 753 << "Old image not a multiple of block size " << kBlockSize; |
689 | 754 |
| 755 // Sanity check kernel partition args |
| 756 TEST_AND_RETURN_FALSE(utils::FileSize(old_kernel_part) >= 0); |
| 757 TEST_AND_RETURN_FALSE(utils::FileSize(new_kernel_part) >= 0); |
| 758 |
690 vector<Block> blocks(min(old_image_stbuf.st_size / kBlockSize, | 759 vector<Block> blocks(min(old_image_stbuf.st_size / kBlockSize, |
691 new_image_stbuf.st_size / kBlockSize)); | 760 new_image_stbuf.st_size / kBlockSize)); |
692 LOG(INFO) << "invalid: " << Vertex::kInvalidIndex; | 761 LOG(INFO) << "invalid: " << Vertex::kInvalidIndex; |
693 LOG(INFO) << "len: " << blocks.size(); | 762 LOG(INFO) << "len: " << blocks.size(); |
694 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { | 763 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { |
695 CHECK(blocks[i].reader == Vertex::kInvalidIndex); | 764 CHECK(blocks[i].reader == Vertex::kInvalidIndex); |
696 CHECK(blocks[i].writer == Vertex::kInvalidIndex); | 765 CHECK(blocks[i].writer == Vertex::kInvalidIndex); |
697 } | 766 } |
698 Graph graph; | 767 Graph graph; |
699 CheckGraph(graph); | 768 CheckGraph(graph); |
700 | 769 |
701 const string kTempFileTemplate("/tmp/CrAU_temp_data.XXXXXX"); | 770 const string kTempFileTemplate("/tmp/CrAU_temp_data.XXXXXX"); |
702 string temp_file_path; | 771 string temp_file_path; |
703 off_t data_file_size = 0; | 772 off_t data_file_size = 0; |
704 | 773 |
705 LOG(INFO) << "Reading files..."; | 774 LOG(INFO) << "Reading files..."; |
706 | 775 |
| 776 vector<DeltaArchiveManifest_InstallOperation> kernel_ops; |
| 777 |
707 DeltaArchiveManifest_InstallOperation final_op; | 778 DeltaArchiveManifest_InstallOperation final_op; |
708 { | 779 { |
709 int fd; | 780 int fd; |
710 TEST_AND_RETURN_FALSE( | 781 TEST_AND_RETURN_FALSE( |
711 utils::MakeTempFile(kTempFileTemplate, &temp_file_path, &fd)); | 782 utils::MakeTempFile(kTempFileTemplate, &temp_file_path, &fd)); |
712 TEST_AND_RETURN_FALSE(fd >= 0); | 783 TEST_AND_RETURN_FALSE(fd >= 0); |
713 ScopedFdCloser fd_closer(&fd); | 784 ScopedFdCloser fd_closer(&fd); |
714 | 785 |
715 TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph, | 786 TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph, |
716 &blocks, | 787 &blocks, |
717 old_root, | 788 old_root, |
718 new_root, | 789 new_root, |
719 fd, | 790 fd, |
720 &data_file_size)); | 791 &data_file_size)); |
721 CheckGraph(graph); | 792 CheckGraph(graph); |
722 | 793 |
723 // TODO(adlr): read all the rest of the blocks in | |
724 TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks, | 794 TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks, |
725 fd, | 795 fd, |
726 &data_file_size, | 796 &data_file_size, |
727 new_image, | 797 new_image, |
728 &final_op)); | 798 &final_op)); |
| 799 |
| 800 // Read kernel partition |
| 801 TEST_AND_RETURN_FALSE(DeltaCompressKernelPartition(old_kernel_part, |
| 802 new_kernel_part, |
| 803 &kernel_ops, |
| 804 fd, |
| 805 &data_file_size)); |
729 } | 806 } |
730 CheckGraph(graph); | 807 CheckGraph(graph); |
731 | 808 |
732 LOG(INFO) << "Creating edges..."; | 809 LOG(INFO) << "Creating edges..."; |
733 CreateEdges(&graph, blocks); | 810 CreateEdges(&graph, blocks); |
734 CheckGraph(graph); | 811 CheckGraph(graph); |
735 | 812 |
736 CycleBreaker cycle_breaker; | 813 CycleBreaker cycle_breaker; |
737 LOG(INFO) << "Finding cycles..."; | 814 LOG(INFO) << "Finding cycles..."; |
738 set<Edge> cut_edges; | 815 set<Edge> cut_edges; |
739 cycle_breaker.BreakCycles(graph, &cut_edges); | 816 cycle_breaker.BreakCycles(graph, &cut_edges); |
740 CheckGraph(graph); | 817 CheckGraph(graph); |
741 | 818 |
742 // Calculate number of scratch blocks needed | 819 // Calculate number of scratch blocks needed |
743 | 820 |
744 LOG(INFO) << "Cutting cycles..."; | 821 LOG(INFO) << "Cutting cycles..."; |
745 TEST_AND_RETURN_FALSE(CutEdges(&graph, blocks, cut_edges)); | 822 TEST_AND_RETURN_FALSE(CutEdges(&graph, blocks, cut_edges)); |
746 CheckGraph(graph); | 823 CheckGraph(graph); |
747 | 824 |
748 vector<Vertex::Index> final_order; | 825 vector<Vertex::Index> final_order; |
749 LOG(INFO) << "Ordering..."; | 826 LOG(INFO) << "Ordering..."; |
750 TopologicalSort(graph, &final_order); | 827 TopologicalSort(graph, &final_order); |
751 CheckGraph(graph); | 828 CheckGraph(graph); |
752 | 829 |
753 // Convert to protobuf Manifest object | 830 // Convert to protobuf Manifest object |
754 DeltaArchiveManifest manifest; | 831 DeltaArchiveManifest manifest; |
755 CheckGraph(graph); | 832 CheckGraph(graph); |
756 InstallOperationsToManifest(graph, final_order, &manifest); | 833 InstallOperationsToManifest(graph, final_order, kernel_ops, &manifest); |
757 { | 834 { |
758 // Write final operation | 835 // Write final operation |
759 DeltaArchiveManifest_InstallOperation* op = | 836 DeltaArchiveManifest_InstallOperation* op = |
760 manifest.add_install_operations(); | 837 manifest.add_install_operations(); |
761 *op = final_op; | 838 *op = final_op; |
762 CHECK(op->has_type()); | 839 CHECK(op->has_type()); |
763 LOG(INFO) << "final op length: " << op->data_length(); | 840 LOG(INFO) << "final op length: " << op->data_length(); |
764 } | 841 } |
| 842 |
765 CheckGraph(graph); | 843 CheckGraph(graph); |
766 manifest.set_block_size(kBlockSize); | 844 manifest.set_block_size(kBlockSize); |
767 // TODO(adlr): set checksums | |
768 | 845 |
769 // Reorder the data blobs with the newly ordered manifest | 846 // Reorder the data blobs with the newly ordered manifest |
770 string ordered_blobs_path; | 847 string ordered_blobs_path; |
771 TEST_AND_RETURN_FALSE(utils::MakeTempFile( | 848 TEST_AND_RETURN_FALSE(utils::MakeTempFile( |
772 "/tmp/CrAU_temp_data.ordered.XXXXXX", | 849 "/tmp/CrAU_temp_data.ordered.XXXXXX", |
773 &ordered_blobs_path, | 850 &ordered_blobs_path, |
774 false)); | 851 false)); |
775 TEST_AND_RETURN_FALSE(ReorderDataBlobs(&manifest, | 852 TEST_AND_RETURN_FALSE(ReorderDataBlobs(&manifest, |
776 temp_file_path, | 853 temp_file_path, |
777 ordered_blobs_path)); | 854 ordered_blobs_path)); |
778 | 855 |
779 // Check that install op blobs are in order and that all blocks are written. | 856 // Check that install op blobs are in order and that all blocks are written. |
780 { | 857 { |
781 vector<uint32_t> written_count(blocks.size(), 0); | 858 vector<uint32_t> written_count(blocks.size(), 0); |
782 uint64_t next_blob_offset = 0; | 859 uint64_t next_blob_offset = 0; |
783 for (int i = 0; i < manifest.install_operations_size(); i++) { | 860 for (int i = 0; i < (manifest.install_operations_size() + |
| 861 manifest.kernel_install_operations_size()); i++) { |
784 const DeltaArchiveManifest_InstallOperation& op = | 862 const DeltaArchiveManifest_InstallOperation& op = |
785 manifest.install_operations(i); | 863 i < manifest.install_operations_size() ? |
| 864 manifest.install_operations(i) : |
| 865 manifest.kernel_install_operations( |
| 866 i - manifest.install_operations_size()); |
786 for (int j = 0; j < op.dst_extents_size(); j++) { | 867 for (int j = 0; j < op.dst_extents_size(); j++) { |
787 const Extent& extent = op.dst_extents(j); | 868 const Extent& extent = op.dst_extents(j); |
788 for (uint64_t block = extent.start_block(); | 869 for (uint64_t block = extent.start_block(); |
789 block < (extent.start_block() + extent.num_blocks()); block++) { | 870 block < (extent.start_block() + extent.num_blocks()); block++) { |
790 written_count[block]++; | 871 written_count[block]++; |
791 } | 872 } |
792 } | 873 } |
793 if (op.has_data_offset()) { | 874 if (op.has_data_offset()) { |
794 if (op.data_offset() != next_blob_offset) { | 875 if (op.data_offset() != next_blob_offset) { |
795 LOG(FATAL) << "bad blob offset! " << op.data_offset() << " != " | 876 LOG(FATAL) << "bad blob offset! " << op.data_offset() << " != " |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
856 | 937 |
857 LOG(INFO) << "All done. Successfully created delta file."; | 938 LOG(INFO) << "All done. Successfully created delta file."; |
858 return true; | 939 return true; |
859 } | 940 } |
860 | 941 |
861 const char* const kBsdiffPath = "/usr/bin/bsdiff"; | 942 const char* const kBsdiffPath = "/usr/bin/bsdiff"; |
862 const char* const kBspatchPath = "/usr/bin/bspatch"; | 943 const char* const kBspatchPath = "/usr/bin/bspatch"; |
863 const char* const kDeltaMagic = "CrAU"; | 944 const char* const kDeltaMagic = "CrAU"; |
864 | 945 |
865 }; // namespace chromeos_update_engine | 946 }; // namespace chromeos_update_engine |
OLD | NEW |