Updated *.pak file format to support both UTF8 and UTF16

ui/base/resource/data_pack.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "ui/base/resource/data_pack.h"
 
 #include <errno.h>
 
 #include "base/file_util.h"
 #include "base/logging.h"
 #include "base/memory/ref_counted_memory.h"
 #include "base/metrics/histogram.h"
 #include "base/string_piece.h"
 
 // For details of the file layout, see
 // http://dev.chromium.org/developers/design-documents/linuxresourcesandlocalizedstrings
 
 namespace {
 
-static const uint32 kFileFormatVersion = 3;
-// Length of file header: version and entry count.
-static const size_t kHeaderLength = 2 * sizeof(uint32);
+static const uint32 kFileFormatVersion = 4;
+// Length of file header: version, entry count and text encoding type.
+static const size_t kHeaderLength = 2 * sizeof(uint32) + sizeof(uint8);
 
 #pragma pack(push,2)
 struct DataPackEntry {
   uint16 resource_id;
   uint32 file_offset;
 
   static int CompareById(const void* void_key, const void* void_entry) {
     uint16 key = *reinterpret_cast<const uint16*>(void_key);
     const DataPackEntry* entry =
         reinterpret_cast<const DataPackEntry*>(void_entry);
@@ skipping 20 matching lines @@
   ENTRY_NOT_FOUND,
 
   LOAD_ERRORS_COUNT,
 };
 
 }  // namespace
 
 namespace ui {
 
 // In .cc for MemoryMappedFile dtor.
-DataPack::DataPack() : resource_count_(0) {
+DataPack::DataPack() : resource_count_(0), text_encoding_type_(BINARY) {
 }
 DataPack::~DataPack() {
 }
 
 bool DataPack::Load(const FilePath& path) {
   mmap_.reset(new file_util::MemoryMappedFile);
   if (!mmap_->Initialize(path)) {
     DLOG(ERROR) << "Failed to mmap datapack";
     UMA_HISTOGRAM_ENUMERATION("DataPack.Load", INIT_FAILED,
                               LOAD_ERRORS_COUNT);
     mmap_.reset();
     return false;
   }
 
   // Parse the header of the file.

[tony, 2011/08/30 21:19:58]

We should probably make sure that kHeaderLength <= mmap_->length() here (a bug
in the existing code).

[marcvs, 2011/08/30 22:17:22]

Done.

-  // First uint32: version; second: resource count.
+  // First uint32: version; second: resource count;
   const uint32* ptr = reinterpret_cast<const uint32*>(mmap_->data());
   uint32 version = ptr[0];
   if (version != kFileFormatVersion) {
     LOG(ERROR) << "Bad data pack version: got " << version << ", expected "
                << kFileFormatVersion;
     UMA_HISTOGRAM_ENUMERATION("DataPack.Load", BAD_VERSION,
                               LOAD_ERRORS_COUNT);
     mmap_.reset();
     return false;
   }
   resource_count_ = ptr[1];
 
+  // third: text encoding.
+  const uint8* ptr_encoding = reinterpret_cast<const uint8*>(ptr + 2);
+  text_encoding_type_ = (TextEncodingType)(*ptr_encoding);

[tony, 2011/08/30 21:19:58]

Nit: static_cast<TextEncodingType>

[marcvs, 2011/08/30 22:17:22]

Done.

+  if (text_encoding_type_ != UTF8 && text_encoding_type_ != UTF16 &&
+      text_encoding_type_ != BINARY) {
+    LOG(ERROR) << "Bad data pack text encoding: got " << text_encoding_type_
+               << ", expected between " << BINARY << " and " << UTF16;
+    mmap_.reset();
+    return false;
+  }
+
   // Sanity check the file.
   // 1) Check we have enough entries.
   if (kHeaderLength + resource_count_ * sizeof(DataPackEntry) >
       mmap_->length()) {
     LOG(ERROR) << "Data pack file corruption: too short for number of "
                   "entries specified.";
     UMA_HISTOGRAM_ENUMERATION("DataPack.Load", INDEX_TRUNCATED,
                               LOAD_ERRORS_COUNT);
     mmap_.reset();
     return false;
@@ skipping 47 matching lines @@
   base::StringPiece piece;
   if (!GetStringPiece(resource_id, &piece))
     return NULL;
 
   return new RefCountedStaticMemory(
       reinterpret_cast<const unsigned char*>(piece.data()), piece.length());
 }
 
 // static
 bool DataPack::WritePack(const FilePath& path,
-                         const std::map<uint16, base::StringPiece>& resources) {
+                         const std::map<uint16, base::StringPiece>& resources,
+                         TextEncodingType textEncodingType) {
   FILE* file = file_util::OpenFile(path, "wb");
   if (!file)
     return false;
 
   if (fwrite(&kFileFormatVersion, sizeof(kFileFormatVersion), 1, file) != 1) {
     LOG(ERROR) << "Failed to write file version";
     file_util::CloseFile(file);
     return false;
   }
 
   // Note: the python version of this function explicitly sorted keys, but
   // std::map is a sorted associative container, we shouldn't have to do that.
   uint32 entry_count = resources.size();
   if (fwrite(&entry_count, sizeof(entry_count), 1, file) != 1) {
     LOG(ERROR) << "Failed to write entry count";
     file_util::CloseFile(file);
     return false;
   }
 
+  if (textEncodingType != UTF8 && textEncodingType != UTF16 &&
+      textEncodingType != BINARY) {
+    LOG(ERROR) << "Invalid text encoding type, got " << textEncodingType
+               << ", expected between " << BINARY << " and " << UTF16;
+    file_util::CloseFile(file);
+    return false;
+  }
+
+  uint8 write_buffer = textEncodingType;
+  if (fwrite(&write_buffer, sizeof(uint8), 1, file) != 1) {
+    LOG(ERROR) << "Failed to write file text resources encoding";
+    file_util::CloseFile(file);
+    return false;
+  }
+
   // Each entry is a uint16 + a uint32. We have an extra entry after the last
   // item so we can compute the size of the list item.
   uint32 index_length = (entry_count + 1) * sizeof(DataPackEntry);
   uint32 data_offset = kHeaderLength + index_length;
   for (std::map<uint16, base::StringPiece>::const_iterator it =
            resources.begin();
        it != resources.end(); ++it) {
     uint16 resource_id = it->first;
     if (fwrite(&resource_id, sizeof(resource_id), 1, file) != 1) {
       LOG(ERROR) << "Failed to write id for " << resource_id;
@@ skipping 34 matching lines @@
       return false;
     }
   }
 
   file_util::CloseFile(file);
 
   return true;
 }
 
 }  // namespace ui