Replace Libarchive with MiniZip.

ui/file_manager/zip_archiver/cpp/volume_archive_libarchive.h

 // Copyright 2014 The Chromium OS Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef VOLUME_ARCHIVE_LIBARCHIVE_H_
 #define VOLUME_ARCHIVE_LIBARCHIVE_H_
 
 #include <string>
 
-#include "archive.h"
+#include "third_party/zlib/contrib/minizip/unzip.h"
+#include "third_party/zlib/contrib/minizip/zip.h"
 
 #include "volume_archive.h"
 
 // A namespace with constants used by VolumeArchiveLibarchive.
 namespace volume_archive_constants {
 
 const char kArchiveReadNewError[] = "Could not allocate archive.";
 const char kFileNotFound[] = "File not found for read data request.";
 const char kVolumeReaderError[] = "VolumeReader failed to retrieve data.";
-const char kArchiveSupportErrorPrefix[] = "Error at support rar/zip format: ";
-const char kArchiveOpenErrorPrefix[] = "Error at open archive: ";
-const char kArchiveNextHeaderErrorPrefix[] =
-    "Error at reading next header for metadata: ";
-const char kArchiveReadDataErrorPrefix[] = "Error at reading data: ";
-const char kArchiveReadFreeErrorPrefix[] = "Error at archive free: ";
+const char kArchiveOpenError[] = "Failed to open archive.";
+const char kArchiveNextHeaderError[] =
+    "Failed to open current file in archive.";
+const char kArchiveReadDataError[] = "Failed to read archive data.";
+const char kArchiveReadFreeError[] = "Failed to close archive.";
 
 // The size of the buffer used to skip unnecessary data.
 // Should be positive and less than size_t maximum.
 const int64_t kDummyBufferSize = 512 * 1024;  // 512 KB
 
 // The size of the buffer used by ReadInProgress to decompress data.
 // Should be positive and less than size_t maximum.
 const int64_t kDecompressBufferSize = 512 * 1024;  // 512 KB.
 
 // The maximum data chunk size for VolumeReader::Read requests.
 // Should be positive.
 const int64_t kMaximumDataChunkSize = 512 * 1024;  // 512 KB.
 
 // The minimum data chunk size for VolumeReader::Read requests.
 // Should be positive.
 const int64_t kMinimumDataChunkSize = 32 * 1024;  // 16 KB.
 
+// Maximum length of filename in zip archive.
+const int kZipMaxPath = 256;
+
+// The size of the static cache. We need at least 64KB to cache whole
+// 'end of central directory' data.
+const int64_t kStaticCacheSize = 128 * 1024;
 }  // namespace volume_archive_constants

[mtomasz, 2017/04/10 07:15:09]

nit: \n missing before } for consistency with \n after namespace {

[takise, 2017/04/11 06:00:52]

Done.

 
+class VolumeArchiveLibarchive;
+
+// A name space with custom functions passed to minizip.

[mtomasz, 2017/04/10 07:15:09]

nit: name space -> namespace

[takise, 2017/04/11 06:00:52]

Done.

+namespace volume_archive_functions {
+  int64_t DynamicCache(VolumeArchiveLibarchive* va, int64_t unz_size);
+  uLong CustomArchiveRead(void* opaque, void* /*stream*/, void* buf, uLong size);
+  // Returns the offset from the beginning of the data.
+  long CustomArchiveTell(void* opaque, void* /*stream*/);

[mtomasz, 2017/04/10 07:15:09]

nit: Can we have an empty line between functions? It's much easier to read.

[takise, 2017/04/11 06:00:52]

Done.

+  // Moves the current offset to the specified position.
+  long CustomArchiveSeek(void* opaque,
+                         void* /*stream*/,
+                         uLong offset,
+                         int origin);
+} // compressor_archive_functions

[mtomasz, 2017/04/10 07:15:09]

nit: \s\s before //

[takise, 2017/04/11 06:00:52]

Done.

+
+
+class VolumeArchiveLibarchive;
+
 // Defines an implementation of VolumeArchive that wraps all libarchive
 // operations.
 class VolumeArchiveLibarchive : public VolumeArchive {
  public:
   explicit VolumeArchiveLibarchive(VolumeReader* reader);
 
   virtual ~VolumeArchiveLibarchive();
 
   // See volume_archive_interface.h.
   virtual bool Init(const std::string& encoding);
 
   // See volume_archive_interface.h.
-  virtual Result GetNextHeader();
-  virtual Result GetNextHeader(const char** path_name,
-                               int64_t* size,
-                               bool* is_directory,
-                               time_t* modification_time);
+  virtual VolumeArchive::Result GetCurrentFileInfo(std::string* path_name,
+                                                   int64_t* size,
+                                                   bool* is_directory,
+                                                   time_t* modification_time);
+
+  virtual VolumeArchive::Result GoToNextFile();
 
   // See volume_archive_interface.h.
-  virtual bool SeekHeader(int64_t index);
+  virtual bool SeekHeader(const std::string& path_name);
 
   // See volume_archive_interface.h.
-  virtual int64_t ReadData(int64_t offset,
-                           int64_t length,
-                           const char** buffer);
+  virtual int64_t ReadData(int64_t offset, int64_t length, const char** buffer);
 
   // See volume_archive_interface.h.
   virtual void MaybeDecompressAhead();
 
   // See volume_archive_interface.h.
   virtual bool Cleanup();
 
   int64_t reader_data_size() const { return reader_data_size_; }
 
+  // Custom functions need to access private variables of
+  // CompressorArchiveLibarchive frequently.
+  friend int64_t volume_archive_functions::DynamicCache(
+      VolumeArchiveLibarchive* va, int64_t unz_size);
+  friend uLong volume_archive_functions::CustomArchiveRead(
+      void* opaque, void* /*stream*/, void* buf, uLong size);

[mtomasz, 2017/04/10 07:15:09]

nit: No need for commenting out arguments in header files. We only do it in cc
files, when body of a function doesn't use them, as some linters warn about
unused variables.

[takise, 2017/04/11 06:00:52]

Done.

+  friend long volume_archive_functions::CustomArchiveTell(
+      void* opaque, void* /*stream*/);
+  friend long volume_archive_functions::CustomArchiveSeek(
+      void* opaque, void* /*stream*/, uLong offset, int origin);
+
  private:
   // Decompress length bytes of data starting from offset.
   void DecompressData(int64_t offset, int64_t length);
 
   // The size of the requested data from VolumeReader.
   int64_t reader_data_size_;
 
-  // The libarchive correspondent archive object.
-  archive* archive_;
+  // The minizip correspondent archive object.
+  zipFile zip_file_;
 
-  // The last reached entry with VolumeArchiveLibarchive::GetNextHeader.
-  archive_entry* current_archive_entry_;
+  // We use two kinds of cache strategies here: dynamic and static.
+  // Dynamic cache is a common cache strategy used in most of IO streams such as
+  // fread. When a file chunk is requested and if the size of the requested
+  // chunk is small, we load larger size of bytes from the archive and cache
+  // them in dynamic_cache_. If the range of the next requested chunk is within
+  // the cache, we don't read the archive and just return the data in the cache.
+  char dynamic_cache_[volume_archive_constants::kMaximumDataChunkSize];
+
+  // The offset from which dynamic_cache_ has the data of the archive.
+  int64_t dynamic_cache_offset_;
+
+  // The size of the data in dynamic_cache_.
+  int64_t dynamic_cache_size_;
+
+  // Although dynamic cache works in most situations, it doesn't work when
+  // MiniZip is looking for the front index of the central directory. Since
+  // MiniZip reads the data little by little backwards from the end to find the
+  // index, dynamic_cache will be reloaded every time. To avoid this, we first
+  // cache a certain length of data from the end into static_cache_. The data
+  // in this buffer is also used when the data in the central directory is

[mtomasz, 2017/04/10 07:15:09]

Is size of this cache enough for large ZIP files, like the mentioned earlier
linux kernel? Can we reload it once reaching the end of the cache? Then it's
also dynamic but going in the other direction, so maybe we could change the
naming.

[takise, 2017/04/11 06:00:52]

Actually, this buffer doesn't have 'central directory', which contains the
matadata
of all the files, but have 'end of central directory header', which follows
'central directory'.
Minizip wants to read the metadata in central directory, but to read the central
directory,
minizip needs to know where central directory starts.
'end of central directory header' holds the first index of 'central directory'.
To find the first index of 'central directory', minizip first tries to look for
the first index of 'end of central directory header'.
The problem is that the size of 'end of central directory header' changes from
archive to archive
because 'end of central directory header' has an field called global comment,
which can be maximum 65536 bytes.
That's why minizip first requests data little by little (256bytes) backwards
from the end to find the first index
of 'end of central directory header'.
It is guaranteed that 'global comment' field has maximum 65536 bytes, so 128 *
1024 bytes are enough
to have whole 'end of central directory header' regardless of the size of the
archive.

Another problem is that minizip assumes zip files are in ZIP64 format and if it
fails,
then minizip considers archives to be in ZIP format. (although most of the zip
files are ZIP format. )
More specifically, Minizip tries to read data little by little backwards to find
the signature of end of central directory in ZIP64 format.
And if it fails, minizip read the same data again little by little to find the
signature in ZIP format.
This is the reason why we use static cache.

Please refer to the function LoadCentralDirectoryRecord() in minizip to see how
it works.
https://cs.chromium.org/chromium/src/third_party/zlib/contrib/minizip/zip.c?s...

We never reload this buffer again. That's why I call this buffer static.
But, I agree there is better name for this. end_of_archive_buffer or something?

I decided to implement the cache like this after I had a few discussions with
satoru-san and yawano-san.
I'm sorry not to sharing the context with you well. Please ask me if the
explanation above doesn't make sense to me.

[mtomasz, 2017/04/11 06:36:45]

Thanks for the explanation. Now I understand very well. Please add some short
comment around the buffer size constant, saying something like "The EOCD (End Of
Central Directory) header is guaranteed to be in the last 64KB of the file, so
cache last 64KB.

Or something like that.

+  // requested by MiniZip later.
+  char static_cache_[volume_archive_constants::kStaticCacheSize];
+
+  // The offset from which static_cache_ has the data of the archive.
+  int64_t static_cache_offset_;
+
+  // The size of the data in static_cache_.
+  int64_t static_cache_size_;
 
   // The data offset, which will be offset + length after last read
   // operation, where offset and length are method parameters for
   // VolumeArchiveLibarchive::ReadData. Data offset is used to improve
   // performance for consecutive calls to VolumeArchiveLibarchive::ReadData.
   //
   // Intead of starting the read from the beginning for every
   // VolumeArchiveLibarchive::ReadData, the next call will start
   // from last_read_data_offset_ in case the offset parameter of
   // VolumeArchiveLibarchive::ReadData has the same value as
@@ skipping 27 matching lines @@
 
   // The size of valid data starting from decompressed_data_ that is stored
   // inside decompressed_data_buffer_.
   int64_t decompressed_data_size_;
 
   // True if VolumeArchiveLibarchive::DecompressData failed.
   bool decompressed_error_;
 };
 
 #endif  // VOLUME_ARCHIVE_LIBARCHIVE_H_