[ZLIB] Add support for windowBits, memLevel, raw

sdk/lib/io/data_transformer.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of dart.io;
 
 
+/// Minimal value for [windowBits]
+const int ZLIB_MIN_WINDOW_BITS = 8;

[Anders Johnsen, 2014/01/27 12:27:45]

If these are moved to ZLibCodec as final members, we can say

ZLIB.MIN_WINDOW_BITS instead.

[vicb, 2014/01/27 15:14:19]

But in the doc they'll be listed in ZLibCodec, not sure it's a good thing ?

+/// Maximal value for [windowBits]
+const int ZLIB_MAX_WINDOW_BITS = 15;
+/// Default value for [windowBits]
+const int ZLIB_DFT_WINDOW_BITS = 15;
+
+/// Minimal value for [level]
+const int  ZLIB_MIN_LEVEL = -1;
+/// Maximal value for [level]
+const int  ZLIB_MAX_LEVEL = 9;
+/// Default value for [level]
+const int  ZLIB_DFT_LEVEL = 6;
+
+/// Minimal value for [memLevel]
+const int  ZLIB_MIN_MEM_LEVEL = 1;
+/// Maximal value for [memLevel]
+const int  ZLIB_MAX_MEM_LEVEL = 9;
+/// Default value for [memLevel]
+const int  ZLIB_DFT_MEM_LEVEL = 8;
+
+/// Compression strategies
+const int ZLIB_STRATEGY_FILTERED = 1;
+const int ZLIB_STRATEGY_HUFFMAN_ONLY = 2;
+const int ZLIB_STRATEGY_RLE = 3;
+const int ZLIB_STRATEGY_FIXED = 4;
+const int ZLIB_STRATEGY_DEFAULT = 0;
+
+
 /**
  * An instance of the default implementation of the [ZLibCodec].
  */
-const ZLibCodec ZLIB = const ZLibCodec();
-
+final ZLibCodec ZLIB = new ZLibCodec();

[Anders Johnsen, 2014/01/27 12:27:45]

make const.

[vicb, 2014/01/27 15:14:19]

Can not because of the required checks

 
 /**
  * The [ZLibCodec] encodes raw bytes to ZLib compressed bytes and decodes ZLib
  * compressed bytes to raw bytes.
  */
 class ZLibCodec extends Codec<List<int>, List<int>> {
   /**
-   * The compression level of the [ZLibCodec].
+   * When true, `GZip` frames will be added to the compressed data.
+   */
+  final bool gzip;
+
+  /**
+   * The compression-[level] can be set in the range of `-1..9`, with `6` being
+   * the default compression level. Levels above `6` will have higher compression
+   * rates at the cost of more CPU and memory usage. Levels below `6` will use
+   * less CPU and memory at the cost of lower compression rates.
    */
   final int level;
 
   /**
+  * Specifies how much memory should be allocated for the internal compression
+  * state. `1` uses minimum memory but is slow and reduces compression ratio;
+  * `9` uses maximum memory for optimal speed. The default value is `8`.
+  *
+  * The memory requirements for deflate are (in bytes):
+  *   (1 << (windowBits + 2)) +  (1 << (memLevel + 9))
+  * that is: 128K for windowBits = 15 + 128K for memLevel = 8 (default values)
+  */
+  final int memLevel;
+
+  /**
+  * Tunes the compression algorithm. Use the value STRATEGY_DEFAULT for normal
+  * data, STRATEGY_FILTERED for data produced by a filter (or predictor),
+  * STRATEGY_HUFFMAN_ONLY to force Huffman encoding only (no string match), or
+  * STRATEGY_RLE to limit match distances to one (run-length encoding).
+  */
+  final int strategy;
+
+  /**
+  * Base two logarithm of the window size (the size of the history buffer). It
+  * should be in the range 8..15. Larger values result in better compression at
+  * the expense of memory usage. The default value is 15
+  */
+  final int windowBits;
+
+  /**
+  * When true, deflate generates raw data with no zlib header or trailer, and
+  * will not compute an adler32 check value
+  */
+  final bool raw;
+
+  /**
+  * Initial compression dictionary.
+  *
+  * It should consist of strings (byte sequences) that are likely to be
+  * encountered later in the data to be compressed, with the most commonly used
+  * strings preferably put towards the end of the dictionary. Using a dictionary
+  * is most useful when the data to be compressed is short and can be predicted
+  * with good accuracy; the data can then be compressed better than with the
+  * default empty dictionary.
+  */
+  final List<int> dictionary;
+
+  ZLibCodec({this.level: ZLIB_DFT_LEVEL,
+            this.windowBits: ZLIB_DFT_WINDOW_BITS,
+            this.memLevel: ZLIB_DFT_MEM_LEVEL,
+            this.strategy: ZLIB_STRATEGY_DEFAULT,
+            this.dictionary: null,
+            this.raw: false}) {
+    validateZLibeLevel(level);

[Anders Johnsen, 2014/01/27 12:27:45]

Move checks to encoder and decoder getters, or better yet, ZLibEncoder/Decoder
constructor.

[vicb, 2014/01/27 15:14:19]

As the decoder & encoder are lazily instantiated the error would not trigger
before actually decoding / encoding which I think is not good.

What's the downside of having ZLIB final (instead of const) ? 

+    validateZLibMemLevel(memLevel);
+    validateZLibStrategy(strategy);
+    validateZLibWindowBits(windowBits);
+  }
+
+  /**
    * Get a [Converter] for encoding to `ZLib` compressed data.
    */
   Converter<List<int>, List<int>> get encoder =>
-      new ZLibEncoder(gzip: false, level: level);
+      new ZLibEncoder(gzip: false, level: level, windowBits: windowBits,
+                      memLevel: memLevel, strategy: strategy,
+                      dictionary: dictionary, raw: raw);
 
   /**
    * Get a [Converter] for decoding `ZLib` compressed data.
    */
-  Converter<List<int>, List<int>> get decoder => const ZLibDecoder();
-
-  /**
-   * The compression-[level] can be set in the range of `1..10`, with `6` being
-   * the default compression level. Levels above 6 will have higher compression
-   * rates at the cost of more CPU and memory usage. Levels below 6 will use
-   * less CPU and memory, but at the cost of lower compression rates.
-   */
-  const ZLibCodec({this.level: 6});
+  Converter<List<int>, List<int>> get decoder => new ZLibDecoder(
+      windowBits: windowBits, dictionary: dictionary, raw: raw);
 }
 
 
 /**
  * An instance of the default implementation of the [GZipCodec].
  */
-const GZipCodec GZIP = const GZipCodec();
+final GZipCodec GZIP = new GZipCodec();
 
 
 /**
- * The [GZipCodec] encodes raw bytes to GZip compressed bytes and decodes GZip
+ * The GZipCodec encodes raw bytes to GZip compressed bytes and decodes GZip
  * compressed bytes to raw bytes.
  *
- * The difference between [ZLibCodec] and [GZipCodec] is that the [GZipCodec]
- * wraps the `ZLib` compressed bytes in `GZip` frames.
+ * The difference between [ZLibCodec] and GZipCodec is that the GZipCodec wraps
+ * the `ZLib` compressed bytes in `GZip` frames.
  */
 class GZipCodec extends Codec<List<int>, List<int>> {
   /**
-   * The compression level of the [ZLibCodec].
+   * When true, `GZip` frames will be added to the compressed data.
+   */
+  final bool gzip;
+
+  /**
+   * The compression-[level] can be set in the range of `-1..9`, with `6` being
+   * the default compression level. Levels above `6` will have higher compression
+   * rates at the cost of more CPU and memory usage. Levels below `6` will use
+   * less CPU and memory at the cost of lower compression rates.
    */
   final int level;
 
   /**
+  * Specifies how much memory should be allocated for the internal compression
+  * state. `1` uses minimum memory but is slow and reduces compression ratio;
+  * `9` uses maximum memory for optimal speed. The default value is `8`.
+  *
+  * The memory requirements for deflate are (in bytes):
+  *   (1 << (windowBits + 2)) +  (1 << (memLevel + 9))
+  * that is: 128K for windowBits = 15 + 128K for memLevel = 8 (default values)
+  */
+  final int memLevel;
+
+  /**
+  * Tunes the compression algorithm. Use the value STRATEGY_DEFAULT for normal
+  * data, STRATEGY_FILTERED for data produced by a filter (or predictor),
+  * STRATEGY_HUFFMAN_ONLY to force Huffman encoding only (no string match), or
+  * STRATEGY_RLE to limit match distances to one (run-length encoding).
+  */
+  final int strategy;
+
+  /**
+  * Base two logarithm of the window size (the size of the history buffer). It
+  * should be in the range 8..15. Larger values result in better compression at
+  * the expense of memory usage. The default value is 15
+  */
+  final int windowBits;
+
+  /**
+  * Initial compression dictionary.
+  *
+  * It should consist of strings (byte sequences) that are likely to be
+  * encountered later in the data to be compressed, with the most commonly used
+  * strings preferably put towards the end of the dictionary. Using a dictionary
+  * is most useful when the data to be compressed is short and can be predicted
+  * with good accuracy; the data can then be compressed better than with the
+  * default empty dictionary.

[Anders Johnsen, 2014/01/27 12:27:45]

Maybe write some more about lifeness - do we take a copy or do we just expect
the list to stay unchanged.

Also, the type is here List<int>. That can be any lists of ints. Look into using
_ensureFastAndSerializableByteData for this list, when passing on to native.

[vicb, 2014/01/27 15:14:19]

I tool models on the current converters. However the base converter is generic,
I'll check _ensureFastAndSerializableByteData and update accordingly. 

+  */
+  final List<int> dictionary;
+
+  /**
+  * When true, deflate generates raw data with no zlib header or trailer, and
+  * will not compute an adler32 check value
+  */
+  final bool raw;
+
+  GZipCodec({this.level: ZLIB_DFT_LEVEL,
+            this.windowBits: ZLIB_DFT_WINDOW_BITS,
+            this.memLevel: ZLIB_DFT_MEM_LEVEL,
+            this.strategy: ZLIB_STRATEGY_DEFAULT,
+            this.dictionary: null,
+            raw: false}) {
+    validateZLibeLevel(level);
+    validateZLibMemLevel(memLevel);
+    validateZLibStrategy(strategy);
+    validateZLibWindowBits(windowBits);
+  }
+
+  /**
    * Get a [Converter] for encoding to `GZip` compressed data.
    */
   Converter<List<int>, List<int>> get encoder =>
-      new ZLibEncoder(gzip: true, level: level);
+      new ZLibEncoder(gzip: true, level: level, windowBits: windowBits,
+                      memLevel: memLevel, strategy: strategy,
+                      dictionary: dictionary, raw: raw);
 
   /**
    * Get a [Converter] for decoding `GZip` compressed data.
    */
-  Converter<List<int>, List<int>> get decoder => const ZLibDecoder();
-
-  /**
-   * The compression-[level] can be set in the range of `1..10`, with `6` being
-   * the default compression level. Levels above 6 will have higher compression
-   * rates at the cost of more CPU and memory usage. Levels below 6 will use
-   * less CPU and memory, but at the cost of lower compression rates.
-   */
-  const GZipCodec({this.level: 6});
+  Converter<List<int>, List<int>> get decoder => new ZLibDecoder(
+      windowBits: windowBits, dictionary: dictionary, raw: raw);
 }
 
-
 /**
- * The [ZLibEncoder] is the encoder used by [ZLibCodec] and [GZipCodec] to
- * compress data.
+ * The ZLibEncoder encoder is used by [ZLibCodec] and [GZipCodec] to compress
+ * data.
  */
 class ZLibEncoder extends Converter<List<int>, List<int>> {
   /**
-   * If [gzip] is true, `GZip` frames will be added to the compressed data.
+   * When true, `GZip` frames will be added to the compressed data.
    */
   final bool gzip;
 
   /**
-   * The compression level used by the encoder.
+   * The compression-[level] can be set in the range of `-1..9`, with `6` being
+   * the default compression level. Levels above `6` will have higher compression
+   * rates at the cost of more CPU and memory usage. Levels below `6` will use
+   * less CPU and memory at the cost of lower compression rates.
    */
   final int level;
 
   /**
-   * Create a new [ZLibEncoder] converter. If the [gzip] flag is set, the
-   * encoder will wrap the encoded ZLib data in GZip frames.
-   */
-  const ZLibEncoder({this.gzip: false, this.level: 6});
-
-
-  /**
-   * Convert a list of bytes using the options given to the [ZLibEncoder]
+  * Specifies how much memory should be allocated for the internal compression
+  * state. `1` uses minimum memory but is slow and reduces compression ratio;
+  * `9` uses maximum memory for optimal speed. The default value is `8`.
+  *
+  * The memory requirements for deflate are (in bytes):
+  *   (1 << (windowBits + 2)) +  (1 << (memLevel + 9))
+  * that is: 128K for windowBits = 15 + 128K for memLevel = 8 (default values)
+  */
+  final int memLevel;
+
+  /**
+  * Tunes the compression algorithm. Use the value STRATEGY_DEFAULT for normal
+  * data, STRATEGY_FILTERED for data produced by a filter (or predictor),
+  * STRATEGY_HUFFMAN_ONLY to force Huffman encoding only (no string match), or
+  * STRATEGY_RLE to limit match distances to one (run-length encoding).
+  */
+  final int strategy;
+
+  /**
+  * Base two logarithm of the window size (the size of the history buffer). It
+  * should be in the range 8..15. Larger values result in better compression at
+  * the expense of memory usage. The default value is 15
+  */
+  final int windowBits;
+
+  /**
+  * Initial compression dictionary.
+  *
+  * It should consist of strings (byte sequences) that are likely to be
+  * encountered later in the data to be compressed, with the most commonly used
+  * strings preferably put towards the end of the dictionary. Using a dictionary
+  * is most useful when the data to be compressed is short and can be predicted
+  * with good accuracy; the data can then be compressed better than with the
+  * default empty dictionary.
+  */
+  final List<int> dictionary;
+
+
+  /**
+<<<<<<< HEAD

[Anders Johnsen, 2014/01/27 12:27:45]

Merge stuff.

[vicb, 2014/01/27 15:14:19]

fixed

+  * When true, deflate generates raw data with no zlib header or trailer, and
+  * will not compute an adler32 check value
+  */
+  final bool raw;
+
+  ZLibEncoder({this.gzip: false,
+              this.level: ZLIB_DFT_LEVEL,
+              this.windowBits: ZLIB_DFT_WINDOW_BITS,
+              this.memLevel: ZLIB_DFT_MEM_LEVEL,
+              this.strategy: ZLIB_STRATEGY_DEFAULT,
+              this.dictionary: null,
+              this.raw: false}) {
+    validateZLibeLevel(level);
+    validateZLibMemLevel(memLevel);
+    validateZLibStrategy(strategy);
+    validateZLibWindowBits(windowBits);
+  }
+
+  /**
+   * Convert a list of bytes using the options given to the ZLibEncoder
    * constructor.
    */
   List<int> convert(List<int> bytes) {
     _BufferSink sink = new _BufferSink();
     startChunkedConversion(sink)
-      ..add(bytes)
-      ..close();
+        ..add(bytes)
+        ..close();
     return sink.builder.takeBytes();
   }
 
   /**
-   * Start a chunked conversion using the options given to the [ZLibEncoder]
+   * Start a chunked conversion using the options given to the ZLibEncoder
    * constructor. While it accepts any [ChunkedConversionSink] taking
    * [List<int>]'s, the optimal sink to be passed as [sink] is a
    * [ByteConversionSink].
    */
   ByteConversionSink startChunkedConversion(
       ChunkedConversionSink<List<int>> sink) {
     if (sink is! ByteConversionSink) {
       sink = new ByteConversionSink.from(sink);
     }
-    return new _ZLibEncoderSink(sink, gzip, level);
+
+    print("Dart _ZLibEncoderSink");

[Anders Johnsen, 2014/01/27 12:27:45]

Debug stuff.

+    return new _ZLibEncoderSink(sink, gzip, level, windowBits, memLevel,
+                                strategy, dictionary, raw);
   }
 }
 
 
 /**
- * The [ZLibDecoder] is the decoder used by [ZLibCodec] and [GZipCodec] to
- * decompress data.
+ * The ZLibDecoder is used by [ZLibCodec] and [GZipCodec] to decompress data.
  */
 class ZLibDecoder extends Converter<List<int>, List<int>> {
+  /**
+  * Base two logarithm of the window size (the size of the history buffer). It
+  * should be in the range 8..15. Larger values result in better compression at
+  * the expense of memory usage. The default value is 15
+  */
+  final int windowBits;
 
   /**
-   * Create a new [ZLibEncoder] converter.
-   */
-  const ZLibDecoder();
+  * Initial compression dictionary.
+  *
+  * It should consist of strings (byte sequences) that are likely to be
+  * encountered later in the data to be compressed, with the most commonly used
+  * strings preferably put towards the end of the dictionary. Using a dictionary
+  * is most useful when the data to be compressed is short and can be predicted
+  * with good accuracy; the data can then be compressed better than with the
+  * default empty dictionary.
+  */
+  final List<int> dictionary;
+
+  /**
+  * When true, deflate generates raw data with no zlib header or trailer, and
+  * will not compute an adler32 check value
+  */
+  final bool raw;
+
+  ZLibDecoder({this.windowBits: ZLIB_DFT_WINDOW_BITS, this.dictionary: null,
+              this.raw: false}) {
+    validateZLibWindowBits(windowBits);
+  }
 
   /**
    * Convert a list of bytes using the options given to the [ZLibDecoder]
    * constructor.
    */
   List<int> convert(List<int> bytes) {
     _BufferSink sink = new _BufferSink();
     startChunkedConversion(sink)
-      ..add(bytes)
-      ..close();
+        ..add(bytes)
+        ..close();
     return sink.builder.takeBytes();
   }
 
   /**
    * Start a chunked conversion. While it accepts any [ChunkedConversionSink]
    * taking [List<int>]'s, the optimal sink to be passed as [sink] is a
    * [ByteConversionSink].
    */
   ByteConversionSink startChunkedConversion(
       ChunkedConversionSink<List<int>> sink) {
     if (sink is! ByteConversionSink) {
       sink = new ByteConversionSink.from(sink);
     }
-    return new _ZLibDecoderSink(sink);
+    print("Dart _ZLibDecoderSink");
+    return new _ZLibDecoderSink(sink, windowBits, dictionary, raw);
   }
 }
 
 
 class _BufferSink extends ByteConversionSink {
   final BytesBuilder builder = new BytesBuilder();
 
   void add(List<int> chunk) {
     builder.add(chunk);
   }
 
   void addSlice(List<int> chunk, int start, int end, bool isLast) {
     if (chunk is Uint8List) {
       Uint8List list = chunk;
       builder.add(new Uint8List.view(list.buffer, start, end - start));
     } else {
       builder.add(chunk.sublist(start, end));
     }
   }
 
   void close() {}
 }
 
 
 class _ZLibEncoderSink extends _FilterSink {
-  _ZLibEncoderSink(ByteConversionSink sink, bool gzip, int level)
-      : super(sink, _Filter.newZLibDeflateFilter(gzip, level));
+  _ZLibEncoderSink(ByteConversionSink sink, bool gzip, int level,
+                   int windowBits, int memLevel, int strategy,
+                   List<int> dictionary, bool raw)
+      : super(sink, _Filter.newZLibDeflateFilter(gzip, level, windowBits,
+                                                 memLevel, strategy,
+                                                 dictionary, raw));
+}
+
+class _ZLibDecoderSink extends _FilterSink {
+  _ZLibDecoderSink(ByteConversionSink sink, int windowBits,
+                   List<int> dictionary, bool raw)
+      : super(sink, _Filter.newZLibInflateFilter(windowBits, dictionary, raw));
 }
 
 
-class _ZLibDecoderSink extends _FilterSink {
-  _ZLibDecoderSink(ByteConversionSink sink)
-      : super(sink, _Filter.newZLibInflateFilter());
-}
-
-
 class _FilterSink extends ByteConversionSink {
   final _Filter _filter;
   final ByteConversionSink _sink;
   bool _closed = false;
   bool _empty = true;
 
-  _FilterSink(ByteConversionSink this._sink, _Filter this._filter);
+  _FilterSink(this._sink, this._filter);
 
   void add(List<int> data) {
     addSlice(data, 0, data.length, false);
   }
 
   void addSlice(List<int> data, int start, int end, bool isLast) {
     if (_closed) return;
     if (start < 0 || start > data.length) {
       throw new ArgumentError("Invalid start position");
     }
@@ skipping 29 matching lines @@
       _closed = true;
       throw e;
     }
     if (!_closed) _filter.end();
     _closed = true;
     _sink.close();
   }
 }
 
 
-
 /**
  * Private helper-class to handle native filters.
  */
 abstract class _Filter {
   /**
    * Call to process a chunk of data. A call to [process] should only be made
    * when [processed] returns [:null:].
    */
   void process(List<int> data, int start, int end);
 
   /**
    * Get a chunk of processed data. When there are no more data available,
    * [processed] will return [:null:]. Set [flush] to [:false:] for non-final
    * calls to improve performance of some filters.
    *
    * The last call to [processed] should have [end] set to [:true:]. This will make
    * sure a 'end' packet is written on the stream.
    */
   List<int> processed({bool flush: true, bool end: false});
 
   /**
    * Mark the filter as closed. Always call this method for any filter created
    * to avoid leaking resources. [end] can be called at any time, but any
    * successive calls to [process] or [processed] will fail.
    */
   void end();
 
-  external static _Filter newZLibDeflateFilter(bool gzip, int level);
-  external static _Filter newZLibInflateFilter();
+  external static _Filter newZLibDeflateFilter(bool gzip, int level,
+                                               int windowBits, int memLevel,
+                                               int strategy,
+                                               List<int> dictionary, bool raw);
+
+  external static _Filter newZLibInflateFilter(int windowBits,
+                                               List<int> dictionary, bool raw);
 }
+
+void validateZLibWindowBits(int windowBits) {

[Anders Johnsen, 2014/01/27 12:27:45]

Make these functions private.

[vicb, 2014/01/27 15:14:19]

fixed

+  if (windowBits is! int ||
+      windowBits < ZLIB_MIN_WINDOW_BITS || windowBits > ZLIB_MAX_WINDOW_BITS) {

[Anders Johnsen, 2014/01/27 12:27:45]

Move last test to next line, as well as 2 places below.

[vicb, 2014/01/27 15:14:19]

fixed

+    throw new ArgumentError("'windowBits' must be in range "
+        "${ZLIB_MIN_WINDOW_BITS}..${ZLIB_MAX_WINDOW_BITS}");
+  }
+}
+
+void validateZLibeLevel(int level) {
+  if (level is! int ||
+      level < ZLIB_MIN_LEVEL || level > ZLIB_MAX_LEVEL) {
+    throw new ArgumentError("'level' must be in range ${ZLIB_MIN_LEVEL}.."
+        "${ZLIB_MAX_LEVEL}");
+  }
+}
+
+void validateZLibMemLevel(int memLevel) {
+  if (memLevel is! int ||
+      memLevel < ZLIB_MIN_MEM_LEVEL || memLevel > ZLIB_MAX_MEM_LEVEL) {
+    throw new ArgumentError("'memLevel' must be in range "
+    "${ZLIB_MIN_MEM_LEVEL}..${ZLIB_MAX_MEM_LEVEL}");
+  }
+}
+
+void validateZLibStrategy(int strategy) {
+  var strategies = <int>[ZLIB_STRATEGY_FILTERED, ZLIB_STRATEGY_HUFFMAN_ONLY,
+      ZLIB_STRATEGY_RLE, ZLIB_STRATEGY_FIXED, ZLIB_STRATEGY_DEFAULT];
+  if (strategies.indexOf(strategy) == -1) {
+    throw new ArgumentError("Unsupported 'strategy'");
+  }
+}