Add ASCII encoding to lib/convert.

sdk/lib/convert/ascii.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of dart.convert;
 
 /**
- * An instance of the default implementation of the [Latin1Codec].
+ * An instance of the default implementation of the [AsciiCodec].
  *
  * This instance provides a convenient access to the most common ISO Latin 1

[floitsch, 2013/08/22 12:34:31]

not ISO Latin 1.

[Lasse Reichstein Nielsen, 2013/08/22 13:04:56]

Done.

  * use cases.
  *
  * Examples:
  *
- *     var encoded = LATIN1.encode("blåbærgrød");
- *     var decoded = LATIN1.decode([0x62, 0x6c, 0xe5, 0x62, 0xe6,
- *                                  0x72, 0x67, 0x72, 0xf8, 0x64]);
+ *     var encoded = LATIN1.encode("This is ASCII!");

[floitsch, 2013/08/22 12:34:31]

ASCII

[Lasse Reichstein Nielsen, 2013/08/22 13:04:56]

Done.

+ *     var decoded = LATIN1.decode([0x62, 0x6c, 0x61, 0x61, 0x62, 0x61, 0x65,

[floitsch, 2013/08/22 12:34:31]

maybe still pick a better ASCII example.

[Lasse Reichstein Nielsen, 2013/08/22 13:04:56]

Pfft!
Done.

+ *                                  0x72, 0x67, 0x72, 0x6f, 0x65, 0x64]);
  */
-const LATIN1 = const Latin1Codec();
+const ASCII = const AsciiCodec();
+
+const int _ASCII_MASK = 0x7F;
 
 /**
- * A [LatinCodec] encodes strings to ISO Latin-1 (aka ISO-8859-1) bytes
- * and decodes Latin-1 bytes to strings.
+ * A [AsciiCodec] encodes strings to ASCII bytes

[floitsch, 2013/08/22 12:34:31]

An

[Lasse Reichstein Nielsen, 2013/08/22 13:04:56]

Done.

+ * and decodes ASCII bytes to strings.
  */
-class Latin1Codec extends _Encoding {
+class AsciiCodec extends _Encoding {
   final bool _allowInvalid;
   /**
-   * Instantiates a new [Latin1Codec].
+   * Instantiates a new [AsciiCodec].
    *
    * If [allowInvalid] is true, the [decode] method and the converter
    * returned by [decoder] will default to allowing invalid values. Invalid
    * values are decoded into the Unicode Replacement character (U+FFFD).
    * Calls to the [decode] method can override this default.

[floitsch, 2013/08/22 12:34:31]

It is not clear that you are talking about the "allowInvalid" default (and not
the replacement character default).

[Lasse Reichstein Nielsen, 2013/08/22 13:04:56]

Done.

    *
    * Encoders will not accept invalid (non Latin-1) characters.
    */
-  const Latin1Codec({bool allowInvalid: false}) : _allowInvalid = allowInvalid;
+  const AsciiCodec({bool allowInvalid: false}) : _allowInvalid = allowInvalid;
 
   /**
-   * Decodes the Latin-1 [bytes] (a list of unsigned 8-bit integers) to the
+   * Decodes the ASCII [bytes] (a list of unsigned 7-bit integers) to the
    * corresponding string.
    *
-   * If [bytes] contains values that are not in the range 0 .. 255, the decoder
+   * If [bytes] contains values that are not in the range 0 .. 127, the decoder
    * will eventually throw a [FormatException].
    *
    * If [allowInvalid] is not provided, it defaults to the value used to create
-   * this [Latin1Codec].
+   * this [AsciiCodec].
    */
   String decode(List<int> bytes, { bool allowInvalid }) {
     if (allowInvalid == null) allowInvalid = _allowInvalid;
     if (allowInvalid) {
-      return const Latin1Decoder(allowInvalid: true).convert(bytes);
+      return const AsciiDecoder(allowInvalid: true).convert(bytes);
     } else {
-      return const Latin1Decoder(allowInvalid: false).convert(bytes);
+      return const AsciiDecoder(allowInvalid: false).convert(bytes);
     }
   }
 
-  Converter<String, List<int>> get encoder => const Latin1Encoder();
+  Converter<String, List<int>> get encoder => const AsciiEncoder();
 
   Converter<List<int>, String> get decoder =>
-      _allowInvalid ? const Latin1Decoder(allowInvalid: true)
-                    : const Latin1Decoder(allowInvalid: false);
+      _allowInvalid ? const AsciiDecoder(allowInvalid: true)
+                    : const AsciiDecoder(allowInvalid: false);
 }
 
-/**
- * This class converts strings of only ISO Latin-1 characters to bytes.
- */
-class Latin1Encoder extends Converter<String, List<int>> {
-  const Latin1Encoder();
+// Superclass for [AsciiEncoder] and [Latin1nocoder].

[floitsch, 2013/08/22 12:34:31]

Latin1Encoder

[Lasse Reichstein Nielsen, 2013/08/22 13:04:56]

Done.

+// Generalizes common operations that only differ by a mask;
+class _UnicodeSubsetEncoder extends Converter<String, List<int>> {
+  final int _subsetMask;
 
-  /**
-   * Converts [string] to its Latin-1 bytes (a list of
-   * unsigned 8-bit integers).
-   */
+  const _UnicodeSubsetEncoder(this._subsetMask);
+
   List<int> convert(String string) {
     // TODO(11971): Use Uint8List when possible.
     List result = new List<int>(string.length);
     for (int i = 0; i < string.length; i++) {
       var codeUnit = string.codeUnitAt(i);
-      if ((codeUnit & ~0xFF) != 0) {
-        throw new ArgumentError("String contains non-Latin-1 characters.");
+      if ((codeUnit & ~_subsetMask) != 0) {
+        throw new ArgumentError("String contains invalid characters.");
       }
       result[i] = codeUnit;
     }
     return result;
   }
 
   /**
    * Starts a chunked conversion.
    *
    * The converter works more efficiently if the given [sink] is a
    * [ByteConversionSink].
    */
   StringConversionSink startChunkedConversion(
       ChunkedConversionSink<List<int>> sink) {
     if (sink is! ByteConversionSink) {
       sink = new ByteConversionSink.from(sink);
     }
-    return new _Latin1EncoderSink(sink);
+    return new _UnicodeSubsetEncoderSink(_subsetMask, sink);
   }
 
   // Override the base-class' bind, to provide a better type.
   Stream<List<int>> bind(Stream<String> stream) => super.bind(stream);
 }
 
 /**
+ * This class converts strings of only ISO Latin-1 characters to bytes.

[floitsch, 2013/08/22 12:34:31]

Ascii

[Lasse Reichstein Nielsen, 2013/08/22 13:04:56]

Done.

+ */
+class AsciiEncoder extends _UnicodeSubsetEncoder {
+  const AsciiEncoder() : super(_ASCII_MASK);
+}
+
+/**
  * This class encodes chunked strings to bytes (unsigned 8-bit
  * integers).
  */
-class _Latin1EncoderSink extends StringConversionSinkBase {
-  static const _DEFAULT_BYTE_BUFFER_SIZE = 1024;
+class _UnicodeSubsetEncoderSink extends StringConversionSinkBase {
   final ByteConversionSink _sink;
+  final int _subsetMask;
 
-  // TODO(11971): Use Uint8List when available.
-  List<int> _buffer = new List<int>(_DEFAULT_BYTE_BUFFER_SIZE);
-  int _bufferIndex = 0;
-
-  _Latin1EncoderSink(this._sink);
+  _UnicodeSubsetEncoderSink(this._subsetMask, this._sink);
 
   void close() {
-    if (_bufferIndex > 0) {
-      _sink.addSlice(_buffer, 0, _bufferIndex, true);
-    } else {
-      _sink.close();
-    }
+    _sink.close();
   }
 
   void addSlice(String source, int start, int end, bool isLast) {
     if (start < 0 || start > source.length) {
       throw new RangeError.range(start, 0, source.length);
     }
     if (end < start || end > source.length) {
       throw new RangeError.range(end, start, source.length);
     }
     for (int i = start; i < end; i++) {
       int codeUnit = source.codeUnitAt(i);
-      if ((codeUnit & ~0xFF) != 0) {
-        throw new ArgumentError("Source contains non-Latin-1 characters.");
-      }
-      _buffer[_bufferIndex] = codeUnit;
-      _bufferIndex++;
-      if (_bufferIndex == _buffer.length) {
-        _sink.addSlice(_buffer, 0, _bufferIndex, false);
-        _bufferIndex = 0;
+      if ((codeUnit & ~_subsetMask) != 0) {
+        throw new ArgumentError(
+            "Source contains invalid character with code point: $codeUnit.");
       }
     }
-    if (isLast) close();
+    _sink.add(source.codeUnits);
+    if (isLast) {
+      close();
+    }
   }
 }
 
 /**
  * This class converts Latin-1 bytes (lists of unsigned 8-bit integers)
  * to a string.
  */
-class Latin1Decoder extends Converter<List<int>, String> {
+abstract class _UnicodeSubsetDecoder extends Converter<List<int>, String> {
   final bool _allowInvalid;
+  final int _subsetMask;
 
   /**
-   * Instantiates a new [Latin1Decoder].
+   * Instantiates a new decoder.
    *
-   * The optional [allowInvalid] argument defines how [convert] deals
+   * The [_allowInvalid] argument defines how [convert] deals
    * with invalid bytes.
    *
-   * If it is `true`, [convert] replaces invalid bytes with the Unicode
-   * Replacement character `U+FFFD` (�).
+   * The [_subsetMask] argument is a bit mask used to define the subset
+   * of Unicode being decoded. Use [_LATIN1_MASK] for Latin-1 (8-bit) or
+   * [_ASCII_MASK] for ASCII (7-bit).
+   *
+   * If [_allowInvalid] is `true`, [convert] replaces invalid bytes with the
+   * Unicode Replacement character `U+FFFD` (�).
    * Otherwise it throws a [FormatException].
    */
-  const Latin1Decoder({ bool allowInvalid: false })
-      : this._allowInvalid = allowInvalid;
+  const _UnicodeSubsetDecoder(this._allowInvalid, this._subsetMask);
 
   /**
-   * Converts the Latin=1 [bytes] (a list of unsigned 8-bit integers) to the
+   * Converts the [bytes] (a list of unsigned 7- or 8-bit integers) to the
    * corresponding string.
    */
   String convert(List<int> bytes) {
     for (int i = 0; i < bytes.length; i++) {
       int byte = bytes[i];
-      if ((byte & ~0xFF) != 0) {
+      if ((byte & ~_subsetMask) != 0) {
         if (!_allowInvalid) {
-          throw new FormatException("Non-byte in byte list");
+          throw new FormatException("Invalid value in input: $byte");
         }
         return _convertInvalid(bytes);
       }
     }
     return new String.fromCharCodes(bytes);
   }
 
   String _convertInvalid(List<int> bytes) {
     StringBuffer buffer = new StringBuffer();
     for (int i = 0; i < bytes.length; i++) {
       int value = bytes[i];
-      if ((value & ~0xFF) != 0) value = 0xFFFD;
+      if ((value & ~_subsetMask) != 0) value = 0xFFFD;
       buffer.writeCharCode(value);
     }
     return buffer.toString();
   }
 
   /**
    * Starts a chunked conversion.
    *
    * The converter works more efficiently if the given [sink] is a
    * [StringConversionSink].
    */
   ByteConversionSink startChunkedConversion(
       ChunkedConversionSink<String> sink) {
     StringConversionSink stringSink;
     if (sink is StringConversionSink) {
       stringSink = sink;
     } else {
       stringSink = new StringConversionSink.from(sink);
     }
     // TODO(lrn): Use stringSink.asUtf16Sink() if it becomes available.
     return new _Latin1DecoderSink(_allowInvalid, stringSink);
   }
 
   // Override the base-class's bind, to provide a better type.
   Stream<String> bind(Stream<List<int>> stream) => super.bind(stream);
 }
 
-class _Latin1DecoderSink extends ByteConversionSinkBase {
+class AsciiDecoder extends _UnicodeSubsetDecoder {
+  const AsciiDecoder({bool allowInvalid: false})
+      : super(allowInvalid, _ASCII_MASK);
+
+  /**
+   * Starts a chunked conversion.
+   *
+   * The converter works more efficiently if the given [sink] is a
+   * [StringConversionSink].
+   */
+  ByteConversionSink startChunkedConversion(
+      ChunkedConversionSink<String> sink) {
+    StringConversionSink stringSink;
+    if (sink is StringConversionSink) {
+      stringSink = sink;
+    } else {
+      stringSink = new StringConversionSink.from(sink);
+    }
+    // TODO(lrn): Use asUtf16Sink when it becomes available. It
+    // works just as well, is likely to have less decoding overhead,
+    // and make adding U+FFFD easier.
+    // At that time, merge this with _Latin1DecoderSink;
+    return new _AsciiDecoderSink(_allowInvalid, stringSink.asUtf8Sink(false));
+  }
+}
+
+class _AsciiDecoderSink extends ByteConversionSinkBase {
   final bool _allowInvalid;
-  StringConversionSink _sink;
-  _Latin1DecoderSink(this._allowInvalid, this._sink);
+  ByteConversionSink _utf8Sink;
+  _AsciiDecoderSink(this._allowInvalid, this._utf8Sink);
 
   void close() {
-    _sink.close();
+    _utf8Sink.close();
   }
 
   void add(List<int> source) {
     addSlice(source, 0, source.length, false);
   }
 
-  void _addSliceToSink(List<int> source, int start, int end, bool isLast) {
-    // If _sink was a UTF-16 conversion sink, just add the slice directly with
-    // _sink.addSlice(source, start, end, isLast).
-    // The code below is an incredibly stupid workaround until a real
-    // solution can be made.
-    _sink.add(new String.fromCharCodes(source.getRange(start, end)));
-    if (isLast) close();
-  }
-
   void addSlice(List<int> source, int start, int end, bool isLast) {
     if (start < 0 || start > source.length) {
       throw new RangeError.range(start, 0, source.length);
     }
     if (end < start || end > source.length) {
       throw new RangeError.range(end, start, source.length);
     }
     for (int i = start; i < end; i++) {
-      if ((source[i] & ~0xFF) != 0) {
+      if ((source[i] & ~_ASCII_MASK) != 0) {
         if (_allowInvalid) {
-          if (i > start) _addSliceToSink(source, start, i);
+          if (i > start) _utf8Sink.addSlice(source, start, i, false);
           // Add UTF-8 encoding of U+FFFD.
-          _addSliceToSink(const[0xFFFD], 0, 1, false);
+          _utf8Sink.add(const<int>[0xEF, 0xBF, 0xBD]);
           start = i + 1;
         } else {
-          throw new FormatException("Source contains non-Latin-1 characters.");
+          throw new FormatException("Source contains non-ASCII bytes.");
         }
       }
     }
     if (start < end) {
-      _addSliceToSink(source, start, end, isLast);
+      _utf8Sink.addSlice(source, start, end, isLast);
     } else if (isLast) {
       close();
     }
   }
 }