Make the parser agnostic to the TokenStream implementation. This is the first step towards compacti…

vm/datastream.h

Index: vm/datastream.h
===================================================================
--- vm/datastream.h	(revision 0)
+++ vm/datastream.h	(revision 0)
@@ -0,0 +1,203 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#ifndef VM_DATASTREAM_H_
+#define VM_DATASTREAM_H_
+
+#include "platform/assert.h"
+#include "vm/allocation.h"
+#include "vm/globals.h"
+
+namespace dart {
+
+static const int8_t kSerializedBitsPerByte = 7;

[hausner, 2012/06/22 00:02:27]

I know you just copied this code from another file, so leaving it unchanged is
fine. I would find it more readable though if the names were shorter and did not
have calculations in them. For example, the mask and end byte marker could be
fixed values just like the min and max values.

[siva, 2012/06/22 17:59:02]

I have renamed the constants removing the Serialized part as that only applied
in the snapshot context.

but I disagree with your comment about not having calculations in them. When one
constant is dependent on another constant it makes sense to use them in that
fashion, otherwise one would have to add a comment explaining that the
subsequent constant is dependent etc. etc.

On 2012/06/22 00:02:27, hausner wrote:

[hausner, 2012/06/22 18:54:37]

Thanks for making the names shorter. I still have a nit though: if you think
that the calculations are the right way, why do you not calculate 63 and -64 and
127? Feel free to ignore the comment though.
 
On 2012/06/22 17:59:02, asiva wrote:

[siva, 2012/06/22 20:36:29]

Used calculations for 63, -64, 127 etc.

On 2012/06/22 18:54:37, hausner wrote:

+static const int8_t kMaxSerializedUnsignedValuePerByte = 127;
+static const int8_t kMaxSerializedValuePerByte = 63;
+static const int8_t kMinSerializedValuePerByte = -64;
+static const uint8_t kEndByteMarker = (255 - kMaxSerializedValuePerByte);
+static const int8_t kSerializedByteMask = (1 << kSerializedBitsPerByte) - 1;
+
+typedef uint8_t* (*ReAlloc)(uint8_t* ptr, intptr_t old_size, intptr_t new_size);
+
+// Stream for reading various types from a buffer.
+class ReadStream : public ValueObject {
+ public:
+  ReadStream(const uint8_t* buffer, intptr_t size) : buffer_(buffer),
+                                                     current_(buffer),
+                                                     end_(buffer + size)  {}
+
+  template<int N, typename T>
+  class Raw { };
+
+  template<typename T>
+  class Raw<1, T> {
+   public:
+    static T Read(ReadStream* st) {
+      return bit_cast<T>(st->ReadByte());
+    }
+  };
+
+  template<typename T>
+  class Raw<2, T> {
+   public:
+    static T Read(ReadStream* st) {
+      return bit_cast<T>(st->Read<int16_t>());
+    }
+  };
+
+  template<typename T>
+  class Raw<4, T> {
+   public:
+    static T Read(ReadStream* st) {
+      return bit_cast<T>(st->Read<int32_t>());
+    }
+  };
+
+  template<typename T>
+  class Raw<8, T> {
+   public:
+    static T Read(ReadStream* st) {
+      return bit_cast<T>(st->Read<int64_t>());
+    }
+  };
+
+  void ReadBytes(uint8_t* addr, intptr_t len) {
+    ASSERT((current_ + len) < end_);
+    memmove(addr, current_, len);
+    current_ += len;
+  }
+
+ private:
+  template<typename T>
+  T Read() {
+    uint8_t b = ReadByte();
+    if (b > kMaxSerializedUnsignedValuePerByte) {
+      return static_cast<T>(b) - kEndByteMarker;
+    }
+    T r = 0;
+    uint8_t s = 0;
+    do {
+      r |= static_cast<T>(b) << s;
+      s += kSerializedBitsPerByte;
+      b = ReadByte();
+    } while (b <= kMaxSerializedUnsignedValuePerByte);
+    return r | ((static_cast<T>(b) - kEndByteMarker) << s);
+  }
+
+  uint8_t ReadByte() {
+    ASSERT(current_ < end_);
+    return *current_++;
+  }
+
+ private:
+  const uint8_t* buffer_;
+  const uint8_t* current_;
+  const uint8_t* end_;
+
+  DISALLOW_COPY_AND_ASSIGN(ReadStream);
+};
+
+
+// Stream for writing various types into a buffer.
+class WriteStream : public ValueObject {
+ public:
+  static const int kBufferIncrementSize = 64 * KB;
+
+  WriteStream(uint8_t** buffer, ReAlloc alloc) :
+      buffer_(buffer),
+      end_(NULL),
+      current_(NULL),
+      current_size_(0),
+      alloc_(alloc) {
+    ASSERT(buffer != NULL);
+    ASSERT(alloc != NULL);
+    *buffer_ = reinterpret_cast<uint8_t*>(alloc_(NULL,
+                                                 0,
+                                                 kBufferIncrementSize));
+    ASSERT(*buffer_ != NULL);
+    current_ = *buffer_;
+    current_size_ = kBufferIncrementSize;
+    end_ = *buffer_ + kBufferIncrementSize;
+  }
+
+  uint8_t* buffer() const { return *buffer_; }
+  int bytes_written() const { return current_ - *buffer_; }
+
+  void set_current(uint8_t* value) { current_ = value; }
+
+  template<int N, typename T>
+  class Raw { };
+
+  template<typename T>
+  class Raw<1, T> {
+   public:
+    static void Write(WriteStream* st, T value) {
+      st->WriteByte(bit_cast<int8_t>(value));
+    }
+  };
+
+  template<typename T>
+  class Raw<2, T> {
+   public:
+    static void Write(WriteStream* st, T value) {
+      st->Write<int16_t>(bit_cast<int16_t>(value));
+    }
+  };
+
+  template<typename T>
+  class Raw<4, T> {
+   public:
+    static void Write(WriteStream* st, T value) {
+      st->Write<int32_t>(bit_cast<int32_t>(value));
+    }
+  };
+
+  template<typename T>
+  class Raw<8, T> {
+   public:
+    static void Write(WriteStream* st, T value) {
+      st->Write<int64_t>(bit_cast<int64_t>(value));
+    }
+  };
+
+ private:
+  template<typename T>
+  void Write(T value) {
+    T v = value;
+    while (v < kMinSerializedValuePerByte ||
+           v > kMaxSerializedValuePerByte) {
+      WriteByte(static_cast<uint8_t>(v & kSerializedByteMask));
+      v = v >> kSerializedBitsPerByte;
+    }
+    WriteByte(static_cast<uint8_t>(v + kEndByteMarker));
+  }
+
+  void WriteByte(uint8_t value) {
+    if (current_ >= end_) {
+      intptr_t new_size = (current_size_ + kBufferIncrementSize);
+      *buffer_ = reinterpret_cast<uint8_t*>(alloc_(*buffer_,
+                                                   current_size_,
+                                                   new_size));
+      ASSERT(*buffer_ != NULL);
+      current_ = *buffer_ + current_size_;
+      current_size_ = new_size;
+      end_ = *buffer_ + new_size;
+    }
+    ASSERT(current_ < end_);
+    *current_++ = value;
+  }
+
+ private:
+  uint8_t** const buffer_;
+  uint8_t* end_;
+  uint8_t* current_;
+  intptr_t current_size_;
+  ReAlloc alloc_;
+
+  DISALLOW_COPY_AND_ASSIGN(WriteStream);
+};
+
+}  // namespace dart
+
+#endif  // VM_DATASTREAM_H_