Introduce (private) ZLibFilter to dart:io.

runtime/bin/filter.cc

+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#include "bin/filter.h"
+#include "bin/dartutils.h"
+
+#include "include/dart_api.h"
+
+const int kZlibFlagMemUsage = 8;
+const int kZLibFlagWindowBits = 15;
+const int kZLibFlagUseGZipHeader = 16;
+const int kZLibFlagAcceptAnyHeader = 32;
+
+const intptr_t kFilterBufferLength = 64 * KB;
+
+static const int kFilterPointerNativeField = 0;
+
+void DeleteBuffer(void* buffer) {
+  delete[] reinterpret_cast<uint8_t*>(buffer);
+}
+
+Filter* GetFilter(Dart_Handle filter_obj) {
+  Filter* filter;
+  if (Dart_IsError(Filter::GetFilterPointerNativeField(filter_obj, &filter))
+      || filter == NULL) {
+    OSError os_error(-1, "Failed to get filter", OSError::kUnknown);
+    Dart_Handle err = DartUtils::NewDartOSError(&os_error);
+    Dart_ThrowException(err);
+  }
+  return filter;
+}
+
+void EndFilter(Dart_Handle filter_obj, Filter* filter) {
+  Filter::SetFilterPointerNativeField(filter_obj, NULL);
+  delete filter;
+}
+
+void FUNCTION_NAME(Filter_CreateZLibInflate)(Dart_NativeArguments args) {
+  Dart_EnterScope();
+  Dart_Handle filter_obj = Dart_GetNativeArgument(args, 0);
+  Filter* filter = new ZLibInflateFilter();
+  if (!filter->Init() ||
+      Dart_IsError(Filter::SetFilterPointerNativeField(filter_obj, filter))) {
+    delete filter;
+    OSError os_error(-1, "Failed to create ZLibInflateFilter",
+                     OSError::kUnknown);
+    Dart_Handle err = DartUtils::NewDartOSError(&os_error);
+    Dart_ThrowException(err);
+  }
+  Dart_ExitScope();
+}
+
+void FUNCTION_NAME(Filter_CreateGZipDeflate)(Dart_NativeArguments args) {
+  Dart_EnterScope();
+  Dart_Handle filter_obj = Dart_GetNativeArgument(args, 0);
+  Filter* filter = new ZLibDeflateFilter(true);
+  if (!filter->Init() ||
+      Dart_IsError(Filter::SetFilterPointerNativeField(filter_obj, filter))) {
+    delete filter;
+    OSError os_error(-1, "Failed to create GZipDeflateFilter",
+                     OSError::kUnknown);
+    Dart_Handle err = DartUtils::NewDartOSError(&os_error);

[Søren Gjesse, 2012/10/25 11:01:44]

I am sorry for leading you down this path, but the errors here are not really OS
errors. We should reserve OS errors for errors returned from "real" OS calls.

We should probably add an InternalError for these kind of errors. You can leave
it as a TODO and open a bug on it.

[Anders Johnsen, 2012/10/25 13:10:56]

I've added an InternalError class and used it accordingly.

+    Dart_ThrowException(err);
+  }
+  Dart_ExitScope();
+}
+
+void FUNCTION_NAME(Filter_Process)(Dart_NativeArguments args) {
+  Dart_EnterScope();
+  Dart_Handle filter_obj = Dart_GetNativeArgument(args, 0);
+  Filter* filter = GetFilter(filter_obj);
+  Dart_Handle data_obj = Dart_GetNativeArgument(args, 1);
+  intptr_t length;
+  if (Dart_IsError(Dart_ListLength(data_obj, &length))) {
+    OSError os_error(-1, "Failed to get data length", OSError::kUnknown);
+    Dart_Handle err = DartUtils::NewDartOSError(&os_error);
+    Dart_ThrowException(err);
+  }
+  uint8_t* data = new uint8_t[length];
+  if (Dart_IsError(Dart_ListGetAsBytes(data_obj, 0, data, length))) {
+    delete[] data;

[Søren Gjesse, 2012/10/25 11:01:44]

Maybe use DeleteBuffer here (and in the other places) to be somewhat consistent.

[Anders Johnsen, 2012/10/25 13:10:56]

Done.

+    OSError os_error(-1, "Failed to get data bytes", OSError::kUnknown);
+    Dart_Handle err = DartUtils::NewDartOSError(&os_error);
+    Dart_ThrowException(err);
+  }
+  if (!filter->Process(data, length)) {
+    OSError os_error(-1,
+                     "Call to Process while still processing data",
+                     OSError::kUnknown);
+    Dart_Handle err = DartUtils::NewDartOSError(&os_error);
+    Dart_ThrowException(err);
+  }
+  Dart_ExitScope();
+}
+
+
+void FUNCTION_NAME(Filter_Processed)(Dart_NativeArguments args) {
+  Dart_EnterScope();
+  Dart_Handle filter_obj = Dart_GetNativeArgument(args, 0);
+  Filter* filter = GetFilter(filter_obj);
+  Dart_Handle flush_obj = Dart_GetNativeArgument(args, 1);
+  bool flush;
+  if (Dart_IsError(Dart_BooleanValue(flush_obj, &flush))) {
+    OSError os_error(-1, "Failed to get 'flush' parameter", OSError::kUnknown);
+    Dart_Handle err = DartUtils::NewDartOSError(&os_error);
+    Dart_ThrowException(err);
+  }
+  uint8_t* buffer = new uint8_t[kFilterBufferLength];
+  intptr_t read = filter->Processed(buffer, kFilterBufferLength, flush);
+  if (read < 0) {
+    // Error, end filter.
+    EndFilter(filter_obj, filter);
+    delete[] buffer;
+    OSError os_error(-1, "Filter error, bad data", OSError::kUnknown);
+    Dart_Handle err = DartUtils::NewDartOSError(&os_error);
+    Dart_ThrowException(err);
+  } else if (read == 0) {
+    delete[] buffer;
+    Dart_SetReturnValue(args, Dart_Null());
+  } else {
+    Dart_Handle result;
+    if (read == kFilterBufferLength) {
+      result = Dart_NewExternalByteArray(buffer, read, buffer, DeleteBuffer);
+    } else {

[Søren Gjesse, 2012/10/25 11:01:44]

Can't you create an external byte array here as well? I don't see any problem of
the buffer being larger than indicated by length. Maybe for very small values of
length you should still fallback to byte arrays in the Dart heap.

[Anders Johnsen, 2012/10/25 13:10:56]

I like this idea! We are currently using buffers with sizes of 64k. That should
be okay. One concern: The GC does not know how much data we actually have
allocated. Should we add a TODO about adding an extra optional parameter to
NewExternalByteArray with the 'real size'?

[Søren Gjesse, 2012/10/25 13:54:45]

At some point we probable need to give the VM some idea of how much external
data is actually retained, but for now we can ignore it.

+      result = Dart_NewByteArray(read);
+      Dart_ListSetAsBytes(result, 0, buffer, read);
+      delete[] buffer;
+    }
+    Dart_SetReturnValue(args, result);
+  }
+  Dart_ExitScope();
+}
+
+
+void FUNCTION_NAME(Filter_End)(Dart_NativeArguments args) {
+  Dart_EnterScope();
+  Dart_Handle filter_obj = Dart_GetNativeArgument(args, 0);
+  Filter* filter = GetFilter(filter_obj);
+  EndFilter(filter_obj, filter);
+  Dart_ExitScope();
+}
+
+
+Dart_Handle Filter::SetFilterPointerNativeField(Dart_Handle filter,
+                                                Filter* filter_pointer) {
+  return Dart_SetNativeInstanceField(filter,
+                                     kFilterPointerNativeField,
+                                     (intptr_t)filter_pointer);
+}
+
+
+Dart_Handle Filter::GetFilterPointerNativeField(Dart_Handle filter,
+                                                Filter** filter_pointer) {
+  return Dart_GetNativeInstanceField(
+      filter,
+      kFilterPointerNativeField,
+      reinterpret_cast<intptr_t*>(filter_pointer));
+}
+
+
+ZLibDeflateFilter::~ZLibDeflateFilter() {
+  delete[] current_buffer;
+  deflateEnd(&stream);
+}
+
+
+bool ZLibDeflateFilter::Init() {
+  stream.zalloc = Z_NULL;
+  stream.zfree = Z_NULL;
+  stream.opaque = Z_NULL;
+  return deflateInit2(&stream,
+                      level,
+                      Z_DEFLATED,
+                      kZLibFlagWindowBits + (gZip ? kZLibFlagUseGZipHeader : 0),

[Søren Gjesse, 2012/10/25 11:01:44]

Maybe | here instead of +

[Anders Johnsen, 2012/10/25 13:10:56]

Done.

+                      kZlibFlagMemUsage,
+                      Z_DEFAULT_STRATEGY) == Z_OK;
+}
+
+
+bool ZLibDeflateFilter::Process(uint8_t* data, intptr_t length) {
+  if (current_buffer != NULL) return false;
+  stream.avail_in = length;
+  stream.next_in = current_buffer = data;
+  return true;
+}
+
+intptr_t ZLibDeflateFilter::Processed(uint8_t* buffer,
+                                      intptr_t length,
+                                      bool flush) {
+  stream.avail_out = length;
+  stream.next_out = buffer;
+  switch (deflate(&stream, flush ? Z_SYNC_FLUSH : Z_NO_FLUSH)) {
+    case Z_OK:
+      // We processed data, should be called again.
+      return length - stream.avail_out;
+
+    case Z_STREAM_END:
+    case Z_BUF_ERROR:
+      // We processed all available input data.
+      delete[] current_buffer;
+      current_buffer = NULL;
+      return 0;
+
+    default:
+    case Z_STREAM_ERROR:
+      // An error occoured.
+      delete[] current_buffer;
+      current_buffer = NULL;
+      return -1;
+  }
+}
+
+
+ZLibInflateFilter::~ZLibInflateFilter() {
+  delete[] current_buffer;
+  inflateEnd(&stream);
+}
+
+
+bool ZLibInflateFilter::Init() {
+  stream.zalloc = Z_NULL;
+  stream.zfree = Z_NULL;
+  stream.opaque = Z_NULL;
+  return inflateInit2(&stream,
+                      kZLibFlagWindowBits + kZLibFlagAcceptAnyHeader) == Z_OK;

[Søren Gjesse, 2012/10/25 11:01:44]

Maybe | here instead of +

[Anders Johnsen, 2012/10/25 13:10:56]

Done.

+}
+
+
+bool ZLibInflateFilter::Process(uint8_t* data, intptr_t length) {
+  if (current_buffer != NULL) return false;
+  stream.avail_in = length;
+  stream.next_in = current_buffer = data;
+  return true;
+}
+
+
+intptr_t ZLibInflateFilter::Processed(uint8_t* buffer,
+                                      intptr_t length,
+                                      bool flush) {
+  stream.avail_out = length;
+  stream.next_out = buffer;
+  switch (inflate(&stream, flush ? Z_SYNC_FLUSH : Z_NO_FLUSH)) {
+    case Z_OK:
+      // We processed data, should be called again.
+      return length - stream.avail_out;
+
+    case Z_STREAM_END:
+    case Z_BUF_ERROR:
+      // We processed all available input data.
+      delete[] current_buffer;
+      current_buffer = NULL;
+      return 0;
+
+    default:
+    case Z_MEM_ERROR:
+    case Z_NEED_DICT:
+    case Z_DATA_ERROR:
+    case Z_STREAM_ERROR:
+      // An error occoured.
+      delete[] current_buffer;
+      current_buffer = NULL;
+      return -1;
+  }
+}
+