Prototype of RAW decoding in Skia.

src/codec/SkRawCodec.cpp

+/*
+ * Copyright 2015 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "SkCodec.h"
+#include "SkCodecPriv.h"
+#include "SkColorPriv.h"
+#include "SkData.h"
+#if !defined(GOOGLE3)
+#include "SkJpegCodec.h"
+#endif
+#include "SkRawCodec.h"
+#include "SkRefCnt.h"
+#include "SkStream.h"
+#include "SkStreamPriv.h"
+#include "SkSwizzler.h"
+#include "SkTemplates.h"
+#include "SkTypes.h"
+
+#include "dng_color_space.h"
+#include "dng_exceptions.h"
+#include "dng_host.h"
+#include "dng_info.h"
+#include "dng_memory.h"
+#include "dng_render.h"
+#include "dng_stream.h"
+
+#include "src/piex.h"
+
+#include <cmath>  // for std::round,floor,ceil
+#include <limits>
+
+namespace {
+
+// T must be unsigned type.
+template <class T>
+bool safe_add_to_size_t(T arg1, T arg2, size_t* result) {
+    SkASSERT(arg1 >= 0);
+    SkASSERT(arg2 >= 0);
+    if (arg1 >= 0 && arg2 <= std::numeric_limits<T>::max() - arg1) {
+        T sum = arg1 + arg2;
+        if (sum <= std::numeric_limits<size_t>::max()) {
+            *result = static_cast<size_t>(sum);
+            return true;
+        }
+    }
+    return false;
+}
+
+class SkDngMemoryAllocator : public dng_memory_allocator {
+public:
+    virtual ~SkDngMemoryAllocator() {}
+
+    dng_memory_block* Allocate(uint32 size) override {
+        // To avoid arbitary allocation requests which might lead to out-of-memory, limit the
+        // amount of memory that can be allocated at once.
+        if (size > 300 * 1024 * 1024) {  // 300 MB

[scroggo, 2016/01/19 21:05:44]

How did you choose this amount? Should it be configurable based on the device?

Do you expect that this will result in failing to decode valid images? Or was
the large request due to the fact that the image file was invalid?

[yujieqin, 2016/01/20 12:53:04]

The memory limit is based on our experiments, it should be sufficient for all
valid DNG images.

We could think about setting this limit based on the device, but currently we
are not doing that.

+            ThrowMemoryFull();
+        }
+        return dng_memory_allocator::Allocate(size);
+    }
+};
+
+}  // namespace
+
+// Note: this class could throw exception if it is used as dng_stream.
+class SkRawStream : public dng_stream, public ::piex::StreamInterface {
+public:
+    // Note that this call will take the ownership of stream.
+    explicit SkRawStream(SkStream* stream)
+        : fStream(stream), fWholeStreamRead(false) {}
+
+    ~SkRawStream() override {}
+
+    /*
+     * Creates an SkMemoryStream from the offset with size.
+     * Note: for performance reason, this function is destructive to the SkRawStream. One should
+     *       abandon current object after the function call.
+     */
+    SkMemoryStream* transferBuffer(size_t offset, size_t size) {
+        SkAutoTUnref<SkData> data(SkData::NewUninitialized(size));
+        if (offset > fStreamBuffer.bytesWritten()) {
+            // If the offset is not buffered, read from fStream directly and skip the buffering.
+            const size_t skipLength = offset - fStreamBuffer.bytesWritten();
+            if (fStream->skip(skipLength) != skipLength) {
+                return nullptr;
+            }
+            const size_t bytesRead = fStream->read(data->writable_data(), size);
+            if (bytesRead < size) {
+                data.reset(SkData::NewSubset(data.get(), 0, bytesRead));
+            }
+        } else {
+            const size_t alreadyBuffered = fStreamBuffer.bytesWritten() - offset;
+            if (!fStreamBuffer.read(data->writable_data(), offset, alreadyBuffered)) {
+                return nullptr;
+            }
+
+            const size_t remaining = size - alreadyBuffered;
+            auto* dst = static_cast<uint8_t*>(data->writable_data()) + alreadyBuffered;

[scroggo, 2016/01/19 21:05:44]

nit: this could go inside the if (remaining) block

[yujieqin, 2016/01/20 12:53:04]

Done.

+            if (remaining) {
+                const size_t bytesRead = fStream->read(dst, remaining);
+                size_t newSize;
+                if (bytesRead < remaining) {
+                    if (safe_add_to_size_t(alreadyBuffered, bytesRead, &newSize)) {

[scroggo, 2016/01/19 21:05:44]

I think this should say if (!safe_add_to_size_t(...))?

FWIW, if this failed, it seems like we would have had a problem on the other
code paths in this method.

[yujieqin, 2016/01/20 12:53:04]

Oh yes, typo :(

But this one could potentially fail, because we got the "offset" and "size" info
(two numbers) directly from the image metadata. E.g. if it is a truncated image,
it can offer a reasonable "offset" but a huge "size".

[scroggo, 2016/01/20 18:13:36]

Agreed. But won't a huge "size" cause problems in the other cases? Like on line
91:

  const size_t bytesRead = fStream->read(data->writable_data(), size);

or I guess we're depending on the stream implementation to handle that case for
us? I don't know that we've ever hardened streams against something like this.
(Although kjlubick@ is adding a fuzzer for SkCodec, which should help in this
case.)

[yujieqin, 2016/01/20 21:42:19]

I don't think there will be issue with a huge "size". As said, the
stream->read() should handle this case correctly. (i.e. if the requested size >
stream size, then fail properly.) If the steam can not guarantee this point, it
will be a general issue for Skia.

[scroggo, 2016/01/21 16:36:09]

Streams are mostly in Skia for images and typefaces. Prior to your changes,
those were always much smaller. So my guess is that we're in new territory for
SkStream dealing with large numbers.

[yujieqin, 2016/01/21 17:44:25]

En... good to know.

Could we maybe leave it as it is, and could you help to create a skia bug to
track the progress of improving the behavior of SkStream.read() for large size
case? As you mentioned, someone (kjlubick@) is already working on something
related, right?

[scroggo, 2016/01/21 17:57:43]

Well, he's just added code to "fuzz" our image decoders, meaning it will modify
existing image files and try decoding them to try to hit error cases. So it's
not aimed at SkStream::read() directly, but it should help.

+                        return nullptr;
+                    }
+                    data.reset(SkData::NewSubset(data.get(), 0, newSize));
+                }
+            }
+        }
+        return new SkMemoryStream(data);
+    }
+
+    // For PIEX
+    ::piex::Error GetData(const size_t offset, const size_t length,
+                          uint8* data) override {
+        if (offset == 0 && length == 0) {
+            return ::piex::Error::kOk;
+        }
+        size_t sum;
+        if (!safe_add_to_size_t(offset, length, &sum) || !this->bufferMoreData(sum)) {
+            return ::piex::Error::kFail;
+        }
+        if (!fStreamBuffer.read(data, offset, length)) {
+            return ::piex::Error::kFail;
+        }
+        return ::piex::Error::kOk;
+    }
+
+protected:
+    // For dng_stream
+    uint64 DoGetLength() override {
+        if (!this->bufferMoreData(kReadToEnd)) {  // read whole stream
+            ThrowReadFile();
+        }
+        return fStreamBuffer.bytesWritten();
+    }
+
+    // For dng_stream
+    void DoRead(void* data, uint32 count, uint64 offset) override {
+        if (count == 0 && offset == 0) {
+            return;
+        }
+        size_t sum;
+        if (!safe_add_to_size_t(static_cast<uint64>(count), offset, &sum) ||
+            !this->bufferMoreData(sum)) {
+            ThrowReadFile();
+        }
+
+        if (!fStreamBuffer.read(data, offset, count)) {
+            ThrowReadFile();
+        }
+    }
+
+private:
+    // Note: if the newSize == kReadToEnd (0), this function will read to the end of stream.
+    bool bufferMoreData(size_t newSize) {
+        if (newSize == kReadToEnd) {
+            if (fWholeStreamRead) {  // already read-to-end.
+                return true;
+            }
+
+            // TODO: optimize for the special case when the input is SkMemoryStream.
+            return SkStreamCopy(&fStreamBuffer, fStream.get());
+        }
+
+        if (newSize <= fStreamBuffer.bytesWritten()) {  // already buffered to newSize
+            return true;
+        }
+        if (fWholeStreamRead) {  // newSize is larger than the whole stream.
+            return false;
+        }
+
+        const size_t sizeToRead = newSize - fStreamBuffer.bytesWritten();
+        SkAutoTMalloc<uint8> tempBuffer(sizeToRead);
+        const size_t bytesRead = fStream->read(tempBuffer.get(), sizeToRead);
+        if (bytesRead != sizeToRead) {
+            return false;
+        }
+        return fStreamBuffer.write(tempBuffer.get(), bytesRead);
+    }
+
+    SkAutoTDelete<SkStream> fStream;
+    bool fWholeStreamRead;
+
+    SkDynamicMemoryWStream fStreamBuffer;
+
+    const size_t kReadToEnd = 0;
+};
+
+class SkDngImage {
+public:
+    static SkDngImage* NewFromStream(SkRawStream* stream) {
+        SkAutoTDelete<SkDngMemoryAllocator> allocator(new SkDngMemoryAllocator);

[scroggo, 2016/01/19 21:05:44]

Is it possible to avoid creating and passing around these objects? It seems
awkward that we "new" all these objects, pass them to a static function, then
hand off their ownership to a new object of the class. It also means that we
have several smaller allocations when we could instead have a single allocation.

Can SkDngImage look more like the following?:

class SkDngImage {
public:
  SkDngImage* NewFromStream(SkRawStream* stream) {
    SkAutoTDelete<SkDngImage> dngImage(new SkDngImage(stream));
    if (!dngImage->readDng()) {
      return nullptr;
    }

    SkASSERT(dngImage->fNegative);
    return dngImage.release();
  }

  dng_image* render(...) {
    // looks mostly the same, but uses member variables directly
    // is it safe to just reuse fHost and fInfo rather than creating new ones?
    // I am not familiar with the DNG SDK, so maybe it's not?
  }
private:
  bool readDng() {
    // looks mostly the same, but uses member variables instead of parameters
    // returns true if there is a dng_negative
  }

  SkDngImage(SkRawStream* stream)
    : fStream(stream)
    , fHost(&fAllocator) {}

  SkAutoTDelete<SkRawStream> fStream;
  SkDngMemoryAllocator fAllocator;
  dng_host fHost;
  dng_info fInfo;
  SkAutoTDelete<dng_negative> fNegative;
};

[yujieqin, 2016/01/20 12:53:04]

* Unfortunately, we can not reuse fHost and fInfo due to the limit of DNG SDK.
And reset there two is not expensive anyway.

* Change the NewFromStream() as you recommend. 

+        SkAutoTDelete<dng_host> host(new dng_host(allocator.get()));
+        SkAutoTDelete<dng_info> info(new dng_info);
+        SkAutoTDelete<dng_negative> negative(SkDngImage::ReadDng(stream, host.get(), info.get()));
+        if (!negative) {
+            return nullptr;
+        }
+        return new SkDngImage(stream, allocator.release(), host.release(), info.release(),
+                              negative.release());
+    }
+
+    /*
+     * Renders the DNG image to the size. The DNG SDK only allows scaling close to integer factors
+     * down to 80 pixels on the short edge. The rendered image will be close to the specified size,
+     * but there is no guarantee that any of the edges will match the requested size. E.g.
+     *   100% size:              4000 x 3000
+     *   requested size:         1600 x 1200
+     *   returned size could be: 2000 x 1500
+     */
+    dng_image* render(int width, int height) {
+        // render() takes ownership of fHost, fInfo and fNegative when available.
+        SkAutoTDelete<dng_host> host;
+        SkAutoTDelete<dng_info> info;
+        SkAutoTDelete<dng_negative> negative;
+        if (!fHost || !fInfo || !fNegative) {
+            SkCodecPrintf("Warning: SkDngImage::render() is called multiple times.");
+            host.reset(new dng_host(fAllocator.get()));
+            info.reset(new dng_info);
+            negative.reset(SkDngImage::ReadDng(fStream.get(), host.get(), info.get()));
+            if (!negative) {
+                return nullptr;
+            }
+        } else {
+            host.reset(fHost.release());
+            info.reset(fInfo.release());
+            negative.reset(fNegative.release());
+        }
+
+        // DNG SDK preserves the aspect ratio, so it only needs to know the longer dimension.
+        const int preferredSize = SkTMax(width, height);
+        try {
+            host->SetPreferredSize(preferredSize);
+            host->ValidateSizes();
+
+            negative->ReadStage1Image(*host, *fStream, *info);
+
+            if (info->fMaskIndex != -1) {
+                negative->ReadTransparencyMask(*host, *fStream, *info);
+            }
+
+            negative->ValidateRawImageDigest(*host);
+            if (negative->IsDamaged()) {
+                return nullptr;
+            }
+
+            const int32 kMosaicPlane = -1;
+            negative->BuildStage2Image(*host);
+            negative->BuildStage3Image(*host, kMosaicPlane);
+
+            dng_render render(*host, *negative);
+            render.SetFinalSpace(dng_space_sRGB::Get());
+            render.SetFinalPixelType(ttByte);
+
+            dng_point stage3_size = negative->Stage3Image()->Size();
+            render.SetMaximumSize(SkTMax(stage3_size.h, stage3_size.v));
+
+            return render.Render();
+        } catch (...) {
+            return nullptr;
+        }
+    }
+
+    const SkImageInfo& getImageInfo() const {
+        return fImageInfo;
+    }
+
+    bool IsXtransImage() const {
+        return fIsXtransImage;
+    }
+
+private:
+    static dng_negative* ReadDng(SkRawStream* stream, dng_host* host, dng_info* info) {
+        try {
+            host->ValidateSizes();
+            info->Parse(*host, *stream);
+            info->PostParse(*host);
+            if (!info->IsValidDNG()) {
+                return nullptr;
+            }
+
+            SkAutoTDelete<dng_negative> negative;
+            negative.reset(host->Make_dng_negative());
+            negative->Parse(*host, *stream, *info);
+            negative->PostParse(*host, *stream, *info);
+            negative->SynchronizeMetadata();
+            return negative.release();
+        } catch (...) {
+            return nullptr;
+        }
+    }
+
+    SkDngImage(SkRawStream* stream, SkDngMemoryAllocator* allocator, dng_host* host,
+               dng_info* info, dng_negative* negative)
+        : fStream(stream)
+        , fAllocator(allocator)
+        , fHost(host)
+        , fInfo(info)
+        , fNegative(negative)
+        , fImageInfo(SkImageInfo::Make(fNegative->DefaultCropSizeH().As_real64(),
+                                       fNegative->DefaultCropSizeV().As_real64(),
+                                       kN32_SkColorType, kOpaque_SkAlphaType))
+        , fIsXtransImage(negative && negative->GetMosaicInfo() != nullptr

[scroggo, 2016/01/19 21:05:44]

I don't think you need to check "negative" here - if it is NULL, we'll crash in
the creation of fImageInfo.

[yujieqin, 2016/01/20 12:53:04]

Done

+                         ? (negative->GetMosaicInfo()->fCFAPatternSize.v == 6
+                            && negative->GetMosaicInfo()->fCFAPatternSize.h == 6)
+                         : false) {}
+
+    SkAutoTDelete<SkRawStream> fStream;
+    SkAutoTDelete<SkDngMemoryAllocator> fAllocator;
+
+    SkAutoTDelete<dng_host> fHost;
+    SkAutoTDelete<dng_info> fInfo;
+    SkAutoTDelete<dng_negative> fNegative;
+
+    const SkImageInfo fImageInfo;
+    const bool fIsXtransImage;
+};
+
+/*
+ * Tries to handle the image with PIEX. If PIEX returns kOk and finds the preview image, create a
+ * SkJpegCodec. If PIEX returns kFail, then the file is invalid, return nullptr. In other cases,
+ * fallback to create SkRawCodec for DNG images.
+ */
+SkCodec* SkRawCodec::NewFromStream(SkStream* stream) {
+    SkAutoTDelete<SkRawStream> rawStream(new SkRawStream(stream));
+    ::piex::PreviewImageData imageData;
+    if (::piex::IsRaw(rawStream.get())) {
+        ::piex::Error error = ::piex::GetPreviewImageData(rawStream.get(), &imageData);
+
+        if (error == ::piex::Error::kOk && imageData.preview_length > 0) {
+#if !defined(GOOGLE3)
+            // transferBuffer() is destructive to the rawStream. Abandon the rawStream after this
+            // function call.
+            SkMemoryStream* memoryStream =
+                    rawStream->transferBuffer(imageData.preview_offset, imageData.preview_length);
+            return memoryStream ? SkJpegCodec::NewFromStream(memoryStream) : nullptr;
+#else
+            return nullptr;
+#endif
+        } else if (error == ::piex::Error::kFail) {
+            return nullptr;
+        }
+    }
+
+    SkAutoTDelete<SkDngImage> dngImage(SkDngImage::NewFromStream(rawStream.release()));
+    if (!dngImage) {
+        return nullptr;
+    }
+
+    return new SkRawCodec(dngImage.release());
+}
+
+SkCodec::Result SkRawCodec::onGetPixels(const SkImageInfo& requestedInfo, void* dst,
+                                        size_t dstRowBytes, const Options& options,
+                                        SkPMColor ctable[], int* ctableCount,
+                                        int* rowsDecoded) {
+    const int width = requestedInfo.width();
+    const int height = requestedInfo.height();
+    SkAutoTDelete<dng_image> image(fDngImage->render(width, height));
+    if (!image) {
+        return kInvalidInput;
+    }
+
+    // Because the DNG SDK can not guarantee to render to requested size, we allow a small
+    // difference. Only the overlapping region will be converted in onGetPixels().
+    const float maxDiffRatio = 1.03f;
+    const dng_point& imageSize = image->Size();
+    if (imageSize.h / width > maxDiffRatio || imageSize.h < width ||
+        imageSize.v / height > maxDiffRatio || imageSize.v < height) {
+        return SkCodec::kInvalidScale;
+    }
+
+    void* dstRow = dst;
+    uint8_t srcRow[width * 3];
+    SkAutoTDelete<SkSwizzler> swizzler(SkSwizzler::CreateSwizzler(
+            SkSwizzler::kRGB, nullptr, requestedInfo, options));
+    SkASSERT(swizzler);
+
+    dng_pixel_buffer buffer;
+    buffer.fData = &srcRow[0];
+    buffer.fPlane = 0;
+    buffer.fPlanes = 3;
+    buffer.fColStep = buffer.fPlanes;
+    buffer.fPlaneStep = 1;
+    buffer.fPixelType = ttByte;
+    buffer.fPixelSize = sizeof(uint8_t);
+    buffer.fRowStep = sizeof(srcRow);
+
+    for (int i = 0; i < height; ++i) {
+        buffer.fArea = dng_rect(i, 0, i + 1, width);
+
+        try {
+            image->Get(buffer, dng_image::edge_zero);
+        } catch (...) {
+            *rowsDecoded = i;
+            return kIncompleteInput; 
+        }
+
+        swizzler->swizzle(dstRow, &srcRow[0]);
+        dstRow = SkTAddOffset<void>(dstRow, dstRowBytes);
+    }
+    return kSuccess;
+}
+
+SkISize SkRawCodec::onGetScaledDimensions(float desiredScale) const {
+    SkASSERT(desiredScale <= 1.f);
+    SkISize dim = this->getInfo().dimensions();
+    const int shortEdge = SkTMin(dim.fWidth, dim.fHeight);

[scroggo, 2016/01/19 21:05:44]

nit: you can cast to float here, simplifying the if statement below.

[yujieqin, 2016/01/20 12:53:04]

Done.

+
+    // Limits the minimun size to be 80 on the short edge.

[scroggo, 2016/01/19 21:05:44]

minimum*

[yujieqin, 2016/01/20 12:53:04]

Done.

+    if (desiredScale < 80.f / static_cast<float>(shortEdge)) {
+        desiredScale = 80.f / static_cast<float>(shortEdge);
+    }
+
+    // For Xtrans images, the integer-factor scaling does not support the half-size scaling case
+    // (stronger downscalings are fine). In this case, returns the non-scaled version.
+    if (fDngImage->IsXtransImage() && desiredScale >= 0.5f) {
+        return dim;
+    }
+
+    // Round to integer-factors.
+    const float finalScale = std::floor(1.f/ desiredScale);
+
+    dim.fWidth = std::round(dim.fWidth / finalScale);
+    dim.fHeight = std::round(dim.fHeight / finalScale);
+    return dim;
+}
+
+bool SkRawCodec::onDimensionsSupported(const SkISize& dim) {
+    const SkISize fullDim = this->getInfo().dimensions();
+    const float fullShortEdge = SkTMin(fullDim.fWidth, fullDim.fHeight);
+    const float shortEdge = SkTMin(dim.fWidth, dim.fHeight);
+
+    SkISize sizeFloor = this->onGetScaledDimensions(1.f / std::floor(fullShortEdge / shortEdge));
+    SkISize sizeCeil = this->onGetScaledDimensions(1.f / std::ceil(fullShortEdge / shortEdge));
+    return sizeFloor == dim || sizeCeil == dim;
+}
+
+SkRawCodec::~SkRawCodec() {}
+
+SkRawCodec::SkRawCodec(SkDngImage* dngImage)
+    : INHERITED(dngImage->getImageInfo(), nullptr)
+    , fDngImage(dngImage) {}