Prototype of RAW decoding in Skia.

src/codec/SkRawCodec.cpp

Index: src/codec/SkRawCodec.cpp
diff --git a/src/codec/SkRawCodec.cpp b/src/codec/SkRawCodec.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b45bf2f61414ab76fb75a67d17c86fdbfa7190c4
--- /dev/null
+++ b/src/codec/SkRawCodec.cpp
@@ -0,0 +1,438 @@
+/*
+ * Copyright 2015 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "SkCodec.h"
+#include "SkCodecPriv.h"
+#include "SkColorPriv.h"
+#include "SkData.h"
+#if !defined(GOOGLE3)
+#include "SkJpegCodec.h"
+#endif
+#include "SkRawCodec.h"
+#include "SkRefCnt.h"
+#include "SkStream.h"
+#include "SkStreamPriv.h"
+#include "SkSwizzler.h"
+#include "SkTemplates.h"
+#include "SkTypes.h"
+
+#include "dng_color_space.h"
+#include "dng_exceptions.h"
+#include "dng_host.h"
+#include "dng_info.h"
+#include "dng_render.h"
+#include "dng_stream.h"
+
+#include "src/piex.h"
+
+#include <cmath>  // for std::round,floor,ceil
+#include <limits>
+
+namespace {
+
+// T must be unsigned type.
+template <class T>
+bool safe_add_to_size_t(T arg1, T arg2, size_t* result) {
+    SkASSERT(arg1 >= 0);
+    SkASSERT(arg2 >= 0);
+    if (arg1 >= 0 && arg2 <= std::numeric_limits<T>::max() - arg1) {
+        T sum = arg1 + arg2;
+        if (sum <= std::numeric_limits<size_t>::max()) {
+            *result = static_cast<size_t>(sum);
+            return true;
+        }
+    }
+    return false;
+}
+
+}  // namespace
+
+// Note: this class could throw exception if it is used as dng_stream.
+class SkRawStream : public dng_stream, public ::piex::StreamInterface {
+public:
+    // Note that this call will take the ownership of stream.
+    explicit SkRawStream(SkStream* stream)
+        : fStream(stream), fWholeStreamRead(false) {}
+
+    ~SkRawStream() override {}
+
+    /*
+     * Creates a SkMemoryStream from the offset with size.
+     * Note: for performance reason, this function is destructive to the SkRawStream. One should
+     *       abandon current object after the function call.
+     */
+    SkMemoryStream* copyBuffer(size_t offset, size_t size) {
+        SkAutoTUnref<SkData> data(SkData::NewUninitialized(size));
+        if (offset > fStreamBuffer.bytesWritten()) {
+            // If the offset is not buffered, read from fStream directly and skip the buffering.
+            // This is destructive to currect object.
+            const size_t skipLength = offset - fStreamBuffer.bytesWritten();
+            if (fStream->skip(skipLength) != skipLength) {
+                return nullptr;
+            }
+            if (fStream->read(data->writable_data(), size) != size) {
+                return nullptr;
+            }
+        } else {
+            // If the buffer is already past the offset, increase the buffer to offset + size,
+            // then copy the buffer to output.
+            size_t sum;
+            if (!safe_add_to_size_t(offset, size, &sum) || !this->bufferMoreData(sum)) {

[scroggo, 2016/01/13 18:24:45]

Again, I don't think you need to do any buffering. Instead, why not do the
following:

const size_t alreadyBuffered = fStreamBuffer.bytesWritten() - offset;
if (!fStreamBuffer.read(data->writable_data(), alreadyBuffered)) {
    return nullptr;
}

const size_t remaining = size - alreadyBuffered;
auto* dst = static_cast<uint8_t*>(data->writable_data()) + alreadyBuffered;
if (remaining && fStream->read(dst, remaining) != remaining) {
    return nullptr;
}

[yujieqin, 2016/01/14 09:33:11]

Done.

+                return nullptr;
+            }
+            if (!fStreamBuffer.read(data->writable_data(), offset, size)) {
+                return nullptr;
+            }
+        }
+        return new SkMemoryStream(data);
+    }
+
+    // For PIEX
+    ::piex::Error GetData(const size_t offset, const size_t length,
+                          uint8* data) override {
+        if (offset == 0 && length == 0) {
+            return ::piex::Error::kOk;
+        }
+        size_t sum;
+        if (!safe_add_to_size_t(offset, length, &sum) || !this->bufferMoreData(sum)) {
+            return ::piex::Error::kFail;
+        }
+        if (!fStreamBuffer.read(data, offset, length)) {
+            return ::piex::Error::kFail;
+        }
+        return ::piex::Error::kOk;
+    }
+
+protected:
+    // For dng_stream
+    uint64 DoGetLength() override {
+        if (!this->bufferMoreData(kReadToEnd)) {  // read whole stream
+            ThrowReadFile();
+        }
+        return fStreamBuffer.bytesWritten();
+    }
+
+    // For dng_stream
+    void DoRead(void* data, uint32 count, uint64 offset) override {
+        if (count == 0 && offset == 0) {
+            return;
+        }
+        size_t sum;
+        if (!safe_add_to_size_t(static_cast<uint64>(count), offset, &sum) ||
+            !this->bufferMoreData(sum)) {
+            ThrowReadFile();
+        }
+
+        if (!fStreamBuffer.read(data, offset, count)) {
+            ThrowReadFile();
+        }
+    }
+
+private:
+    // Note: if the newSize == kReadToEnd (0), this function will read to the end of stream.
+    bool bufferMoreData(size_t newSize) {
+        if (newSize == kReadToEnd) {
+            if (fWholeStreamRead) {  // already read-to-end.
+                return true;
+            }
+
+            // TODO: optimize for the special case when the input is SkMemoryStream.
+            return SkStreamCopy(&fStreamBuffer, fStream.get());
+        }
+
+        if (newSize <= fStreamBuffer.bytesWritten()) {  // already buffered to newSize
+            return true;
+        }
+        if (fWholeStreamRead) {  // newSize is larger than the whole stream.
+            return false;
+        }
+
+        const size_t sizeToRead = newSize - fStreamBuffer.bytesWritten();
+        SkAutoTMalloc<uint8> tempBuffer(sizeToRead);
+        const size_t bytesRead = fStream->read(tempBuffer.get(), sizeToRead);
+        if (bytesRead != sizeToRead) {
+            return false;
+        }
+        return fStreamBuffer.write(tempBuffer.get(), bytesRead);
+    }
+
+    SkAutoTDelete<SkStream> fStream;
+    bool fWholeStreamRead;
+
+    SkDynamicMemoryWStream fStreamBuffer;
+
+    const size_t kReadToEnd = 0;
+};
+
+class SkDngImage {
+public:
+    static dng_negative* ReadDng(SkRawStream* stream, dng_host* host, dng_info* info) {

[scroggo, 2016/01/13 18:24:45]

I think this does not need to be public?

[yujieqin, 2016/01/14 09:33:11]

Done.

+        try {
+            host->ValidateSizes();
+            info->Parse(*host, *stream);
+            info->PostParse(*host);
+            if (!info->IsValidDNG()) {
+                return nullptr;
+            }
+
+            SkAutoTDelete<dng_negative> negative;
+            negative.reset(host->Make_dng_negative());
+            negative->Parse(*host, *stream, *info);
+            negative->PostParse(*host, *stream, *info);
+            negative->SynchronizeMetadata();
+            return negative.release();
+        } catch (...) {
+            return nullptr;
+        }
+    }
+
+    static SkDngImage* NewFromStream(SkRawStream* stream) {
+        SkAutoTDelete<dng_host> host(new dng_host);
+        SkAutoTDelete<dng_info> info(new dng_info);
+        SkAutoTDelete<dng_negative> negative(SkDngImage::ReadDng(stream, host.get(), info.get()));
+        if (!negative) {
+            return nullptr;
+        }
+        return new SkDngImage(stream, host.release(), info.release(), negative.release());
+    }
+
+    /*
+     * Renders the DNG image to the size.
+     * The rendered image will be close to the specified size, but there is no guarantee that any
+     * of the edges will match the requested size. E.g.
+     *   100% size:              4000 x 3000
+     *   requested size:         1600 x 1200
+     *   returned size could be: 2000 x 1500
+     *
+     */
+    SkCodec::Result render(int width, int height) {

[scroggo, 2016/01/13 18:24:45]

Can this just return a boolean for success or failure?

[yujieqin, 2016/01/14 09:33:11]

Changed this method to return pointer.

+        if (!fImage ||

[scroggo, 2016/01/13 18:24:45]

I think saving the image across calls to onGetPixels is not necessary. (It will
run faster, but at the cost of using more memory.)

[yujieqin, 2016/01/14 09:33:11]

Changed the logic to return dng_image directly and clean up host, info,
negative.

+            (fImage->Size().h != width || fImage->Size().v != height))
+        {
+            // Reset the DNG image, so that we can rerender it for different size.
+            if (fImage) {
+                SkCodecPrintf("Warning: For RAW images, calling render() with a size that does \
+                              not match the existing image size forces the decoder to render the \
+                              image multiple times.");
+                fImage.reset();
+                fHost.reset(new dng_host);
+                fInfo.reset(new dng_info);
+                fNegative.reset(SkDngImage::ReadDng(fStream, fHost.get(), fInfo.get()));
+                if (!fNegative) {
+                    return SkCodec::kInvalidInput;
+                }
+            }
+
+            // DNG SDK preserves the aspect ratio, so it only needs to know the longer dimension.
+            const int preferredSize = SkTMax(width, height);
+            try {
+                fHost->SetPreferredSize(preferredSize);
+                fHost->ValidateSizes();
+
+                fNegative->ReadStage1Image(*fHost, *fStream, *fInfo);
+
+                if (fInfo->fMaskIndex != -1) {
+                    fNegative->ReadTransparencyMask(*fHost, *fStream, *fInfo);
+                }
+
+                fNegative->ValidateRawImageDigest(*fHost);
+                if (fNegative->IsDamaged()) {
+                    return SkCodec::kInvalidInput;
+                }
+
+                const int32 kMosaicPlane = -1;
+                fNegative->BuildStage2Image(*fHost);
+                fNegative->BuildStage3Image(*fHost, kMosaicPlane);
+
+                dng_render render(*fHost, *fNegative);
+                render.SetFinalSpace(dng_space_sRGB::Get());
+                render.SetFinalPixelType(ttByte);
+
+                dng_point stage3_size = fNegative->Stage3Image()->Size();
+                render.SetMaximumSize(SkTMax(stage3_size.h, stage3_size.v));
+
+                fImage.reset(render.Render());
+
+                // Because the DNG SDK can not gaurantee to render to requested size, we allow a small

[scroggo, 2016/01/13 18:24:45]

nit: 100 chars

[yujieqin, 2016/01/14 09:33:11]

Done.

+                // difference. Only the overlapping region will be copied in onGetPixels().
+                const float maxDiffRatio = 1.02f;
+                const dng_point& imageSize = fImage->Size();
+                if (SkTMax(width, imageSize.h) / SkTMin(width, imageSize.h) > maxDiffRatio ||
+                    SkTMax(height, imageSize.v) / SkTMin(height, imageSize.v) > maxDiffRatio) {
+                    return SkCodec::kInvalidScale;
+                }
+                return SkCodec::kSuccess;
+            } catch (...) {
+                fImage.reset();
+                return SkCodec::kInvalidInput;
+            }
+        }
+        return SkCodec::kSuccess;
+    }
+
+    SkCodec::Result writeToBuffer(dng_pixel_buffer* buffer) {

[scroggo, 2016/01/13 18:24:45]

This only returns two values, which are essentially success or failure. Why not
return a bool?

Also, naming this "readRow" would align with some of our other codecs.

[yujieqin, 2016/01/14 09:33:11]

Done.

+        if (!fImage) {
+            return SkCodec::kInvalidInput;
+        }
+
+        try {
+            fImage->Get(*buffer, dng_image::edge_zero);
+            return SkCodec::kSuccess;
+        } catch (...) {
+            return SkCodec::kInvalidInput;
+        }
+    }
+
+    bool hasRenderedResult() const {
+        return fImage != nullptr;
+    }
+
+    SkISize getRenderedSize() const {
+        if (!this->hasRenderedResult()) {
+            return SkISize::Make(0, 0);
+        }
+        SkISize dim;
+        dng_point imageSize = fImage->Size();
+        dim.fWidth = imageSize.h;
+        dim.fHeight = imageSize.v;
+        return dim;
+    }
+
+    SkImageInfo getImageInfo() const {
+        SkASSERT(fNegative);
+        return SkImageInfo::Make(fNegative->DefaultCropSizeH().As_real64(),
+                                 fNegative->DefaultCropSizeV().As_real64(),
+                                 kN32_SkColorType, kOpaque_SkAlphaType);
+    }
+
+private:
+    SkDngImage(SkRawStream* stream, dng_host* host, dng_info* info, dng_negative* negative)
+        : fStream(stream)
+        , fHost(host)
+        , fInfo(info)
+        , fNegative(negative) {}
+
+    SkAutoTDelete<SkRawStream> fStream;
+    SkAutoTDelete<dng_host> fHost;
+    SkAutoTDelete<dng_info> fInfo;
+    SkAutoTDelete<dng_negative> fNegative;
+    SkAutoTDelete<dng_image> fImage;
+};
+
+/*
+ * Tries to handle the image with PIEX. If PIEX returns kOk and finds the preview image, create a
+ * SkJpegCodec. If PIEX returns kFail, then the file is invalid, return nullptr. In other cases,
+ * fallback to create SkRawCodec for DNG images.
+ */
+SkCodec* SkRawCodec::NewFromStream(SkStream* stream) {
+    SkAutoTDelete<SkRawStream> rawStream(new SkRawStream(stream));
+    ::piex::PreviewImageData imageData;
+    if (::piex::IsRaw(rawStream.get())) {
+        ::piex::Error error = ::piex::GetPreviewImageData(rawStream.get(), &imageData);
+
+        if (error == ::piex::Error::kOk && imageData.preview_length > 0) {
+            // copyBuffer() is destructive to the rawStream. Abandon the rawStream after this
+            // function call.
+            SkMemoryStream* memoryStream =

[scroggo, 2016/01/13 18:24:45]

This will leak in GOOGLE3. Instead, I think you want to move your #if !defined
above this line.

[yujieqin, 2016/01/14 09:33:11]

Done.

+                    rawStream->copyBuffer(imageData.preview_offset, imageData.preview_length);
+#if !defined(GOOGLE3)
+            return memoryStream ? SkJpegCodec::NewFromStream(memoryStream) : nullptr;
+#else
+            return nullptr;
+#endif
+        } else if (error == ::piex::Error::kFail) {
+            return nullptr;
+        }
+    }
+
+    SkAutoTDelete<SkDngImage> dngImage(SkDngImage::NewFromStream(rawStream.release()));
+    if (!dngImage) {
+        return nullptr;
+    }
+
+    return new SkRawCodec(dngImage.release());
+}
+
+SkCodec::Result SkRawCodec::onGetPixels(const SkImageInfo& requestedInfo, void* dst,
+                                        size_t dstRowBytes, const Options& options,
+                                        SkPMColor ctable[], int* ctableCount,
+                                        int* rowsDecoded) {
+    int width = requestedInfo.width();
+    int height = requestedInfo.height();
+    SkCodec::Result result = fDngImage->render(width, height);
+    if (result != kSuccess) {
+        return result;
+    }
+
+    // Only copy the top-left of the overlapping region.

[scroggo, 2016/01/13 18:24:45]

In practice, how much will be "cropped" here? If the client used values that
came from getScaledDimensions, will any cropping be necessary?

[yujieqin, 2016/01/14 09:33:11]

Our experiment shows that the cropping will be less than 2%, even the client
used the value from getScaledDimensions(). See comment in onGetPiexels() and the
check for the 2%.

+    width = SkTMin(width, fDngImage->getRenderedSize().fWidth);
+    height = SkTMin(height, fDngImage->getRenderedSize().fHeight);
+
+    void* dstRow = dst;
+    uint8 srcRow[width * 3];
+    SkAutoTDelete<SkSwizzler> swizzler(SkSwizzler::CreateSwizzler(
+            SkSwizzler::kRGB, nullptr, requestedInfo, options));
+    SkASSERT(swizzler);
+    for (int i = 0; i < height; ++i) {
+        dng_pixel_buffer buffer;
+        buffer.fData = &srcRow[0];
+        buffer.fArea = dng_rect(i, 0, i + 1, width);

[scroggo, 2016/01/13 18:24:45]

Every setup assignment for buffer is the same for each iteration. I don't know
anything about dng_pixel_buffer; would it be possible to do the following?

dng_pixel_buffer buffer;
buffer.fData = &srcRow[0];
buffer.fPlane = 0;
buffer.fColoStep = buffer.fPlanes;
buffer.fPlaneStep = 1;
buffer.fPixelType = ttByte;
buffer.fPixelSize = sizeof(uint8); // (uint8_t?)
buffer.fRowStep = sizeof(srcRow);

for (int i = 0; i < height; ++i) {
    buffer.fArea = dng_rect(i, 0, i + 1, width);
}

(Or do you think the compiler will handle this for us?)

[yujieqin, 2016/01/14 09:33:11]

Done.

+        buffer.fPlane = 0;
+        buffer.fPlanes = 3;
+        buffer.fColStep = buffer.fPlanes;
+        buffer.fPlaneStep = 1;
+        buffer.fPixelType = ttByte;
+        buffer.fPixelSize = sizeof(uint8);

[scroggo, 2016/01/13 18:24:45]

Should this be uint8_t? (Actually, I just realized you're using "uint8" (no
"_t") everywhere. Where is this defined? Any reason to use this rather than
"uint8_t", as we do in the rest of Skia?

[yujieqin, 2016/01/14 09:33:11]

Done.

+        buffer.fRowStep = sizeof(srcRow);
+
+        result = fDngImage->writeToBuffer(&buffer);

[scroggo, 2016/01/13 18:24:45]

if (!fDngImage->readRow(&buffer)) {
    *rowsDecoded = y;
    return kIncompleteInput;
}

(I recommended the name/signature change at the method's declaration.)

By setting rowsDecoded and returning kIncompleteInput, we can return a partial
image which will get the rest of its rows filled by the base class.

[yujieqin, 2016/01/14 09:33:11]

Done.

+        if (result != kSuccess) {
+            return result;
+        }
+
+        swizzler->swizzle(dstRow, &srcRow[0]);
+        dstRow = SkTAddOffset<void>(dstRow, dstRowBytes);
+    }
+    return kSuccess;
+}
+
+SkISize SkRawCodec::onGetScaledDimensions(float desiredScale) const {
+    SkASSERT(desiredScale <= 1.f);
+    SkISize dim = this->getInfo().dimensions();
+    const int longEdge = SkTMax(dim.fWidth, dim.fHeight);
+
+    // Limits the minimun size to be 128 on the long edge.

[msarett, 2016/01/13 17:15:14]

This limitation seems reasonable.  Is this a limit imposed by the DNG lib or
arbitrarily by us?

My concern is that this is a different behavior than the other Codecs.

On the other hand, will RAW support all integer downscales (1, 2, 3, 5, 6 ...)? 
Because if not, we already have a difference in behavior anyway.

[scroggo, 2016/01/13 18:24:45]

I think that's okay. I believe we've had other cases where we did not exactly
match the downscale.

[yujieqin, 2016/01/14 09:33:11]

Our experiment shows that the DNG SDK does not scale properly when the size is
smaller than 128.

[scroggo, 2016/01/14 15:26:38]

Can you add a comment explaining that? Also, is it worth filing a bug against
the DNG SDK? (And referencing that bug here?)

If we need to be able to get smaller images, we can scale afterwards. (I almost
suggested scanline decoding, but since we'll need to have the entire dng_image
for the duration of the scanline decode, we might as well use the drawing code.
The drawbacks to the drawing code are that we cannot have an output of index8 or
unpremul, but we won't return index8 here anyway, and these images are opaque,
so they can be treated as unpremul.)

+    if (desiredScale < 128.f / static_cast<float>(longEdge)) {
+        desiredScale = 128.f / static_cast<float>(longEdge);
+    }
+
+    const float finalScale = (int)(1.f/ desiredScale);
+
+    dim.fWidth = std::round(dim.fWidth / finalScale);
+    dim.fHeight = std::round(dim.fHeight / finalScale);
+    return dim;
+}
+
+bool SkRawCodec::onDimensionsSupported(const SkISize& dim) {
+    const SkImageInfo& info = this->getInfo();
+    return dim.width() >= 1 && dim.width() <= info.width()

[msarett, 2016/01/13 17:15:14]

Everything below this line is dead code?

[yujieqin, 2016/01/14 09:33:11]

Done.

+            && dim.height() >= 1 && dim.height() <= info.height();
+
+    const SkISize fullDim = this->getInfo().dimensions();
+    const float fullLongEdge = SkTMax(fullDim.fWidth, fullDim.fHeight);
+    const float longEdge = SkTMax(dim.fWidth, dim.fHeight);
+
+    SkISize sizeFloor = this->onGetScaledDimensions(1.f / std::floor(fullLongEdge / longEdge));
+    SkISize sizeCeil = this->onGetScaledDimensions(1.f / std::ceil(fullLongEdge / longEdge));
+    return sizeFloor == dim || sizeCeil == dim;
+}
+
+SkRawCodec::~SkRawCodec() {}
+
+SkRawCodec::SkRawCodec(SkDngImage* dngImage)
+    : INHERITED(dngImage->getImageInfo(), nullptr)
+    , fDngImage(dngImage) {}